From 4cdb8f78cbb1035e88174ca432e5e7dfedb4eb44 Mon Sep 17 00:00:00 2001 From: decoy-walrus Date: Mon, 7 Feb 2022 14:01:05 -0500 Subject: [PATCH 01/17] Add new endpoint for original resolution images This change is to work around the issue that chromium based browsers have handling the "name=orig" parameter appended to URLs. This parameter is needed to retrieve the full resolution image from twitter, but causes those browsers to fill in "jpg_name=orig" as the extension on the filename. This change adds a new endpoint, "/pic/orig/". This new endpoint will internally fetch the URL with ":orig" appended on the end for the full res image. Externally, the endpoint will serve the image without the extra parameter to expose the real extension to the browser. This new endpoint is used when rendering tweets with attached images. The old endpoint is still in place for all other proxied images, and for any legacy links. I also updated the "?name=small" parameter to ":small" since that seems to be the new pattern for image sizing. This should fix issue #458. --- src/routes/media.nim | 14 ++++++++++++++ src/utils.nim | 6 ++++++ src/views/tweet.nim | 6 +++--- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/routes/media.nim b/src/routes/media.nim index c953a93..95446a1 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -88,6 +88,20 @@ proc createMediaRouter*(cfg: Config) = get "/pic/?": resp Http404 + get re"^\/pic\/orig\/(enc)?\/?(.+)": + var url = decoded(request, 1) + if "twimg.com" notin url: + url.insert(twimg) + if not url.startsWith(https): + url.insert(https) + url.add(":orig") + + let uri = parseUri(url) + cond isTwitterUrl(uri) == true + + let code = await proxyMedia(request, url) + check code + get re"^\/pic\/(enc)?\/?(.+)": var url = decoded(request, 1) if "twimg.com" notin url: diff --git a/src/utils.nim b/src/utils.nim index 9c8414d..9002bbf 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -42,6 +42,12 @@ proc getPicUrl*(link: string): string = else: &"/pic/{encodeUrl(link)}" +proc getOrigPicUrl*(link: string): string = + if base64Media: + &"/pic/orig/enc/{encode(link, safe=true)}" + else: + &"/pic/orig/{encodeUrl(link)}" + proc filterParams*(params: Table): seq[(string, string)] = for p in params.pairs(): if p[1].len > 0 and p[0] notin nitterParams: diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 8b712a6..e64f959 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -57,9 +57,9 @@ proc renderAlbum(tweet: Tweet): VNode = tdiv(class="attachment image"): let named = "name=" in photo - orig = if named: photo else: photo & "?name=orig" - small = if named: photo else: photo & "?name=small" - a(href=getPicUrl(orig), class="still-image", target="_blank"): + orig = photo + small = if named: photo else: photo & ":small" + a(href=getOrigPicUrl(orig), class="still-image", target="_blank"): genImg(small) proc isPlaybackEnabled(prefs: Prefs; video: Video): bool = From 644fe41a08e41313894a017b934e5d89889202ac Mon Sep 17 00:00:00 2001 From: decoy-walrus Date: Tue, 8 Feb 2022 14:50:56 -0500 Subject: [PATCH 02/17] Use the correct format string for fetching files from twitter. Per their docs https://developer.twitter.com/en/docs/twitter-api/v1/data-dictionary/object-model/entities#photo_format --- src/routes/media.nim | 2 +- src/views/tweet.nim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/media.nim b/src/routes/media.nim index 95446a1..e63a0f8 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -94,7 +94,7 @@ proc createMediaRouter*(cfg: Config) = url.insert(twimg) if not url.startsWith(https): url.insert(https) - url.add(":orig") + url.add("?name=orig") let uri = parseUri(url) cond isTwitterUrl(uri) == true diff --git a/src/views/tweet.nim b/src/views/tweet.nim index e64f959..51d427f 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -10,7 +10,7 @@ import general proc getSmallPic(url: string): string = result = url if "?" notin url and not url.endsWith("placeholder.png"): - result &= ":small" + result &= "?name=small" result = getPicUrl(result) proc renderMiniAvatar(user: User; prefs: Prefs): VNode = @@ -58,7 +58,7 @@ proc renderAlbum(tweet: Tweet): VNode = let named = "name=" in photo orig = photo - small = if named: photo else: photo & ":small" + small = if named: photo else: photo & "?name=small" a(href=getOrigPicUrl(orig), class="still-image", target="_blank"): genImg(small) From 0633ec2c3935aae068bfa6867a0547bbe28e685d Mon Sep 17 00:00:00 2001 From: girst Date: Fri, 25 Feb 2022 19:26:24 +0100 Subject: [PATCH 03/17] Prefer mp4 to m3u8 for Video Playback if proxyVideos is off m3u8 videos only work when the proxy is enabled. Further, this allows video playback without Javascript. This is only done when proxying is disabled to avoid excessive memory usage on the nitter instance that would result from loading longer videos in a single chunk. --- src/views/tweet.nim | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 6e6f3af..73548a1 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -62,11 +62,14 @@ proc renderAlbum(tweet: Tweet): VNode = a(href=getPicUrl(orig), class="still-image", target="_blank"): genImg(small) -proc isPlaybackEnabled(prefs: Prefs; video: Video): bool = - case video.playbackType +proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool = + case playbackType of mp4: prefs.mp4Playback of m3u8, vmap: prefs.hlsPlayback +proc hasMp4Url(video: Video): bool = + video.variants.anyIt(it.contentType == mp4) + proc renderVideoDisabled(video: Video; path: string): VNode = buildHtml(tdiv(class="video-overlay")): case video.playbackType @@ -87,6 +90,9 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode = let container = if video.description.len > 0 or video.title.len > 0: " card-container" else: "" + let playbackType = + if not prefs.proxyVideos and video.hasMp4Url: mp4 + else: video.playbackType buildHtml(tdiv(class="attachments card")): tdiv(class="gallery-video" & container): @@ -95,13 +101,13 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode = if not video.available: img(src=thumb) renderVideoUnavailable(video) - elif not prefs.isPlaybackEnabled(video): + elif not prefs.isPlaybackEnabled(playbackType): img(src=thumb) renderVideoDisabled(video, path) else: - let vid = video.variants.filterIt(it.contentType == video.playbackType) - let source = getVidUrl(vid[0].url) - case video.playbackType + let vid = video.variants.filterIt(it.contentType == playbackType) + let source = if prefs.proxyVideos: getVidUrl(vid[0].url) else: vid[0].url + case playbackType of mp4: if prefs.muteVideos: video(poster=thumb, controls="", muted=""): From e2b8e17f857f3a1bcfdc7942e58f3e4070ea8533 Mon Sep 17 00:00:00 2001 From: girst Date: Wed, 18 May 2022 19:47:03 +0200 Subject: [PATCH 04/17] use largest resolution mp4 video available --- src/parser.nim | 9 +++++++-- src/parserutils.nim | 15 +++++++++++++++ src/types.nim | 1 + src/views/tweet.nim | 9 ++++++--- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/parser.nim b/src/parser.nim index 1a7c073..fc8dc3b 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -87,10 +87,15 @@ proc parseVideo(js: JsonNode): Video = result.description = description.getStr for v in js{"video_info", "variants"}: + let + contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary")) + url = v{"url"}.getStr + resolution = getMp4Resolution(url) # only available if contentType == mp4 result.variants.add VideoVariant( - contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")), + contentType: contentType, bitrate: v{"bitrate"}.getInt, - url: v{"url"}.getStr + url: url, + resolution: resolution ) proc parsePromoVideo(js: JsonNode): Video = diff --git a/src/parserutils.nim b/src/parserutils.nim index 4929c2a..51ccea5 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -137,6 +137,21 @@ proc getSource*(js: JsonNode): string = let src = js{"source"}.getStr result = src.substr(src.find('>') + 1, src.rfind('<') - 1) +proc getMp4Resolution*(url: string): int = + # parses the height out of a URL like this one: + # https://video.twimg.com/ext_tw_video//pu/vid/720x1280/.mp4 + const vidSep = "/vid/" + let + vidIdx = url.find(vidSep) + vidSep.len + resIdx = url.find('x', vidIdx) + 1 + res = url[resIdx ..< url.find("/", resIdx)] + + try: + return parseInt(res) + except ValueError: + # cannot determine resolution (e.g. m3u8/non-mp4 video) + return 0 + proc extractSlice(js: JsonNode): Slice[int] = result = js["indices"][0].getInt ..< js["indices"][1].getInt diff --git a/src/types.nim b/src/types.nim index 98433aa..061ec8a 100644 --- a/src/types.nim +++ b/src/types.nim @@ -75,6 +75,7 @@ type contentType*: VideoType url*: string bitrate*: int + resolution*: int Video* = object durationMs*: int diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 73548a1..d07ca06 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, sequtils, strformat, options +import strutils, sequtils, strformat, options, algorithm import karax/[karaxdsl, vdom, vstyles] from jester import Request @@ -105,8 +105,11 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode = img(src=thumb) renderVideoDisabled(video, path) else: - let vid = video.variants.filterIt(it.contentType == playbackType) - let source = if prefs.proxyVideos: getVidUrl(vid[0].url) else: vid[0].url + let + vars = video.variants.filterIt(it.contentType == playbackType) + vidUrl = vars.sortedByIt(it.resolution)[^1].url + source = if prefs.proxyVideos: getVidUrl(vidUrl) + else: vidUrl case playbackType of mp4: if prefs.muteVideos: From 6c83e872923c6d0eff9be0d74c2066f3b702eeed Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 4 Jun 2022 00:52:28 +0200 Subject: [PATCH 05/17] Update outdated tests --- .gitignore | 4 +++- tests/test_card.py | 4 ++-- tests/test_tweet.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 4d742bb..d43cc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,11 @@ nitter *.db /tests/__pycache__ /tests/geckodriver.log -/tests/downloaded_files/* +/tests/downloaded_files +/tests/latest_logs /tools/gencss /tools/rendermd /public/css/style.css /public/md/*.html nitter.conf +dump.rdb diff --git a/tests/test_card.py b/tests/test_card.py index b4b0998..ab79e9c 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -4,8 +4,8 @@ from parameterized import parameterized card = [ ['Thom_Wolf/status/1122466524860702729', - 'pytorch/fairseq', - 'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - pytorch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.', + 'facebookresearch/fairseq', + 'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.', 'github.com', True], ['nim_lang/status/1136652293510717440', diff --git a/tests/test_tweet.py b/tests/test_tweet.py index e8ba0f7..9209e70 100644 --- a/tests/test_tweet.py +++ b/tests/test_tweet.py @@ -16,7 +16,7 @@ timeline = [ ] status = [ - [20, 'jack⚡️', 'jack', '21 Mar 2006', 'just setting up my twttr'], + [20, 'jack', 'jack', '21 Mar 2006', 'just setting up my twttr'], [134849778302464000, 'The Twoffice', 'TheTwoffice', '11 Nov 2011', 'test'], [105685475985080322, 'The Twoffice', 'TheTwoffice', '22 Aug 2011', 'regular tweet'], [572593440719912960, 'Test account', 'mobile_test', '3 Mar 2015', 'testing test'] @@ -71,7 +71,7 @@ emoji = [ retweet = [ [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'], - [3, 'mobile_test_8', 'mobile test 8', 'jack⚡️', '@jack', 'twttr'] + [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr'] ] reply = [ From 93f605f4fe9f0787e3dd618cba85130b9cba2438 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 4 Jun 2022 01:11:35 +0200 Subject: [PATCH 06/17] Update deps --- nitter.nimble | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nitter.nimble b/nitter.nimble index 78c1254..f24d08e 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -12,15 +12,15 @@ bin = @["nitter"] requires "nim >= 1.4.8" requires "jester >= 0.5.0" -requires "karax#fa4a2dc" +requires "karax#5498909" requires "sass#e683aa1" requires "nimcrypto#a5742a9" -requires "markdown#abdbe5e" +requires "markdown#a661c26" requires "packedjson#d11d167" requires "supersnappy#2.1.1" requires "redpool#8b7c1db" requires "https://github.com/zedeus/redis#d0a0e6f" -requires "zippy#0.7.3" +requires "zippy#0.9.9" requires "flatty#0.2.3" requires "jsony#d0e69bd" From 6709f6f1b52ed5ecdbbd791d570d9eaaa1e0ac00 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 4 Jun 2022 01:32:02 +0200 Subject: [PATCH 07/17] Fix "playback disabled" message --- src/views/tweet.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/views/tweet.nim b/src/views/tweet.nim index fe96c8e..671ff36 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -70,9 +70,9 @@ proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool = proc hasMp4Url(video: Video): bool = video.variants.anyIt(it.contentType == mp4) -proc renderVideoDisabled(video: Video; path: string): VNode = +proc renderVideoDisabled(playbackType: VideoType; path: string): VNode = buildHtml(tdiv(class="video-overlay")): - case video.playbackType + case playbackType of mp4: p: text "mp4 playback disabled in preferences" of m3u8, vmap: @@ -102,7 +102,7 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode = renderVideoUnavailable(video) elif not prefs.isPlaybackEnabled(playbackType): img(src=thumb) - renderVideoDisabled(video, path) + renderVideoDisabled(playbackType, path) else: let vars = video.variants.filterIt(it.contentType == playbackType) From 21e8f04fa47cbef5332210eb650ef7819cf9bca8 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 4 Jun 2022 02:18:26 +0200 Subject: [PATCH 08/17] Use strformat more --- src/formatters.nim | 6 +++--- src/query.nim | 6 +++--- src/routes/embed.nim | 4 ++-- src/routes/list.nim | 19 ++++++++++--------- src/routes/rss.nim | 20 ++++++++++---------- src/routes/search.nim | 4 ++-- src/routes/timeline.nim | 4 ++-- src/views/general.nim | 2 +- src/views/renderutils.nim | 4 ++-- src/views/rss.nimf | 4 ++-- src/views/tweet.nim | 2 +- 11 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/formatters.nim b/src/formatters.nim index bb8698c..28bfa1c 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -60,11 +60,11 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string = result = result.replace("/c/", "/") if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body): - result = result.replace(tco, https & prefs.replaceTwitter & "/t.co") + result = result.replace(tco, &"{https}{prefs.replaceTwitter}/t.co") result = result.replace(cards, prefs.replaceTwitter & "/cards") result = result.replace(twRegex, prefs.replaceTwitter) result = result.replacef(twLinkRegex, a( - prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1")) + prefs.replaceTwitter & "$2", href = &"{https}{prefs.replaceTwitter}$1")) if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result): result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/") @@ -76,7 +76,7 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string = result = result.replace(igRegex, prefs.replaceInstagram) if absolute.len > 0 and "href" in result: - result = result.replace("href=\"/", "href=\"" & absolute & "/") + result = result.replace("href=\"/", &"href=\"{absolute}/") proc getM3u8Url*(content: string): string = var matches: array[1, string] diff --git a/src/query.nim b/src/query.nim index cf9b0e6..d128f6f 100644 --- a/src/query.nim +++ b/src/query.nim @@ -93,11 +93,11 @@ proc genQueryUrl*(query: Query): string = if query.text.len > 0: params.add "q=" & encodeUrl(query.text) for f in query.filters: - params.add "f-" & f & "=on" + params.add &"f-{f}=on" for e in query.excludes: - params.add "e-" & e & "=on" + params.add &"e-{e}=on" for i in query.includes.filterIt(it != "nativeretweets"): - params.add "i-" & i & "=on" + params.add &"i-{i}=on" if query.since.len > 0: params.add "since=" & query.since diff --git a/src/routes/embed.nim b/src/routes/embed.nim index 1a93d40..8690357 100644 --- a/src/routes/embed.nim +++ b/src/routes/embed.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, strutils, options +import asyncdispatch, strutils, strformat, options import jester, karax/vdom import ".."/[types, api] import ../views/[embed, tweet, general] @@ -31,6 +31,6 @@ proc createEmbedRouter*(cfg: Config) = let id = @"id" if id.len > 0: - redirect("/i/status/" & id & "/embed") + redirect(&"/i/status/{id}/embed") else: resp Http404 diff --git a/src/routes/list.nim b/src/routes/list.nim index d466080..c97b1c1 100644 --- a/src/routes/list.nim +++ b/src/routes/list.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, uri +import strutils, strformat, uri import jester @@ -10,14 +10,17 @@ export getListTimeline, getGraphList template respList*(list, timeline, title, vnode: typed) = if list.id.len == 0 or list.name.len == 0: - resp Http404, showError("List " & @"id" & " not found", cfg) + resp Http404, showError(&"""List "{@"id"}" not found""", cfg) let html = renderList(vnode, timeline.query, list) - rss = "/i/lists/$1/rss" % [@"id"] + rss = &"""/i/lists/{@"id"}/rss""" resp renderMain(html, request, cfg, prefs, titleText=title, rss=rss, banner=list.banner) +proc title*(list: List): string = + &"@{list.username}/{list.name}" + proc createListRouter*(cfg: Config) = router list: get "/@name/lists/@slug/?": @@ -28,24 +31,22 @@ proc createListRouter*(cfg: Config) = slug = decodeUrl(@"slug") list = await getCachedList(@"name", slug) if list.id.len == 0: - resp Http404, showError("List \"" & @"slug" & "\" not found", cfg) - redirect("/i/lists/" & list.id) + resp Http404, showError(&"""List "{@"slug"}" not found""", cfg) + redirect(&"/i/lists/{list.id}") get "/i/lists/@id/?": cond '.' notin @"id" let prefs = cookiePrefs() list = await getCachedList(id=(@"id")) - title = "@" & list.username & "/" & list.name timeline = await getListTimeline(list.id, getCursor()) vnode = renderTimelineTweets(timeline, prefs, request.path) - respList(list, timeline, title, vnode) + respList(list, timeline, list.title, vnode) get "/i/lists/@id/members": cond '.' notin @"id" let prefs = cookiePrefs() list = await getCachedList(id=(@"id")) - title = "@" & list.username & "/" & list.name members = await getGraphListMembers(list, getCursor()) - respList(list, members, title, renderTimelineUsers(members, prefs, request.path)) + respList(list, members, list.title, renderTimelineUsers(members, prefs, request.path)) diff --git a/src/routes/rss.nim b/src/routes/rss.nim index 40aa6a7..700c215 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, strutils, tables, times, hashes, uri +import asyncdispatch, strutils, strformat, tables, times, hashes, uri import jester @@ -42,8 +42,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. template respRss*(rss, page) = if rss.cursor.len == 0: let info = case page - of "User": " \"$1\" " % @"name" - of "List": " $1 " % @"id" + of "User": &""" "{@"name"}" """ + of "List": &""" "{@"id"}" """ else: " " resp Http404, showError(page & info & "not found", cfg) @@ -67,7 +67,7 @@ proc createRssRouter*(cfg: Config) = let cursor = getCursor() - key = "search:" & $hash(genQueryUrl(query)) & ":" & cursor + key = &"search:{hash(genQueryUrl(query))}:cursor" var rss = await getCachedRss(key) if rss.cursor.len > 0: @@ -86,7 +86,7 @@ proc createRssRouter*(cfg: Config) = let cursor = getCursor() name = @"name" - key = "twitter:" & name & ":" & cursor + key = &"twitter:{name}:{cursor}" var rss = await getCachedRss(key) if rss.cursor.len > 0: @@ -109,7 +109,7 @@ proc createRssRouter*(cfg: Config) = of "search": initQuery(params(request), name=name) else: Query(fromUser: @[name]) - var key = @"tab" & ":" & @"name" & ":" + var key = &"""{@"tab"}:{@"name"}:""" if @"tab" == "search": key &= $hash(genQueryUrl(query)) & ":" key &= getCursor() @@ -132,11 +132,11 @@ proc createRssRouter*(cfg: Config) = cursor = getCursor() if list.id.len == 0: - resp Http404, showError("List \"" & @"slug" & "\" not found", cfg) + resp Http404, showError(&"""List "{@"slug"}" not found""", cfg) - let url = "/i/lists/" & list.id & "/rss" + let url = &"/i/lists/{list.id}/rss" if cursor.len > 0: - redirect(url & "?cursor=" & encodeUrl(cursor, false)) + redirect(&"{url}?cursor={encodeUrl(cursor, false)}") else: redirect(url) @@ -146,7 +146,7 @@ proc createRssRouter*(cfg: Config) = cursor = getCursor() key = if cursor.len == 0: "lists:" & @"id" - else: "lists:" & @"id" & ":" & cursor + else: &"""lists:{@"id"}:{cursor}""" var rss = await getCachedRss(key) if rss.cursor.len > 0: diff --git a/src/routes/search.nim b/src/routes/search.nim index 3fc44a9..554f2f6 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, uri +import strutils, strformat, uri import jester @@ -37,7 +37,7 @@ proc createSearchRouter*(cfg: Config) = resp Http404, showError("Invalid search", cfg) get "/hashtag/@hash": - redirect("/search?q=" & encodeUrl("#" & @"hash")) + redirect(&"""/search?q={encodeUrl("#" & @"hash")}""") get "/opensearch": let url = getUrlPrefix(cfg) & "/search?q=" diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index a0a6e21..9d97c29 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, strutils, sequtils, uri, options, times +import asyncdispatch, strutils, strformat, sequtils, uri, options, times import jester, karax/vdom import router_utils @@ -102,7 +102,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; template respTimeline*(timeline: typed) = let t = timeline if t.len == 0: - resp Http404, showError("User \"" & @"name" & "\" not found", cfg) + resp Http404, showError(&"""User "{@"name"}" not found""", cfg) resp t template respUserId*() = diff --git a/src/views/general.nim b/src/views/general.nim index b18dae5..f242e66 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -81,7 +81,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; title: if titleText.len > 0: - text titleText & " | " & cfg.title + text &"{titleText}|{cfg.title}" else: text cfg.title diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 3e0cd19..bab01cd 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -1,11 +1,11 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils +import strutils, strformat import karax/[karaxdsl, vdom, vstyles] import ".."/[types, utils] proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = var c = "icon-" & icon - if class.len > 0: c = c & " " & class + if class.len > 0: c = &"{c} {class}" buildHtml(tdiv(class="icon-container")): if href.len > 0: a(class=c, title=title, href=href) diff --git a/src/views/rss.nimf b/src/views/rss.nimf index f910f92..96f6466 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -117,7 +117,7 @@ ${renderRssTweets(profile.tweets.content, cfg)} ${xmltree.escape(list.name)} / @${list.username} ${link} - ${getDescription(list.name & " by @" & list.username, cfg)} + ${getDescription(&"{list.name} by @{list.username}", cfg)} en-us 40 ${renderRssTweets(tweets, cfg)} @@ -135,7 +135,7 @@ ${renderRssTweets(tweets, cfg)} Search results for "${escName}" ${link} - ${getDescription("Search \"" & escName & "\"", cfg)} + ${getDescription(&"Search \"{escName}\"", cfg)} en-us 40 ${renderRssTweets(tweets, cfg)} diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 671ff36..5bee864 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -154,7 +154,7 @@ proc renderPoll(poll: Poll): VNode = span(class="poll-choice-value"): text percStr span(class="poll-choice-option"): text poll.options[i] span(class="poll-info"): - text insertSep($poll.votes, ',') & " votes • " & poll.status + text &"{insertSep($poll.votes, ',')} votes • {poll.status}" proc renderCardImage(card: Card): VNode = buildHtml(tdiv(class="card-image-container")): From 778c6c64cb93a56faade09b55df25c8c82b793e4 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 4 Jun 2022 02:24:43 +0200 Subject: [PATCH 09/17] Use a different quote for testing --- tests/test_quote.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_quote.py b/tests/test_quote.py index 82ca385..e1ee728 100644 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -8,9 +8,13 @@ text = [ What are we doing wrong? reuters.com/article/us-norwa…"""], - ['nim_lang/status/924694255364341760', - 'Hacker News', '@newsycombinator', - 'Why Rust fails hard at scientific computing andre-ratsimbazafy.com/why-r…'] + ['nim_lang/status/1491461266849808397#m', + 'Nim language', '@nim_lang', + """What's better than Nim 1.6.0? + +Nim 1.6.2 :) + +nim-lang.org/blog/2021/12/17…"""] ] image = [ From c543a1df8cf438d04ed47936069639c0543fbece Mon Sep 17 00:00:00 2001 From: minus Date: Sat, 4 Jun 2022 15:48:25 +0000 Subject: [PATCH 10/17] Block search engines via robots.txt (#631) Prevents instances from being rate limited due to being senselessly crawled by search engines. Since there is no reason to index Nitter instances, simply block all robots. Notably, this does *not* affect link previews (e.g. in various chat software). --- public/robots.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 public/robots.txt diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / From dfb2519870acf99d9e09a864910a6590552ff08a Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 4 Jun 2022 17:52:47 +0200 Subject: [PATCH 11/17] Explicitly allow Twitterbot to generate previews --- public/robots.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/robots.txt b/public/robots.txt index 1f53798..4c2b2a6 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,2 +1,4 @@ +User-agent: Twitterbot +Allow: / User-agent: * Disallow: / From 138826fb4fbdec73fc6fee2e025fda88f7f2fb49 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 4 Jun 2022 17:55:35 +0200 Subject: [PATCH 12/17] Fix Twitterbot rule in robots.txt --- public/robots.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 4c2b2a6..7a74568 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,4 +1,4 @@ -User-agent: Twitterbot -Allow: / User-agent: * Disallow: / +User-agent: Twitterbot +Disallow: From adaa94d998f727f5820643f3bf64d02a10f2517a Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 5 Jun 2022 21:47:25 +0200 Subject: [PATCH 13/17] Add more logging to the token pool --- src/tokens.nim | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/tokens.nim b/src/tokens.nim index 8a68e46..4bbcd81 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -14,6 +14,10 @@ var clientPool: HttpPool tokenPool: seq[Token] lastFailed: Time + enableLogging = false + +template log(str) = + if enableLogging: echo "[tokens] ", str proc getPoolJson*(): JsonNode = var @@ -77,8 +81,10 @@ proc fetchToken(): Future[Token] {.async.} = return Token(tok: tok, init: time, lastUse: time) except Exception as e: - lastFailed = getTime() - echo "fetching token failed: ", e.msg + echo "[tokens] fetching token failed: ", e.msg + if "Try again" notin e.msg: + echo "[tokens] fetching tokens paused, resuming in 30 minutes" + lastFailed = getTime() proc expired(token: Token): bool = let time = getTime() @@ -100,6 +106,9 @@ proc isReady(token: Token; api: Api): bool = proc release*(token: Token; used=false; invalid=false) = if token.isNil: return if invalid or token.expired: + if invalid: log "discarding invalid token" + elif token.expired: log "discarding expired token" + let idx = tokenPool.find(token) if idx > -1: tokenPool.delete(idx) elif used: @@ -115,6 +124,7 @@ proc getToken*(api: Api): Future[Token] {.async.} = if not result.isReady(api): release(result) result = await fetchToken() + log "added new token to pool" tokenPool.add result if not result.isNil: @@ -143,10 +153,12 @@ proc poolTokens*(amount: int) {.async.} = except: discard if not newToken.isNil: + log "added new token to pool" tokenPool.add newToken proc initTokenPool*(cfg: Config) {.async.} = clientPool = HttpPool() + enableLogging = cfg.enableDebug while true: if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens: From 38bbc677574a6e33409b830d6b66a15c49853a22 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 5 Jun 2022 22:27:22 +0200 Subject: [PATCH 14/17] Remove old unnecessary rate limit error log --- src/nitter.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nitter.nim b/src/nitter.nim index 9f8fcb7..d743599 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -85,7 +85,6 @@ routes: &"An error occurred, please {link} with the URL you tried to visit.", cfg) error RateLimitError: - echo error.exc.name, ": ", error.exc.msg const link = a("another instance", href = instancesUrl) resp Http429, showError( &"Instance has been rate limited.
Use {link} or try again later.", cfg) From d407051b66c3a7d0ed6a9aed16ad1a844af44398 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 5 Jun 2022 22:40:10 +0200 Subject: [PATCH 15/17] Downgrade zippy library to fix checksum error --- nitter.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitter.nimble b/nitter.nimble index f24d08e..d364562 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -20,7 +20,7 @@ requires "packedjson#d11d167" requires "supersnappy#2.1.1" requires "redpool#8b7c1db" requires "https://github.com/zedeus/redis#d0a0e6f" -requires "zippy#0.9.9" +requires "zippy#0.7.3" requires "flatty#0.2.3" requires "jsony#d0e69bd" From 5cceca4e9396e760517f616ae45d01ea37b704a8 Mon Sep 17 00:00:00 2001 From: zedeus Date: Thu, 9 Jun 2022 09:25:45 +0200 Subject: [PATCH 16/17] Bump zippy dependency --- nitter.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitter.nimble b/nitter.nimble index d364562..90a7921 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -20,7 +20,7 @@ requires "packedjson#d11d167" requires "supersnappy#2.1.1" requires "redpool#8b7c1db" requires "https://github.com/zedeus/redis#d0a0e6f" -requires "zippy#0.7.3" +requires "zippy#0.9.11" requires "flatty#0.2.3" requires "jsony#d0e69bd" From 74c13b372dfe43014385e2ce165fd01d5d656fae Mon Sep 17 00:00:00 2001 From: zedeus Date: Thu, 9 Jun 2022 09:34:06 +0200 Subject: [PATCH 17/17] Format readme --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 460ce21..286bea7 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscW ## Resources -The wiki contains +The wiki contains [a list of instances](https://github.com/zedeus/nitter/wiki/Instances) and [browser extensions](https://github.com/zedeus/nitter/wiki/Extensions) maintained by the community. @@ -67,9 +67,10 @@ Twitter account. ## Installation ### Dependencies -* libpcre -* libsass -* redis + +- libpcre +- libsass +- redis To compile Nitter you need a Nim installation, see [nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to @@ -115,18 +116,21 @@ before you can run the container. See below for how to also run Redis using Docker. To build and run Nitter in Docker: + ```bash docker build -t nitter:latest . docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest ``` A prebuilt Docker image is provided as well: + ```bash docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest ``` Using docker-compose to run both Nitter and Redis as different containers: Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run: + ```bash docker-compose up -d ```