From 2224605d5f9e3561b8067a3b219cf0ae6f3a1bdc Mon Sep 17 00:00:00 2001 From: Mitarashi Date: Tue, 18 Jan 2022 19:44:49 +0200 Subject: [PATCH 1/8] changed jester version for samesite cookie fix --- nitter.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitter.nimble b/nitter.nimble index 7c49fdd..f6fa583 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -11,7 +11,7 @@ bin = @["nitter"] # Dependencies requires "nim >= 1.4.8" -requires "jester >= 0.5.0" +requires "jester#3e128fe" requires "karax#c71bc92" requires "sass#e683aa1" requires "nimcrypto#a5742a9" From 575cb590d70a9a0cf6e1773b9387b3c8e7a0726c Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 18 Jan 2022 19:01:48 +0100 Subject: [PATCH 2/8] Revert "changed jester version for samesite cookie fix" This reverts commit 2224605d5f9e3561b8067a3b219cf0ae6f3a1bdc. --- nitter.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitter.nimble b/nitter.nimble index f6fa583..7c49fdd 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -11,7 +11,7 @@ bin = @["nitter"] # Dependencies requires "nim >= 1.4.8" -requires "jester#3e128fe" +requires "jester >= 0.5.0" requires "karax#c71bc92" requires "sass#e683aa1" requires "nimcrypto#a5742a9" From a76ca3d60f04616e4b8189b40d48e1b09bd27394 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 20 Jan 2022 02:22:32 +0100 Subject: [PATCH 3/8] Remove Redis pipelining --- src/redis_cache.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/redis_cache.nim b/src/redis_cache.nim index fe622f9..f272d3e 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -83,11 +83,9 @@ proc cache*(data: Profile) {.async.} = if data.username.len == 0: return let name = toLower(data.username) pool.withAcquire(r): - r.startPipelining() dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data))) if data.id.len > 0: dawait r.hSet(name.pidKey, name, data.id) - dawait r.flushPipeline() proc cacheProfileId(username, id: string) {.async.} = if username.len == 0 or id.len == 0: return @@ -98,11 +96,9 @@ proc cacheProfileId(username, id: string) {.async.} = proc cacheRss*(query: string; rss: Rss) {.async.} = let key = "rss:" & query pool.withAcquire(r): - r.startPipelining() dawait r.hSet(key, "rss", rss.feed) dawait r.hSet(key, "min", rss.cursor) dawait r.expire(key, rssCacheTime) - dawait r.flushPipeline() proc getProfileId*(username: string): Future[string] {.async.} = let name = toLower(username) From 637168ecc66073a0738709e3bbf5ab1b9da4da7f Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 20 Jan 2022 02:30:05 +0100 Subject: [PATCH 4/8] Bump Redis pool commit --- nitter.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitter.nimble b/nitter.nimble index 7c49fdd..e78ca8b 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -18,7 +18,7 @@ requires "nimcrypto#a5742a9" requires "markdown#abdbe5e" requires "packedjson#d11d167" requires "supersnappy#2.1.1" -requires "redpool#f880f49" +requires "redpool#499396d" requires "https://github.com/zedeus/redis#d0a0e6f" requires "zippy#0.7.3" requires "flatty#0.2.3" From 29a53de6916696c2118995791453b57b87f93e54 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 20 Jan 2022 06:03:08 +0100 Subject: [PATCH 5/8] Update test to reflect layout change --- tests/test_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_profile.py b/tests/test_profile.py index 1a7fe84..d9aa4fa 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -78,7 +78,7 @@ class ProfileTest(BaseTestCase): @parameterized.expand(banner_color) def test_banner_color(self, username, color): self.open_nitter(username) - banner = self.find_element(Profile.banner + '-color') + banner = self.find_element(Profile.banner + ' a') self.assertIn(color, banner.value_of_css_property('background-color')) @parameterized.expand(banner_image) From d77d5911b0561de01c4dbb5d01830c9df64683a3 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 20 Jan 2022 11:54:36 +0100 Subject: [PATCH 6/8] Bump Redis pool tag --- nitter.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nitter.nimble b/nitter.nimble index e78ca8b..d0eb890 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -18,7 +18,7 @@ requires "nimcrypto#a5742a9" requires "markdown#abdbe5e" requires "packedjson#d11d167" requires "supersnappy#2.1.1" -requires "redpool#499396d" +requires "redpool#8b7c1db" requires "https://github.com/zedeus/redis#d0a0e6f" requires "zippy#0.7.3" requires "flatty#0.2.3" From cdb4efadfeb5102b501c7ff79261fefc7327edb9 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 21 Jan 2022 09:17:18 +0100 Subject: [PATCH 7/8] Temporary fix to Twitter's global timeline error --- src/apiutils.nim | 16 ++++++++++++---- src/http_pool.nim | 22 ++++++++++++---------- src/routes/timeline.nim | 9 +++++++-- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index f6a4b47..21c25df 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -58,11 +58,17 @@ template fetchImpl(result, fetchBody) {.dirty.} = if token.tok.len == 0: raise rateLimitError() + var + client = pool.acquire(genHeaders(token)) + badClient = false + try: - var resp: AsyncResponse - result = pool.use(genHeaders(token)): - resp = await c.get($url) - await resp.body + let resp = await client.get($url) + result = await resp.body + + if resp.status == $Http503: + badClient = true + raise newException(InternalError, result) if result.len > 0: if resp.headers.getOrDefault("content-encoding") == "gzip": @@ -83,6 +89,8 @@ template fetchImpl(result, fetchBody) {.dirty.} = if "length" notin e.msg and "descriptor" notin e.msg: release(token, invalid=true) raise rateLimitError() + finally: + pool.release(client, badClient=badClient) proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = var body: string diff --git a/src/http_pool.nim b/src/http_pool.nim index 8f4bcda..a39918b 100644 --- a/src/http_pool.nim +++ b/src/http_pool.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, httpclient +import httpclient type HttpPool* = ref object @@ -17,20 +17,22 @@ proc setHttpProxy*(url: string; auth: string) = else: proxy = nil -proc release*(pool: HttpPool; client: AsyncHttpClient) = - if pool.conns.len >= maxConns: - client.close() +proc release*(pool: HttpPool; client: AsyncHttpClient; badClient=false) = + if pool.conns.len >= maxConns or badClient: + try: client.close() + except: discard elif client != nil: pool.conns.insert(client) -template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped = - var c {.inject.}: AsyncHttpClient - +proc acquire*(pool: HttpPool; heads: HttpHeaders): AsyncHttpClient = if pool.conns.len == 0: - c = newAsyncHttpClient(headers=heads, proxy=proxy) + result = newAsyncHttpClient(headers=heads, proxy=proxy) else: - c = pool.conns.pop() - c.headers = heads + result = pool.conns.pop() + result.headers = heads + +template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped = + let c {.inject.} = pool.acquire(heads) try: body diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 9eefc4e..815e08e 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -45,10 +45,15 @@ proc fetchTimeline*(after: string; query: Query; skipRail=false): else: rail = getCachedPhotoRail(name) + # var timeline = + # case query.kind + # of posts: await getTimeline(profileId, after) + # of replies: await getTimeline(profileId, after, replies=true) + # of media: await getMediaTimeline(profileId, after) + # else: await getSearch[Tweet](query, after) + var timeline = case query.kind - of posts: await getTimeline(profileId, after) - of replies: await getTimeline(profileId, after, replies=true) of media: await getMediaTimeline(profileId, after) else: await getSearch[Tweet](query, after) From ae7d2eafb8246485d1772dc48e18956431b1878f Mon Sep 17 00:00:00 2001 From: somini Date: Fri, 21 Jan 2022 23:01:38 +0000 Subject: [PATCH 8/8] Fix special tasks I get errors like this: ``` nitter-git/tools/gencss.nim(1, 8) Error: cannot open file: sass stack trace: (most recent call last) /tmp/nimblecache-798796522/nimscriptapi_3049515398.nim(187, 16) nitter-git/nitter.nimble(31, 8) scssTask /usr/lib/nim/system/nimscript.nim(260, 7) exec /usr/lib/nim/system/nimscript.nim(260, 7) Error: unhandled exception: FAILED: nim c -r tools/gencss [OSError] Tip: 2 messages have been suppressed, use --verbose to show them. Error: Exception raised during nimble script execution ``` --- nitter.nimble | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nitter.nimble b/nitter.nimble index d0eb890..ea2436c 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -28,7 +28,7 @@ requires "jsony#d0e69bd" # Tasks task scss, "Generate css": - exec "nim c --hint[Processing]:off -d:danger -r tools/gencss" + exec "nimble c --hint[Processing]:off -d:danger -r tools/gencss" task md, "Render md": - exec "nim c --hint[Processing]:off -d:danger -r tools/rendermd" + exec "nimble c --hint[Processing]:off -d:danger -r tools/rendermd"