diff --git a/nitter.nimble b/nitter.nimble index 7c49fdd..ea2436c 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -18,7 +18,7 @@ requires "nimcrypto#a5742a9" requires "markdown#abdbe5e" requires "packedjson#d11d167" requires "supersnappy#2.1.1" -requires "redpool#f880f49" +requires "redpool#8b7c1db" requires "https://github.com/zedeus/redis#d0a0e6f" requires "zippy#0.7.3" requires "flatty#0.2.3" @@ -28,7 +28,7 @@ requires "jsony#d0e69bd" # Tasks task scss, "Generate css": - exec "nim c --hint[Processing]:off -d:danger -r tools/gencss" + exec "nimble c --hint[Processing]:off -d:danger -r tools/gencss" task md, "Render md": - exec "nim c --hint[Processing]:off -d:danger -r tools/rendermd" + exec "nimble c --hint[Processing]:off -d:danger -r tools/rendermd" diff --git a/src/apiutils.nim b/src/apiutils.nim index f6a4b47..21c25df 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -58,11 +58,17 @@ template fetchImpl(result, fetchBody) {.dirty.} = if token.tok.len == 0: raise rateLimitError() + var + client = pool.acquire(genHeaders(token)) + badClient = false + try: - var resp: AsyncResponse - result = pool.use(genHeaders(token)): - resp = await c.get($url) - await resp.body + let resp = await client.get($url) + result = await resp.body + + if resp.status == $Http503: + badClient = true + raise newException(InternalError, result) if result.len > 0: if resp.headers.getOrDefault("content-encoding") == "gzip": @@ -83,6 +89,8 @@ template fetchImpl(result, fetchBody) {.dirty.} = if "length" notin e.msg and "descriptor" notin e.msg: release(token, invalid=true) raise rateLimitError() + finally: + pool.release(client, badClient=badClient) proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = var body: string diff --git a/src/http_pool.nim b/src/http_pool.nim index 8f4bcda..a39918b 100644 --- a/src/http_pool.nim +++ b/src/http_pool.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, httpclient +import httpclient type HttpPool* = ref object @@ -17,20 +17,22 @@ proc setHttpProxy*(url: string; auth: string) = else: proxy = nil -proc release*(pool: HttpPool; client: AsyncHttpClient) = - if pool.conns.len >= maxConns: - client.close() +proc release*(pool: HttpPool; client: AsyncHttpClient; badClient=false) = + if pool.conns.len >= maxConns or badClient: + try: client.close() + except: discard elif client != nil: pool.conns.insert(client) -template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped = - var c {.inject.}: AsyncHttpClient - +proc acquire*(pool: HttpPool; heads: HttpHeaders): AsyncHttpClient = if pool.conns.len == 0: - c = newAsyncHttpClient(headers=heads, proxy=proxy) + result = newAsyncHttpClient(headers=heads, proxy=proxy) else: - c = pool.conns.pop() - c.headers = heads + result = pool.conns.pop() + result.headers = heads + +template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped = + let c {.inject.} = pool.acquire(heads) try: body diff --git a/src/redis_cache.nim b/src/redis_cache.nim index fe622f9..f272d3e 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -83,11 +83,9 @@ proc cache*(data: Profile) {.async.} = if data.username.len == 0: return let name = toLower(data.username) pool.withAcquire(r): - r.startPipelining() dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data))) if data.id.len > 0: dawait r.hSet(name.pidKey, name, data.id) - dawait r.flushPipeline() proc cacheProfileId(username, id: string) {.async.} = if username.len == 0 or id.len == 0: return @@ -98,11 +96,9 @@ proc cacheProfileId(username, id: string) {.async.} = proc cacheRss*(query: string; rss: Rss) {.async.} = let key = "rss:" & query pool.withAcquire(r): - r.startPipelining() dawait r.hSet(key, "rss", rss.feed) dawait r.hSet(key, "min", rss.cursor) dawait r.expire(key, rssCacheTime) - dawait r.flushPipeline() proc getProfileId*(username: string): Future[string] {.async.} = let name = toLower(username) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 9eefc4e..815e08e 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -45,10 +45,15 @@ proc fetchTimeline*(after: string; query: Query; skipRail=false): else: rail = getCachedPhotoRail(name) + # var timeline = + # case query.kind + # of posts: await getTimeline(profileId, after) + # of replies: await getTimeline(profileId, after, replies=true) + # of media: await getMediaTimeline(profileId, after) + # else: await getSearch[Tweet](query, after) + var timeline = case query.kind - of posts: await getTimeline(profileId, after) - of replies: await getTimeline(profileId, after, replies=true) of media: await getMediaTimeline(profileId, after) else: await getSearch[Tweet](query, after) diff --git a/tests/test_profile.py b/tests/test_profile.py index 1a7fe84..d9aa4fa 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -78,7 +78,7 @@ class ProfileTest(BaseTestCase): @parameterized.expand(banner_color) def test_banner_color(self, username, color): self.open_nitter(username) - banner = self.find_element(Profile.banner + '-color') + banner = self.find_element(Profile.banner + ' a') self.assertIn(color, banner.value_of_css_property('background-color')) @parameterized.expand(banner_image)