Merge branch 'master' of https://github.com/zedeus/nitter

nitter.nimble

@@ -18,7 +18,7 @@ requires "nimcrypto#a5742a9"
 requires "markdown#abdbe5e"
 requires "packedjson#d11d167"
 requires "supersnappy#2.1.1"
-requires "redpool#f880f49"
+requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#0.7.3"
 requires "flatty#0.2.3"
@@ -28,7 +28,7 @@ requires "jsony#d0e69bd"
 # Tasks
 
 task scss, "Generate css":
-  exec "nim c --hint[Processing]:off -d:danger -r tools/gencss"
+  exec "nimble c --hint[Processing]:off -d:danger -r tools/gencss"
 
 task md, "Render md":
-  exec "nim c --hint[Processing]:off -d:danger -r tools/rendermd"
+  exec "nimble c --hint[Processing]:off -d:danger -r tools/rendermd"

src/apiutils.nim

@@ -58,11 +58,17 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   if token.tok.len == 0:
     raise rateLimitError()
 
+  var
+    client = pool.acquire(genHeaders(token))
+    badClient = false
+
   try:
-    var resp: AsyncResponse
-    result = pool.use(genHeaders(token)):
-      resp = await c.get($url)
-      await resp.body
+    let resp = await client.get($url)
+    result = await resp.body
+
+    if resp.status == $Http503:
+      badClient = true
+      raise newException(InternalError, result)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -83,6 +89,8 @@ template fetchImpl(result, fetchBody) {.dirty.} =
     if "length" notin e.msg and "descriptor" notin e.msg:
       release(token, invalid=true)
     raise rateLimitError()
+  finally:
+    pool.release(client, badClient=badClient)
 
 proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
   var body: string

src/http_pool.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient
+import httpclient
 
 type
   HttpPool* = ref object
@@ -17,20 +17,22 @@ proc setHttpProxy*(url: string; auth: string) =
   else:
     proxy = nil
 
-proc release*(pool: HttpPool; client: AsyncHttpClient) =
-  if pool.conns.len >= maxConns:
-    client.close()
+proc release*(pool: HttpPool; client: AsyncHttpClient; badClient=false) =
+  if pool.conns.len >= maxConns or badClient:
+    try: client.close()
+    except: discard
   elif client != nil:
     pool.conns.insert(client)
 
-template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
-  var c {.inject.}: AsyncHttpClient
-
+proc acquire*(pool: HttpPool; heads: HttpHeaders): AsyncHttpClient =
   if pool.conns.len == 0:
-    c = newAsyncHttpClient(headers=heads, proxy=proxy)
+    result = newAsyncHttpClient(headers=heads, proxy=proxy)
   else:
-    c = pool.conns.pop()
-    c.headers = heads
+    result = pool.conns.pop()
+    result.headers = heads
+
+template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
+  let c {.inject.} = pool.acquire(heads)
 
   try:
     body

src/redis_cache.nim

@@ -83,11 +83,9 @@ proc cache*(data: Profile) {.async.} =
   if data.username.len == 0: return
   let name = toLower(data.username)
   pool.withAcquire(r):
-    r.startPipelining()
     dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
     if data.id.len > 0:
       dawait r.hSet(name.pidKey, name, data.id)
-    dawait r.flushPipeline()
 
 proc cacheProfileId(username, id: string) {.async.} =
   if username.len == 0 or id.len == 0: return
@@ -98,11 +96,9 @@ proc cacheProfileId(username, id: string) {.async.} =
 proc cacheRss*(query: string; rss: Rss) {.async.} =
   let key = "rss:" & query
   pool.withAcquire(r):
-    r.startPipelining()
     dawait r.hSet(key, "rss", rss.feed)
     dawait r.hSet(key, "min", rss.cursor)
     dawait r.expire(key, rssCacheTime)
-    dawait r.flushPipeline()
 
 proc getProfileId*(username: string): Future[string] {.async.} =
   let name = toLower(username)

src/routes/timeline.nim

@@ -45,10 +45,15 @@ proc fetchTimeline*(after: string; query: Query; skipRail=false):
   else:
     rail = getCachedPhotoRail(name)
 
+  # var timeline =
+  #   case query.kind
+  #   of posts: await getTimeline(profileId, after)
+  #   of replies: await getTimeline(profileId, after, replies=true)
+  #   of media: await getMediaTimeline(profileId, after)
+  #   else: await getSearch[Tweet](query, after)
+
   var timeline =
     case query.kind
-    of posts: await getTimeline(profileId, after)
-    of replies: await getTimeline(profileId, after, replies=true)
     of media: await getMediaTimeline(profileId, after)
     else: await getSearch[Tweet](query, after)
 

tests/test_profile.py

@@ -78,7 +78,7 @@ class ProfileTest(BaseTestCase):
     @parameterized.expand(banner_color)
     def test_banner_color(self, username, color):
         self.open_nitter(username)
-        banner = self.find_element(Profile.banner + '-color')
+        banner = self.find_element(Profile.banner + ' a')
         self.assertIn(color, banner.value_of_css_property('background-color'))
 
     @parameterized.expand(banner_image)