Merge branch 'master' of https://github.com/zedeus/nitter
このコミットが含まれているのは:
コミット
59828d2651
|
@ -18,7 +18,7 @@ requires "nimcrypto#a5742a9"
|
||||||
requires "markdown#abdbe5e"
|
requires "markdown#abdbe5e"
|
||||||
requires "packedjson#d11d167"
|
requires "packedjson#d11d167"
|
||||||
requires "supersnappy#2.1.1"
|
requires "supersnappy#2.1.1"
|
||||||
requires "redpool#f880f49"
|
requires "redpool#8b7c1db"
|
||||||
requires "https://github.com/zedeus/redis#d0a0e6f"
|
requires "https://github.com/zedeus/redis#d0a0e6f"
|
||||||
requires "zippy#0.7.3"
|
requires "zippy#0.7.3"
|
||||||
requires "flatty#0.2.3"
|
requires "flatty#0.2.3"
|
||||||
|
@ -28,7 +28,7 @@ requires "jsony#d0e69bd"
|
||||||
# Tasks
|
# Tasks
|
||||||
|
|
||||||
task scss, "Generate css":
|
task scss, "Generate css":
|
||||||
exec "nim c --hint[Processing]:off -d:danger -r tools/gencss"
|
exec "nimble c --hint[Processing]:off -d:danger -r tools/gencss"
|
||||||
|
|
||||||
task md, "Render md":
|
task md, "Render md":
|
||||||
exec "nim c --hint[Processing]:off -d:danger -r tools/rendermd"
|
exec "nimble c --hint[Processing]:off -d:danger -r tools/rendermd"
|
||||||
|
|
|
@ -58,11 +58,17 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
||||||
if token.tok.len == 0:
|
if token.tok.len == 0:
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
|
|
||||||
|
var
|
||||||
|
client = pool.acquire(genHeaders(token))
|
||||||
|
badClient = false
|
||||||
|
|
||||||
try:
|
try:
|
||||||
var resp: AsyncResponse
|
let resp = await client.get($url)
|
||||||
result = pool.use(genHeaders(token)):
|
result = await resp.body
|
||||||
resp = await c.get($url)
|
|
||||||
await resp.body
|
if resp.status == $Http503:
|
||||||
|
badClient = true
|
||||||
|
raise newException(InternalError, result)
|
||||||
|
|
||||||
if result.len > 0:
|
if result.len > 0:
|
||||||
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
||||||
|
@ -83,6 +89,8 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
||||||
if "length" notin e.msg and "descriptor" notin e.msg:
|
if "length" notin e.msg and "descriptor" notin e.msg:
|
||||||
release(token, invalid=true)
|
release(token, invalid=true)
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
|
finally:
|
||||||
|
pool.release(client, badClient=badClient)
|
||||||
|
|
||||||
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
|
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
|
||||||
var body: string
|
var body: string
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
import asyncdispatch, httpclient
|
import httpclient
|
||||||
|
|
||||||
type
|
type
|
||||||
HttpPool* = ref object
|
HttpPool* = ref object
|
||||||
|
@ -17,20 +17,22 @@ proc setHttpProxy*(url: string; auth: string) =
|
||||||
else:
|
else:
|
||||||
proxy = nil
|
proxy = nil
|
||||||
|
|
||||||
proc release*(pool: HttpPool; client: AsyncHttpClient) =
|
proc release*(pool: HttpPool; client: AsyncHttpClient; badClient=false) =
|
||||||
if pool.conns.len >= maxConns:
|
if pool.conns.len >= maxConns or badClient:
|
||||||
client.close()
|
try: client.close()
|
||||||
|
except: discard
|
||||||
elif client != nil:
|
elif client != nil:
|
||||||
pool.conns.insert(client)
|
pool.conns.insert(client)
|
||||||
|
|
||||||
template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
|
proc acquire*(pool: HttpPool; heads: HttpHeaders): AsyncHttpClient =
|
||||||
var c {.inject.}: AsyncHttpClient
|
|
||||||
|
|
||||||
if pool.conns.len == 0:
|
if pool.conns.len == 0:
|
||||||
c = newAsyncHttpClient(headers=heads, proxy=proxy)
|
result = newAsyncHttpClient(headers=heads, proxy=proxy)
|
||||||
else:
|
else:
|
||||||
c = pool.conns.pop()
|
result = pool.conns.pop()
|
||||||
c.headers = heads
|
result.headers = heads
|
||||||
|
|
||||||
|
template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
|
||||||
|
let c {.inject.} = pool.acquire(heads)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
body
|
body
|
||||||
|
|
|
@ -83,11 +83,9 @@ proc cache*(data: Profile) {.async.} =
|
||||||
if data.username.len == 0: return
|
if data.username.len == 0: return
|
||||||
let name = toLower(data.username)
|
let name = toLower(data.username)
|
||||||
pool.withAcquire(r):
|
pool.withAcquire(r):
|
||||||
r.startPipelining()
|
|
||||||
dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
|
dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
|
||||||
if data.id.len > 0:
|
if data.id.len > 0:
|
||||||
dawait r.hSet(name.pidKey, name, data.id)
|
dawait r.hSet(name.pidKey, name, data.id)
|
||||||
dawait r.flushPipeline()
|
|
||||||
|
|
||||||
proc cacheProfileId(username, id: string) {.async.} =
|
proc cacheProfileId(username, id: string) {.async.} =
|
||||||
if username.len == 0 or id.len == 0: return
|
if username.len == 0 or id.len == 0: return
|
||||||
|
@ -98,11 +96,9 @@ proc cacheProfileId(username, id: string) {.async.} =
|
||||||
proc cacheRss*(query: string; rss: Rss) {.async.} =
|
proc cacheRss*(query: string; rss: Rss) {.async.} =
|
||||||
let key = "rss:" & query
|
let key = "rss:" & query
|
||||||
pool.withAcquire(r):
|
pool.withAcquire(r):
|
||||||
r.startPipelining()
|
|
||||||
dawait r.hSet(key, "rss", rss.feed)
|
dawait r.hSet(key, "rss", rss.feed)
|
||||||
dawait r.hSet(key, "min", rss.cursor)
|
dawait r.hSet(key, "min", rss.cursor)
|
||||||
dawait r.expire(key, rssCacheTime)
|
dawait r.expire(key, rssCacheTime)
|
||||||
dawait r.flushPipeline()
|
|
||||||
|
|
||||||
proc getProfileId*(username: string): Future[string] {.async.} =
|
proc getProfileId*(username: string): Future[string] {.async.} =
|
||||||
let name = toLower(username)
|
let name = toLower(username)
|
||||||
|
|
|
@ -45,10 +45,15 @@ proc fetchTimeline*(after: string; query: Query; skipRail=false):
|
||||||
else:
|
else:
|
||||||
rail = getCachedPhotoRail(name)
|
rail = getCachedPhotoRail(name)
|
||||||
|
|
||||||
|
# var timeline =
|
||||||
|
# case query.kind
|
||||||
|
# of posts: await getTimeline(profileId, after)
|
||||||
|
# of replies: await getTimeline(profileId, after, replies=true)
|
||||||
|
# of media: await getMediaTimeline(profileId, after)
|
||||||
|
# else: await getSearch[Tweet](query, after)
|
||||||
|
|
||||||
var timeline =
|
var timeline =
|
||||||
case query.kind
|
case query.kind
|
||||||
of posts: await getTimeline(profileId, after)
|
|
||||||
of replies: await getTimeline(profileId, after, replies=true)
|
|
||||||
of media: await getMediaTimeline(profileId, after)
|
of media: await getMediaTimeline(profileId, after)
|
||||||
else: await getSearch[Tweet](query, after)
|
else: await getSearch[Tweet](query, after)
|
||||||
|
|
||||||
|
|
|
@ -78,7 +78,7 @@ class ProfileTest(BaseTestCase):
|
||||||
@parameterized.expand(banner_color)
|
@parameterized.expand(banner_color)
|
||||||
def test_banner_color(self, username, color):
|
def test_banner_color(self, username, color):
|
||||||
self.open_nitter(username)
|
self.open_nitter(username)
|
||||||
banner = self.find_element(Profile.banner + '-color')
|
banner = self.find_element(Profile.banner + ' a')
|
||||||
self.assertIn(color, banner.value_of_css_property('background-color'))
|
self.assertIn(color, banner.value_of_css_property('background-color'))
|
||||||
|
|
||||||
@parameterized.expand(banner_image)
|
@parameterized.expand(banner_image)
|
||||||
|
|
新しいイシューから参照