diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
index d30c0db..30a209b 100644
--- a/.github/workflows/build-docker.yml
+++ b/.github/workflows/build-docker.yml
@@ -33,6 +33,6 @@ jobs:
with:
context: .
file: ./Dockerfile
- platforms: linux/amd64,linux/arm64
+ platforms: linux/amd64
push: true
tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
diff --git a/config.nims b/config.nims
index b74d70e..ee77289 100644
--- a/config.nims
+++ b/config.nims
@@ -7,6 +7,7 @@
# disable annoying warnings
warning("GcUnsafe2", off)
hint("XDeclaredButNotUsed", off)
+hint("XCannotRaiseY", off)
hint("User", off)
const
diff --git a/nitter.nimble b/nitter.nimble
index a2c0615..7c49fdd 100644
--- a/nitter.nimble
+++ b/nitter.nimble
@@ -22,7 +22,7 @@ requires "redpool#f880f49"
requires "https://github.com/zedeus/redis#d0a0e6f"
requires "zippy#0.7.3"
requires "flatty#0.2.3"
-requires "jsony#1.1.3"
+requires "jsony#d0e69bd"
# Tasks
diff --git a/src/agents.nim b/src/agents.nim
deleted file mode 100644
index a907e7b..0000000
--- a/src/agents.nim
+++ /dev/null
@@ -1,70 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import random, strformat, strutils, sequtils
-
-randomize()
-
-const rvs = [
- "11.0", "40.0", "42.0", "43.0", "47.0", "50.0", "52.0", "53.0", "54.0",
- "61.0", "66.0", "67.0", "69.0", "70.0"
-]
-
-proc rv(): string =
- if rand(10) < 1: ""
- else: "; rv:" & sample(rvs)
-
-# OS
-
-const enc = ["; U", "; N", "; I", ""]
-
-proc linux(): string =
- const
- window = ["X11", "Wayland", "Unknown"]
- arch = ["i686", "x86_64", "arm"]
- distro = ["", "; Ubuntu/14.10", "; Ubuntu/16.10", "; Ubuntu/19.10",
- "; Ubuntu", "; Fedora"]
- sample(window) & sample(enc) & "; Linux " & sample(arch) & sample(distro)
-
-proc windows(): string =
- const
- nt = ["5.1", "5.2", "6.0", "6.1", "6.2", "6.3", "6.4", "9.0", "10.0"]
- arch = ["; WOW64", "; Win64; x64", "; ARM", ""]
- trident = ["", "; Trident/5.0", "; Trident/6.0", "; Trident/7.0"]
- "Windows " & sample(nt) & sample(enc) & sample(arch) & sample(trident)
-
-const macs = toSeq(6..15).mapIt($it) & @["14_4", "10_1", "9_3"]
-
-proc mac(): string =
- "Macintosh; Intel Mac OS X 10_" & sample(macs) & sample(enc)
-
-# Browser
-
-proc presto(): string =
- const p = ["2.12.388", "2.12.407", "22.9.168", "2.9.201", "2.8.131", "2.7.62",
- "2.6.30", "2.5.24"]
- const v = ["10.0", "11.0", "11.1", "11.5", "11.6", "12.00", "12.14", "12.16"]
- &"Presto/{sample(p)} Version/{sample(v)}"
-
-# Samples
-
-proc product(): string =
- const opera = ["Opera/9.80", "Opera/12.0"]
- if rand(20) < 1: "Mozilla/5.0"
- else: sample(opera)
-
-proc os(): string =
- let r = rand(10)
- let os =
- if r < 6: windows()
- elif r < 9: linux()
- else: mac()
- &"({os}{rv()})"
-
-proc browser(prod: string): string =
- if "Opera" in prod: presto()
- else: "like Gecko"
-
-# Agent
-
-proc getAgent*(): string =
- let prod = product()
- &"{prod} {os()} {browser(prod)}"
diff --git a/src/api.nim b/src/api.nim
index 9a6e70a..b1ecc57 100644
--- a/src/api.nim
+++ b/src/api.nim
@@ -2,6 +2,7 @@
import asyncdispatch, httpclient, uri, strutils
import packedjson
import types, query, formatters, consts, apiutils, parser
+import experimental/parser/user
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
@@ -32,14 +33,14 @@ proc getListMembers*(list: List; after=""): Future[Result[Profile]] {.async.} =
proc getProfile*(username: string): Future[Profile] {.async.} =
let
ps = genParams({"screen_name": username})
- js = await fetch(userShow ? ps, Api.userShow)
- result = parseUserShow(js, username=username)
+ json = await fetchRaw(userShow ? ps, Api.userShow)
+ result = parseUser(json, username)
proc getProfileById*(userId: string): Future[Profile] {.async.} =
let
ps = genParams({"user_id": userId})
- js = await fetch(userShow ? ps, Api.userShow)
- result = parseUserShow(js, id=userId)
+ json = await fetchRaw(userShow ? ps, Api.userShow)
+ result = parseUser(json)
proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
let
diff --git a/src/apiutils.nim b/src/apiutils.nim
index e287dc5..f6a4b47 100644
--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, times, strutils, uri
-import packedjson, zippy
+import httpclient, asyncdispatch, options, strutils, uri
+import jsony, packedjson, zippy
import types, tokens, consts, parserutils, http_pool
+import experimental/types/common
const
rlRemaining = "x-rate-limit-remaining"
@@ -16,6 +17,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
result &= p
if ext:
result &= ("ext", "mediaStats")
+ result &= ("include_ext_alt_text", "true")
+ result &= ("include_ext_media_availability", "true")
if count.len > 0:
result &= ("count", count)
if cursor.len > 0:
@@ -40,7 +43,14 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
"DNT": "1"
})
-proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+template updateToken() =
+ if api != Api.search and resp.headers.hasKey(rlRemaining):
+ let
+ remaining = parseInt(resp.headers[rlRemaining])
+ reset = parseInt(resp.headers[rlReset])
+ token.setRateLimit(api, remaining, reset)
+
+template fetchImpl(result, fetchBody) {.dirty.} =
once:
pool = HttpPool()
@@ -48,37 +58,21 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
if token.tok.len == 0:
raise rateLimitError()
- let headers = genHeaders(token)
try:
var resp: AsyncResponse
- var body = pool.use(headers):
+ result = pool.use(genHeaders(token)):
resp = await c.get($url)
await resp.body
- if body.len > 0:
+ if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip":
- body = uncompress(body, dfGzip)
+ result = uncompress(result, dfGzip)
else:
- echo "non-gzip body, url: ", url, ", body: ", body
+ echo "non-gzip body, url: ", url, ", body: ", result
- if body.startsWith('{') or body.startsWith('['):
- result = parseJson(body)
- else:
- echo resp.status, ": ", body
- result = newJNull()
+ fetchBody
- if api != Api.search and resp.headers.hasKey(rlRemaining):
- let
- remaining = parseInt(resp.headers[rlRemaining])
- reset = parseInt(resp.headers[rlReset])
- token.setRateLimit(api, remaining, reset)
-
- if result.getError notin {invalidToken, forbidden, badToken}:
- release(token, used=true)
- else:
- echo "fetch error: ", result.getError
- release(token, invalid=true)
- raise rateLimitError()
+ release(token, used=true)
if resp.status == $Http400:
raise newException(InternalError, $url)
@@ -89,3 +83,35 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
if "length" notin e.msg and "descriptor" notin e.msg:
release(token, invalid=true)
raise rateLimitError()
+
+proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+ var body: string
+ fetchImpl body:
+ if body.startsWith('{') or body.startsWith('['):
+ result = parseJson(body)
+ else:
+ echo resp.status, ": ", body
+ result = newJNull()
+
+ updateToken()
+
+ let error = result.getError
+ if error in {invalidToken, forbidden, badToken}:
+ echo "fetch error: ", result.getError
+ release(token, invalid=true)
+ raise rateLimitError()
+
+proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
+ fetchImpl result:
+ if not (result.startsWith('{') or result.startsWith('[')):
+ echo resp.status, ": ", result
+ result.setLen(0)
+
+ updateToken()
+
+ if result.startsWith("{\"errors"):
+ let errors = result.fromJson(Errors)
+ if errors in {invalidToken, forbidden, badToken}:
+ echo "fetch error: ", errors
+ release(token, invalid=true)
+ raise rateLimitError()
diff --git a/src/consts.nim b/src/consts.nim
index efa0420..c77ebef 100644
--- a/src/consts.nim
+++ b/src/consts.nim
@@ -35,16 +35,13 @@ const
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "false",
- "include_ext_alt_text": "true",
"include_reply_count": "1",
"tweet_mode": "extended",
"include_entities": "true",
"include_user_entities": "true",
"include_ext_media_color": "false",
- "include_ext_media_availability": "true",
"send_error_codes": "true",
"simple_quoted_tweet": "true",
- "ext": "mediaStats",
"include_quote_count": "true"
}.toSeq
diff --git a/src/experimental/parser/slices.nim b/src/experimental/parser/slices.nim
new file mode 100644
index 0000000..45e6e1d
--- /dev/null
+++ b/src/experimental/parser/slices.nim
@@ -0,0 +1,67 @@
+import std/[macros, htmlgen, unicode]
+import ../types/common
+import ".."/../[formatters, utils]
+
+type
+ ReplaceSliceKind = enum
+ rkRemove, rkUrl, rkHashtag, rkMention
+
+ ReplaceSlice* = object
+ slice: Slice[int]
+ kind: ReplaceSliceKind
+ url, display: string
+
+proc cmp*(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
+
+proc dedupSlices*(s: var seq[ReplaceSlice]) =
+ var
+ len = s.len
+ i = 0
+ while i < len:
+ var j = i + 1
+ while j < len:
+ if s[i].slice.a == s[j].slice.a:
+ s.del j
+ dec len
+ else:
+ inc j
+ inc i
+
+proc extractUrls*(result: var seq[ReplaceSlice]; url: Url;
+ textLen: int; hideTwitter = false) =
+ let
+ link = url.expandedUrl
+ slice = url.indices[0] ..< url.indices[1]
+
+ if hideTwitter and slice.b.succ >= textLen and link.isTwitterUrl:
+ if slice.a < textLen:
+ result.add ReplaceSlice(kind: rkRemove, slice: slice)
+ else:
+ result.add ReplaceSlice(kind: rkUrl, url: link,
+ display: link.shortLink, slice: slice)
+
+proc replacedWith*(runes: seq[Rune]; repls: openArray[ReplaceSlice];
+ textSlice: Slice[int]): string =
+ template extractLowerBound(i: int; idx): int =
+ if i > 0: repls[idx].slice.b.succ else: textSlice.a
+
+ result = newStringOfCap(runes.len)
+
+ for i, rep in repls:
+ result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
+ case rep.kind
+ of rkHashtag:
+ let
+ name = $runes[rep.slice.a.succ .. rep.slice.b]
+ symbol = $runes[rep.slice.a]
+ result.add a(symbol & name, href = "/search?q=%23" & name)
+ of rkMention:
+ result.add a($runes[rep.slice], href = rep.url, title = rep.display)
+ of rkUrl:
+ result.add a(rep.display, href = rep.url)
+ of rkRemove:
+ discard
+
+ let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
+ if rest.a <= rest.b:
+ result.add $runes[rest]
diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim
new file mode 100644
index 0000000..de55ed8
--- /dev/null
+++ b/src/experimental/parser/user.nim
@@ -0,0 +1,67 @@
+import std/[algorithm, unicode, re, strutils]
+import jsony
+import utils, slices
+import ../types/user as userType
+from ../../types import Profile, Error
+
+let
+ unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
+ unReplace = "$1@$2"
+
+ htRegex = re"(^|[^\w-_./?])([##$])([\w_]+)"
+ htReplace = "$1$2$3"
+
+proc expandProfileEntities(profile: var Profile; user: User) =
+ let
+ orig = profile.bio.toRunes
+ ent = user.entities
+
+ if ent.url.urls.len > 0:
+ profile.website = ent.url.urls[0].expandedUrl
+
+ var replacements = newSeq[ReplaceSlice]()
+
+ for u in ent.description.urls:
+ replacements.extractUrls(u, orig.high)
+
+ replacements.dedupSlices
+ replacements.sort(cmp)
+
+ profile.bio = orig.replacedWith(replacements, 0 .. orig.len)
+ .replacef(unRegex, unReplace)
+ .replacef(htRegex, htReplace)
+
+proc getBanner(user: User): string =
+ if user.profileBannerUrl.len > 0:
+ return user.profileBannerUrl & "/1500x500"
+ if user.profileLinkColor.len > 0:
+ return '#' & user.profileLinkColor
+
+proc parseUser*(json: string; username=""): Profile =
+ handleErrors:
+ case error.code
+ of suspended: return Profile(username: username, suspended: true)
+ of userNotFound: return
+ else: echo "[error - parseUser]: ", error
+
+ let user = json.fromJson(User)
+
+ result = Profile(
+ id: user.idStr,
+ username: user.screenName,
+ fullname: user.name,
+ location: user.location,
+ bio: user.description,
+ following: user.friendsCount,
+ followers: user.followersCount,
+ tweets: user.statusesCount,
+ likes: user.favouritesCount,
+ media: user.mediaCount,
+ verified: user.verified,
+ protected: user.protected,
+ joinDate: parseTwitterDate(user.createdAt),
+ banner: getBanner(user),
+ userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
+ )
+
+ result.expandProfileEntities(user)
diff --git a/src/experimental/parser/utils.nim b/src/experimental/parser/utils.nim
new file mode 100644
index 0000000..999683d
--- /dev/null
+++ b/src/experimental/parser/utils.nim
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import std/[sugar, strutils, times]
+import ../types/common
+import ../../utils as uutils
+
+template parseTime(time: string; f: static string; flen: int): DateTime =
+ if time.len != flen: return
+ parse(time, f, utc())
+
+proc parseIsoDate*(date: string): DateTime =
+ date.parseTime("yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
+
+proc parseTwitterDate*(date: string): DateTime =
+ date.parseTime("ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
+
+proc getImageUrl*(url: string): string =
+ url.dup(removePrefix(twimg), removePrefix(https))
+
+template handleErrors*(body) =
+ if json.startsWith("{\"errors"):
+ for error {.inject.} in json.fromJson(Errors).errors:
+ body
diff --git a/src/experimental/types/common.nim b/src/experimental/types/common.nim
new file mode 100644
index 0000000..e979015
--- /dev/null
+++ b/src/experimental/types/common.nim
@@ -0,0 +1,20 @@
+from ../../types import Error
+
+type
+ Url* = object
+ url*: string
+ expandedUrl*: string
+ displayUrl*: string
+ indices*: array[2, int]
+
+ ErrorObj* = object
+ code*: Error
+ message*: string
+
+ Errors* = object
+ errors*: seq[ErrorObj]
+
+proc contains*(codes: set[Error]; errors: Errors): bool =
+ for e in errors.errors:
+ if e.code in codes:
+ return true
diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim
new file mode 100644
index 0000000..1f31318
--- /dev/null
+++ b/src/experimental/types/user.nim
@@ -0,0 +1,28 @@
+import common
+
+type
+ User* = object
+ idStr*: string
+ name*: string
+ screenName*: string
+ location*: string
+ description*: string
+ entities*: Entities
+ createdAt*: string
+ followersCount*: int
+ friendsCount*: int
+ favouritesCount*: int
+ statusesCount*: int
+ mediaCount*: int
+ verified*: bool
+ protected*: bool
+ profileBannerUrl*: string
+ profileImageUrlHttps*: string
+ profileLinkColor*: string
+
+ Entities* = object
+ url*: Urls
+ description*: Urls
+
+ Urls* = object
+ urls*: seq[Url]
diff --git a/src/formatters.nim b/src/formatters.nim
index e572dec..d61fd85 100644
--- a/src/formatters.nim
+++ b/src/formatters.nim
@@ -33,23 +33,25 @@ proc getUrlPrefix*(cfg: Config): string =
if cfg.useHttps: https & cfg.hostname
else: "http://" & cfg.hostname
-proc stripHtml*(text: string): string =
+proc shortLink*(text: string; length=28): string =
+ result = text.replace(wwwRegex, "")
+ if result.len > length:
+ result = result[0 ..< length] & "…"
+
+proc stripHtml*(text: string; shorten=false): string =
var html = parseHtml(text)
for el in html.findAll("a"):
let link = el.attr("href")
if "http" in link:
if el.len == 0: continue
- el[0].text = link
+ el[0].text =
+ if shorten: link.shortLink
+ else: link
html.innerText()
proc sanitizeXml*(text: string): string =
text.replace(illegalXmlRegex, "")
-proc shortLink*(text: string; length=28): string =
- result = text.replace(wwwRegex, "")
- if result.len > length:
- result = result[0 ..< length] & "…"
-
proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
result = body
diff --git a/src/parser.nim b/src/parser.nim
index 6aa6a7a..0991091 100644
--- a/src/parser.nim
+++ b/src/parser.nim
@@ -14,11 +14,11 @@ proc parseProfile(js: JsonNode; id=""): Profile =
bio: js{"description"}.getStr,
userPic: js{"profile_image_url_https"}.getImageStr.replace("_normal", ""),
banner: js.getBanner,
- following: $js{"friends_count"}.getInt,
- followers: $js{"followers_count"}.getInt,
- tweets: $js{"statuses_count"}.getInt,
- likes: $js{"favourites_count"}.getInt,
- media: $js{"media_count"}.getInt,
+ following: js{"friends_count"}.getInt,
+ followers: js{"followers_count"}.getInt,
+ tweets: js{"statuses_count"}.getInt,
+ likes: js{"favourites_count"}.getInt,
+ media: js{"media_count"}.getInt,
verified: js{"verified"}.getBool,
protected: js{"protected"}.getBool,
joinDate: js{"created_at"}.getTime
@@ -26,21 +26,6 @@ proc parseProfile(js: JsonNode; id=""): Profile =
result.expandProfileEntities(js)
-proc parseUserShow*(js: JsonNode; username=""; id=""): Profile =
- if id.len > 0:
- result = Profile(id: id)
- else:
- result = Profile(username: username)
-
- if js.isNull: return
-
- with error, js{"errors"}:
- if error.getError == suspended:
- result.suspended = true
- return
-
- result = parseProfile(js)
-
proc parseGraphList*(js: JsonNode): List =
if js.isNull: return
diff --git a/src/parserutils.nim b/src/parserutils.nim
index aae3dfc..4b18202 100644
--- a/src/parserutils.nim
+++ b/src/parserutils.nim
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
-import strutils, times, macros, htmlgen, unicode, options, algorithm
-import std/re
+import std/[strutils, times, macros, htmlgen, options, algorithm, re]
+import std/unicode except strip
import packedjson
import types, utils, formatters
@@ -119,18 +119,6 @@ proc getBanner*(js: JsonNode): string =
if color.len > 0:
return '#' & color
- # use primary color from profile picture color histogram
- with p, js{"profile_image_extensions", "mediaColor", "r", "ok", "palette"}:
- if p.len > 0:
- let pal = p[0]{"rgb"}
- result = "#"
- result.add toHex(pal{"red"}.getInt, 2)
- result.add toHex(pal{"green"}.getInt, 2)
- result.add toHex(pal{"blue"}.getInt, 2)
- return
-
- return "#161616"
-
proc getTombstone*(js: JsonNode): string =
result = js{"tombstoneInfo", "richText", "text"}.getStr
result.removeSuffix(" Learn more")
@@ -275,3 +263,4 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
replacements.sort(cmp)
tweet.text = orig.replacedWith(replacements, textSlice)
+ .strip(leading=false)
diff --git a/src/redis_cache.nim b/src/redis_cache.nim
index dbe917c..fe622f9 100644
--- a/src/redis_cache.nim
+++ b/src/redis_cache.nim
@@ -13,6 +13,9 @@ var
rssCacheTime: int
listCacheTime*: int
+template dawait(future) =
+ discard await future
+
# flatty can't serialize DateTime, so we need to define this
proc toFlatty*(s: var string, x: DateTime) =
s.toFlatty(x.toTime().toUnix())
@@ -33,9 +36,9 @@ proc migrate*(key, match: string) {.async.} =
let list = await r.scan(newCursor(0), match, 100000)
r.startPipelining()
for item in list:
- discard await r.del(item)
+ dawait r.del(item)
await r.setk(key, "true")
- discard await r.flushPipeline()
+ dawait r.flushPipeline()
proc initRedisPool*(cfg: Config) {.async.} =
try:
@@ -47,6 +50,7 @@ proc initRedisPool*(cfg: Config) {.async.} =
await migrate("snappyRss", "rss:*")
await migrate("userBuckets", "p:*")
await migrate("profileDates", "p:*")
+ await migrate("profileStats", "p:*")
pool.withAcquire(r):
# optimize memory usage for profile ID buckets
@@ -67,7 +71,7 @@ proc get(query: string): Future[string] {.async.} =
proc setEx(key: string; time: int; data: string) {.async.} =
pool.withAcquire(r):
- discard await r.setEx(key, time, data)
+ dawait r.setEx(key, time, data)
proc cache*(data: List) {.async.} =
await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
@@ -76,29 +80,29 @@ proc cache*(data: PhotoRail; name: string) {.async.} =
await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
proc cache*(data: Profile) {.async.} =
- if data.username.len == 0 or data.id.len == 0: return
+ if data.username.len == 0: return
let name = toLower(data.username)
pool.withAcquire(r):
r.startPipelining()
- discard await r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
- discard await r.setEx("i:" & data.id , baseCacheTime, data.username)
- discard await r.hSet(name.pidKey, name, data.id)
- discard await r.flushPipeline()
+ dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
+ if data.id.len > 0:
+ dawait r.hSet(name.pidKey, name, data.id)
+ dawait r.flushPipeline()
-proc cacheProfileId*(username, id: string) {.async.} =
+proc cacheProfileId(username, id: string) {.async.} =
if username.len == 0 or id.len == 0: return
let name = toLower(username)
pool.withAcquire(r):
- discard await r.hSet(name.pidKey, name, id)
+ dawait r.hSet(name.pidKey, name, id)
proc cacheRss*(query: string; rss: Rss) {.async.} =
let key = "rss:" & query
pool.withAcquire(r):
r.startPipelining()
- discard await r.hSet(key, "rss", rss.feed)
- discard await r.hSet(key, "min", rss.cursor)
- discard await r.expire(key, rssCacheTime)
- discard await r.flushPipeline()
+ dawait r.hSet(key, "rss", rss.feed)
+ dawait r.hSet(key, "min", rss.cursor)
+ dawait r.expire(key, rssCacheTime)
+ dawait r.flushPipeline()
proc getProfileId*(username: string): Future[string] {.async.} =
let name = toLower(username)
@@ -113,15 +117,21 @@ proc getCachedProfile*(username: string; fetch=true): Future[Profile] {.async.}
result = fromFlatty(uncompress(prof), Profile)
elif fetch:
result = await getProfile(username)
+ await cacheProfileId(result.username, result.id)
+ if result.suspended:
+ await cache(result)
proc getCachedProfileUsername*(userId: string): Future[string] {.async.} =
- let username = await get("i:" & userId)
+ let
+ key = "i:" & userId
+ username = await get(key)
+
if username != redisNil:
result = username
else:
let profile = await getProfileById(userId)
result = profile.username
- await cache(profile)
+ await setEx(key, baseCacheTime, result)
proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
diff --git a/src/routes/embed.nim b/src/routes/embed.nim
index d9a9ee9..1a93d40 100644
--- a/src/routes/embed.nim
+++ b/src/routes/embed.nim
@@ -1,9 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, strutils, options
-import jester
-import ".."/[types, api], ../views/embed
+import jester, karax/vdom
+import ".."/[types, api]
+import ../views/[embed, tweet, general]
+import router_utils
-export api, embed
+export api, embed, vdom, tweet, general, router_utils
proc createEmbedRouter*(cfg: Config) =
router embed:
@@ -12,4 +14,23 @@ proc createEmbedRouter*(cfg: Config) =
if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
resp Http404
- resp renderVideoEmbed(cfg, convo.tweet)
+ resp renderVideoEmbed(convo.tweet, cfg, request)
+
+ get "/@user/status/@id/embed":
+ let
+ convo = await getTweet(@"id")
+ prefs = cookiePrefs()
+ path = getPath()
+
+ if convo == nil or convo.tweet == nil:
+ resp Http404
+
+ resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
+
+ get "/embed/Tweet.html":
+ let id = @"id"
+
+ if id.len > 0:
+ redirect("/i/status/" & id & "/embed")
+ else:
+ resp Http404
diff --git a/src/routes/media.nim b/src/routes/media.nim
index a2a6369..c953a93 100644
--- a/src/routes/media.nim
+++ b/src/routes/media.nim
@@ -5,7 +5,7 @@ import asynchttpserver, asyncstreams, asyncfile, asyncnet
import jester
import router_utils
-import ".."/[types, formatters, agents, utils]
+import ".."/[types, formatters, utils]
export asynchttpserver, asyncstreams, asyncfile, asyncnet
export httpclient, os, strutils, asyncstreams, base64, re
@@ -14,10 +14,8 @@ const
m3u8Mime* = "application/vnd.apple.mpegurl"
maxAge* = "max-age=604800"
-let mediaAgent* = getAgent()
-
-proc safeFetch*(url, agent: string): Future[string] {.async.} =
- let client = newAsyncHttpClient(userAgent=agent)
+proc safeFetch*(url: string): Future[string] {.async.} =
+ let client = newAsyncHttpClient()
try: result = await client.getContent(url)
except: discard
finally: client.close()
@@ -34,7 +32,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
result = Http200
let
request = req.getNativeReq()
- client = newAsyncHttpClient(userAgent=mediaAgent)
+ client = newAsyncHttpClient()
try:
let res = await client.get(url)
@@ -116,14 +114,14 @@ proc createMediaRouter*(cfg: Config) =
var content: string
if ".vmap" in url:
- let m3u8 = getM3u8Url(await safeFetch(url, mediaAgent))
+ let m3u8 = getM3u8Url(await safeFetch(url))
if m3u8.len > 0:
- content = await safeFetch(url, mediaAgent)
+ content = await safeFetch(url)
else:
resp Http404
if ".m3u8" in url:
- let vid = await safeFetch(url, mediaAgent)
+ let vid = await safeFetch(url)
content = proxifyVideo(vid, cookiePref(proxyVideos))
resp content, m3u8Mime
diff --git a/src/routes/router_utils.nim b/src/routes/router_utils.nim
index 7159890..a071a0d 100644
--- a/src/routes/router_utils.nim
+++ b/src/routes/router_utils.nim
@@ -4,12 +4,12 @@ from jester import Request, cookies
import ../views/general
import ".."/[utils, prefs, types]
-export utils, prefs, types
+export utils, prefs, types, uri
template savePref*(pref, value: string; req: Request; expire=false) =
if not expire or pref in cookies(req):
setCookie(pref, value, daysForward(when expire: -10 else: 360),
- httpOnly=true, secure=cfg.useHttps)
+ httpOnly=true, secure=cfg.useHttps, sameSite=None)
template cookiePrefs*(): untyped {.dirty.} =
getPrefs(cookies(request))
diff --git a/src/routes/rss.nim b/src/routes/rss.nim
index 5deb1cf..771a3ad 100644
--- a/src/routes/rss.nim
+++ b/src/routes/rss.nim
@@ -19,8 +19,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
names = getNames(name)
if names.len == 1:
- (profile, timeline) =
- await fetchSingleTimeline(after, query, skipRail=true)
+ (profile, timeline) = await fetchTimeline(after, query, skipRail=true)
else:
var q = query
q.fromUser = names
diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim
index 2992888..9eefc4e 100644
--- a/src/routes/timeline.nim
+++ b/src/routes/timeline.nim
@@ -19,8 +19,8 @@ proc getQuery*(request: Request; tab, name: string): Query =
of "search": initQuery(params(request), name=name)
else: Query(fromUser: @[name])
-proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
- Future[(Profile, Timeline, PhotoRail)] {.async.} =
+proc fetchTimeline*(after: string; query: Query; skipRail=false):
+ Future[(Profile, Timeline, PhotoRail)] {.async.} =
let name = query.fromUser[0]
var
@@ -30,20 +30,13 @@ proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
if profileId.len == 0:
profile = await getCachedProfile(name)
- profileId = if profile.suspended: "s"
- else: profile.id
-
- if profileId.len > 0:
- await cacheProfileId(profile.username, profileId)
-
+ profileId = profile.id
fetched = true
- if profileId.len == 0 or profile.protected:
- result[0] = profile
- return
- elif profileId == "s":
- result[0] = Profile(username: name, suspended: true)
- return
+ if profile.protected or profile.suspended:
+ return (profile, Timeline(), @[])
+ elif profileId.len == 0:
+ return (Profile(username: name), Timeline(), @[])
var rail: Future[PhotoRail]
if skipRail or profile.protected or query.kind == media:
@@ -86,7 +79,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
html = renderTweetSearch(timeline, prefs, getPath())
return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
- var (p, t, r) = await fetchSingleTimeline(after, query)
+ var (p, t, r) = await fetchTimeline(after, query)
if p.suspended: return showError(getSuspended(p.username), cfg)
if p.id.len == 0: return
@@ -139,7 +132,7 @@ proc createTimelineRouter*(cfg: Config) =
timeline.beginning = true
resp $renderTweetSearch(timeline, prefs, getPath())
else:
- var (_, timeline, _) = await fetchSingleTimeline(after, query, skipRail=true)
+ var (_, timeline, _) = await fetchTimeline(after, query, skipRail=true)
if timeline.content.len == 0: resp Http404
timeline.beginning = true
resp $renderTimelineTweets(timeline, prefs, getPath())
diff --git a/src/sass/profile/_base.scss b/src/sass/profile/_base.scss
index 6a2cfec..ae6b801 100644
--- a/src/sass/profile/_base.scss
+++ b/src/sass/profile/_base.scss
@@ -15,23 +15,22 @@
}
.profile-banner {
- padding-bottom: 4px;
+ margin-bottom: 4px;
+ background-color: var(--bg_panel);
a {
- display: inherit;
- line-height: 0;
+ display: block;
+ position: relative;
+ padding: 33.34% 0 0 0;
}
img {
- width: 100%;
+ max-width: 100%;
+ position: absolute;
+ top: 0;
}
}
-.profile-banner-color {
- width: 100%;
- padding-bottom: 25%;
-}
-
.profile-tab {
padding: 0 4px 0 0;
box-sizing: border-box;
diff --git a/src/sass/profile/card.scss b/src/sass/profile/card.scss
index 9475fbf..cc68d7d 100644
--- a/src/sass/profile/card.scss
+++ b/src/sass/profile/card.scss
@@ -35,19 +35,25 @@
}
.profile-card-avatar {
- display: block;
+ display: inline-block;
+ position: relative;
width: 100%;
- padding-bottom: 6px;
margin-right: 4px;
+ margin-bottom: 6px;
+
+ &:after {
+ content: '';
+ display: block;
+ margin-top: 100%;
+ }
img {
- display: block;
box-sizing: border-box;
+ position: absolute;
width: 100%;
height: 100%;
- margin: 0;
border: 4px solid var(--darker_grey);
- background: var(--bg_color);
+ background: var(--bg_panel);
}
}
@@ -113,8 +119,8 @@
}
.profile-card-avatar {
- height: 60px;
- width: unset;
+ width: 80px;
+ height: 80px;
img {
border-width: 2px;
diff --git a/src/sass/profile/photo-rail.scss b/src/sass/profile/photo-rail.scss
index 314eadf..f32d22a 100644
--- a/src/sass/profile/photo-rail.scss
+++ b/src/sass/profile/photo-rail.scss
@@ -32,6 +32,7 @@
a {
position: relative;
border-radius: 5px;
+ background-color: var(--darker_grey);
&:before {
content: "";
diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss
index 80a1171..7c1196a 100644
--- a/src/sass/tweet/_base.scss
+++ b/src/sass/tweet/_base.scss
@@ -98,11 +98,14 @@
}
.avatar {
+ position: absolute;
+
&.round {
border-radius: 50%;
}
&.mini {
+ position: unset;
margin-right: 5px;
margin-top: -1px;
width: 20px;
@@ -110,6 +113,24 @@
}
}
+.tweet-embed {
+ display: flex;
+ flex-direction: column;
+ justify-content: center;
+ height: 100%;
+ background-color: var(--bg_panel);
+
+ .tweet-content {
+ font-size: 18px
+ }
+
+ .tweet-body {
+ display: flex;
+ flex-direction: column;
+ max-height: calc(100vh - 0.75em * 2);
+ }
+}
+
.attribution {
display: flex;
pointer-events: all;
diff --git a/src/tokens.nim b/src/tokens.nim
index 929a984..d864a4c 100644
--- a/src/tokens.nim
+++ b/src/tokens.nim
@@ -2,12 +2,12 @@
import asyncdispatch, httpclient, times, sequtils, json, random
import strutils, tables
import zippy
-import types, agents, consts, http_pool
+import types, consts, http_pool
const
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
- maxAge = 3.hours # tokens expire after 3 hours
maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
+ maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
failDelay = initDuration(minutes=30)
var
@@ -65,7 +65,6 @@ proc fetchToken(): Future[Token] {.async.} =
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.5",
"connection": "keep-alive",
- "user-agent": getAgent(),
"authorization": auth
})
diff --git a/src/types.nim b/src/types.nim
index 1903cf8..5ee2c28 100644
--- a/src/types.nim
+++ b/src/types.nim
@@ -48,17 +48,16 @@ type
id*: string
username*: string
fullname*: string
- lowername*: string
location*: string
website*: string
bio*: string
userPic*: string
banner*: string
- following*: string
- followers*: string
- tweets*: string
- likes*: string
- media*: string
+ following*: int
+ followers*: int
+ tweets*: int
+ likes*: int
+ media*: int
verified*: bool
protected*: bool
suspended*: bool
diff --git a/src/views/embed.nim b/src/views/embed.nim
index 4c2f7b3..e6afffd 100644
--- a/src/views/embed.nim
+++ b/src/views/embed.nim
@@ -1,18 +1,19 @@
# SPDX-License-Identifier: AGPL-3.0-only
import options
import karax/[karaxdsl, vdom]
+from jester import Request
import ".."/[types, formatters]
import general, tweet
const doctype = "\n"
-proc renderVideoEmbed*(cfg: Config; tweet: Tweet): string =
+proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
let thumb = get(tweet.video).thumb
let vidUrl = getVideoEmbed(cfg, tweet.id)
let prefs = Prefs(hlsPlayback: true)
let node = buildHtml(html(lang="en")):
- renderHead(prefs, cfg, video=vidUrl, images=(@[thumb]))
+ renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
tdiv(class="embed-video"):
renderVideo(get(tweet.video), prefs, "")
diff --git a/src/views/general.nim b/src/views/general.nim
index 4a8b4a3..7054fd4 100644
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -11,6 +11,9 @@ const
doctype = "\n"
lp = readFile("public/lp.svg")
+proc toTheme(theme: string): string =
+ theme.toLowerAscii.replace(" ", "_")
+
proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
var path = req.params.getOrDefault("referer")
if path.len == 0:
@@ -33,9 +36,13 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
icon "info", title="About", href="/about"
icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
-proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
- images: seq[string] = @[]; banner=""; ogTitle=""; theme="";
+proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
+ video=""; images: seq[string] = @[]; banner=""; ogTitle="";
rss=""; canonical=""): VNode =
+ var theme = prefs.theme.toTheme
+ if "theme" in req.params:
+ theme = req.params["theme"].toTheme
+
let ogType =
if video.len > 0: "video"
elif rss.len > 0: "object"
@@ -45,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
buildHtml(head):
- link(rel="stylesheet", type="text/css", href="/css/style.css?v=10")
+ link(rel="stylesheet", type="text/css", href="/css/style.css?v=15")
link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
if theme.len > 0:
@@ -118,15 +125,12 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
titleText=""; desc=""; ogTitle=""; rss=""; video="";
images: seq[string] = @[]; banner=""): string =
- var theme = toLowerAscii(prefs.theme).replace(" ", "_")
- if "theme" in req.params:
- theme = toLowerAscii(req.params["theme"]).replace(" ", "_")
let canonical = getTwitterLink(req.path, req.params)
let node = buildHtml(html(lang="en")):
- renderHead(prefs, cfg, titleText, desc, video, images, banner, ogTitle,
- theme, rss, canonical)
+ renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
+ rss, canonical)
body:
renderNavbar(cfg, req, rss, canonical)
diff --git a/src/views/profile.nim b/src/views/profile.nim
index 9481faf..e44b8a2 100644
--- a/src/views/profile.nim
+++ b/src/views/profile.nim
@@ -5,12 +5,12 @@ import karax/[karaxdsl, vdom, vstyles]
import renderutils, search
import ".."/[types, utils, formatters]
-proc renderStat(num, class: string; text=""): VNode =
+proc renderStat(num: int; class: string; text=""): VNode =
let t = if text.len > 0: text else: class
buildHtml(li(class=class)):
span(class="profile-stat-header"): text capitalizeAscii(t)
span(class="profile-stat-num"):
- text if num.len == 0: "?" else: insertSep(num, ',')
+ text insertSep($num, ',')
proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
buildHtml(tdiv(class="profile-card")):
@@ -78,18 +78,16 @@ proc renderPhotoRail(profile: Profile; photoRail: PhotoRail): VNode =
tdiv(class="photo-rail-grid"):
for i, photo in photoRail:
if i == 16: break
- let col = if photo.color.len > 0: photo.color else: "#161616"
- a(href=(&"/{profile.username}/status/{photo.tweetId}#m"),
- style={backgroundColor: col}):
+ a(href=(&"/{profile.username}/status/{photo.tweetId}#m")):
genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
-proc renderBanner(profile: Profile): VNode =
+proc renderBanner(banner: string): VNode =
buildHtml():
- if "#" in profile.banner:
- tdiv(class="profile-banner-color", style={backgroundColor: profile.banner})
+ if banner.startsWith('#'):
+ a(style={backgroundColor: banner})
else:
- a(href=getPicUrl(profile.banner), target="_blank"):
- genImg(profile.banner)
+ a(href=getPicUrl(banner), target="_blank"):
+ genImg(banner)
proc renderProtected(username: string): VNode =
buildHtml(tdiv(class="timeline-container")):
@@ -103,7 +101,8 @@ proc renderProfile*(profile: Profile; timeline: var Timeline;
buildHtml(tdiv(class="profile-tabs")):
if not prefs.hideBanner:
tdiv(class="profile-banner"):
- renderBanner(profile)
+ if profile.banner.len > 0:
+ renderBanner(profile.banner)
let sticky = if prefs.stickyProfile: " sticky" else: ""
tdiv(class=(&"profile-tab{sticky}")):
diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index aa376b9..5180eef 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -1,9 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, sequtils, strformat, options
import karax/[karaxdsl, vdom, vstyles]
+from jester import Request
import renderutils
import ".."/[types, utils, formatters]
+import general
proc getSmallPic(url: string): string =
result = url
@@ -353,3 +355,8 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
if showThread:
a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
text "Show this thread"
+
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
+ buildHtml(tdiv(class="tweet-embed")):
+ renderHead(prefs, cfg, req)
+ renderTweet(tweet, prefs, path, mainTweet=true)