Merge branch 'master' of https://github.com/zedeus/nitter

Fixes #167

.github/workflows/build-docker.yml

@@ -33,6 +33,6 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}

config.nims

@@ -7,6 +7,7 @@
 # disable annoying warnings
 warning("GcUnsafe2", off)
 hint("XDeclaredButNotUsed", off)
+hint("XCannotRaiseY", off)
 hint("User", off)
 
 const

nitter.nimble

@@ -22,7 +22,7 @@ requires "redpool#f880f49"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#0.7.3"
 requires "flatty#0.2.3"
-requires "jsony#1.1.3"
+requires "jsony#d0e69bd"
 
 
 # Tasks

src/agents.nim

@@ -1,70 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import random, strformat, strutils, sequtils
-
-randomize()
-
-const rvs = [
-  "11.0", "40.0", "42.0", "43.0", "47.0", "50.0", "52.0", "53.0", "54.0",
-  "61.0", "66.0", "67.0", "69.0", "70.0"
-]
-
-proc rv(): string =
-  if rand(10) < 1: ""
-  else: "; rv:" & sample(rvs)
-
-# OS
-
-const enc = ["; U", "; N", "; I", ""]
-
-proc linux(): string =
-  const
-    window = ["X11", "Wayland", "Unknown"]
-    arch = ["i686", "x86_64", "arm"]
-    distro = ["", "; Ubuntu/14.10", "; Ubuntu/16.10", "; Ubuntu/19.10",
-              "; Ubuntu", "; Fedora"]
-  sample(window) & sample(enc) & "; Linux " & sample(arch) & sample(distro)
-
-proc windows(): string =
-  const
-    nt = ["5.1", "5.2", "6.0", "6.1", "6.2", "6.3", "6.4", "9.0", "10.0"]
-    arch = ["; WOW64", "; Win64; x64", "; ARM", ""]
-    trident = ["", "; Trident/5.0", "; Trident/6.0", "; Trident/7.0"]
-  "Windows " & sample(nt) & sample(enc) & sample(arch) & sample(trident)
-
-const macs = toSeq(6..15).mapIt($it) & @["14_4", "10_1", "9_3"]
-
-proc mac(): string =
-  "Macintosh; Intel Mac OS X 10_" & sample(macs) & sample(enc)
-
-# Browser
-
-proc presto(): string =
-  const p = ["2.12.388", "2.12.407", "22.9.168", "2.9.201", "2.8.131", "2.7.62",
-             "2.6.30", "2.5.24"]
-  const v = ["10.0", "11.0", "11.1", "11.5", "11.6", "12.00", "12.14", "12.16"]
-  &"Presto/{sample(p)} Version/{sample(v)}"
-
-# Samples
-
-proc product(): string =
-  const opera = ["Opera/9.80", "Opera/12.0"]
-  if rand(20) < 1: "Mozilla/5.0"
-  else: sample(opera)
-
-proc os(): string =
-  let r = rand(10)
-  let os =
-    if r < 6: windows()
-    elif r < 9: linux()
-    else: mac()
-  &"({os}{rv()})"
-
-proc browser(prod: string): string =
-  if "Opera" in prod: presto()
-  else: "like Gecko"
-
-# Agent
-
-proc getAgent*(): string =
-  let prod = product()
-  &"{prod} {os()} {browser(prod)}"

src/api.nim

@@ -2,6 +2,7 @@
 import asyncdispatch, httpclient, uri, strutils
 import packedjson
 import types, query, formatters, consts, apiutils, parser
+import experimental/parser/user
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
@@ -32,14 +33,14 @@ proc getListMembers*(list: List; after=""): Future[Result[Profile]] {.async.} =
 proc getProfile*(username: string): Future[Profile] {.async.} =
   let
     ps = genParams({"screen_name": username})
-    js = await fetch(userShow ? ps, Api.userShow)
-  result = parseUserShow(js, username=username)
+    json = await fetchRaw(userShow ? ps, Api.userShow)
+  result = parseUser(json, username)
 
 proc getProfileById*(userId: string): Future[Profile] {.async.} =
   let
     ps = genParams({"user_id": userId})
-    js = await fetch(userShow ? ps, Api.userShow)
-  result = parseUserShow(js, id=userId)
+    json = await fetchRaw(userShow ? ps, Api.userShow)
+  result = parseUser(json)
 
 proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
   let

src/apiutils.nim

@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, times, strutils, uri
-import packedjson, zippy
+import httpclient, asyncdispatch, options, strutils, uri
+import jsony, packedjson, zippy
 import types, tokens, consts, parserutils, http_pool
+import experimental/types/common
 
 const
   rlRemaining = "x-rate-limit-remaining"
@@ -16,6 +17,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
     result &= p
   if ext:
     result &= ("ext", "mediaStats")
+    result &= ("include_ext_alt_text", "true")
+    result &= ("include_ext_media_availability", "true")
   if count.len > 0:
     result &= ("count", count)
   if cursor.len > 0:
@@ -40,7 +43,14 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
     "DNT": "1"
   })
 
-proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+template updateToken() =
+  if api != Api.search and resp.headers.hasKey(rlRemaining):
+    let
+      remaining = parseInt(resp.headers[rlRemaining])
+      reset = parseInt(resp.headers[rlReset])
+    token.setRateLimit(api, remaining, reset)
+
+template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
@@ -48,37 +58,21 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
   if token.tok.len == 0:
     raise rateLimitError()
 
-  let headers = genHeaders(token)
   try:
     var resp: AsyncResponse
-    var body = pool.use(headers):
+    result = pool.use(genHeaders(token)):
       resp = await c.get($url)
       await resp.body
 
-    if body.len > 0:
+    if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
-        body = uncompress(body, dfGzip)
+        result = uncompress(result, dfGzip)
       else:
-        echo "non-gzip body, url: ", url, ", body: ", body
+        echo "non-gzip body, url: ", url, ", body: ", result
 
-    if body.startsWith('{') or body.startsWith('['):
-      result = parseJson(body)
-    else:
-      echo resp.status, ": ", body
-      result = newJNull()
+    fetchBody
 
-    if api != Api.search and resp.headers.hasKey(rlRemaining):
-      let
-        remaining = parseInt(resp.headers[rlRemaining])
-        reset = parseInt(resp.headers[rlReset])
-      token.setRateLimit(api, remaining, reset)
-
-    if result.getError notin {invalidToken, forbidden, badToken}:
-      release(token, used=true)
-    else:
-      echo "fetch error: ", result.getError
-      release(token, invalid=true)
-      raise rateLimitError()
+    release(token, used=true)
 
     if resp.status == $Http400:
       raise newException(InternalError, $url)
@@ -89,3 +83,35 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
     if "length" notin e.msg and "descriptor" notin e.msg:
       release(token, invalid=true)
     raise rateLimitError()
+
+proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+  var body: string
+  fetchImpl body:
+    if body.startsWith('{') or body.startsWith('['):
+      result = parseJson(body)
+    else:
+      echo resp.status, ": ", body
+      result = newJNull()
+
+    updateToken()
+
+    let error = result.getError
+    if error in {invalidToken, forbidden, badToken}:
+      echo "fetch error: ", result.getError
+      release(token, invalid=true)
+      raise rateLimitError()
+
+proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
+  fetchImpl result:
+    if not (result.startsWith('{') or result.startsWith('[')):
+      echo resp.status, ": ", result
+      result.setLen(0)
+
+    updateToken()
+
+    if result.startsWith("{\"errors"):
+      let errors = result.fromJson(Errors)
+      if errors in {invalidToken, forbidden, badToken}:
+        echo "fetch error: ", errors
+        release(token, invalid=true)
+        raise rateLimitError()

src/consts.nim

@@ -35,16 +35,13 @@ const
     "cards_platform": "Web-12",
     "include_cards": "1",
     "include_composer_source": "false",
-    "include_ext_alt_text": "true",
     "include_reply_count": "1",
     "tweet_mode": "extended",
     "include_entities": "true",
     "include_user_entities": "true",
     "include_ext_media_color": "false",
-    "include_ext_media_availability": "true",
     "send_error_codes": "true",
     "simple_quoted_tweet": "true",
-    "ext": "mediaStats",
     "include_quote_count": "true"
   }.toSeq
 

src/experimental/parser/slices.nim

@@ -0,0 +1,67 @@
+import std/[macros, htmlgen, unicode]
+import ../types/common
+import ".."/../[formatters, utils]
+
+type
+  ReplaceSliceKind = enum
+    rkRemove, rkUrl, rkHashtag, rkMention
+
+  ReplaceSlice* = object
+    slice: Slice[int]
+    kind: ReplaceSliceKind
+    url, display: string
+
+proc cmp*(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
+
+proc dedupSlices*(s: var seq[ReplaceSlice]) =
+  var
+    len = s.len
+    i = 0
+  while i < len:
+    var j = i + 1
+    while j < len:
+      if s[i].slice.a == s[j].slice.a:
+        s.del j
+        dec len
+      else:
+        inc j
+    inc i
+
+proc extractUrls*(result: var seq[ReplaceSlice]; url: Url;
+                  textLen: int; hideTwitter = false) =
+  let
+    link = url.expandedUrl
+    slice = url.indices[0] ..< url.indices[1]
+
+  if hideTwitter and slice.b.succ >= textLen and link.isTwitterUrl:
+    if slice.a < textLen:
+      result.add ReplaceSlice(kind: rkRemove, slice: slice)
+  else:
+    result.add ReplaceSlice(kind: rkUrl, url: link,
+                            display: link.shortLink, slice: slice)
+
+proc replacedWith*(runes: seq[Rune]; repls: openArray[ReplaceSlice];
+                   textSlice: Slice[int]): string =
+  template extractLowerBound(i: int; idx): int =
+    if i > 0: repls[idx].slice.b.succ else: textSlice.a
+
+  result = newStringOfCap(runes.len)
+
+  for i, rep in repls:
+    result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
+    case rep.kind
+    of rkHashtag:
+      let
+        name = $runes[rep.slice.a.succ .. rep.slice.b]
+        symbol = $runes[rep.slice.a]
+      result.add a(symbol & name, href = "/search?q=%23" & name)
+    of rkMention:
+      result.add a($runes[rep.slice], href = rep.url, title = rep.display)
+    of rkUrl:
+      result.add a(rep.display, href = rep.url)
+    of rkRemove:
+      discard
+
+  let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
+  if rest.a <= rest.b:
+    result.add $runes[rest]

src/experimental/parser/user.nim

@@ -0,0 +1,67 @@
+import std/[algorithm, unicode, re, strutils]
+import jsony
+import utils, slices
+import ../types/user as userType
+from ../../types import Profile, Error
+
+let
+  unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
+  unReplace = "$1<a href=\"/$2\">@$2</a>"
+
+  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
+  htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
+
+proc expandProfileEntities(profile: var Profile; user: User) =
+  let
+    orig = profile.bio.toRunes
+    ent = user.entities
+
+  if ent.url.urls.len > 0:
+    profile.website = ent.url.urls[0].expandedUrl
+
+  var replacements = newSeq[ReplaceSlice]()
+
+  for u in ent.description.urls:
+    replacements.extractUrls(u, orig.high)
+
+  replacements.dedupSlices
+  replacements.sort(cmp)
+
+  profile.bio = orig.replacedWith(replacements, 0 .. orig.len)
+                    .replacef(unRegex, unReplace)
+                    .replacef(htRegex, htReplace)
+
+proc getBanner(user: User): string =
+  if user.profileBannerUrl.len > 0:
+    return user.profileBannerUrl & "/1500x500"
+  if user.profileLinkColor.len > 0:
+    return '#' & user.profileLinkColor
+
+proc parseUser*(json: string; username=""): Profile =
+  handleErrors:
+    case error.code
+    of suspended: return Profile(username: username, suspended: true)
+    of userNotFound: return
+    else: echo "[error - parseUser]: ", error
+
+  let user = json.fromJson(User)
+
+  result = Profile(
+    id: user.idStr,
+    username: user.screenName,
+    fullname: user.name,
+    location: user.location,
+    bio: user.description,
+    following: user.friendsCount,
+    followers: user.followersCount,
+    tweets: user.statusesCount,
+    likes: user.favouritesCount,
+    media: user.mediaCount,
+    verified: user.verified,
+    protected: user.protected,
+    joinDate: parseTwitterDate(user.createdAt),
+    banner: getBanner(user),
+    userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
+  )
+
+  result.expandProfileEntities(user)

src/experimental/parser/utils.nim

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import std/[sugar, strutils, times]
+import ../types/common
+import ../../utils as uutils
+
+template parseTime(time: string; f: static string; flen: int): DateTime =
+  if time.len != flen: return
+  parse(time, f, utc())
+
+proc parseIsoDate*(date: string): DateTime =
+  date.parseTime("yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
+
+proc parseTwitterDate*(date: string): DateTime =
+  date.parseTime("ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
+
+proc getImageUrl*(url: string): string =
+  url.dup(removePrefix(twimg), removePrefix(https))
+
+template handleErrors*(body) =
+  if json.startsWith("{\"errors"):
+    for error {.inject.} in json.fromJson(Errors).errors:
+      body

src/experimental/types/common.nim

@@ -0,0 +1,20 @@
+from ../../types import Error
+
+type
+  Url* = object
+    url*: string
+    expandedUrl*: string
+    displayUrl*: string
+    indices*: array[2, int]
+
+  ErrorObj* = object
+    code*: Error
+    message*: string
+
+  Errors* = object
+    errors*: seq[ErrorObj]
+
+proc contains*(codes: set[Error]; errors: Errors): bool =
+  for e in errors.errors:
+    if e.code in codes:
+      return true

src/experimental/types/user.nim

@@ -0,0 +1,28 @@
+import common
+
+type
+  User* = object
+    idStr*: string
+    name*: string
+    screenName*: string
+    location*: string
+    description*: string
+    entities*: Entities
+    createdAt*: string
+    followersCount*: int
+    friendsCount*: int
+    favouritesCount*: int
+    statusesCount*: int
+    mediaCount*: int
+    verified*: bool
+    protected*: bool
+    profileBannerUrl*: string
+    profileImageUrlHttps*: string
+    profileLinkColor*: string
+
+  Entities* = object
+    url*: Urls
+    description*: Urls
+
+  Urls* = object
+    urls*: seq[Url]

src/formatters.nim

@@ -33,23 +33,25 @@ proc getUrlPrefix*(cfg: Config): string =
   if cfg.useHttps: https & cfg.hostname
   else: "http://" & cfg.hostname
 
-proc stripHtml*(text: string): string =
+proc shortLink*(text: string; length=28): string =
+  result = text.replace(wwwRegex, "")
+  if result.len > length:
+    result = result[0 ..< length] & "…"
+    
+proc stripHtml*(text: string; shorten=false): string =
   var html = parseHtml(text)
   for el in html.findAll("a"):
     let link = el.attr("href")
     if "http" in link:
       if el.len == 0: continue
-      el[0].text = link
+      el[0].text =
+        if shorten: link.shortLink
+        else: link
   html.innerText()
 
 proc sanitizeXml*(text: string): string =
   text.replace(illegalXmlRegex, "")
 
-proc shortLink*(text: string; length=28): string =
-  result = text.replace(wwwRegex, "")
-  if result.len > length:
-    result = result[0 ..< length] & "…"
-
 proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   result = body
 

src/parser.nim

@@ -14,11 +14,11 @@ proc parseProfile(js: JsonNode; id=""): Profile =
     bio: js{"description"}.getStr,
     userPic: js{"profile_image_url_https"}.getImageStr.replace("_normal", ""),
     banner: js.getBanner,
-    following: $js{"friends_count"}.getInt,
-    followers: $js{"followers_count"}.getInt,
-    tweets: $js{"statuses_count"}.getInt,
-    likes: $js{"favourites_count"}.getInt,
-    media: $js{"media_count"}.getInt,
+    following: js{"friends_count"}.getInt,
+    followers: js{"followers_count"}.getInt,
+    tweets: js{"statuses_count"}.getInt,
+    likes: js{"favourites_count"}.getInt,
+    media: js{"media_count"}.getInt,
     verified: js{"verified"}.getBool,
     protected: js{"protected"}.getBool,
     joinDate: js{"created_at"}.getTime
@@ -26,21 +26,6 @@ proc parseProfile(js: JsonNode; id=""): Profile =
 
   result.expandProfileEntities(js)
 
-proc parseUserShow*(js: JsonNode; username=""; id=""): Profile =
-  if id.len > 0:
-    result = Profile(id: id)
-  else:
-    result = Profile(username: username)
-
-  if js.isNull: return
-
-  with error, js{"errors"}:
-    if error.getError == suspended:
-      result.suspended = true
-    return
-
-  result = parseProfile(js)
-
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 

src/parserutils.nim

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, times, macros, htmlgen, unicode, options, algorithm
-import std/re
+import std/[strutils, times, macros, htmlgen, options, algorithm, re]
+import std/unicode except strip
 import packedjson
 import types, utils, formatters
 
@@ -119,18 +119,6 @@ proc getBanner*(js: JsonNode): string =
   if color.len > 0:
     return '#' & color
 
-  # use primary color from profile picture color histogram
-  with p, js{"profile_image_extensions", "mediaColor", "r", "ok", "palette"}:
-    if p.len > 0:
-      let pal = p[0]{"rgb"}
-      result = "#"
-      result.add toHex(pal{"red"}.getInt, 2)
-      result.add toHex(pal{"green"}.getInt, 2)
-      result.add toHex(pal{"blue"}.getInt, 2)
-      return
-
-  return "#161616"
-
 proc getTombstone*(js: JsonNode): string =
   result = js{"tombstoneInfo", "richText", "text"}.getStr
   result.removeSuffix(" Learn more")
@@ -275,3 +263,4 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   replacements.sort(cmp)
 
   tweet.text = orig.replacedWith(replacements, textSlice)
+                   .strip(leading=false)

src/redis_cache.nim

@@ -13,6 +13,9 @@ var
   rssCacheTime: int
   listCacheTime*: int
 
+template dawait(future) =
+  discard await future
+
 # flatty can't serialize DateTime, so we need to define this
 proc toFlatty*(s: var string, x: DateTime) =
   s.toFlatty(x.toTime().toUnix())
@@ -33,9 +36,9 @@ proc migrate*(key, match: string) {.async.} =
       let list = await r.scan(newCursor(0), match, 100000)
       r.startPipelining()
       for item in list:
-        discard await r.del(item)
+        dawait r.del(item)
       await r.setk(key, "true")
-      discard await r.flushPipeline()
+      dawait r.flushPipeline()
 
 proc initRedisPool*(cfg: Config) {.async.} =
   try:
@@ -47,6 +50,7 @@ proc initRedisPool*(cfg: Config) {.async.} =
     await migrate("snappyRss", "rss:*")
     await migrate("userBuckets", "p:*")
     await migrate("profileDates", "p:*")
+    await migrate("profileStats", "p:*")
 
     pool.withAcquire(r):
       # optimize memory usage for profile ID buckets
@@ -67,7 +71,7 @@ proc get(query: string): Future[string] {.async.} =
 
 proc setEx(key: string; time: int; data: string) {.async.} =
   pool.withAcquire(r):
-    discard await r.setEx(key, time, data)
+    dawait r.setEx(key, time, data)
 
 proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
@@ -76,29 +80,29 @@ proc cache*(data: PhotoRail; name: string) {.async.} =
   await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: Profile) {.async.} =
-  if data.username.len == 0 or data.id.len == 0: return
+  if data.username.len == 0: return
   let name = toLower(data.username)
   pool.withAcquire(r):
     r.startPipelining()
-    discard await r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
-    discard await r.setEx("i:" & data.id , baseCacheTime, data.username)
-    discard await r.hSet(name.pidKey, name, data.id)
-    discard await r.flushPipeline()
+    dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
+    if data.id.len > 0:
+      dawait r.hSet(name.pidKey, name, data.id)
+    dawait r.flushPipeline()
 
-proc cacheProfileId*(username, id: string) {.async.} =
+proc cacheProfileId(username, id: string) {.async.} =
   if username.len == 0 or id.len == 0: return
   let name = toLower(username)
   pool.withAcquire(r):
-    discard await r.hSet(name.pidKey, name, id)
+    dawait r.hSet(name.pidKey, name, id)
 
 proc cacheRss*(query: string; rss: Rss) {.async.} =
   let key = "rss:" & query
   pool.withAcquire(r):
     r.startPipelining()
-    discard await r.hSet(key, "rss", rss.feed)
-    discard await r.hSet(key, "min", rss.cursor)
-    discard await r.expire(key, rssCacheTime)
-    discard await r.flushPipeline()
+    dawait r.hSet(key, "rss", rss.feed)
+    dawait r.hSet(key, "min", rss.cursor)
+    dawait r.expire(key, rssCacheTime)
+    dawait r.flushPipeline()
 
 proc getProfileId*(username: string): Future[string] {.async.} =
   let name = toLower(username)
@@ -113,15 +117,21 @@ proc getCachedProfile*(username: string; fetch=true): Future[Profile] {.async.}
     result = fromFlatty(uncompress(prof), Profile)
   elif fetch:
     result = await getProfile(username)
+    await cacheProfileId(result.username, result.id)
+    if result.suspended:
+      await cache(result)
 
 proc getCachedProfileUsername*(userId: string): Future[string] {.async.} =
-  let username = await get("i:" & userId)
+  let
+    key = "i:" & userId
+    username = await get(key)
+
   if username != redisNil:
     result = username
   else:
     let profile = await getProfileById(userId)
     result = profile.username
-    await cache(profile)
+    await setEx(key, baseCacheTime, result)
 
 proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
   if name.len == 0: return

src/routes/embed.nim

@@ -1,9 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, strutils, options
-import jester
-import ".."/[types, api], ../views/embed
+import jester, karax/vdom
+import ".."/[types, api]
+import ../views/[embed, tweet, general]
+import router_utils
 
-export api, embed
+export api, embed, vdom, tweet, general, router_utils
 
 proc createEmbedRouter*(cfg: Config) =
   router embed:
@@ -12,4 +14,23 @@ proc createEmbedRouter*(cfg: Config) =
       if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
         resp Http404
 
-      resp renderVideoEmbed(cfg, convo.tweet)
+      resp renderVideoEmbed(convo.tweet, cfg, request)
+
+    get "/@user/status/@id/embed":
+      let
+        convo = await getTweet(@"id")
+        prefs = cookiePrefs()
+        path = getPath()
+
+      if convo == nil or convo.tweet == nil:
+        resp Http404
+
+      resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
+
+    get "/embed/Tweet.html":
+      let id = @"id"
+
+      if id.len > 0:
+        redirect("/i/status/" & id & "/embed")
+      else:
+        resp Http404

src/routes/media.nim

@@ -5,7 +5,7 @@ import asynchttpserver, asyncstreams, asyncfile, asyncnet
 import jester
 
 import router_utils
-import ".."/[types, formatters, agents, utils]
+import ".."/[types, formatters, utils]
 
 export asynchttpserver, asyncstreams, asyncfile, asyncnet
 export httpclient, os, strutils, asyncstreams, base64, re
@@ -14,10 +14,8 @@ const
   m3u8Mime* = "application/vnd.apple.mpegurl"
   maxAge* = "max-age=604800"
 
-let mediaAgent* = getAgent()
-
-proc safeFetch*(url, agent: string): Future[string] {.async.} =
-  let client = newAsyncHttpClient(userAgent=agent)
+proc safeFetch*(url: string): Future[string] {.async.} =
+  let client = newAsyncHttpClient()
   try: result = await client.getContent(url)
   except: discard
   finally: client.close()
@@ -34,7 +32,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
   result = Http200
   let
     request = req.getNativeReq()
-    client = newAsyncHttpClient(userAgent=mediaAgent)
+    client = newAsyncHttpClient()
 
   try:
     let res = await client.get(url)
@@ -116,14 +114,14 @@ proc createMediaRouter*(cfg: Config) =
 
       var content: string
       if ".vmap" in url:
-        let m3u8 = getM3u8Url(await safeFetch(url, mediaAgent))
+        let m3u8 = getM3u8Url(await safeFetch(url))
         if m3u8.len > 0:
-          content = await safeFetch(url, mediaAgent)
+          content = await safeFetch(url)
         else:
           resp Http404
 
       if ".m3u8" in url:
-        let vid = await safeFetch(url, mediaAgent)
+        let vid = await safeFetch(url)
         content = proxifyVideo(vid, cookiePref(proxyVideos))
 
       resp content, m3u8Mime

src/routes/router_utils.nim

@@ -4,12 +4,12 @@ from jester import Request, cookies
 
 import ../views/general
 import ".."/[utils, prefs, types]
-export utils, prefs, types
+export utils, prefs, types, uri
 
 template savePref*(pref, value: string; req: Request; expire=false) =
   if not expire or pref in cookies(req):
     setCookie(pref, value, daysForward(when expire: -10 else: 360),
-              httpOnly=true, secure=cfg.useHttps)
+              httpOnly=true, secure=cfg.useHttps, sameSite=None)
 
 template cookiePrefs*(): untyped {.dirty.} =
   getPrefs(cookies(request))

src/routes/rss.nim

@@ -19,8 +19,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
     names = getNames(name)
 
   if names.len == 1:
-    (profile, timeline) =
-      await fetchSingleTimeline(after, query, skipRail=true)
+    (profile, timeline) = await fetchTimeline(after, query, skipRail=true)
   else:
     var q = query
     q.fromUser = names

src/routes/timeline.nim

@@ -19,8 +19,8 @@ proc getQuery*(request: Request; tab, name: string): Query =
   of "search": initQuery(params(request), name=name)
   else: Query(fromUser: @[name])
 
-proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
-                        Future[(Profile, Timeline, PhotoRail)] {.async.} =
+proc fetchTimeline*(after: string; query: Query; skipRail=false):
+                  Future[(Profile, Timeline, PhotoRail)] {.async.} =
   let name = query.fromUser[0]
 
   var
@@ -30,20 +30,13 @@ proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
 
   if profileId.len == 0:
     profile = await getCachedProfile(name)
-    profileId = if profile.suspended: "s"
-                else: profile.id
-
-    if profileId.len > 0:
-       await cacheProfileId(profile.username, profileId)
-
+    profileId = profile.id
     fetched = true
 
-  if profileId.len == 0 or profile.protected:
-    result[0] = profile
-    return
-  elif profileId == "s":
-    result[0] = Profile(username: name, suspended: true)
-    return
+  if profile.protected or profile.suspended:
+    return (profile, Timeline(), @[])
+  elif profileId.len == 0:
+    return (Profile(username: name), Timeline(), @[])
 
   var rail: Future[PhotoRail]
   if skipRail or profile.protected or query.kind == media:
@@ -86,7 +79,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
-  var (p, t, r) = await fetchSingleTimeline(after, query)
+  var (p, t, r) = await fetchTimeline(after, query)
 
   if p.suspended: return showError(getSuspended(p.username), cfg)
   if p.id.len == 0: return
@@ -139,7 +132,7 @@ proc createTimelineRouter*(cfg: Config) =
           timeline.beginning = true
           resp $renderTweetSearch(timeline, prefs, getPath())
         else:
-          var (_, timeline, _) = await fetchSingleTimeline(after, query, skipRail=true)
+          var (_, timeline, _) = await fetchTimeline(after, query, skipRail=true)
           if timeline.content.len == 0: resp Http404
           timeline.beginning = true
           resp $renderTimelineTweets(timeline, prefs, getPath())

src/sass/profile/_base.scss

@@ -15,23 +15,22 @@
 }
 
 .profile-banner {
-    padding-bottom: 4px;
+    margin-bottom: 4px;
+    background-color: var(--bg_panel);
 
     a {
-        display: inherit;
-        line-height: 0;
+        display: block;
+        position: relative;
+        padding: 33.34% 0 0 0;
     }
 
     img {
-        width: 100%;
+        max-width: 100%;
+        position: absolute;
+        top: 0;
     }
 }
 
-.profile-banner-color {
-    width: 100%;
-    padding-bottom: 25%;
-}
-
 .profile-tab {
     padding: 0 4px 0 0;
     box-sizing: border-box;

src/sass/profile/card.scss

@@ -35,19 +35,25 @@
 }
 
 .profile-card-avatar {
-    display: block;
+    display: inline-block;
+    position: relative;
     width: 100%;
-    padding-bottom: 6px;
     margin-right: 4px;
+    margin-bottom: 6px;
+
+    &:after {
+        content: '';
+        display: block;
+        margin-top: 100%;
+    }
 
     img {
-        display: block;
         box-sizing: border-box;
+        position: absolute;
         width: 100%;
         height: 100%;
-        margin: 0;
         border: 4px solid var(--darker_grey);
-        background: var(--bg_color);
+        background: var(--bg_panel);
     }
 }
 
@@ -113,8 +119,8 @@
     }
 
     .profile-card-avatar {
-        height: 60px;
-        width: unset;
+        width: 80px;
+        height: 80px;
 
         img {
             border-width: 2px;

src/sass/profile/photo-rail.scss

@@ -32,6 +32,7 @@
         a {
             position: relative;
             border-radius: 5px;
+            background-color: var(--darker_grey);
 
             &:before {
                 content: "";

src/sass/tweet/_base.scss

@@ -98,11 +98,14 @@
 }
 
 .avatar {
+    position: absolute;
+
     &.round {
         border-radius: 50%;
     }
     
     &.mini {
+        position: unset;
         margin-right: 5px;
         margin-top: -1px;
         width: 20px;
@@ -110,6 +113,24 @@
     }
 }
 
+.tweet-embed {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    height: 100%;
+    background-color: var(--bg_panel);
+
+    .tweet-content { 
+        font-size: 18px 
+    }
+    
+    .tweet-body { 
+        display: flex;
+        flex-direction: column;
+        max-height: calc(100vh - 0.75em * 2);
+    }
+}
+
 .attribution {
     display: flex;
     pointer-events: all;

src/tokens.nim

@@ -2,12 +2,12 @@
 import asyncdispatch, httpclient, times, sequtils, json, random
 import strutils, tables
 import zippy
-import types, agents, consts, http_pool
+import types, consts, http_pool
 
 const
   maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
-  maxAge = 3.hours       # tokens expire after 3 hours
   maxLastUse = 1.hours   # if a token is unused for 60 minutes, it expires
+  maxAge = 2.hours + 55.minutes  # tokens expire after 3 hours
   failDelay = initDuration(minutes=30)
 
 var
@@ -65,7 +65,6 @@ proc fetchToken(): Future[Token] {.async.} =
     "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.5",
     "connection": "keep-alive",
-    "user-agent": getAgent(),
     "authorization": auth
   })
 

src/types.nim

@@ -48,17 +48,16 @@ type
     id*: string
     username*: string
     fullname*: string
-    lowername*: string
     location*: string
     website*: string
     bio*: string
     userPic*: string
     banner*: string
-    following*: string
-    followers*: string
-    tweets*: string
-    likes*: string
-    media*: string
+    following*: int
+    followers*: int
+    tweets*: int
+    likes*: int
+    media*: int
     verified*: bool
     protected*: bool
     suspended*: bool

src/views/embed.nim

@@ -1,18 +1,19 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import options
 import karax/[karaxdsl, vdom]
+from jester import Request
 
 import ".."/[types, formatters]
 import general, tweet
 
 const doctype = "<!DOCTYPE html>\n"
 
-proc renderVideoEmbed*(cfg: Config; tweet: Tweet): string =
+proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
   let thumb = get(tweet.video).thumb
   let vidUrl = getVideoEmbed(cfg, tweet.id)
   let prefs = Prefs(hlsPlayback: true)
   let node = buildHtml(html(lang="en")):
-    renderHead(prefs, cfg, video=vidUrl, images=(@[thumb]))
+    renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
 
     tdiv(class="embed-video"):
       renderVideo(get(tweet.video), prefs, "")

src/views/general.nim

@@ -11,6 +11,9 @@ const
   doctype = "<!DOCTYPE html>\n"
   lp = readFile("public/lp.svg")
 
+proc toTheme(theme: string): string =
+  theme.toLowerAscii.replace(" ", "_")
+
 proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
   var path = req.params.getOrDefault("referer")
   if path.len == 0:
@@ -33,9 +36,13 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
         icon "info", title="About", href="/about"
         icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
 
-proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
-                 images: seq[string] = @[]; banner=""; ogTitle=""; theme="";
+proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
+                 video=""; images: seq[string] = @[]; banner=""; ogTitle="";
                  rss=""; canonical=""): VNode =
+  var theme = prefs.theme.toTheme
+  if "theme" in req.params:
+    theme = req.params["theme"].toTheme
+    
   let ogType =
     if video.len > 0: "video"
     elif rss.len > 0: "object"
@@ -45,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=10")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=15")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:
@@ -118,15 +125,12 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
 proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
                  titleText=""; desc=""; ogTitle=""; rss=""; video="";
                  images: seq[string] = @[]; banner=""): string =
-  var theme = toLowerAscii(prefs.theme).replace(" ", "_")
-  if "theme" in req.params:
-    theme = toLowerAscii(req.params["theme"]).replace(" ", "_")
 
   let canonical = getTwitterLink(req.path, req.params)
 
   let node = buildHtml(html(lang="en")):
-    renderHead(prefs, cfg, titleText, desc, video, images, banner, ogTitle,
-               theme, rss, canonical)
+    renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
+               rss, canonical)
 
     body:
       renderNavbar(cfg, req, rss, canonical)

src/views/profile.nim

@@ -5,12 +5,12 @@ import karax/[karaxdsl, vdom, vstyles]
 import renderutils, search
 import ".."/[types, utils, formatters]
 
-proc renderStat(num, class: string; text=""): VNode =
+proc renderStat(num: int; class: string; text=""): VNode =
   let t = if text.len > 0: text else: class
   buildHtml(li(class=class)):
     span(class="profile-stat-header"): text capitalizeAscii(t)
     span(class="profile-stat-num"):
-      text if num.len == 0: "?" else: insertSep(num, ',')
+      text insertSep($num, ',')
 
 proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
   buildHtml(tdiv(class="profile-card")):
@@ -78,18 +78,16 @@ proc renderPhotoRail(profile: Profile; photoRail: PhotoRail): VNode =
     tdiv(class="photo-rail-grid"):
       for i, photo in photoRail:
         if i == 16: break
-        let col = if photo.color.len > 0: photo.color else: "#161616"
-        a(href=(&"/{profile.username}/status/{photo.tweetId}#m"),
-          style={backgroundColor: col}):
+        a(href=(&"/{profile.username}/status/{photo.tweetId}#m")):
           genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
 
-proc renderBanner(profile: Profile): VNode =
+proc renderBanner(banner: string): VNode =
   buildHtml():
-    if "#" in profile.banner:
-      tdiv(class="profile-banner-color", style={backgroundColor: profile.banner})
+    if banner.startsWith('#'):
+      a(style={backgroundColor: banner})
     else:
-      a(href=getPicUrl(profile.banner), target="_blank"):
-        genImg(profile.banner)
+      a(href=getPicUrl(banner), target="_blank"):
+        genImg(banner)
 
 proc renderProtected(username: string): VNode =
   buildHtml(tdiv(class="timeline-container")):
@@ -103,7 +101,8 @@ proc renderProfile*(profile: Profile; timeline: var Timeline;
   buildHtml(tdiv(class="profile-tabs")):
     if not prefs.hideBanner:
       tdiv(class="profile-banner"):
-        renderBanner(profile)
+        if profile.banner.len > 0:
+          renderBanner(profile.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
     tdiv(class=(&"profile-tab{sticky}")):

src/views/tweet.nim

@@ -1,9 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import strutils, sequtils, strformat, options
 import karax/[karaxdsl, vdom, vstyles]
+from jester import Request
 
 import renderutils
 import ".."/[types, utils, formatters]
+import general
 
 proc getSmallPic(url: string): string =
   result = url
@@ -353,3 +355,8 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if showThread:
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
+
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
+  buildHtml(tdiv(class="tweet-embed")):
+    renderHead(prefs, cfg, req)
+    renderTweet(tweet, prefs, path, mainTweet=true)