From 7f15993a74c8dacd8177801ee46fde7e7e666e32 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 15:23:53 +0200
Subject: [PATCH 01/28] crude implementation of embedding tweets

---
 src/nitter.nim            |  4 +++-
 src/routes/embedtweet.nim | 22 ++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 src/routes/embedtweet.nim

diff --git a/src/nitter.nim b/src/nitter.nim
index 9f8fcb7..411c9a3 100644
--- a/src/nitter.nim
+++ b/src/nitter.nim
@@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, tokens
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
-  unsupported, embed, resolver, router_utils]
+  unsupported, embed, resolver, router_utils, embedtweet]
 
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
@@ -51,6 +51,7 @@ createMediaRouter(cfg)
 createEmbedRouter(cfg)
 createRssRouter(cfg)
 createDebugRouter(cfg)
+createEmbedTweetRouter(cfg)
 
 settings:
   port = Port(cfg.port)
@@ -101,3 +102,4 @@ routes:
   extend media, ""
   extend embed, ""
   extend debug, ""
+  extend embedtweet, ""
diff --git a/src/routes/embedtweet.nim b/src/routes/embedtweet.nim
new file mode 100644
index 0000000..d318e67
--- /dev/null
+++ b/src/routes/embedtweet.nim
@@ -0,0 +1,22 @@
+import asyncdispatch, strutils, uri, options
+import jester, karax/vdom
+
+import router_utils
+import ".."/views/[general, tweet]
+import ".."/[types, api]
+
+export vdom
+export router_utils
+export api, tweet, general
+
+proc createEmbedTweetRouter*(cfg: Config) =
+  router embedtweet:
+    get "/embed/Tweet.html":
+      let
+        prefs = cookiePrefs() 
+        t = (await getTweet(@"id")).tweet
+
+      resp ($renderHead(prefs, cfg) & $renderTweet(t, prefs, getPath(), mainTweet=true))
+
+ 
+

From 0d3469df667818005f60da81beb5310fe229a7bf Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 19:01:47 +0200
Subject: [PATCH 02/28] changed code to be not shit

---
 src/formatters.nim          | 16 ++++++-----
 src/nitter.nim              |  4 +--
 src/routes/embed.nim        | 17 +++++++++---
 src/routes/embedtweet.nim   | 22 ---------------
 src/routes/router_utils.nim |  4 +--
 src/views/tweet.nim         | 54 +++++++++++++++++++++++++++++++++++++
 6 files changed, 80 insertions(+), 37 deletions(-)
 delete mode 100644 src/routes/embedtweet.nim

diff --git a/src/formatters.nim b/src/formatters.nim
index b251ccf..2ae3077 100644
--- a/src/formatters.nim
+++ b/src/formatters.nim
@@ -32,23 +32,25 @@ proc getUrlPrefix*(cfg: Config): string =
   if cfg.useHttps: https & cfg.hostname
   else: "http://" & cfg.hostname
 
-proc stripHtml*(text: string): string =
+proc shortLink*(text: string; length=28): string =
+  result = text.replace(wwwRegex, "")
+  if result.len > length:
+    result = result[0 ..< length] & "…"
+    
+proc stripHtml*(text: string; shorten=false): string =
   var html = parseHtml(text)
   for el in html.findAll("a"):
     let link = el.attr("href")
     if "http" in link:
       if el.len == 0: continue
-      el[0].text = link
+      el[0].text =
+        if shorten: link.shortLink
+        else: link
   html.innerText()
 
 proc sanitizeXml*(text: string): string =
   text.replace(illegalXmlRegex, "")
 
-proc shortLink*(text: string; length=28): string =
-  result = text.replace(wwwRegex, "")
-  if result.len > length:
-    result = result[0 ..< length] & "…"
-
 proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   result = body
 
diff --git a/src/nitter.nim b/src/nitter.nim
index 411c9a3..9f8fcb7 100644
--- a/src/nitter.nim
+++ b/src/nitter.nim
@@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, tokens
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
-  unsupported, embed, resolver, router_utils, embedtweet]
+  unsupported, embed, resolver, router_utils]
 
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
@@ -51,7 +51,6 @@ createMediaRouter(cfg)
 createEmbedRouter(cfg)
 createRssRouter(cfg)
 createDebugRouter(cfg)
-createEmbedTweetRouter(cfg)
 
 settings:
   port = Port(cfg.port)
@@ -102,4 +101,3 @@ routes:
   extend media, ""
   extend embed, ""
   extend debug, ""
-  extend embedtweet, ""
diff --git a/src/routes/embed.nim b/src/routes/embed.nim
index d9a9ee9..9a4c85b 100644
--- a/src/routes/embed.nim
+++ b/src/routes/embed.nim
@@ -1,9 +1,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, strutils, options
-import jester
-import ".."/[types, api], ../views/embed
+import jester, karax/vdom
+import ".."/[types, api], ../views/[embed, tweet, general]
+import router_utils
 
-export api, embed
+export api, embed, vdom
+export tweet, general
+export router_utils
 
 proc createEmbedRouter*(cfg: Config) =
   router embed:
@@ -13,3 +16,11 @@ proc createEmbedRouter*(cfg: Config) =
         resp Http404
 
       resp renderVideoEmbed(cfg, convo.tweet)
+
+    get "/@user/status/@id/embedded":
+      let
+        tweet = (await getTweet(@"id")).tweet
+        prefs = cookiePrefs()
+        path = getPath()
+
+      resp $renderEmbeddedTweet(tweet, cfg, prefs, path)
diff --git a/src/routes/embedtweet.nim b/src/routes/embedtweet.nim
deleted file mode 100644
index d318e67..0000000
--- a/src/routes/embedtweet.nim
+++ /dev/null
@@ -1,22 +0,0 @@
-import asyncdispatch, strutils, uri, options
-import jester, karax/vdom
-
-import router_utils
-import ".."/views/[general, tweet]
-import ".."/[types, api]
-
-export vdom
-export router_utils
-export api, tweet, general
-
-proc createEmbedTweetRouter*(cfg: Config) =
-  router embedtweet:
-    get "/embed/Tweet.html":
-      let
-        prefs = cookiePrefs() 
-        t = (await getTweet(@"id")).tweet
-
-      resp ($renderHead(prefs, cfg) & $renderTweet(t, prefs, getPath(), mainTweet=true))
-
- 
-
diff --git a/src/routes/router_utils.nim b/src/routes/router_utils.nim
index 7159890..a071a0d 100644
--- a/src/routes/router_utils.nim
+++ b/src/routes/router_utils.nim
@@ -4,12 +4,12 @@ from jester import Request, cookies
 
 import ../views/general
 import ".."/[utils, prefs, types]
-export utils, prefs, types
+export utils, prefs, types, uri
 
 template savePref*(pref, value: string; req: Request; expire=false) =
   if not expire or pref in cookies(req):
     setCookie(pref, value, daysForward(when expire: -10 else: 360),
-              httpOnly=true, secure=cfg.useHttps)
+              httpOnly=true, secure=cfg.useHttps, sameSite=None)
 
 template cookiePrefs*(): untyped {.dirty.} =
   getPrefs(cookies(request))
diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index 7494685..3d05993 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -4,6 +4,7 @@ import karax/[karaxdsl, vdom, vstyles]
 
 import renderutils
 import ".."/[types, utils, formatters]
+import general
 
 proc getSmallPic(url: string): string =
   result = url
@@ -275,6 +276,59 @@ proc renderLocation*(tweet: Tweet): string =
       text place
   return $node
 
+proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; prefs: Prefs; path: string): VNode =
+  let fullTweet = tweet
+  var retweet: string
+  var tweet = fullTweet
+  if tweet.retweet.isSome:
+    tweet = tweet.retweet.get
+    retweet = fullTweet.profile.fullname
+
+  # handle unavailable
+
+  buildHtml(tdiv(class="timeline-item")):
+    renderHead(prefs, cfg)
+    tdiv(class="tweet-body"):
+      var views = ""
+      renderHeader(tweet, retweet, prefs)
+
+      var tweetClass = "tweet-content media-body"
+      if prefs.bidiSupport:
+        tweetClass &= " tweet-bidi"
+
+      tdiv(class=tweetClass, dir="auto"):
+        verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet)
+
+      if tweet.attribution.isSome:
+        renderAttribution(tweet.attribution.get(), prefs)
+
+      if tweet.card.isSome:
+        renderCard(tweet.card.get(), prefs, path)
+
+      if tweet.photos.len > 0:
+        renderAlbum(tweet)
+      elif tweet.video.isSome:
+        renderVideo(tweet.video.get(), prefs, path)
+        views = tweet.video.get().views
+      elif tweet.gif.isSome:
+        renderGif(tweet.gif.get(), prefs)
+        views = "GIF"
+
+      if tweet.poll.isSome:
+        renderPoll(tweet.poll.get())
+
+      if tweet.quote.isSome:
+        renderQuote(tweet.quote.get(), prefs, path)
+
+      p(class="tweet-published"): text getTime(tweet)
+
+      if tweet.mediaTags.len > 0:
+        renderMediaTags(tweet.mediaTags)
+
+      if not prefs.hideTweetStats:
+        renderStats(tweet.stats, views)
+  
+
 proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
                   last=false; showThread=false; mainTweet=false; afterTweet=false): VNode =
   var divClass = class

From 3579bd8e3036d165efdcd794b20f8afc14bc39c4 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 19:17:10 +0200
Subject: [PATCH 03/28] handled unavailable in renderEmbeddedTweet

---
 src/views/tweet.nim | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index 3d05993..f8a570a 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -284,7 +284,18 @@ proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; prefs: Prefs; path: string)
     tweet = tweet.retweet.get
     retweet = fullTweet.profile.fullname
 
-  # handle unavailable
+  if not tweet.available:
+    return buildHtml(tdiv(class="unavailable timeline-item")):
+      tdiv(class="unavailable-box"):
+        if tweet.tombstone.len > 0:
+          text tweet.tombstone
+        elif tweet.text.len > 0:
+          text tweet.text
+        else:
+          text "This tweet is unavailable"
+
+      if tweet.quote.isSome:
+        renderQuote(tweet.quote.get(), prefs, path)
 
   buildHtml(tdiv(class="timeline-item")):
     renderHead(prefs, cfg)

From 875a2c538796924e7cd8e03aff9b0930aac42178 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 19:34:10 +0200
Subject: [PATCH 04/28] moved themes to be handled in renderHead and changed
 path to /embed

---
 src/routes/embed.nim  | 11 +++++++----
 src/views/embed.nim   |  5 +++--
 src/views/general.nim | 18 +++++++++++-------
 src/views/tweet.nim   |  5 +++--
 4 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/src/routes/embed.nim b/src/routes/embed.nim
index 9a4c85b..2db9a41 100644
--- a/src/routes/embed.nim
+++ b/src/routes/embed.nim
@@ -15,12 +15,15 @@ proc createEmbedRouter*(cfg: Config) =
       if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
         resp Http404
 
-      resp renderVideoEmbed(cfg, convo.tweet)
+      resp renderVideoEmbed(cfg, request, convo.tweet)
 
-    get "/@user/status/@id/embedded":
+    get "/@user/status/@id/embed":
       let
-        tweet = (await getTweet(@"id")).tweet
+        convo = await getTweet(@"id")
         prefs = cookiePrefs()
         path = getPath()
 
-      resp $renderEmbeddedTweet(tweet, cfg, prefs, path)
+      if convo == nil or convo.tweet == nil:
+        resp Http404
+
+      resp $renderEmbeddedTweet(convo.tweet, cfg, request, prefs, path)
diff --git a/src/views/embed.nim b/src/views/embed.nim
index 4c2f7b3..edce944 100644
--- a/src/views/embed.nim
+++ b/src/views/embed.nim
@@ -1,18 +1,19 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import options
 import karax/[karaxdsl, vdom]
+from jester import Request
 
 import ".."/[types, formatters]
 import general, tweet
 
 const doctype = "<!DOCTYPE html>\n"
 
-proc renderVideoEmbed*(cfg: Config; tweet: Tweet): string =
+proc renderVideoEmbed*(cfg: Config; req: Request; tweet: Tweet): string =
   let thumb = get(tweet.video).thumb
   let vidUrl = getVideoEmbed(cfg, tweet.id)
   let prefs = Prefs(hlsPlayback: true)
   let node = buildHtml(html(lang="en")):
-    renderHead(prefs, cfg, video=vidUrl, images=(@[thumb]))
+    renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
 
     tdiv(class="embed-video"):
       renderVideo(get(tweet.video), prefs, "")
diff --git a/src/views/general.nim b/src/views/general.nim
index 4a8b4a3..e514ab8 100644
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -11,6 +11,9 @@ const
   doctype = "<!DOCTYPE html>\n"
   lp = readFile("public/lp.svg")
 
+proc toTheme(theme: string): string =
+  theme.toLowerAscii.replace(" ", "_")
+
 proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
   var path = req.params.getOrDefault("referer")
   if path.len == 0:
@@ -33,9 +36,13 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
         icon "info", title="About", href="/about"
         icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
 
-proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
-                 images: seq[string] = @[]; banner=""; ogTitle=""; theme="";
+proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
+                 video=""; images: seq[string] = @[]; banner=""; ogTitle="";
                  rss=""; canonical=""): VNode =
+  var theme = prefs.theme.toTheme
+  if "theme" in req.params:
+    theme = req.params["theme"].toTheme
+    
   let ogType =
     if video.len > 0: "video"
     elif rss.len > 0: "object"
@@ -118,15 +125,12 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
 proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
                  titleText=""; desc=""; ogTitle=""; rss=""; video="";
                  images: seq[string] = @[]; banner=""): string =
-  var theme = toLowerAscii(prefs.theme).replace(" ", "_")
-  if "theme" in req.params:
-    theme = toLowerAscii(req.params["theme"]).replace(" ", "_")
 
   let canonical = getTwitterLink(req.path, req.params)
 
   let node = buildHtml(html(lang="en")):
-    renderHead(prefs, cfg, titleText, desc, video, images, banner, ogTitle,
-               theme, rss, canonical)
+    renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
+               rss, canonical)
 
     body:
       renderNavbar(cfg, req, rss, canonical)
diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index f8a570a..f96d978 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import strutils, sequtils, strformat, options
 import karax/[karaxdsl, vdom, vstyles]
+from jester import Request
 
 import renderutils
 import ".."/[types, utils, formatters]
@@ -276,7 +277,7 @@ proc renderLocation*(tweet: Tweet): string =
       text place
   return $node
 
-proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; prefs: Prefs; path: string): VNode =
+proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; req: Request; prefs: Prefs; path: string): VNode =
   let fullTweet = tweet
   var retweet: string
   var tweet = fullTweet
@@ -298,7 +299,7 @@ proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; prefs: Prefs; path: string)
         renderQuote(tweet.quote.get(), prefs, path)
 
   buildHtml(tdiv(class="timeline-item")):
-    renderHead(prefs, cfg)
+    renderHead(prefs, cfg, req)
     tdiv(class="tweet-body"):
       var views = ""
       renderHeader(tweet, retweet, prefs)

From 817501a5166e39c6ebea1c661210f651d0d361c2 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 19:44:09 +0200
Subject: [PATCH 05/28] wrapped embedded tweet in div and changed css also
 bumped css version

---
 src/sass/tweet/_base.scss | 15 +++++++++++++++
 src/views/general.nim     |  2 +-
 src/views/tweet.nim       |  5 ++++-
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss
index 80a1171..d624fb8 100644
--- a/src/sass/tweet/_base.scss
+++ b/src/sass/tweet/_base.scss
@@ -98,6 +98,8 @@
 }
 
 .avatar {
+    position: absolute;
+
     &.round {
         border-radius: 50%;
     }
@@ -110,6 +112,19 @@
     }
 }
 
+.tweet-embed {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    height: 100%;
+    
+    .tweet-body { 
+        display: flex;
+        flex-direction: column;
+        max-height: calc(100vh - 0.75em * 2);
+    }
+}
+
 .attribution {
     display: flex;
     pointer-events: all;
diff --git a/src/views/general.nim b/src/views/general.nim
index e514ab8..7267852 100644
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=10")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=11")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:
diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index f96d978..667ca16 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -298,7 +298,7 @@ proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; req: Request; prefs: Prefs;
       if tweet.quote.isSome:
         renderQuote(tweet.quote.get(), prefs, path)
 
-  buildHtml(tdiv(class="timeline-item")):
+  let body = buildHtml(tdiv(class="timeline-item")):
     renderHead(prefs, cfg, req)
     tdiv(class="tweet-body"):
       var views = ""
@@ -339,6 +339,9 @@ proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; req: Request; prefs: Prefs;
 
       if not prefs.hideTweetStats:
         renderStats(tweet.stats, views)
+
+  return buildHtml(tdiv(class="tweet-embed")):
+    body
   
 
 proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;

From 784d0d42acffebc047b88e483231419b54af20c1 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 19:49:36 +0200
Subject: [PATCH 06/28] minor css change and version bump

---
 src/sass/tweet/_base.scss | 2 ++
 src/views/general.nim     | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss
index d624fb8..e9965da 100644
--- a/src/sass/tweet/_base.scss
+++ b/src/sass/tweet/_base.scss
@@ -117,6 +117,8 @@
     flex-direction: column;
     justify-content: center;
     height: 100%;
+
+    .tweet-content { font-size: 18px }
     
     .tweet-body { 
         display: flex;
diff --git a/src/views/general.nim b/src/views/general.nim
index 7267852..f909eb0 100644
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=11")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=12")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:

From 90eae2669b66090a401309cdf030112b0a407788 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 20:11:51 +0200
Subject: [PATCH 07/28] fixed stupid code (sorry)

---
 src/views/tweet.nim | 72 ++++-----------------------------------------
 1 file changed, 5 insertions(+), 67 deletions(-)

diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index 667ca16..e211128 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -277,73 +277,6 @@ proc renderLocation*(tweet: Tweet): string =
       text place
   return $node
 
-proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; req: Request; prefs: Prefs; path: string): VNode =
-  let fullTweet = tweet
-  var retweet: string
-  var tweet = fullTweet
-  if tweet.retweet.isSome:
-    tweet = tweet.retweet.get
-    retweet = fullTweet.profile.fullname
-
-  if not tweet.available:
-    return buildHtml(tdiv(class="unavailable timeline-item")):
-      tdiv(class="unavailable-box"):
-        if tweet.tombstone.len > 0:
-          text tweet.tombstone
-        elif tweet.text.len > 0:
-          text tweet.text
-        else:
-          text "This tweet is unavailable"
-
-      if tweet.quote.isSome:
-        renderQuote(tweet.quote.get(), prefs, path)
-
-  let body = buildHtml(tdiv(class="timeline-item")):
-    renderHead(prefs, cfg, req)
-    tdiv(class="tweet-body"):
-      var views = ""
-      renderHeader(tweet, retweet, prefs)
-
-      var tweetClass = "tweet-content media-body"
-      if prefs.bidiSupport:
-        tweetClass &= " tweet-bidi"
-
-      tdiv(class=tweetClass, dir="auto"):
-        verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet)
-
-      if tweet.attribution.isSome:
-        renderAttribution(tweet.attribution.get(), prefs)
-
-      if tweet.card.isSome:
-        renderCard(tweet.card.get(), prefs, path)
-
-      if tweet.photos.len > 0:
-        renderAlbum(tweet)
-      elif tweet.video.isSome:
-        renderVideo(tweet.video.get(), prefs, path)
-        views = tweet.video.get().views
-      elif tweet.gif.isSome:
-        renderGif(tweet.gif.get(), prefs)
-        views = "GIF"
-
-      if tweet.poll.isSome:
-        renderPoll(tweet.poll.get())
-
-      if tweet.quote.isSome:
-        renderQuote(tweet.quote.get(), prefs, path)
-
-      p(class="tweet-published"): text getTime(tweet)
-
-      if tweet.mediaTags.len > 0:
-        renderMediaTags(tweet.mediaTags)
-
-      if not prefs.hideTweetStats:
-        renderStats(tweet.stats, views)
-
-  return buildHtml(tdiv(class="tweet-embed")):
-    body
-  
-
 proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
                   last=false; showThread=false; mainTweet=false; afterTweet=false): VNode =
   var divClass = class
@@ -422,3 +355,8 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if showThread:
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
+
+proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; req: Request; prefs: Prefs; path: string): VNode =
+  return buildHtml(tdiv(class="tweet-embed")):
+    renderHead(prefs, cfg, req)
+    renderTweet(tweet, prefs, path)

From a6bd05bca684eba2de14807246387c4b0c32db81 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 20:14:06 +0200
Subject: [PATCH 08/28] fixed more stupid code

---
 src/views/tweet.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index e211128..b7d18f0 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -357,6 +357,6 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
           text "Show this thread"
 
 proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; req: Request; prefs: Prefs; path: string): VNode =
-  return buildHtml(tdiv(class="tweet-embed")):
+  buildHtml(tdiv(class="tweet-embed")):
     renderHead(prefs, cfg, req)
-    renderTweet(tweet, prefs, path)
+    renderTweet(tweet, prefs, path, mainTweet=true)

From ac0edc0a41f13e9cce725901eff2ad65f98d5193 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 20:24:06 +0200
Subject: [PATCH 09/28] made twitter embed links redirect to nitter ones

---
 src/routes/embed.nim | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/routes/embed.nim b/src/routes/embed.nim
index 2db9a41..b758b52 100644
--- a/src/routes/embed.nim
+++ b/src/routes/embed.nim
@@ -27,3 +27,11 @@ proc createEmbedRouter*(cfg: Config) =
         resp Http404
 
       resp $renderEmbeddedTweet(convo.tweet, cfg, request, prefs, path)
+
+    get "/embed/Tweet.html":
+      let id = @"id"
+
+      if id.len > 0:
+        redirect("/i/status/" & id & "/embed")
+      else:
+        resp Http404

From 74fcc071a3ff811cf10c0b31a74be27397d8a2f0 Mon Sep 17 00:00:00 2001
From: alqeeu <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 20:32:50 +0200
Subject: [PATCH 10/28] Update src/sass/tweet/_base.scss

Co-authored-by: Zed <zedeus@pm.me>
---
 src/sass/tweet/_base.scss | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss
index e9965da..aa87ff1 100644
--- a/src/sass/tweet/_base.scss
+++ b/src/sass/tweet/_base.scss
@@ -118,7 +118,9 @@
     justify-content: center;
     height: 100%;
 
-    .tweet-content { font-size: 18px }
+    .tweet-content { 
+        font-size: 18px 
+    }
     
     .tweet-body { 
         display: flex;

From 1e027f5edf21f77ccebe3eaf2ea47f616889b7e3 Mon Sep 17 00:00:00 2001
From: alqeeu <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 20:33:01 +0200
Subject: [PATCH 11/28] Update src/routes/embed.nim

Co-authored-by: Zed <zedeus@pm.me>
---
 src/routes/embed.nim | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/routes/embed.nim b/src/routes/embed.nim
index b758b52..31e1bc3 100644
--- a/src/routes/embed.nim
+++ b/src/routes/embed.nim
@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, strutils, options
 import jester, karax/vdom
-import ".."/[types, api], ../views/[embed, tweet, general]
+import ".."/[types, api]
+import ../views/[embed, tweet, general]
 import router_utils
 
 export api, embed, vdom

From d29186bf8f08b47a0d29f83da16185626b4c8cab Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 20:35:01 +0200
Subject: [PATCH 12/28] stylistic changes

---
 src/routes/embed.nim | 6 ++----
 src/views/tweet.nim  | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/routes/embed.nim b/src/routes/embed.nim
index b758b52..ec914f0 100644
--- a/src/routes/embed.nim
+++ b/src/routes/embed.nim
@@ -4,9 +4,7 @@ import jester, karax/vdom
 import ".."/[types, api], ../views/[embed, tweet, general]
 import router_utils
 
-export api, embed, vdom
-export tweet, general
-export router_utils
+export api, embed, vdom, tweet, general, router_utils
 
 proc createEmbedRouter*(cfg: Config) =
   router embed:
@@ -26,7 +24,7 @@ proc createEmbedRouter*(cfg: Config) =
       if convo == nil or convo.tweet == nil:
         resp Http404
 
-      resp $renderEmbeddedTweet(convo.tweet, cfg, request, prefs, path)
+      resp $renderTweetEmbed(convo.tweet, prefs, path, cfg, request)
 
     get "/embed/Tweet.html":
       let id = @"id"
diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index b7d18f0..88edac4 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -356,7 +356,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
 
-proc renderEmbeddedTweet*(tweet: Tweet; cfg: Config; req: Request; prefs: Prefs; path: string): VNode =
+proc renderTweetEmbed*(tweet: Tweet; prefs: Prefs; path: string; cfg: Config; req: Request): VNode =
   buildHtml(tdiv(class="tweet-embed")):
     renderHead(prefs, cfg, req)
     renderTweet(tweet, prefs, path, mainTweet=true)

From eff098003f3d3bebdb64ee7feef3cec026c3d4c5 Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 20:45:02 +0200
Subject: [PATCH 13/28] unified function call styles

---
 src/routes/embed.nim | 4 ++--
 src/views/embed.nim  | 2 +-
 src/views/tweet.nim  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/routes/embed.nim b/src/routes/embed.nim
index 7057f67..1a93d40 100644
--- a/src/routes/embed.nim
+++ b/src/routes/embed.nim
@@ -14,7 +14,7 @@ proc createEmbedRouter*(cfg: Config) =
       if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
         resp Http404
 
-      resp renderVideoEmbed(cfg, request, convo.tweet)
+      resp renderVideoEmbed(convo.tweet, cfg, request)
 
     get "/@user/status/@id/embed":
       let
@@ -25,7 +25,7 @@ proc createEmbedRouter*(cfg: Config) =
       if convo == nil or convo.tweet == nil:
         resp Http404
 
-      resp $renderTweetEmbed(convo.tweet, prefs, path, cfg, request)
+      resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
 
     get "/embed/Tweet.html":
       let id = @"id"
diff --git a/src/views/embed.nim b/src/views/embed.nim
index edce944..e6afffd 100644
--- a/src/views/embed.nim
+++ b/src/views/embed.nim
@@ -8,7 +8,7 @@ import general, tweet
 
 const doctype = "<!DOCTYPE html>\n"
 
-proc renderVideoEmbed*(cfg: Config; req: Request; tweet: Tweet): string =
+proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
   let thumb = get(tweet.video).thumb
   let vidUrl = getVideoEmbed(cfg, tweet.id)
   let prefs = Prefs(hlsPlayback: true)
diff --git a/src/views/tweet.nim b/src/views/tweet.nim
index 88edac4..e8dfc83 100644
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@@ -356,7 +356,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
 
-proc renderTweetEmbed*(tweet: Tweet; prefs: Prefs; path: string; cfg: Config; req: Request): VNode =
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
   buildHtml(tdiv(class="tweet-embed")):
     renderHead(prefs, cfg, req)
     renderTweet(tweet, prefs, path, mainTweet=true)

From 62f8d48c5a1f07c09ebd14fb37abf03d95a73b52 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Fri, 14 Jan 2022 19:34:26 +0100
Subject: [PATCH 14/28] Bump jsony version to fix unified card unicode

---
 nitter.nimble | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nitter.nimble b/nitter.nimble
index a2c0615..7c49fdd 100644
--- a/nitter.nimble
+++ b/nitter.nimble
@@ -22,7 +22,7 @@ requires "redpool#f880f49"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#0.7.3"
 requires "flatty#0.2.3"
-requires "jsony#1.1.3"
+requires "jsony#d0e69bd"
 
 
 # Tasks

From 6ebfafde80f2c53481888da171dcd8019e4c31ff Mon Sep 17 00:00:00 2001
From: Mitarashi <doggymangc@gmail.com>
Date: Fri, 14 Jan 2022 21:07:02 +0200
Subject: [PATCH 15/28] added tweet background and bumped css

---
 src/sass/tweet/_base.scss | 1 +
 src/views/general.nim     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss
index aa87ff1..0b04930 100644
--- a/src/sass/tweet/_base.scss
+++ b/src/sass/tweet/_base.scss
@@ -117,6 +117,7 @@
     flex-direction: column;
     justify-content: center;
     height: 100%;
+    background-color: var(--bg_panel);
 
     .tweet-content { 
         font-size: 18px 
diff --git a/src/views/general.nim b/src/views/general.nim
index f909eb0..83208eb 100644
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=12")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=13")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:

From 54330f0b0c31f3302ad34191da39f2c991880fc6 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Fri, 14 Jan 2022 23:12:33 +0100
Subject: [PATCH 16/28] Fix quote avatar css

---
 src/sass/tweet/_base.scss | 1 +
 src/views/general.nim     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss
index 0b04930..7c1196a 100644
--- a/src/sass/tweet/_base.scss
+++ b/src/sass/tweet/_base.scss
@@ -105,6 +105,7 @@
     }
     
     &.mini {
+        position: unset;
         margin-right: 5px;
         margin-top: -1px;
         width: 20px;
diff --git a/src/views/general.nim b/src/views/general.nim
index 83208eb..010e2df 100644
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=13")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=14")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:

From fcfc1ef497ad48d3c63fb6527c6932616de0f29d Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 03:32:18 +0100
Subject: [PATCH 17/28] Parse user stats as ints, not strings, cleanup

---
 src/parser.nim          | 10 +++++-----
 src/redis_cache.nim     |  1 +
 src/routes/rss.nim      |  3 +--
 src/routes/timeline.nim |  8 ++++----
 src/types.nim           | 11 +++++------
 src/views/profile.nim   |  4 ++--
 6 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/parser.nim b/src/parser.nim
index 6aa6a7a..011798f 100644
--- a/src/parser.nim
+++ b/src/parser.nim
@@ -14,11 +14,11 @@ proc parseProfile(js: JsonNode; id=""): Profile =
     bio: js{"description"}.getStr,
     userPic: js{"profile_image_url_https"}.getImageStr.replace("_normal", ""),
     banner: js.getBanner,
-    following: $js{"friends_count"}.getInt,
-    followers: $js{"followers_count"}.getInt,
-    tweets: $js{"statuses_count"}.getInt,
-    likes: $js{"favourites_count"}.getInt,
-    media: $js{"media_count"}.getInt,
+    following: js{"friends_count"}.getInt,
+    followers: js{"followers_count"}.getInt,
+    tweets: js{"statuses_count"}.getInt,
+    likes: js{"favourites_count"}.getInt,
+    media: js{"media_count"}.getInt,
     verified: js{"verified"}.getBool,
     protected: js{"protected"}.getBool,
     joinDate: js{"created_at"}.getTime
diff --git a/src/redis_cache.nim b/src/redis_cache.nim
index dbe917c..7a921c8 100644
--- a/src/redis_cache.nim
+++ b/src/redis_cache.nim
@@ -47,6 +47,7 @@ proc initRedisPool*(cfg: Config) {.async.} =
     await migrate("snappyRss", "rss:*")
     await migrate("userBuckets", "p:*")
     await migrate("profileDates", "p:*")
+    await migrate("profileStats", "p:*")
 
     pool.withAcquire(r):
       # optimize memory usage for profile ID buckets
diff --git a/src/routes/rss.nim b/src/routes/rss.nim
index 5deb1cf..771a3ad 100644
--- a/src/routes/rss.nim
+++ b/src/routes/rss.nim
@@ -19,8 +19,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
     names = getNames(name)
 
   if names.len == 1:
-    (profile, timeline) =
-      await fetchSingleTimeline(after, query, skipRail=true)
+    (profile, timeline) = await fetchTimeline(after, query, skipRail=true)
   else:
     var q = query
     q.fromUser = names
diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim
index 2992888..12c2baa 100644
--- a/src/routes/timeline.nim
+++ b/src/routes/timeline.nim
@@ -19,8 +19,8 @@ proc getQuery*(request: Request; tab, name: string): Query =
   of "search": initQuery(params(request), name=name)
   else: Query(fromUser: @[name])
 
-proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
-                        Future[(Profile, Timeline, PhotoRail)] {.async.} =
+proc fetchTimeline*(after: string; query: Query; skipRail=false):
+                  Future[(Profile, Timeline, PhotoRail)] {.async.} =
   let name = query.fromUser[0]
 
   var
@@ -86,7 +86,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
-  var (p, t, r) = await fetchSingleTimeline(after, query)
+  var (p, t, r) = await fetchTimeline(after, query)
 
   if p.suspended: return showError(getSuspended(p.username), cfg)
   if p.id.len == 0: return
@@ -139,7 +139,7 @@ proc createTimelineRouter*(cfg: Config) =
           timeline.beginning = true
           resp $renderTweetSearch(timeline, prefs, getPath())
         else:
-          var (_, timeline, _) = await fetchSingleTimeline(after, query, skipRail=true)
+          var (_, timeline, _) = await fetchTimeline(after, query, skipRail=true)
           if timeline.content.len == 0: resp Http404
           timeline.beginning = true
           resp $renderTimelineTweets(timeline, prefs, getPath())
diff --git a/src/types.nim b/src/types.nim
index 8bb1956..ce3446b 100644
--- a/src/types.nim
+++ b/src/types.nim
@@ -48,17 +48,16 @@ type
     id*: string
     username*: string
     fullname*: string
-    lowername*: string
     location*: string
     website*: string
     bio*: string
     userPic*: string
     banner*: string
-    following*: string
-    followers*: string
-    tweets*: string
-    likes*: string
-    media*: string
+    following*: int
+    followers*: int
+    tweets*: int
+    likes*: int
+    media*: int
     verified*: bool
     protected*: bool
     suspended*: bool
diff --git a/src/views/profile.nim b/src/views/profile.nim
index 9481faf..a0ef40f 100644
--- a/src/views/profile.nim
+++ b/src/views/profile.nim
@@ -5,12 +5,12 @@ import karax/[karaxdsl, vdom, vstyles]
 import renderutils, search
 import ".."/[types, utils, formatters]
 
-proc renderStat(num, class: string; text=""): VNode =
+proc renderStat(num: int; class: string; text=""): VNode =
   let t = if text.len > 0: text else: class
   buildHtml(li(class=class)):
     span(class="profile-stat-header"): text capitalizeAscii(t)
     span(class="profile-stat-num"):
-      text if num.len == 0: "?" else: insertSep(num, ',')
+      text insertSep($num, ',')
 
 proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
   buildHtml(tdiv(class="profile-card")):

From cdf49dcdddf25b746550dd7dc3602cbfff6178dc Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 06:00:11 +0100
Subject: [PATCH 18/28] Add experimental user parser

---
 config.nims                        |  1 +
 src/api.nim                        |  9 ++--
 src/apiutils.nim                   | 74 ++++++++++++++++++++----------
 src/experimental/parser/slices.nim | 67 +++++++++++++++++++++++++++
 src/experimental/parser/user.nim   | 68 +++++++++++++++++++++++++++
 src/experimental/parser/utils.nim  | 22 +++++++++
 src/experimental/types/common.nim  | 30 ++++++++++++
 src/experimental/types/user.nim    | 28 +++++++++++
 8 files changed, 270 insertions(+), 29 deletions(-)
 create mode 100644 src/experimental/parser/slices.nim
 create mode 100644 src/experimental/parser/user.nim
 create mode 100644 src/experimental/parser/utils.nim
 create mode 100644 src/experimental/types/common.nim
 create mode 100644 src/experimental/types/user.nim

diff --git a/config.nims b/config.nims
index b74d70e..ee77289 100644
--- a/config.nims
+++ b/config.nims
@@ -7,6 +7,7 @@
 # disable annoying warnings
 warning("GcUnsafe2", off)
 hint("XDeclaredButNotUsed", off)
+hint("XCannotRaiseY", off)
 hint("User", off)
 
 const
diff --git a/src/api.nim b/src/api.nim
index 9a6e70a..50771b7 100644
--- a/src/api.nim
+++ b/src/api.nim
@@ -2,6 +2,7 @@
 import asyncdispatch, httpclient, uri, strutils
 import packedjson
 import types, query, formatters, consts, apiutils, parser
+import experimental/parser/user
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
@@ -32,14 +33,14 @@ proc getListMembers*(list: List; after=""): Future[Result[Profile]] {.async.} =
 proc getProfile*(username: string): Future[Profile] {.async.} =
   let
     ps = genParams({"screen_name": username})
-    js = await fetch(userShow ? ps, Api.userShow)
-  result = parseUserShow(js, username=username)
+    json = await fetchRaw(userShow ? ps, Api.userShow)
+  result = parseUser(json)
 
 proc getProfileById*(userId: string): Future[Profile] {.async.} =
   let
     ps = genParams({"user_id": userId})
-    js = await fetch(userShow ? ps, Api.userShow)
-  result = parseUserShow(js, id=userId)
+    json = await fetchRaw(userShow ? ps, Api.userShow)
+  result = parseUser(json)
 
 proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
   let
diff --git a/src/apiutils.nim b/src/apiutils.nim
index e287dc5..85d789b 100644
--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, times, strutils, uri
-import packedjson, zippy
+import httpclient, asyncdispatch, options, sequtils, strutils, uri
+import jsony, packedjson, zippy
 import types, tokens, consts, parserutils, http_pool
+from experimental/types/common import Errors, ErrorObj
 
 const
   rlRemaining = "x-rate-limit-remaining"
@@ -40,7 +41,14 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
     "DNT": "1"
   })
 
-proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+template updateToken() =
+  if api != Api.search and resp.headers.hasKey(rlRemaining):
+    let
+      remaining = parseInt(resp.headers[rlRemaining])
+      reset = parseInt(resp.headers[rlReset])
+    token.setRateLimit(api, remaining, reset)
+
+template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
@@ -48,37 +56,21 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
   if token.tok.len == 0:
     raise rateLimitError()
 
-  let headers = genHeaders(token)
   try:
     var resp: AsyncResponse
-    var body = pool.use(headers):
+    result = pool.use(genHeaders(token)):
       resp = await c.get($url)
       await resp.body
 
-    if body.len > 0:
+    if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
-        body = uncompress(body, dfGzip)
+        result = uncompress(result, dfGzip)
       else:
-        echo "non-gzip body, url: ", url, ", body: ", body
+        echo "non-gzip body, url: ", url, ", body: ", result
 
-    if body.startsWith('{') or body.startsWith('['):
-      result = parseJson(body)
-    else:
-      echo resp.status, ": ", body
-      result = newJNull()
+    fetchBody
 
-    if api != Api.search and resp.headers.hasKey(rlRemaining):
-      let
-        remaining = parseInt(resp.headers[rlRemaining])
-        reset = parseInt(resp.headers[rlReset])
-      token.setRateLimit(api, remaining, reset)
-
-    if result.getError notin {invalidToken, forbidden, badToken}:
-      release(token, used=true)
-    else:
-      echo "fetch error: ", result.getError
-      release(token, invalid=true)
-      raise rateLimitError()
+    release(token, used=true)
 
     if resp.status == $Http400:
       raise newException(InternalError, $url)
@@ -89,3 +81,35 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
     if "length" notin e.msg and "descriptor" notin e.msg:
       release(token, invalid=true)
     raise rateLimitError()
+
+proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+  var body: string
+  fetchImpl body:
+    if body.startsWith('{') or body.startsWith('['):
+      result = parseJson(body)
+    else:
+      echo resp.status, ": ", body
+      result = newJNull()
+
+    updateToken()
+
+    let error = result.getError
+    if error in {invalidToken, forbidden, badToken}:
+      echo "fetch error: ", result.getError
+      release(token, invalid=true)
+      raise rateLimitError()
+
+proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
+  fetchImpl result:
+    if not (result.startsWith('{') or result.startsWith('[')):
+      echo resp.status, ": ", result
+      result.setLen(0)
+
+    updateToken()
+
+    if result.startsWith("{\"errors"):
+      let errors = result.fromJson(Errors).errors
+      if errors.anyIt(it.code in {invalidToken, forbidden, badToken}):
+        echo "fetch error: ", errors
+        release(token, invalid=true)
+        raise rateLimitError()
diff --git a/src/experimental/parser/slices.nim b/src/experimental/parser/slices.nim
new file mode 100644
index 0000000..45e6e1d
--- /dev/null
+++ b/src/experimental/parser/slices.nim
@@ -0,0 +1,67 @@
+import std/[macros, htmlgen, unicode]
+import ../types/common
+import ".."/../[formatters, utils]
+
+type
+  ReplaceSliceKind = enum
+    rkRemove, rkUrl, rkHashtag, rkMention
+
+  ReplaceSlice* = object
+    slice: Slice[int]
+    kind: ReplaceSliceKind
+    url, display: string
+
+proc cmp*(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
+
+proc dedupSlices*(s: var seq[ReplaceSlice]) =
+  var
+    len = s.len
+    i = 0
+  while i < len:
+    var j = i + 1
+    while j < len:
+      if s[i].slice.a == s[j].slice.a:
+        s.del j
+        dec len
+      else:
+        inc j
+    inc i
+
+proc extractUrls*(result: var seq[ReplaceSlice]; url: Url;
+                  textLen: int; hideTwitter = false) =
+  let
+    link = url.expandedUrl
+    slice = url.indices[0] ..< url.indices[1]
+
+  if hideTwitter and slice.b.succ >= textLen and link.isTwitterUrl:
+    if slice.a < textLen:
+      result.add ReplaceSlice(kind: rkRemove, slice: slice)
+  else:
+    result.add ReplaceSlice(kind: rkUrl, url: link,
+                            display: link.shortLink, slice: slice)
+
+proc replacedWith*(runes: seq[Rune]; repls: openArray[ReplaceSlice];
+                   textSlice: Slice[int]): string =
+  template extractLowerBound(i: int; idx): int =
+    if i > 0: repls[idx].slice.b.succ else: textSlice.a
+
+  result = newStringOfCap(runes.len)
+
+  for i, rep in repls:
+    result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
+    case rep.kind
+    of rkHashtag:
+      let
+        name = $runes[rep.slice.a.succ .. rep.slice.b]
+        symbol = $runes[rep.slice.a]
+      result.add a(symbol & name, href = "/search?q=%23" & name)
+    of rkMention:
+      result.add a($runes[rep.slice], href = rep.url, title = rep.display)
+    of rkUrl:
+      result.add a(rep.display, href = rep.url)
+    of rkRemove:
+      discard
+
+  let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
+  if rest.a <= rest.b:
+    result.add $runes[rest]
diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim
new file mode 100644
index 0000000..8a77aca
--- /dev/null
+++ b/src/experimental/parser/user.nim
@@ -0,0 +1,68 @@
+import std/[algorithm, unicode, re, strutils]
+import jsony
+import utils, slices
+import ../types/user as userType
+from ../../types import Profile, Error
+
+let
+  unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
+  unReplace = "$1<a href=\"/$2\">@$2</a>"
+
+  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
+  htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
+
+proc expandProfileEntities(profile: var Profile; user: User) =
+  let
+    orig = profile.bio.toRunes
+    ent = user.entities
+
+  if ent.url.urls.len > 0:
+    profile.website = ent.url.urls[0].expandedUrl
+
+  var replacements = newSeq[ReplaceSlice]()
+
+  for u in ent.description.urls:
+    replacements.extractUrls(u, orig.high)
+
+  replacements.dedupSlices
+  replacements.sort(cmp)
+
+  profile.bio = orig.replacedWith(replacements, 0 .. orig.len)
+                    .replacef(unRegex, unReplace)
+                    .replacef(htRegex, htReplace)
+
+proc getBanner(user: User): string =
+  if user.profileBannerUrl.len > 0:
+    return user.profileBannerUrl & "/1500x500"
+  if user.profileLinkColor.len > 0:
+    return '#' & user.profileLinkColor
+  return "#161616"
+
+proc parseUser*(json: string): Profile =
+  handleErrors:
+    case error
+    of suspended: return Profile(suspended: true)
+    of userNotFound: return
+    else: echo "[error - parseUser]: ", error
+
+  let user = json.fromJson(User)
+
+  result = Profile(
+    id: user.idStr,
+    username: user.screenName,
+    fullname: user.name,
+    location: user.location,
+    bio: user.description,
+    following: user.friendsCount,
+    followers: user.followersCount,
+    tweets: user.statusesCount,
+    likes: user.favouritesCount,
+    media: user.mediaCount,
+    verified: user.verified,
+    protected: user.protected,
+    joinDate: parseTwitterDate(user.createdAt),
+    banner: getBanner(user),
+    userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
+  )
+
+  result.expandProfileEntities(user)
diff --git a/src/experimental/parser/utils.nim b/src/experimental/parser/utils.nim
new file mode 100644
index 0000000..1614093
--- /dev/null
+++ b/src/experimental/parser/utils.nim
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import std/[sugar, strutils, times]
+import ../types/common
+import ../../utils as uutils
+
+template parseTime(time: string; f: static string; flen: int): DateTime =
+  if time.len != flen: return
+  parse(time, f, utc())
+
+proc parseIsoDate*(date: string): DateTime =
+  date.parseTime("yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
+
+proc parseTwitterDate*(date: string): DateTime =
+  date.parseTime("ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
+
+proc getImageUrl*(url: string): string =
+  url.dup(removePrefix(twimg), removePrefix(https))
+
+template handleErrors*(body) =
+  if json.startsWith("{\"errors"):
+    let error {.inject.} = json.fromJson(Errors).errors[0].code
+    body
diff --git a/src/experimental/types/common.nim b/src/experimental/types/common.nim
new file mode 100644
index 0000000..1d3b30b
--- /dev/null
+++ b/src/experimental/types/common.nim
@@ -0,0 +1,30 @@
+from ../../types import Error
+
+type
+  Url* = object
+    url*: string
+    expandedUrl*: string
+    displayUrl*: string
+    indices*: array[2, int]
+
+  ErrorCode* = enum
+    null = 0
+    noUserMatches = 17
+    protectedUser = 22
+    couldntAuth = 32
+    doesntExist = 34
+    userNotFound = 50
+    suspended = 63
+    rateLimited = 88
+    invalidToken = 89
+    listIdOrSlug = 112
+    forbidden = 200
+    badToken = 239
+    noCsrf = 353
+
+  ErrorObj* = object
+    code*: Error
+    message*: string
+
+  Errors* = object
+    errors*: seq[ErrorObj]
diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim
new file mode 100644
index 0000000..1f31318
--- /dev/null
+++ b/src/experimental/types/user.nim
@@ -0,0 +1,28 @@
+import common
+
+type
+  User* = object
+    idStr*: string
+    name*: string
+    screenName*: string
+    location*: string
+    description*: string
+    entities*: Entities
+    createdAt*: string
+    followersCount*: int
+    friendsCount*: int
+    favouritesCount*: int
+    statusesCount*: int
+    mediaCount*: int
+    verified*: bool
+    protected*: bool
+    profileBannerUrl*: string
+    profileImageUrlHttps*: string
+    profileLinkColor*: string
+
+  Entities* = object
+    url*: Urls
+    description*: Urls
+
+  Urls* = object
+    urls*: seq[Url]

From 6f348f2f2e30a056829af16e2f28f6bcd45dc014 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 06:18:01 +0100
Subject: [PATCH 19/28] Strip trailing newlines from tweets

---
 src/parserutils.nim | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/parserutils.nim b/src/parserutils.nim
index aae3dfc..3291955 100644
--- a/src/parserutils.nim
+++ b/src/parserutils.nim
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, times, macros, htmlgen, unicode, options, algorithm
-import std/re
+import std/[strutils, times, macros, htmlgen, options, algorithm, re]
+import std/unicode except strip
 import packedjson
 import types, utils, formatters
 
@@ -275,3 +275,4 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   replacements.sort(cmp)
 
   tweet.text = orig.replacedWith(replacements, textSlice)
+                   .strip(leading=false)

From 3ab778b49c7689ba80131894036a88e09a5f3285 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 06:34:38 +0100
Subject: [PATCH 20/28] Remove old parseUserShow proc

---
 src/parser.nim | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/parser.nim b/src/parser.nim
index 011798f..0991091 100644
--- a/src/parser.nim
+++ b/src/parser.nim
@@ -26,21 +26,6 @@ proc parseProfile(js: JsonNode; id=""): Profile =
 
   result.expandProfileEntities(js)
 
-proc parseUserShow*(js: JsonNode; username=""; id=""): Profile =
-  if id.len > 0:
-    result = Profile(id: id)
-  else:
-    result = Profile(username: username)
-
-  if js.isNull: return
-
-  with error, js{"errors"}:
-    if error.getError == suspended:
-      result.suspended = true
-    return
-
-  result = parseProfile(js)
-
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 

From 3d91ae0256d899da5346449c26380fa269011b6e Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 17:56:45 +0100
Subject: [PATCH 21/28] Set tokens to expire 5 minutes early

Prevents occasional usage of tokens the very second they expire
---
 src/experimental/types/common.nim | 15 ---------------
 src/tokens.nim                    |  2 +-
 2 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/src/experimental/types/common.nim b/src/experimental/types/common.nim
index 1d3b30b..90667dc 100644
--- a/src/experimental/types/common.nim
+++ b/src/experimental/types/common.nim
@@ -7,21 +7,6 @@ type
     displayUrl*: string
     indices*: array[2, int]
 
-  ErrorCode* = enum
-    null = 0
-    noUserMatches = 17
-    protectedUser = 22
-    couldntAuth = 32
-    doesntExist = 34
-    userNotFound = 50
-    suspended = 63
-    rateLimited = 88
-    invalidToken = 89
-    listIdOrSlug = 112
-    forbidden = 200
-    badToken = 239
-    noCsrf = 353
-
   ErrorObj* = object
     code*: Error
     message*: string
diff --git a/src/tokens.nim b/src/tokens.nim
index 929a984..bc7ea55 100644
--- a/src/tokens.nim
+++ b/src/tokens.nim
@@ -6,8 +6,8 @@ import types, agents, consts, http_pool
 
 const
   maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
-  maxAge = 3.hours       # tokens expire after 3 hours
   maxLastUse = 1.hours   # if a token is unused for 60 minutes, it expires
+  maxAge = 2.hours + 55.minutes  # tokens expire after 3 hours
   failDelay = initDuration(minutes=30)
 
 var

From fff04de24bbe704c02e35b68f9cdb178c6e9b4cb Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 18:28:40 +0100
Subject: [PATCH 22/28] Simplify new error handling

---
 src/apiutils.nim                  | 8 ++++----
 src/experimental/parser/user.nim  | 2 +-
 src/experimental/parser/utils.nim | 4 ++--
 src/experimental/types/common.nim | 5 +++++
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/apiutils.nim b/src/apiutils.nim
index 85d789b..7873c22 100644
--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, sequtils, strutils, uri
+import httpclient, asyncdispatch, options, strutils, uri
 import jsony, packedjson, zippy
 import types, tokens, consts, parserutils, http_pool
-from experimental/types/common import Errors, ErrorObj
+import experimental/types/common
 
 const
   rlRemaining = "x-rate-limit-remaining"
@@ -108,8 +108,8 @@ proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
     updateToken()
 
     if result.startsWith("{\"errors"):
-      let errors = result.fromJson(Errors).errors
-      if errors.anyIt(it.code in {invalidToken, forbidden, badToken}):
+      let errors = result.fromJson(Errors)
+      if errors in {invalidToken, forbidden, badToken}:
         echo "fetch error: ", errors
         release(token, invalid=true)
         raise rateLimitError()
diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim
index 8a77aca..f5b91ac 100644
--- a/src/experimental/parser/user.nim
+++ b/src/experimental/parser/user.nim
@@ -40,7 +40,7 @@ proc getBanner(user: User): string =
 
 proc parseUser*(json: string): Profile =
   handleErrors:
-    case error
+    case error.code
     of suspended: return Profile(suspended: true)
     of userNotFound: return
     else: echo "[error - parseUser]: ", error
diff --git a/src/experimental/parser/utils.nim b/src/experimental/parser/utils.nim
index 1614093..999683d 100644
--- a/src/experimental/parser/utils.nim
+++ b/src/experimental/parser/utils.nim
@@ -18,5 +18,5 @@ proc getImageUrl*(url: string): string =
 
 template handleErrors*(body) =
   if json.startsWith("{\"errors"):
-    let error {.inject.} = json.fromJson(Errors).errors[0].code
-    body
+    for error {.inject.} in json.fromJson(Errors).errors:
+      body
diff --git a/src/experimental/types/common.nim b/src/experimental/types/common.nim
index 90667dc..e979015 100644
--- a/src/experimental/types/common.nim
+++ b/src/experimental/types/common.nim
@@ -13,3 +13,8 @@ type
 
   Errors* = object
     errors*: seq[ErrorObj]
+
+proc contains*(codes: set[Error]; errors: Errors): bool =
+  for e in errors.errors:
+    if e.code in codes:
+      return true

From 2aa07e7395f370fff6128cab68b5995feb09e156 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 18:53:30 +0100
Subject: [PATCH 23/28] Remove broken ARM64 Docker image

---
 .github/workflows/build-docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
index d30c0db..30a209b 100644
--- a/.github/workflows/build-docker.yml
+++ b/.github/workflows/build-docker.yml
@@ -33,6 +33,6 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}

From 23f87c115a2fc8c14b2dbb735426bb50d078daa8 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 19:22:27 +0100
Subject: [PATCH 24/28] Add template to make Redis usage cleaner

---
 src/redis_cache.nim | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/src/redis_cache.nim b/src/redis_cache.nim
index 7a921c8..ea1df26 100644
--- a/src/redis_cache.nim
+++ b/src/redis_cache.nim
@@ -13,6 +13,9 @@ var
   rssCacheTime: int
   listCacheTime*: int
 
+template dawait(future) =
+  discard await future
+
 # flatty can't serialize DateTime, so we need to define this
 proc toFlatty*(s: var string, x: DateTime) =
   s.toFlatty(x.toTime().toUnix())
@@ -33,9 +36,9 @@ proc migrate*(key, match: string) {.async.} =
       let list = await r.scan(newCursor(0), match, 100000)
       r.startPipelining()
       for item in list:
-        discard await r.del(item)
+        dawait r.del(item)
       await r.setk(key, "true")
-      discard await r.flushPipeline()
+      dawait r.flushPipeline()
 
 proc initRedisPool*(cfg: Config) {.async.} =
   try:
@@ -68,7 +71,7 @@ proc get(query: string): Future[string] {.async.} =
 
 proc setEx(key: string; time: int; data: string) {.async.} =
   pool.withAcquire(r):
-    discard await r.setEx(key, time, data)
+    dawait r.setEx(key, time, data)
 
 proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
@@ -81,25 +84,25 @@ proc cache*(data: Profile) {.async.} =
   let name = toLower(data.username)
   pool.withAcquire(r):
     r.startPipelining()
-    discard await r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
-    discard await r.setEx("i:" & data.id , baseCacheTime, data.username)
-    discard await r.hSet(name.pidKey, name, data.id)
-    discard await r.flushPipeline()
+    dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
+    dawait r.setEx("i:" & data.id , baseCacheTime, data.username)
+    dawait r.hSet(name.pidKey, name, data.id)
+    dawait r.flushPipeline()
 
 proc cacheProfileId*(username, id: string) {.async.} =
   if username.len == 0 or id.len == 0: return
   let name = toLower(username)
   pool.withAcquire(r):
-    discard await r.hSet(name.pidKey, name, id)
+    dawait r.hSet(name.pidKey, name, id)
 
 proc cacheRss*(query: string; rss: Rss) {.async.} =
   let key = "rss:" & query
   pool.withAcquire(r):
     r.startPipelining()
-    discard await r.hSet(key, "rss", rss.feed)
-    discard await r.hSet(key, "min", rss.cursor)
-    discard await r.expire(key, rssCacheTime)
-    discard await r.flushPipeline()
+    dawait r.hSet(key, "rss", rss.feed)
+    dawait r.hSet(key, "min", rss.cursor)
+    dawait r.expire(key, rssCacheTime)
+    dawait r.flushPipeline()
 
 proc getProfileId*(username: string): Future[string] {.async.} =
   let name = toLower(username)

From f3d6f53f6dcf882c79453e18b469b47b5fd01f38 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 16 Jan 2022 20:32:45 +0100
Subject: [PATCH 25/28] Rework profile cache behavior, fix suspended cache

Fixes #480
---
 src/api.nim                      |  2 +-
 src/experimental/parser/user.nim |  4 ++--
 src/redis_cache.nim              | 18 ++++++++++++------
 src/routes/timeline.nim          | 17 +++++------------
 4 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/src/api.nim b/src/api.nim
index 50771b7..b1ecc57 100644
--- a/src/api.nim
+++ b/src/api.nim
@@ -34,7 +34,7 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
   let
     ps = genParams({"screen_name": username})
     json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json)
+  result = parseUser(json, username)
 
 proc getProfileById*(userId: string): Future[Profile] {.async.} =
   let
diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim
index f5b91ac..af8cb82 100644
--- a/src/experimental/parser/user.nim
+++ b/src/experimental/parser/user.nim
@@ -38,10 +38,10 @@ proc getBanner(user: User): string =
     return '#' & user.profileLinkColor
   return "#161616"
 
-proc parseUser*(json: string): Profile =
+proc parseUser*(json: string; username=""): Profile =
   handleErrors:
     case error.code
-    of suspended: return Profile(suspended: true)
+    of suspended: return Profile(username: username, suspended: true)
     of userNotFound: return
     else: echo "[error - parseUser]: ", error
 
diff --git a/src/redis_cache.nim b/src/redis_cache.nim
index ea1df26..fe622f9 100644
--- a/src/redis_cache.nim
+++ b/src/redis_cache.nim
@@ -80,16 +80,16 @@ proc cache*(data: PhotoRail; name: string) {.async.} =
   await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: Profile) {.async.} =
-  if data.username.len == 0 or data.id.len == 0: return
+  if data.username.len == 0: return
   let name = toLower(data.username)
   pool.withAcquire(r):
     r.startPipelining()
     dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
-    dawait r.setEx("i:" & data.id , baseCacheTime, data.username)
-    dawait r.hSet(name.pidKey, name, data.id)
+    if data.id.len > 0:
+      dawait r.hSet(name.pidKey, name, data.id)
     dawait r.flushPipeline()
 
-proc cacheProfileId*(username, id: string) {.async.} =
+proc cacheProfileId(username, id: string) {.async.} =
   if username.len == 0 or id.len == 0: return
   let name = toLower(username)
   pool.withAcquire(r):
@@ -117,15 +117,21 @@ proc getCachedProfile*(username: string; fetch=true): Future[Profile] {.async.}
     result = fromFlatty(uncompress(prof), Profile)
   elif fetch:
     result = await getProfile(username)
+    await cacheProfileId(result.username, result.id)
+    if result.suspended:
+      await cache(result)
 
 proc getCachedProfileUsername*(userId: string): Future[string] {.async.} =
-  let username = await get("i:" & userId)
+  let
+    key = "i:" & userId
+    username = await get(key)
+
   if username != redisNil:
     result = username
   else:
     let profile = await getProfileById(userId)
     result = profile.username
-    await cache(profile)
+    await setEx(key, baseCacheTime, result)
 
 proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
   if name.len == 0: return
diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim
index 12c2baa..9eefc4e 100644
--- a/src/routes/timeline.nim
+++ b/src/routes/timeline.nim
@@ -30,20 +30,13 @@ proc fetchTimeline*(after: string; query: Query; skipRail=false):
 
   if profileId.len == 0:
     profile = await getCachedProfile(name)
-    profileId = if profile.suspended: "s"
-                else: profile.id
-
-    if profileId.len > 0:
-       await cacheProfileId(profile.username, profileId)
-
+    profileId = profile.id
     fetched = true
 
-  if profileId.len == 0 or profile.protected:
-    result[0] = profile
-    return
-  elif profileId == "s":
-    result[0] = Profile(username: name, suspended: true)
-    return
+  if profile.protected or profile.suspended:
+    return (profile, Timeline(), @[])
+  elif profileId.len == 0:
+    return (Profile(username: name), Timeline(), @[])
 
   var rail: Future[PhotoRail]
   if skipRail or profile.protected or query.kind == media:

From e0b141daf9c6bef7d4fc685e35d6cb17f0dfa497 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Mon, 17 Jan 2022 03:21:38 +0100
Subject: [PATCH 26/28] Small optimization for photo rail request size

---
 src/apiutils.nim | 2 ++
 src/consts.nim   | 3 ---
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/apiutils.nim b/src/apiutils.nim
index 7873c22..f6a4b47 100644
--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@@ -17,6 +17,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
     result &= p
   if ext:
     result &= ("ext", "mediaStats")
+    result &= ("include_ext_alt_text", "true")
+    result &= ("include_ext_media_availability", "true")
   if count.len > 0:
     result &= ("count", count)
   if cursor.len > 0:
diff --git a/src/consts.nim b/src/consts.nim
index efa0420..c77ebef 100644
--- a/src/consts.nim
+++ b/src/consts.nim
@@ -35,16 +35,13 @@ const
     "cards_platform": "Web-12",
     "include_cards": "1",
     "include_composer_source": "false",
-    "include_ext_alt_text": "true",
     "include_reply_count": "1",
     "tweet_mode": "extended",
     "include_entities": "true",
     "include_user_entities": "true",
     "include_ext_media_color": "false",
-    "include_ext_media_availability": "true",
     "send_error_codes": "true",
     "simple_quoted_tweet": "true",
-    "ext": "mediaStats",
     "include_quote_count": "true"
   }.toSeq
 

From 43b0bdc08a64554f9a3cf3f922e65a4e7d09bbdf Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Mon, 17 Jan 2022 04:13:27 +0100
Subject: [PATCH 27/28] Remove user agents

---
 src/agents.nim       | 70 --------------------------------------------
 src/routes/media.nim | 16 +++++-----
 src/tokens.nim       |  3 +-
 3 files changed, 8 insertions(+), 81 deletions(-)
 delete mode 100644 src/agents.nim

diff --git a/src/agents.nim b/src/agents.nim
deleted file mode 100644
index a907e7b..0000000
--- a/src/agents.nim
+++ /dev/null
@@ -1,70 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import random, strformat, strutils, sequtils
-
-randomize()
-
-const rvs = [
-  "11.0", "40.0", "42.0", "43.0", "47.0", "50.0", "52.0", "53.0", "54.0",
-  "61.0", "66.0", "67.0", "69.0", "70.0"
-]
-
-proc rv(): string =
-  if rand(10) < 1: ""
-  else: "; rv:" & sample(rvs)
-
-# OS
-
-const enc = ["; U", "; N", "; I", ""]
-
-proc linux(): string =
-  const
-    window = ["X11", "Wayland", "Unknown"]
-    arch = ["i686", "x86_64", "arm"]
-    distro = ["", "; Ubuntu/14.10", "; Ubuntu/16.10", "; Ubuntu/19.10",
-              "; Ubuntu", "; Fedora"]
-  sample(window) & sample(enc) & "; Linux " & sample(arch) & sample(distro)
-
-proc windows(): string =
-  const
-    nt = ["5.1", "5.2", "6.0", "6.1", "6.2", "6.3", "6.4", "9.0", "10.0"]
-    arch = ["; WOW64", "; Win64; x64", "; ARM", ""]
-    trident = ["", "; Trident/5.0", "; Trident/6.0", "; Trident/7.0"]
-  "Windows " & sample(nt) & sample(enc) & sample(arch) & sample(trident)
-
-const macs = toSeq(6..15).mapIt($it) & @["14_4", "10_1", "9_3"]
-
-proc mac(): string =
-  "Macintosh; Intel Mac OS X 10_" & sample(macs) & sample(enc)
-
-# Browser
-
-proc presto(): string =
-  const p = ["2.12.388", "2.12.407", "22.9.168", "2.9.201", "2.8.131", "2.7.62",
-             "2.6.30", "2.5.24"]
-  const v = ["10.0", "11.0", "11.1", "11.5", "11.6", "12.00", "12.14", "12.16"]
-  &"Presto/{sample(p)} Version/{sample(v)}"
-
-# Samples
-
-proc product(): string =
-  const opera = ["Opera/9.80", "Opera/12.0"]
-  if rand(20) < 1: "Mozilla/5.0"
-  else: sample(opera)
-
-proc os(): string =
-  let r = rand(10)
-  let os =
-    if r < 6: windows()
-    elif r < 9: linux()
-    else: mac()
-  &"({os}{rv()})"
-
-proc browser(prod: string): string =
-  if "Opera" in prod: presto()
-  else: "like Gecko"
-
-# Agent
-
-proc getAgent*(): string =
-  let prod = product()
-  &"{prod} {os()} {browser(prod)}"
diff --git a/src/routes/media.nim b/src/routes/media.nim
index a2a6369..c953a93 100644
--- a/src/routes/media.nim
+++ b/src/routes/media.nim
@@ -5,7 +5,7 @@ import asynchttpserver, asyncstreams, asyncfile, asyncnet
 import jester
 
 import router_utils
-import ".."/[types, formatters, agents, utils]
+import ".."/[types, formatters, utils]
 
 export asynchttpserver, asyncstreams, asyncfile, asyncnet
 export httpclient, os, strutils, asyncstreams, base64, re
@@ -14,10 +14,8 @@ const
   m3u8Mime* = "application/vnd.apple.mpegurl"
   maxAge* = "max-age=604800"
 
-let mediaAgent* = getAgent()
-
-proc safeFetch*(url, agent: string): Future[string] {.async.} =
-  let client = newAsyncHttpClient(userAgent=agent)
+proc safeFetch*(url: string): Future[string] {.async.} =
+  let client = newAsyncHttpClient()
   try: result = await client.getContent(url)
   except: discard
   finally: client.close()
@@ -34,7 +32,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
   result = Http200
   let
     request = req.getNativeReq()
-    client = newAsyncHttpClient(userAgent=mediaAgent)
+    client = newAsyncHttpClient()
 
   try:
     let res = await client.get(url)
@@ -116,14 +114,14 @@ proc createMediaRouter*(cfg: Config) =
 
       var content: string
       if ".vmap" in url:
-        let m3u8 = getM3u8Url(await safeFetch(url, mediaAgent))
+        let m3u8 = getM3u8Url(await safeFetch(url))
         if m3u8.len > 0:
-          content = await safeFetch(url, mediaAgent)
+          content = await safeFetch(url)
         else:
           resp Http404
 
       if ".m3u8" in url:
-        let vid = await safeFetch(url, mediaAgent)
+        let vid = await safeFetch(url)
         content = proxifyVideo(vid, cookiePref(proxyVideos))
 
       resp content, m3u8Mime
diff --git a/src/tokens.nim b/src/tokens.nim
index bc7ea55..d864a4c 100644
--- a/src/tokens.nim
+++ b/src/tokens.nim
@@ -2,7 +2,7 @@
 import asyncdispatch, httpclient, times, sequtils, json, random
 import strutils, tables
 import zippy
-import types, agents, consts, http_pool
+import types, consts, http_pool
 
 const
   maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
@@ -65,7 +65,6 @@ proc fetchToken(): Future[Token] {.async.} =
     "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.5",
     "connection": "keep-alive",
-    "user-agent": getAgent(),
     "authorization": auth
   })
 

From b01810e2613491236c3b46624490d779b680a7a6 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Mon, 17 Jan 2022 05:50:45 +0100
Subject: [PATCH 28/28] Improve profile page elements, reduce jank

Fixes #167
---
 src/experimental/parser/user.nim |  1 -
 src/parserutils.nim              | 12 ------------
 src/sass/profile/_base.scss      | 17 ++++++++---------
 src/sass/profile/card.scss       | 20 +++++++++++++-------
 src/sass/profile/photo-rail.scss |  1 +
 src/views/general.nim            |  2 +-
 src/views/profile.nim            | 17 ++++++++---------
 7 files changed, 31 insertions(+), 39 deletions(-)

diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim
index af8cb82..de55ed8 100644
--- a/src/experimental/parser/user.nim
+++ b/src/experimental/parser/user.nim
@@ -36,7 +36,6 @@ proc getBanner(user: User): string =
     return user.profileBannerUrl & "/1500x500"
   if user.profileLinkColor.len > 0:
     return '#' & user.profileLinkColor
-  return "#161616"
 
 proc parseUser*(json: string; username=""): Profile =
   handleErrors:
diff --git a/src/parserutils.nim b/src/parserutils.nim
index 3291955..4b18202 100644
--- a/src/parserutils.nim
+++ b/src/parserutils.nim
@@ -119,18 +119,6 @@ proc getBanner*(js: JsonNode): string =
   if color.len > 0:
     return '#' & color
 
-  # use primary color from profile picture color histogram
-  with p, js{"profile_image_extensions", "mediaColor", "r", "ok", "palette"}:
-    if p.len > 0:
-      let pal = p[0]{"rgb"}
-      result = "#"
-      result.add toHex(pal{"red"}.getInt, 2)
-      result.add toHex(pal{"green"}.getInt, 2)
-      result.add toHex(pal{"blue"}.getInt, 2)
-      return
-
-  return "#161616"
-
 proc getTombstone*(js: JsonNode): string =
   result = js{"tombstoneInfo", "richText", "text"}.getStr
   result.removeSuffix(" Learn more")
diff --git a/src/sass/profile/_base.scss b/src/sass/profile/_base.scss
index 6a2cfec..ae6b801 100644
--- a/src/sass/profile/_base.scss
+++ b/src/sass/profile/_base.scss
@@ -15,23 +15,22 @@
 }
 
 .profile-banner {
-    padding-bottom: 4px;
+    margin-bottom: 4px;
+    background-color: var(--bg_panel);
 
     a {
-        display: inherit;
-        line-height: 0;
+        display: block;
+        position: relative;
+        padding: 33.34% 0 0 0;
     }
 
     img {
-        width: 100%;
+        max-width: 100%;
+        position: absolute;
+        top: 0;
     }
 }
 
-.profile-banner-color {
-    width: 100%;
-    padding-bottom: 25%;
-}
-
 .profile-tab {
     padding: 0 4px 0 0;
     box-sizing: border-box;
diff --git a/src/sass/profile/card.scss b/src/sass/profile/card.scss
index 9475fbf..cc68d7d 100644
--- a/src/sass/profile/card.scss
+++ b/src/sass/profile/card.scss
@@ -35,19 +35,25 @@
 }
 
 .profile-card-avatar {
-    display: block;
+    display: inline-block;
+    position: relative;
     width: 100%;
-    padding-bottom: 6px;
     margin-right: 4px;
+    margin-bottom: 6px;
+
+    &:after {
+        content: '';
+        display: block;
+        margin-top: 100%;
+    }
 
     img {
-        display: block;
         box-sizing: border-box;
+        position: absolute;
         width: 100%;
         height: 100%;
-        margin: 0;
         border: 4px solid var(--darker_grey);
-        background: var(--bg_color);
+        background: var(--bg_panel);
     }
 }
 
@@ -113,8 +119,8 @@
     }
 
     .profile-card-avatar {
-        height: 60px;
-        width: unset;
+        width: 80px;
+        height: 80px;
 
         img {
             border-width: 2px;
diff --git a/src/sass/profile/photo-rail.scss b/src/sass/profile/photo-rail.scss
index 314eadf..f32d22a 100644
--- a/src/sass/profile/photo-rail.scss
+++ b/src/sass/profile/photo-rail.scss
@@ -32,6 +32,7 @@
         a {
             position: relative;
             border-radius: 5px;
+            background-color: var(--darker_grey);
 
             &:before {
                 content: "";
diff --git a/src/views/general.nim b/src/views/general.nim
index 010e2df..7054fd4 100644
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=14")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=15")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:
diff --git a/src/views/profile.nim b/src/views/profile.nim
index a0ef40f..e44b8a2 100644
--- a/src/views/profile.nim
+++ b/src/views/profile.nim
@@ -78,18 +78,16 @@ proc renderPhotoRail(profile: Profile; photoRail: PhotoRail): VNode =
     tdiv(class="photo-rail-grid"):
       for i, photo in photoRail:
         if i == 16: break
-        let col = if photo.color.len > 0: photo.color else: "#161616"
-        a(href=(&"/{profile.username}/status/{photo.tweetId}#m"),
-          style={backgroundColor: col}):
+        a(href=(&"/{profile.username}/status/{photo.tweetId}#m")):
           genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
 
-proc renderBanner(profile: Profile): VNode =
+proc renderBanner(banner: string): VNode =
   buildHtml():
-    if "#" in profile.banner:
-      tdiv(class="profile-banner-color", style={backgroundColor: profile.banner})
+    if banner.startsWith('#'):
+      a(style={backgroundColor: banner})
     else:
-      a(href=getPicUrl(profile.banner), target="_blank"):
-        genImg(profile.banner)
+      a(href=getPicUrl(banner), target="_blank"):
+        genImg(banner)
 
 proc renderProtected(username: string): VNode =
   buildHtml(tdiv(class="timeline-container")):
@@ -103,7 +101,8 @@ proc renderProfile*(profile: Profile; timeline: var Timeline;
   buildHtml(tdiv(class="profile-tabs")):
     if not prefs.hideBanner:
       tdiv(class="profile-banner"):
-        renderBanner(profile)
+        if profile.banner.len > 0:
+          renderBanner(profile.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
     tdiv(class=(&"profile-tab{sticky}")):