Merge branch 'master' of https://github.com/zedeus/nitter

Fixes #597

Dockerfile

@@ -1,4 +1,4 @@
-FROM nimlang/nim:1.6.2-alpine-regular as nim
+FROM nimlang/nim:1.6.10-alpine-regular as nim
 LABEL maintainer="setenforce@protonmail.com"
 
 RUN apk --no-cache add libsass-dev pcre

nitter.example.conf

@@ -38,7 +38,7 @@ tokenCount = 10
 theme = "Loli"
 replaceTwitter = "twitter.076.ne.jp"
 replaceYouTube = "youtube.076.ne.jp"
-replaceReddit = ""
+replaceReddit = "teddit.net"
 replaceInstagram = ""
 replaceOdysee = "odysee.076.ne.jp"
 proxyVideos = true

nitter.nimble

@@ -11,17 +11,17 @@ bin           = @["nitter"]
 # Dependencies
 
 requires "nim >= 1.4.8"
-requires "jester >= 0.5.0"
+requires "jester#baca3f"
 requires "karax#6abcb77"
 requires "sass#e683aa1"
-requires "nimcrypto#a5742a9"
+requires "nimcrypto#b41129f"
 requires "markdown#a661c26"
 requires "packedjson#9e6fbb6"
-requires "supersnappy#2.1.1"
+requires "supersnappy#6c94198"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#0.9.11"
-requires "flatty#0.2.3"
+requires "zippy#61922b9"
+requires "flatty#9f885d7"
 requires "jsony#d0e69bd"
 
 

src/experimental/parser/unifiedcard.nim

@@ -66,6 +66,8 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
       durationMs: videoInfo.durationMillis,
       variants: videoInfo.variants
     )
+  of model3d:
+    result.title = "Unsupported 3D model ad"
 
 proc parseUnifiedCard*(json: string): Card =
   let card = json.fromJson(UnifiedCard)
@@ -82,6 +84,8 @@ proc parseUnifiedCard*(json: string): Card =
       component.parseMedia(card, result)
     of buttonGroup:
       discard
+    of ComponentType.unknown:
+      echo "ERROR: Unknown component type: ", json
 
     case component.kind
     of twitterListDetails:

src/experimental/types/unifiedcard.nim

@@ -17,6 +17,7 @@ type
     twitterListDetails
     communityDetails
     mediaWithDetailsHorizontal
+    unknown
 
   Component* = object
     kind*: ComponentType
@@ -47,7 +48,7 @@ type
     vanity*: string
 
   MediaType* = enum
-    photo, video
+    photo, video, model3d
 
   MediaEntity* = object
     kind*: MediaType
@@ -77,3 +78,29 @@ converter fromText*(text: Text): string = text.content
 proc renameHook*(v: var HasTypeField; fieldName: var string) =
   if fieldName == "type":
     fieldName = "kind"
+
+proc enumHook*(s: string; v: var ComponentType) =
+  v = case s
+      of "details": details
+      of "media": media
+      of "swipeable_media": swipeableMedia
+      of "button_group": buttonGroup
+      of "app_store_details": appStoreDetails
+      of "twitter_list_details": twitterListDetails
+      of "community_details": communityDetails
+      of "media_with_details_horizontal": mediaWithDetailsHorizontal
+      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
+
+proc enumHook*(s: string; v: var AppType) =
+  v = case s
+      of "android_app": androidApp
+      of "iphone_app": iPhoneApp
+      of "ipad_app": iPadApp
+      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
+
+proc enumHook*(s: string; v: var MediaType) =
+  v = case s
+      of "video": video
+      of "photo": photo
+      of "model3d": model3d
+      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo

src/formatters.nim

@@ -12,7 +12,7 @@ let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
   twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
 
-  ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
+  ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
   igRegex = re"(www\.)?instagram\.com"
   odRegex = re"(www\.)?(odysee\.com|lbry\.tv|open\.lbry\.com)"
 
@@ -57,8 +57,6 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
 
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
     result = result.replace(ytRegex, prefs.replaceYouTube)
-    if prefs.replaceYouTube in result:
-      result = result.replace("/c/", "/")
 
   if prefs.replaceInstagram.len > 0:
     result = result.replace(igRegex, prefs.replaceInstagram)
@@ -67,11 +65,11 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
     result = result.replace(odRegex, prefs.replaceOdysee)
 
   if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
-    result = result.replace(tco, &"{https}{prefs.replaceTwitter}/t.co")
+    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
     result = result.replace(cards, prefs.replaceTwitter & "/cards")
     result = result.replace(twRegex, prefs.replaceTwitter)
     result = result.replacef(twLinkRegex, a(
-      prefs.replaceTwitter & "$2", href = &"{https}{prefs.replaceTwitter}$1"))
+      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
     result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")

src/parser.nim

@@ -45,7 +45,6 @@ proc parseGraphList*(js: JsonNode): List =
     banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
   )
 
-
 proc parsePoll(js: JsonNode): Poll =
   let vals = js{"binding_values"}
   # name format is pollNchoice_*
@@ -206,6 +205,10 @@ proc parseTweet(js: JsonNode): Tweet =
     )
   )
 
+  # fix for pinned threads
+  if result.hasThread and result.threadId == 0:
+    result.threadId = js{"self_thread", "id_str"}.getId
+
   result.expandTweetEntities(js)
 
   if js{"is_quote_status"}.getBool:

src/parserutils.nim

@@ -28,13 +28,13 @@ template `?`*(js: JsonNode): untyped =
   if j.isNull: return
   j
 
-template `with`*(ident, value, body): untyped =
-  block:
+template with*(ident, value, body): untyped =
+  if true:
     let ident {.inject.} = value
     if ident != nil: body
 
-template `with`*(ident; value: JsonNode; body): untyped =
-  block:
+template with*(ident; value: JsonNode; body): untyped =
+  if true:
     let ident {.inject.} = value
     if value.notNull: body
 

src/routes/rss.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, tables, times, hashes, uri
+import asyncdispatch, tables, times, hashes, uri
 
 import jester
 
@@ -10,6 +10,11 @@ include "../views/rss.nimf"
 
 export times, hashes
 
+proc redisKey*(page, name, cursor: string): string =
+  result = page & ":" & name
+  if cursor.len > 0:
+    result &= ":" & cursor
+
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
   var profile: Profile
   let
@@ -42,8 +47,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
   if rss.cursor.len == 0:
     let info = case page
-               of "User": &""" "{@"name"}" """
-               of "List": &""" "{@"id"}" """
+               of "User": " \"" & @"name" & "\" "
+               of "List": " \"" & @"id" & "\" "
                else: " "
 
     resp Http404, showError(page & info & "not found", cfg)
@@ -67,7 +72,7 @@ proc createRssRouter*(cfg: Config) =
 
       let
         cursor = getCursor()
-        key = &"search:{hash(genQueryUrl(query))}:cursor"
+        key = redisKey("search", $hash(genQueryUrl(query)), cursor)
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -84,9 +89,8 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       let
-        cursor = getCursor()
         name = @"name"
-        key = &"twitter:{name}:{cursor}"
+        key = redisKey("twitter", name, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -101,18 +105,20 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       cond @"tab" in ["with_replies", "media", "search"]
-      let name = @"name"
-      let query =
-        case @"tab"
-        of "with_replies": getReplyQuery(name)
-        of "media": getMediaQuery(name)
-        of "search": initQuery(params(request), name=name)
-        else: Query(fromUser: @[name])
+      let
+        name = @"name"
+        tab = @"tab"
+        query =
+          case tab
+          of "with_replies": getReplyQuery(name)
+          of "media": getMediaQuery(name)
+          of "search": initQuery(params(request), name=name)
+          else: Query(fromUser: @[name])
 
-      var key = &"""{@"tab"}:{@"name"}:"""
-      if @"tab" == "search":
-        key &= $hash(genQueryUrl(query)) & ":"
-      key &= getCursor()
+      let searchKey = if tab != "search": ""
+                      else: ":" & $hash(genQueryUrl(query))
+
+      let key = redisKey(tab, name & searchKey, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -132,28 +138,27 @@ proc createRssRouter*(cfg: Config) =
         cursor = getCursor()
 
       if list.id.len == 0:
-        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
+        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
 
-      let url = &"/i/lists/{list.id}/rss"
+      let url = "/i/lists/" & list.id & "/rss"
       if cursor.len > 0:
-        redirect(&"{url}?cursor={encodeUrl(cursor, false)}")
+        redirect(url & "?cursor=" & encodeUrl(cursor, false))
       else:
         redirect(url)
 
     get "/i/lists/@id/rss":
       cond cfg.enableRss
       let
+        id = @"id"
         cursor = getCursor()
-        key =
-          if cursor.len == 0: "lists:" & @"id"
-          else: &"""lists:{@"id"}:{cursor}"""
+        key = redisKey("lists", id, cursor)
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "List")
 
       let
-        list = await getCachedList(id=(@"id"))
+        list = await getCachedList(id=id)
         timeline = await getListTimeline(list.id, cursor)
       rss.cursor = timeline.bottom
       rss.feed = renderListRss(timeline.content, list, cfg)

src/routes/search.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, uri
+import strutils, uri
 
 import jester
 
@@ -14,32 +14,34 @@ export search
 proc createSearchRouter*(cfg: Config) =
   router search:
     get "/search/?":
-      if @"q".len > 500:
+      let q = @"q"
+      if q.len > 500:
         resp Http400, showError("Search input too long.", cfg)
 
       let
         prefs = cookiePrefs()
         query = initQuery(params(request))
+        title = "Search" & (if q.len > 0: " (" & q & ")" else: "")
 
       case query.kind
       of users:
-        if "," in @"q":
-          redirect("/" & @"q")
+        if "," in q:
+          redirect("/" & q)
         let users = await getSearch[User](query, getCursor())
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
+        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
       of tweets:
         let
           tweets = await getSearch[Tweet](query, getCursor())
           rss = "/search/rss?" & genQueryUrl(query)
         resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
-                        request, cfg, prefs, rss=rss)
+                        request, cfg, prefs, title, rss=rss)
       else:
         resp Http404, showError("Invalid search", cfg)
 
     get "/hashtag/@hash":
-      redirect(&"""/search?q={encodeUrl("#" & @"hash")}""")
+      redirect("/search?q=" & encodeUrl("#" & @"hash"))
 
     get "/opensearch":
       let url = getUrlPrefix(cfg) & "/search?q="
       resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                    generateOpenSearchXML(cfg.title, cfg.hostname, url)
+                     generateOpenSearchXML(cfg.title, cfg.hostname, url)

src/routes/timeline.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, sequtils, uri, options, times
+import asyncdispatch, strutils, sequtils, uri, options, times
 import jester, karax/vdom
 
 import router_utils
@@ -102,7 +102,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
 template respTimeline*(timeline: typed) =
   let t = timeline
   if t.len == 0:
-    resp Http404, showError(&"""User "{@"name"}" not found""", cfg)
+    resp Http404, showError("User \"" & @"name" & "\" not found", cfg)
   resp t
 
 template respUserId*() =

src/sass/tweet/_base.scss

@@ -138,7 +138,6 @@
     }
 }
 
-
 .attribution {
     display: flex;
     pointer-events: all;

src/sass/tweet/thread.scss

@@ -23,7 +23,6 @@
     font-size: 18px;
 }
 
-
 @media(max-width: 600px) {
     .main-tweet .tweet-content {
         font-size: 16px;

src/views/general.nim

@@ -81,7 +81,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
 
     title:
       if titleText.len > 0:
-        text &"{titleText}|{cfg.title}"
+        text titleText & " | " & cfg.title
       else:
         text cfg.title
 

src/views/profile.nim

@@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
           span:
             let url = replaceUrls(user.website, prefs)
             icon "link"
-            a(href=url): text shortLink(url)
+            a(href=url): text url.shortLink
 
       tdiv(class="profile-joindate"):
         span(title=getJoinDateFull(user)):
@@ -108,7 +108,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
         renderBanner(profile.user.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
-    tdiv(class=(&"profile-tab{sticky}")):
+    tdiv(class=("profile-tab" & sticky)):
       renderUserCard(profile.user, prefs)
       if profile.photoRail.len > 0:
         renderPhotoRail(profile)

src/views/renderutils.nim

@@ -55,12 +55,12 @@ proc genCheckbox*(pref, label: string; state: bool): VNode =
     else: input(name=pref, `type`="checkbox")
     span(class="checkbox")
 
-proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
+proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
   let p = placeholder
   buildHtml(tdiv(class=("pref-group pref-input " & class))):
     if label.len > 0:
       label(`for`=pref): text label
-    if state.len == 0:
+    if autofocus and state.len == 0:
       input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
     else:
       input(name=pref, `type`="text", placeholder=p, value=state)

src/views/search.nim

@@ -87,7 +87,7 @@ proc renderSearchPanel*(query: Query): VNode =
             genDate("until", query.until)
         tdiv:
           span(class="search-title"): text "Near"
-          genInput("near", "", query.near, placeholder="地域…")
+          genInput("near", "", query.near, "地域…", autofocus=false)
 
 proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
                         pinned=none(Tweet)): VNode =

tests/test_quote.py

@@ -3,7 +3,7 @@ from parameterized import parameterized
 
 text = [
     ['elonmusk/status/1138136540096319488',
-     'Trev Page', '@Model3Owners',
+     'TREV PAGE', '@Model3Owners',
      """As of March 58.4% of new car sales in Norway are electric. 
 
 What are we doing wrong? reuters.com/article/us-norwa…"""],