マージ

Dockerfile

@@ -4,10 +4,13 @@ EXPOSE 8080
 
 RUN apk --no-cache add libsass-dev pcre
 
-COPY . /src/nitter
 WORKDIR /src/nitter
 
-RUN nimble build -y -d:danger -d:lto -d:strip \
+COPY nitter.nimble .
+RUN nimble install -y --depsOnly
+
+COPY . .
+RUN nimble build -d:danger -d:lto -d:strip \
     && nimble scss \
     && nimble md
 

src/api.nim

@@ -2,14 +2,14 @@
 import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
 import packedjson
 import types, query, formatters, consts, apiutils, parser
-import experimental/parser/user
+import experimental/parser as newParser
 
 proc getGraphUser*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
     variables = %*{"userId": id, "withSuperFollowsUserFields": true}
-    js = await fetch(graphUser ? {"variables": $variables}, Api.userRestId)
-  result = parseGraphUser(js, id)
+    js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
+  result = parseGraphUser(js)
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
@@ -37,7 +37,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
       "withSuperFollowsTweetFields": false
     }
     url = graphListMembers ? {"variables": $variables}
-  result = parseGraphListMembers(await fetch(url, Api.listMembers), after)
+  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
 proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
@@ -85,10 +85,12 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
     const
       searchMode = ("result_filter", "user")
       parse = parseUsers
+      fetchFunc = fetchRaw
   else:
     const
       searchMode = ("tweet_search_mode", "live")
       parse = parseTimeline
+      fetchFunc = fetch
 
   let q = genQueryParam(query)
   if q.len == 0 or q == emptyQuery:
@@ -96,7 +98,7 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
 
   let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
   try:
-    result = parse(await fetch(url, Api.search), after)
+    result = parse(await fetchFunc(url, Api.search), after)
     result.query = query
   except InternalError:
     return Result[T](beginning: true, query: query)

src/experimental/parser.nim

@@ -0,0 +1,2 @@
+import parser/[user, graphql, timeline]
+export user, graphql, timeline

src/experimental/parser/graphql.nim

@@ -0,0 +1,27 @@
+import jsony
+import user, ../types/[graphuser, graphlistmembers]
+from ../../types import User, Result, Query, QueryKind
+
+proc parseGraphUser*(json: string): User =
+  let raw = json.fromJson(GraphUser)
+  result = toUser raw.data.user.result.legacy
+  result.id = raw.data.user.result.restId
+
+proc parseGraphListMembers*(json, cursor: string): Result[User] =
+  result = Result[User](
+    beginning: cursor.len == 0,
+    query: Query(kind: userList)
+  )
+
+  let raw = json.fromJson(GraphListMembers)
+  for instruction in raw.data.list.membersTimeline.timeline.instructions:
+    if instruction.kind == "TimelineAddEntries":
+      for entry in instruction.entries:
+        case entry.content.entryType
+        of TimelineTimelineItem:
+          let userResult = entry.content.itemContent.userResults.result
+          if userResult.restId.len > 0:
+            result.content.add toUser userResult.legacy
+        of TimelineTimelineCursor:
+          if entry.content.cursorType == "Bottom":
+            result.bottom = entry.content.value

src/experimental/parser/timeline.nim

@@ -0,0 +1,28 @@
+import std/[strutils, tables]
+import jsony
+import user, ../types/timeline
+from ../../types import Result, User
+
+proc getId(id: string): string {.inline.} =
+  let start = id.rfind("-")
+  if start < 0: return id
+  id[start + 1 ..< id.len]
+
+proc parseUsers*(json: string; after=""): Result[User] =
+  result = Result[User](beginning: after.len == 0)
+
+  let raw = json.fromJson(Search)
+  if raw.timeline.instructions.len == 0:
+    return
+
+  for e in raw.timeline.instructions[0].addEntries.entries:
+    let id = e.entryId.getId
+    if e.entryId.startsWith("user"):
+      if id in raw.globalObjects.users:
+        result.content.add toUser raw.globalObjects.users[id]
+    elif e.entryId.startsWith("cursor"):
+      let cursor = e.content.operation.cursor
+      if cursor.cursorType == "Top":
+        result.top = cursor.value
+      elif cursor.cursorType == "Bottom":
+        result.bottom = cursor.value

src/experimental/parser/user.nim

@@ -1,4 +1,4 @@
-import std/[algorithm, unicode, re, strutils]
+import std/[algorithm, unicode, re, strutils, strformat, options]
 import jsony
 import utils, slices
 import ../types/user as userType
@@ -34,9 +34,40 @@ proc expandUserEntities(user: var User; raw: RawUser) =
 proc getBanner(user: RawUser): string =
   if user.profileBannerUrl.len > 0:
     return user.profileBannerUrl & "/1500x500"
+
   if user.profileLinkColor.len > 0:
     return '#' & user.profileLinkColor
 
+  if user.profileImageExtensions.isSome:
+    let ext = get(user.profileImageExtensions)
+    if ext.mediaColor.r.ok.palette.len > 0:
+      let color = ext.mediaColor.r.ok.palette[0].rgb
+      return &"#{color.red:02x}{color.green:02x}{color.blue:02x}"
+
+proc toUser*(raw: RawUser): User =
+  result = User(
+    id: raw.idStr,
+    username: raw.screenName,
+    fullname: raw.name,
+    location: raw.location,
+    bio: raw.description,
+    following: raw.friendsCount,
+    followers: raw.followersCount,
+    tweets: raw.statusesCount,
+    likes: raw.favouritesCount,
+    media: raw.mediaCount,
+    verified: raw.verified,
+    protected: raw.protected,
+    joinDate: parseTwitterDate(raw.createdAt),
+    banner: getBanner(raw),
+    userPic: getImageUrl(raw.profileImageUrlHttps).replace("_normal", "")
+  )
+
+  if raw.pinnedTweetIdsStr.len > 0:
+    result.pinnedTweet = parseBiggestInt(raw.pinnedTweetIdsStr[0])
+
+  result.expandUserEntities(raw)
+
 proc parseUser*(json: string; username=""): User =
   handleErrors:
     case error.code
@@ -44,24 +75,4 @@ proc parseUser*(json: string; username=""): User =
     of userNotFound: return
     else: echo "[error - parseUser]: ", error
 
-  let user = json.fromJson(RawUser)
-
-  result = User(
-    id: user.idStr,
-    username: user.screenName,
-    fullname: user.name,
-    location: user.location,
-    bio: user.description,
-    following: user.friendsCount,
-    followers: user.followersCount,
-    tweets: user.statusesCount,
-    likes: user.favouritesCount,
-    media: user.mediaCount,
-    verified: user.verified,
-    protected: user.protected,
-    joinDate: parseTwitterDate(user.createdAt),
-    banner: getBanner(user),
-    userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
-  )
-
-  result.expandUserEntities(user)
+  result = toUser json.fromJson(RawUser)

src/experimental/types/graphlistmembers.nim

@@ -0,0 +1,31 @@
+import graphuser
+
+type
+  GraphListMembers* = object
+    data*: tuple[list: List]
+
+  List = object
+    membersTimeline*: tuple[timeline: Timeline]
+
+  Timeline = object
+    instructions*: seq[Instruction]
+
+  Instruction = object
+    kind*: string
+    entries*: seq[tuple[content: Content]]
+
+  ContentEntryType* = enum
+    TimelineTimelineItem
+    TimelineTimelineCursor
+
+  Content = object
+    case entryType*: ContentEntryType
+    of TimelineTimelineItem:
+      itemContent*: tuple[userResults: UserData]
+    of TimelineTimelineCursor:
+      value*: string
+      cursorType*: string
+
+proc renameHook*(v: var Instruction; fieldName: var string) =
+  if fieldName == "type":
+    fieldName = "kind"

src/experimental/types/graphuser.nim

@@ -0,0 +1,12 @@
+import user
+
+type
+  GraphUser* = object
+    data*: tuple[user: UserData]
+
+  UserData* = object
+    result*: UserResult
+
+  UserResult = object
+    legacy*: RawUser
+    restId*: string

src/experimental/types/timeline.nim

@@ -0,0 +1,23 @@
+import std/tables
+import user
+
+type
+  Search* = object
+    globalObjects*: GlobalObjects
+    timeline*: Timeline
+
+  GlobalObjects = object
+    users*: Table[string, RawUser]
+
+  Timeline = object
+    instructions*: seq[Instructions]
+
+  Instructions = object
+    addEntries*: tuple[entries: seq[Entry]]
+
+  Entry = object
+    entryId*: string
+    content*: tuple[operation: Operation]
+
+  Operation = object
+    cursor*: tuple[value, cursorType: string]

src/experimental/types/user.nim

@@ -1,3 +1,4 @@
+import options
 import common
 
 type
@@ -16,9 +17,11 @@ type
     mediaCount*: int
     verified*: bool
     protected*: bool
+    profileLinkColor*: string
     profileBannerUrl*: string
     profileImageUrlHttps*: string
-    profileLinkColor*: string
+    profileImageExtensions*: Option[ImageExtensions]
+    pinnedTweetIdsStr*: seq[string]
 
   Entities* = object
     url*: Urls
@@ -26,3 +29,15 @@ type
 
   Urls* = object
     urls*: seq[Url]
+
+  ImageExtensions = object
+    mediaColor*: tuple[r: Ok]
+
+  Ok = object
+    ok*: Palette
+
+  Palette = object
+    palette*: seq[tuple[rgb: Color]]
+
+  Color* = object
+    red*, green*, blue*: int

src/parser.nim

@@ -26,16 +26,6 @@ proc parseUser(js: JsonNode; id=""): User =
 
   result.expandUserEntities(js)
 
-proc parseGraphUser*(js: JsonNode; id: string): User =
-  if js.isNull: return
-
-  with user, js{"data", "user", "result", "legacy"}:
-    result = parseUser(user, id)
-
-    with pinned, user{"pinned_tweet_ids_str"}:
-      if pinned.kind == JArray and pinned.len > 0:
-        result.pinnedTweet = parseBiggestInt(pinned[0].getStr)
-
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 
@@ -55,25 +45,6 @@ proc parseGraphList*(js: JsonNode): List =
     banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
   )
 
-proc parseGraphListMembers*(js: JsonNode; cursor: string): Result[User] =
-  result = Result[User](
-    beginning: cursor.len == 0,
-    query: Query(kind: userList)
-  )
-
-  if js.isNull: return
-
-  let root = js{"data", "list", "members_timeline", "timeline", "instructions"}
-  for instruction in root:
-    if instruction{"type"}.getStr == "TimelineAddEntries":
-      for entry in instruction{"entries"}:
-        let content = entry{"content"}
-        if content{"entryType"}.getStr == "TimelineTimelineItem":
-          with legacy, content{"itemContent", "user_results", "result", "legacy"}:
-            result.content.add parseUser(legacy)
-        elif content{"cursorType"}.getStr == "Bottom":
-          result.bottom = content{"value"}.getStr
-
 
 proc parsePoll(js: JsonNode): Poll =
   let vals = js{"binding_values"}
@@ -395,26 +366,6 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod
       elif "bottom" in r{"entryId"}.getStr:
         res.bottom = r.getCursor
 
-proc parseUsers*(js: JsonNode; after=""): Result[User] =
-  result = Result[User](beginning: after.len == 0)
-  let global = parseGlobalObjects(? js)
-
-  let instructions = ? js{"timeline", "instructions"}
-  if instructions.len == 0: return
-
-  result.parseInstructions(global, instructions)
-
-  for e in instructions[0]{"addEntries", "entries"}:
-    let entry = e{"entryId"}.getStr
-    if "user-" in entry:
-      let id = entry.getId
-      if id in global.users:
-        result.content.add global.users[id]
-    elif "cursor-top" in entry:
-      result.top = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.bottom = e.getCursor
-
 proc parseTimeline*(js: JsonNode; after=""): Timeline =
   result = Timeline(beginning: after.len == 0)
   let global = parseGlobalObjects(? js)

src/parserutils.nim

@@ -8,7 +8,7 @@ let
   unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
 
-  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
+  htRegex = re"(^|[^\w-_./?])([#$]|＃)([\w_]+)"
   htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
 
 type

src/redis_cache.nim

@@ -102,8 +102,9 @@ proc cache*(data: Tweet) {.async.} =
 proc cacheRss*(query: string; rss: Rss) {.async.} =
   let key = "rss:" & query
   pool.withAcquire(r):
-    dawait r.hSet(key, "rss", rss.feed)
     dawait r.hSet(key, "min", rss.cursor)
+    if rss.cursor != "suspended":
+      dawait r.hSet(key, "rss", compress(rss.feed))
     dawait r.expire(key, rssCacheTime)
 
 template deserialize(data, T) =
@@ -182,6 +183,10 @@ proc getCachedRss*(key: string): Future[Rss] {.async.} =
   pool.withAcquire(r):
     result.cursor = await r.hGet(k, "min")
     if result.cursor.len > 2:
-      result.feed = await r.hGet(k, "rss")
+      if result.cursor != "suspended":
+        let feed = await r.hGet(k, "rss")
+        if feed.len > 0 and feed != redisNil:
+          try: result.feed = uncompress feed
+          except: echo "Decompressing RSS failed: ", feed
     else:
       result.cursor.setLen 0

src/routes/rss.nim

@@ -1,14 +1,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, strutils, tables, times, hashes, uri
 
-import jester, supersnappy
+import jester
 
 import router_utils, timeline
 import ../query
 
 include "../views/rss.nimf"
 
-export times, hashes, supersnappy
+export times, hashes
 
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
   var profile: Profile
@@ -36,7 +36,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
     return Rss(feed: profile.user.username, cursor: "suspended")
 
   if profile.user.fullname.len > 0:
-    let rss = compress renderTimelineRss(profile, cfg, multi=(names.len > 1))
+    let rss = renderTimelineRss(profile, cfg, multi=(names.len > 1))
     return Rss(feed: rss, cursor: profile.tweets.bottom)
 
 template respRss*(rss, page) =
@@ -48,11 +48,11 @@ template respRss*(rss, page) =
 
     resp Http404, showError(page & info & "not found", cfg)
   elif rss.cursor.len == 9 and rss.cursor == "suspended":
-    resp Http404, showError(getSuspended(rss.feed), cfg)
+    resp Http404, showError(getSuspended(@"name"), cfg)
 
   let headers = {"Content-Type": "application/rss+xml; charset=utf-8",
                  "Min-Id": rss.cursor}
-  resp Http200, headers, uncompress rss.feed
+  resp Http200, headers, rss.feed
 
 proc createRssRouter*(cfg: Config) =
   router rss:
@@ -75,8 +75,7 @@ proc createRssRouter*(cfg: Config) =
 
       let tweets = await getSearch[Tweet](query, cursor)
       rss.cursor = tweets.bottom
-      rss.feed = compress renderSearchRss(tweets.content, query.text,
-                                          genQueryUrl(query), cfg)
+      rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
 
       await cacheRss(key, rss)
       respRss(rss, "Search")
@@ -157,7 +156,7 @@ proc createRssRouter*(cfg: Config) =
         list = await getCachedList(id=(@"id"))
         timeline = await getListTimeline(list.id, cursor)
       rss.cursor = timeline.bottom
-      rss.feed = compress renderListRss(timeline.content, list, cfg)
+      rss.feed = renderListRss(timeline.content, list, cfg)
 
       await cacheRss(key, rss)
       respRss(rss, "List")

src/routes/timeline.nim

@@ -86,7 +86,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
-  var profile = await fetchProfile(after, query)
+  var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins)
   template u: untyped = profile.user
 
   if u.suspended:

src/sass/profile/_base.scss

@@ -42,12 +42,16 @@
     top: 50px;
 }
 
-.profile-result .username {
-    margin: 0 !important;
-}
+.profile-result {
+    min-height: 54px;
 
-.profile-result .tweet-header {
-    margin-bottom: unset;
+    .username {
+        margin: 0 !important;
+    }
+
+    .tweet-header {
+        margin-bottom: unset;
+    }
 }
 
 @media(max-width: 700px) {

src/views/general.nim

@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=15")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=16")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:

src/views/preferences.nim

@@ -35,7 +35,7 @@ macro renderPrefs*(): untyped =
 proc renderPreferences*(prefs: Prefs; path: string; themes: seq[string]): VNode =
   buildHtml(tdiv(class="overlay-panel")):
     fieldset(class="preferences"):
-      form(`method`="post", action="/saveprefs"):
+      form(`method`="post", action="/saveprefs", autocomplete="off"):
         refererField path
 
         renderPrefs()

src/views/search.nim

@@ -23,7 +23,7 @@ const toggles = {
 proc renderSearch*(): VNode =
   buildHtml(tdiv(class="panel-container")):
     tdiv(class="search-bar"):
-      form(`method`="get", action="/search"):
+      form(`method`="get", action="/search", autocomplete="off"):
         hiddenField("f", "users")
         input(`type`="text", name="q", autofocus="", placeholder="ユーザー名の入力…", dir="auto")
         button(`type`="submit"): icon "search"
@@ -57,7 +57,8 @@ proc isPanelOpen(q: Query): bool =
 proc renderSearchPanel*(query: Query): VNode =
   let user = query.fromUser.join(",")
   let action = if user.len > 0: &"/{user}/search" else: "/search"
-  buildHtml(form(`method`="get", action=action, class="search-field")):
+  buildHtml(form(`method`="get", action=action,
+                 class="search-field", autocomplete="off")):
     hiddenField("f", "tweets")
     genInput("q", "", query.text, "検索…", class="pref-inline")
     button(`type`="submit"): icon "search"
@@ -111,7 +112,7 @@ proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
 proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode =
   buildHtml(tdiv(class="timeline-container")):
     tdiv(class="timeline-header"):
-      form(`method`="get", action="/search", class="search-field"):
+      form(`method`="get", action="/search", class="search-field", autocomplete="off"):
         hiddenField("f", "users")
         genInput("q", "", results.query.text, "ユーザー名の入力…", class="pref-inline")
         button(`type`="submit"): icon "search"

src/views/timeline.nim

@@ -95,7 +95,7 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
     if not results.beginning:
       renderNewer(results.query, parseUri(path).path)
 
-    if pinned.isSome:
+    if not prefs.hidePins and pinned.isSome:
       let tweet = get pinned
       renderTweet(tweet, prefs, path, showThread=tweet.hasThread)