diff --git a/Dockerfile b/Dockerfile
index 60aac63..29e5d4e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,10 +4,13 @@ EXPOSE 8080
RUN apk --no-cache add libsass-dev pcre
-COPY . /src/nitter
WORKDIR /src/nitter
-RUN nimble build -y -d:danger -d:lto -d:strip \
+COPY nitter.nimble .
+RUN nimble install -y --depsOnly
+
+COPY . .
+RUN nimble build -d:danger -d:lto -d:strip \
&& nimble scss \
&& nimble md
diff --git a/src/api.nim b/src/api.nim
index 503af9e..708b72f 100644
--- a/src/api.nim
+++ b/src/api.nim
@@ -2,14 +2,14 @@
import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
import packedjson
import types, query, formatters, consts, apiutils, parser
-import experimental/parser/user
+import experimental/parser as newParser
proc getGraphUser*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return
let
variables = %*{"userId": id, "withSuperFollowsUserFields": true}
- js = await fetch(graphUser ? {"variables": $variables}, Api.userRestId)
- result = parseGraphUser(js, id)
+ js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
+ result = parseGraphUser(js)
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
@@ -37,7 +37,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
"withSuperFollowsTweetFields": false
}
url = graphListMembers ? {"variables": $variables}
- result = parseGraphListMembers(await fetch(url, Api.listMembers), after)
+ result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
@@ -85,10 +85,12 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
const
searchMode = ("result_filter", "user")
parse = parseUsers
+ fetchFunc = fetchRaw
else:
const
searchMode = ("tweet_search_mode", "live")
parse = parseTimeline
+ fetchFunc = fetch
let q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
@@ -96,7 +98,7 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
try:
- result = parse(await fetch(url, Api.search), after)
+ result = parse(await fetchFunc(url, Api.search), after)
result.query = query
except InternalError:
return Result[T](beginning: true, query: query)
diff --git a/src/experimental/parser.nim b/src/experimental/parser.nim
new file mode 100644
index 0000000..98ce7df
--- /dev/null
+++ b/src/experimental/parser.nim
@@ -0,0 +1,2 @@
+import parser/[user, graphql, timeline]
+export user, graphql, timeline
diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim
new file mode 100644
index 0000000..b00ab24
--- /dev/null
+++ b/src/experimental/parser/graphql.nim
@@ -0,0 +1,27 @@
+import jsony
+import user, ../types/[graphuser, graphlistmembers]
+from ../../types import User, Result, Query, QueryKind
+
+proc parseGraphUser*(json: string): User =
+ let raw = json.fromJson(GraphUser)
+ result = toUser raw.data.user.result.legacy
+ result.id = raw.data.user.result.restId
+
+proc parseGraphListMembers*(json, cursor: string): Result[User] =
+ result = Result[User](
+ beginning: cursor.len == 0,
+ query: Query(kind: userList)
+ )
+
+ let raw = json.fromJson(GraphListMembers)
+ for instruction in raw.data.list.membersTimeline.timeline.instructions:
+ if instruction.kind == "TimelineAddEntries":
+ for entry in instruction.entries:
+ case entry.content.entryType
+ of TimelineTimelineItem:
+ let userResult = entry.content.itemContent.userResults.result
+ if userResult.restId.len > 0:
+ result.content.add toUser userResult.legacy
+ of TimelineTimelineCursor:
+ if entry.content.cursorType == "Bottom":
+ result.bottom = entry.content.value
diff --git a/src/experimental/parser/timeline.nim b/src/experimental/parser/timeline.nim
new file mode 100644
index 0000000..351ca85
--- /dev/null
+++ b/src/experimental/parser/timeline.nim
@@ -0,0 +1,28 @@
+import std/[strutils, tables]
+import jsony
+import user, ../types/timeline
+from ../../types import Result, User
+
+proc getId(id: string): string {.inline.} =
+ let start = id.rfind("-")
+ if start < 0: return id
+ id[start + 1 ..< id.len]
+
+proc parseUsers*(json: string; after=""): Result[User] =
+ result = Result[User](beginning: after.len == 0)
+
+ let raw = json.fromJson(Search)
+ if raw.timeline.instructions.len == 0:
+ return
+
+ for e in raw.timeline.instructions[0].addEntries.entries:
+ let id = e.entryId.getId
+ if e.entryId.startsWith("user"):
+ if id in raw.globalObjects.users:
+ result.content.add toUser raw.globalObjects.users[id]
+ elif e.entryId.startsWith("cursor"):
+ let cursor = e.content.operation.cursor
+ if cursor.cursorType == "Top":
+ result.top = cursor.value
+ elif cursor.cursorType == "Bottom":
+ result.bottom = cursor.value
diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim
index 095d25c..dc760f0 100644
--- a/src/experimental/parser/user.nim
+++ b/src/experimental/parser/user.nim
@@ -1,4 +1,4 @@
-import std/[algorithm, unicode, re, strutils]
+import std/[algorithm, unicode, re, strutils, strformat, options]
import jsony
import utils, slices
import ../types/user as userType
@@ -34,9 +34,40 @@ proc expandUserEntities(user: var User; raw: RawUser) =
proc getBanner(user: RawUser): string =
if user.profileBannerUrl.len > 0:
return user.profileBannerUrl & "/1500x500"
+
if user.profileLinkColor.len > 0:
return '#' & user.profileLinkColor
+ if user.profileImageExtensions.isSome:
+ let ext = get(user.profileImageExtensions)
+ if ext.mediaColor.r.ok.palette.len > 0:
+ let color = ext.mediaColor.r.ok.palette[0].rgb
+ return &"#{color.red:02x}{color.green:02x}{color.blue:02x}"
+
+proc toUser*(raw: RawUser): User =
+ result = User(
+ id: raw.idStr,
+ username: raw.screenName,
+ fullname: raw.name,
+ location: raw.location,
+ bio: raw.description,
+ following: raw.friendsCount,
+ followers: raw.followersCount,
+ tweets: raw.statusesCount,
+ likes: raw.favouritesCount,
+ media: raw.mediaCount,
+ verified: raw.verified,
+ protected: raw.protected,
+ joinDate: parseTwitterDate(raw.createdAt),
+ banner: getBanner(raw),
+ userPic: getImageUrl(raw.profileImageUrlHttps).replace("_normal", "")
+ )
+
+ if raw.pinnedTweetIdsStr.len > 0:
+ result.pinnedTweet = parseBiggestInt(raw.pinnedTweetIdsStr[0])
+
+ result.expandUserEntities(raw)
+
proc parseUser*(json: string; username=""): User =
handleErrors:
case error.code
@@ -44,24 +75,4 @@ proc parseUser*(json: string; username=""): User =
of userNotFound: return
else: echo "[error - parseUser]: ", error
- let user = json.fromJson(RawUser)
-
- result = User(
- id: user.idStr,
- username: user.screenName,
- fullname: user.name,
- location: user.location,
- bio: user.description,
- following: user.friendsCount,
- followers: user.followersCount,
- tweets: user.statusesCount,
- likes: user.favouritesCount,
- media: user.mediaCount,
- verified: user.verified,
- protected: user.protected,
- joinDate: parseTwitterDate(user.createdAt),
- banner: getBanner(user),
- userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
- )
-
- result.expandUserEntities(user)
+ result = toUser json.fromJson(RawUser)
diff --git a/src/experimental/types/graphlistmembers.nim b/src/experimental/types/graphlistmembers.nim
new file mode 100644
index 0000000..4cb3757
--- /dev/null
+++ b/src/experimental/types/graphlistmembers.nim
@@ -0,0 +1,31 @@
+import graphuser
+
+type
+ GraphListMembers* = object
+ data*: tuple[list: List]
+
+ List = object
+ membersTimeline*: tuple[timeline: Timeline]
+
+ Timeline = object
+ instructions*: seq[Instruction]
+
+ Instruction = object
+ kind*: string
+ entries*: seq[tuple[content: Content]]
+
+ ContentEntryType* = enum
+ TimelineTimelineItem
+ TimelineTimelineCursor
+
+ Content = object
+ case entryType*: ContentEntryType
+ of TimelineTimelineItem:
+ itemContent*: tuple[userResults: UserData]
+ of TimelineTimelineCursor:
+ value*: string
+ cursorType*: string
+
+proc renameHook*(v: var Instruction; fieldName: var string) =
+ if fieldName == "type":
+ fieldName = "kind"
diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim
new file mode 100644
index 0000000..dded4eb
--- /dev/null
+++ b/src/experimental/types/graphuser.nim
@@ -0,0 +1,12 @@
+import user
+
+type
+ GraphUser* = object
+ data*: tuple[user: UserData]
+
+ UserData* = object
+ result*: UserResult
+
+ UserResult = object
+ legacy*: RawUser
+ restId*: string
diff --git a/src/experimental/types/timeline.nim b/src/experimental/types/timeline.nim
new file mode 100644
index 0000000..28239ad
--- /dev/null
+++ b/src/experimental/types/timeline.nim
@@ -0,0 +1,23 @@
+import std/tables
+import user
+
+type
+ Search* = object
+ globalObjects*: GlobalObjects
+ timeline*: Timeline
+
+ GlobalObjects = object
+ users*: Table[string, RawUser]
+
+ Timeline = object
+ instructions*: seq[Instructions]
+
+ Instructions = object
+ addEntries*: tuple[entries: seq[Entry]]
+
+ Entry = object
+ entryId*: string
+ content*: tuple[operation: Operation]
+
+ Operation = object
+ cursor*: tuple[value, cursorType: string]
diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim
index e3afaf0..1c8a5c3 100644
--- a/src/experimental/types/user.nim
+++ b/src/experimental/types/user.nim
@@ -1,3 +1,4 @@
+import options
import common
type
@@ -16,9 +17,11 @@ type
mediaCount*: int
verified*: bool
protected*: bool
+ profileLinkColor*: string
profileBannerUrl*: string
profileImageUrlHttps*: string
- profileLinkColor*: string
+ profileImageExtensions*: Option[ImageExtensions]
+ pinnedTweetIdsStr*: seq[string]
Entities* = object
url*: Urls
@@ -26,3 +29,15 @@ type
Urls* = object
urls*: seq[Url]
+
+ ImageExtensions = object
+ mediaColor*: tuple[r: Ok]
+
+ Ok = object
+ ok*: Palette
+
+ Palette = object
+ palette*: seq[tuple[rgb: Color]]
+
+ Color* = object
+ red*, green*, blue*: int
diff --git a/src/parser.nim b/src/parser.nim
index 4f3cc04..ae5e505 100644
--- a/src/parser.nim
+++ b/src/parser.nim
@@ -26,16 +26,6 @@ proc parseUser(js: JsonNode; id=""): User =
result.expandUserEntities(js)
-proc parseGraphUser*(js: JsonNode; id: string): User =
- if js.isNull: return
-
- with user, js{"data", "user", "result", "legacy"}:
- result = parseUser(user, id)
-
- with pinned, user{"pinned_tweet_ids_str"}:
- if pinned.kind == JArray and pinned.len > 0:
- result.pinnedTweet = parseBiggestInt(pinned[0].getStr)
-
proc parseGraphList*(js: JsonNode): List =
if js.isNull: return
@@ -55,25 +45,6 @@ proc parseGraphList*(js: JsonNode): List =
banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
)
-proc parseGraphListMembers*(js: JsonNode; cursor: string): Result[User] =
- result = Result[User](
- beginning: cursor.len == 0,
- query: Query(kind: userList)
- )
-
- if js.isNull: return
-
- let root = js{"data", "list", "members_timeline", "timeline", "instructions"}
- for instruction in root:
- if instruction{"type"}.getStr == "TimelineAddEntries":
- for entry in instruction{"entries"}:
- let content = entry{"content"}
- if content{"entryType"}.getStr == "TimelineTimelineItem":
- with legacy, content{"itemContent", "user_results", "result", "legacy"}:
- result.content.add parseUser(legacy)
- elif content{"cursorType"}.getStr == "Bottom":
- result.bottom = content{"value"}.getStr
-
proc parsePoll(js: JsonNode): Poll =
let vals = js{"binding_values"}
@@ -395,26 +366,6 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod
elif "bottom" in r{"entryId"}.getStr:
res.bottom = r.getCursor
-proc parseUsers*(js: JsonNode; after=""): Result[User] =
- result = Result[User](beginning: after.len == 0)
- let global = parseGlobalObjects(? js)
-
- let instructions = ? js{"timeline", "instructions"}
- if instructions.len == 0: return
-
- result.parseInstructions(global, instructions)
-
- for e in instructions[0]{"addEntries", "entries"}:
- let entry = e{"entryId"}.getStr
- if "user-" in entry:
- let id = entry.getId
- if id in global.users:
- result.content.add global.users[id]
- elif "cursor-top" in entry:
- result.top = e.getCursor
- elif "cursor-bottom" in entry:
- result.bottom = e.getCursor
-
proc parseTimeline*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
let global = parseGlobalObjects(? js)
diff --git a/src/parserutils.nim b/src/parserutils.nim
index 0e7c8b5..a605ea4 100644
--- a/src/parserutils.nim
+++ b/src/parserutils.nim
@@ -8,7 +8,7 @@ let
unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
unReplace = "$1@$2"
- htRegex = re"(^|[^\w-_./?])([##$])([\w_]+)"
+ htRegex = re"(^|[^\w-_./?])([#$]|#)([\w_]+)"
htReplace = "$1$2$3"
type
diff --git a/src/redis_cache.nim b/src/redis_cache.nim
index 206bb51..469157a 100644
--- a/src/redis_cache.nim
+++ b/src/redis_cache.nim
@@ -102,8 +102,9 @@ proc cache*(data: Tweet) {.async.} =
proc cacheRss*(query: string; rss: Rss) {.async.} =
let key = "rss:" & query
pool.withAcquire(r):
- dawait r.hSet(key, "rss", rss.feed)
dawait r.hSet(key, "min", rss.cursor)
+ if rss.cursor != "suspended":
+ dawait r.hSet(key, "rss", compress(rss.feed))
dawait r.expire(key, rssCacheTime)
template deserialize(data, T) =
@@ -182,6 +183,10 @@ proc getCachedRss*(key: string): Future[Rss] {.async.} =
pool.withAcquire(r):
result.cursor = await r.hGet(k, "min")
if result.cursor.len > 2:
- result.feed = await r.hGet(k, "rss")
+ if result.cursor != "suspended":
+ let feed = await r.hGet(k, "rss")
+ if feed.len > 0 and feed != redisNil:
+ try: result.feed = uncompress feed
+ except: echo "Decompressing RSS failed: ", feed
else:
result.cursor.setLen 0
diff --git a/src/routes/rss.nim b/src/routes/rss.nim
index af7312d..40aa6a7 100644
--- a/src/routes/rss.nim
+++ b/src/routes/rss.nim
@@ -1,14 +1,14 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, strutils, tables, times, hashes, uri
-import jester, supersnappy
+import jester
import router_utils, timeline
import ../query
include "../views/rss.nimf"
-export times, hashes, supersnappy
+export times, hashes
proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
var profile: Profile
@@ -36,7 +36,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
return Rss(feed: profile.user.username, cursor: "suspended")
if profile.user.fullname.len > 0:
- let rss = compress renderTimelineRss(profile, cfg, multi=(names.len > 1))
+ let rss = renderTimelineRss(profile, cfg, multi=(names.len > 1))
return Rss(feed: rss, cursor: profile.tweets.bottom)
template respRss*(rss, page) =
@@ -48,11 +48,11 @@ template respRss*(rss, page) =
resp Http404, showError(page & info & "not found", cfg)
elif rss.cursor.len == 9 and rss.cursor == "suspended":
- resp Http404, showError(getSuspended(rss.feed), cfg)
+ resp Http404, showError(getSuspended(@"name"), cfg)
let headers = {"Content-Type": "application/rss+xml; charset=utf-8",
"Min-Id": rss.cursor}
- resp Http200, headers, uncompress rss.feed
+ resp Http200, headers, rss.feed
proc createRssRouter*(cfg: Config) =
router rss:
@@ -75,8 +75,7 @@ proc createRssRouter*(cfg: Config) =
let tweets = await getSearch[Tweet](query, cursor)
rss.cursor = tweets.bottom
- rss.feed = compress renderSearchRss(tweets.content, query.text,
- genQueryUrl(query), cfg)
+ rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
await cacheRss(key, rss)
respRss(rss, "Search")
@@ -157,7 +156,7 @@ proc createRssRouter*(cfg: Config) =
list = await getCachedList(id=(@"id"))
timeline = await getListTimeline(list.id, cursor)
rss.cursor = timeline.bottom
- rss.feed = compress renderListRss(timeline.content, list, cfg)
+ rss.feed = renderListRss(timeline.content, list, cfg)
await cacheRss(key, rss)
respRss(rss, "List")
diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim
index 3df67f4..a0a6e21 100644
--- a/src/routes/timeline.nim
+++ b/src/routes/timeline.nim
@@ -86,7 +86,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
html = renderTweetSearch(timeline, prefs, getPath())
return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
- var profile = await fetchProfile(after, query)
+ var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins)
template u: untyped = profile.user
if u.suspended:
diff --git a/src/sass/profile/_base.scss b/src/sass/profile/_base.scss
index ae6b801..b7f33e6 100644
--- a/src/sass/profile/_base.scss
+++ b/src/sass/profile/_base.scss
@@ -42,12 +42,16 @@
top: 50px;
}
-.profile-result .username {
- margin: 0 !important;
-}
+.profile-result {
+ min-height: 54px;
-.profile-result .tweet-header {
- margin-bottom: unset;
+ .username {
+ margin: 0 !important;
+ }
+
+ .tweet-header {
+ margin-bottom: unset;
+ }
}
@media(max-width: 700px) {
diff --git a/src/views/general.nim b/src/views/general.nim
index 7054fd4..82902d4 100644
--- a/src/views/general.nim
+++ b/src/views/general.nim
@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
buildHtml(head):
- link(rel="stylesheet", type="text/css", href="/css/style.css?v=15")
+ link(rel="stylesheet", type="text/css", href="/css/style.css?v=16")
link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
if theme.len > 0:
diff --git a/src/views/preferences.nim b/src/views/preferences.nim
index ba1bcc0..0ce4687 100644
--- a/src/views/preferences.nim
+++ b/src/views/preferences.nim
@@ -35,7 +35,7 @@ macro renderPrefs*(): untyped =
proc renderPreferences*(prefs: Prefs; path: string; themes: seq[string]): VNode =
buildHtml(tdiv(class="overlay-panel")):
fieldset(class="preferences"):
- form(`method`="post", action="/saveprefs"):
+ form(`method`="post", action="/saveprefs", autocomplete="off"):
refererField path
renderPrefs()
diff --git a/src/views/search.nim b/src/views/search.nim
index 5f831ca..f4510d4 100644
--- a/src/views/search.nim
+++ b/src/views/search.nim
@@ -23,7 +23,7 @@ const toggles = {
proc renderSearch*(): VNode =
buildHtml(tdiv(class="panel-container")):
tdiv(class="search-bar"):
- form(`method`="get", action="/search"):
+ form(`method`="get", action="/search", autocomplete="off"):
hiddenField("f", "users")
input(`type`="text", name="q", autofocus="", placeholder="ユーザー名の入力…", dir="auto")
button(`type`="submit"): icon "search"
@@ -57,7 +57,8 @@ proc isPanelOpen(q: Query): bool =
proc renderSearchPanel*(query: Query): VNode =
let user = query.fromUser.join(",")
let action = if user.len > 0: &"/{user}/search" else: "/search"
- buildHtml(form(`method`="get", action=action, class="search-field")):
+ buildHtml(form(`method`="get", action=action,
+ class="search-field", autocomplete="off")):
hiddenField("f", "tweets")
genInput("q", "", query.text, "検索…", class="pref-inline")
button(`type`="submit"): icon "search"
@@ -111,7 +112,7 @@ proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"):
- form(`method`="get", action="/search", class="search-field"):
+ form(`method`="get", action="/search", class="search-field", autocomplete="off"):
hiddenField("f", "users")
genInput("q", "", results.query.text, "ユーザー名の入力…", class="pref-inline")
button(`type`="submit"): icon "search"
diff --git a/src/views/timeline.nim b/src/views/timeline.nim
index 3456bdb..afe35dc 100644
--- a/src/views/timeline.nim
+++ b/src/views/timeline.nim
@@ -95,7 +95,7 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
if not results.beginning:
renderNewer(results.query, parseUri(path).path)
- if pinned.isSome:
+ if not prefs.hidePins and pinned.isSome:
let tweet = get pinned
renderTweet(tweet, prefs, path, showThread=tweet.hasThread)