diff --git a/Dockerfile b/Dockerfile index 60aac63..29e5d4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,13 @@ EXPOSE 8080 RUN apk --no-cache add libsass-dev pcre -COPY . /src/nitter WORKDIR /src/nitter -RUN nimble build -y -d:danger -d:lto -d:strip \ +COPY nitter.nimble . +RUN nimble install -y --depsOnly + +COPY . . +RUN nimble build -d:danger -d:lto -d:strip \ && nimble scss \ && nimble md diff --git a/src/api.nim b/src/api.nim index 503af9e..708b72f 100644 --- a/src/api.nim +++ b/src/api.nim @@ -2,14 +2,14 @@ import asyncdispatch, httpclient, uri, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser -import experimental/parser/user +import experimental/parser as newParser proc getGraphUser*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return let variables = %*{"userId": id, "withSuperFollowsUserFields": true} - js = await fetch(graphUser ? {"variables": $variables}, Api.userRestId) - result = parseGraphUser(js, id) + js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId) + result = parseGraphUser(js) proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = let @@ -37,7 +37,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} "withSuperFollowsTweetFields": false } url = graphListMembers ? {"variables": $variables} - result = parseGraphListMembers(await fetch(url, Api.listMembers), after) + result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after) proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return @@ -85,10 +85,12 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = const searchMode = ("result_filter", "user") parse = parseUsers + fetchFunc = fetchRaw else: const searchMode = ("tweet_search_mode", "live") parse = parseTimeline + fetchFunc = fetch let q = genQueryParam(query) if q.len == 0 or q == emptyQuery: @@ -96,7 +98,7 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = let url = search ? genParams(searchParams & @[("q", q), searchMode], after) try: - result = parse(await fetch(url, Api.search), after) + result = parse(await fetchFunc(url, Api.search), after) result.query = query except InternalError: return Result[T](beginning: true, query: query) diff --git a/src/experimental/parser.nim b/src/experimental/parser.nim new file mode 100644 index 0000000..98ce7df --- /dev/null +++ b/src/experimental/parser.nim @@ -0,0 +1,2 @@ +import parser/[user, graphql, timeline] +export user, graphql, timeline diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim new file mode 100644 index 0000000..b00ab24 --- /dev/null +++ b/src/experimental/parser/graphql.nim @@ -0,0 +1,27 @@ +import jsony +import user, ../types/[graphuser, graphlistmembers] +from ../../types import User, Result, Query, QueryKind + +proc parseGraphUser*(json: string): User = + let raw = json.fromJson(GraphUser) + result = toUser raw.data.user.result.legacy + result.id = raw.data.user.result.restId + +proc parseGraphListMembers*(json, cursor: string): Result[User] = + result = Result[User]( + beginning: cursor.len == 0, + query: Query(kind: userList) + ) + + let raw = json.fromJson(GraphListMembers) + for instruction in raw.data.list.membersTimeline.timeline.instructions: + if instruction.kind == "TimelineAddEntries": + for entry in instruction.entries: + case entry.content.entryType + of TimelineTimelineItem: + let userResult = entry.content.itemContent.userResults.result + if userResult.restId.len > 0: + result.content.add toUser userResult.legacy + of TimelineTimelineCursor: + if entry.content.cursorType == "Bottom": + result.bottom = entry.content.value diff --git a/src/experimental/parser/timeline.nim b/src/experimental/parser/timeline.nim new file mode 100644 index 0000000..351ca85 --- /dev/null +++ b/src/experimental/parser/timeline.nim @@ -0,0 +1,28 @@ +import std/[strutils, tables] +import jsony +import user, ../types/timeline +from ../../types import Result, User + +proc getId(id: string): string {.inline.} = + let start = id.rfind("-") + if start < 0: return id + id[start + 1 ..< id.len] + +proc parseUsers*(json: string; after=""): Result[User] = + result = Result[User](beginning: after.len == 0) + + let raw = json.fromJson(Search) + if raw.timeline.instructions.len == 0: + return + + for e in raw.timeline.instructions[0].addEntries.entries: + let id = e.entryId.getId + if e.entryId.startsWith("user"): + if id in raw.globalObjects.users: + result.content.add toUser raw.globalObjects.users[id] + elif e.entryId.startsWith("cursor"): + let cursor = e.content.operation.cursor + if cursor.cursorType == "Top": + result.top = cursor.value + elif cursor.cursorType == "Bottom": + result.bottom = cursor.value diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim index 095d25c..dc760f0 100644 --- a/src/experimental/parser/user.nim +++ b/src/experimental/parser/user.nim @@ -1,4 +1,4 @@ -import std/[algorithm, unicode, re, strutils] +import std/[algorithm, unicode, re, strutils, strformat, options] import jsony import utils, slices import ../types/user as userType @@ -34,9 +34,40 @@ proc expandUserEntities(user: var User; raw: RawUser) = proc getBanner(user: RawUser): string = if user.profileBannerUrl.len > 0: return user.profileBannerUrl & "/1500x500" + if user.profileLinkColor.len > 0: return '#' & user.profileLinkColor + if user.profileImageExtensions.isSome: + let ext = get(user.profileImageExtensions) + if ext.mediaColor.r.ok.palette.len > 0: + let color = ext.mediaColor.r.ok.palette[0].rgb + return &"#{color.red:02x}{color.green:02x}{color.blue:02x}" + +proc toUser*(raw: RawUser): User = + result = User( + id: raw.idStr, + username: raw.screenName, + fullname: raw.name, + location: raw.location, + bio: raw.description, + following: raw.friendsCount, + followers: raw.followersCount, + tweets: raw.statusesCount, + likes: raw.favouritesCount, + media: raw.mediaCount, + verified: raw.verified, + protected: raw.protected, + joinDate: parseTwitterDate(raw.createdAt), + banner: getBanner(raw), + userPic: getImageUrl(raw.profileImageUrlHttps).replace("_normal", "") + ) + + if raw.pinnedTweetIdsStr.len > 0: + result.pinnedTweet = parseBiggestInt(raw.pinnedTweetIdsStr[0]) + + result.expandUserEntities(raw) + proc parseUser*(json: string; username=""): User = handleErrors: case error.code @@ -44,24 +75,4 @@ proc parseUser*(json: string; username=""): User = of userNotFound: return else: echo "[error - parseUser]: ", error - let user = json.fromJson(RawUser) - - result = User( - id: user.idStr, - username: user.screenName, - fullname: user.name, - location: user.location, - bio: user.description, - following: user.friendsCount, - followers: user.followersCount, - tweets: user.statusesCount, - likes: user.favouritesCount, - media: user.mediaCount, - verified: user.verified, - protected: user.protected, - joinDate: parseTwitterDate(user.createdAt), - banner: getBanner(user), - userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "") - ) - - result.expandUserEntities(user) + result = toUser json.fromJson(RawUser) diff --git a/src/experimental/types/graphlistmembers.nim b/src/experimental/types/graphlistmembers.nim new file mode 100644 index 0000000..4cb3757 --- /dev/null +++ b/src/experimental/types/graphlistmembers.nim @@ -0,0 +1,31 @@ +import graphuser + +type + GraphListMembers* = object + data*: tuple[list: List] + + List = object + membersTimeline*: tuple[timeline: Timeline] + + Timeline = object + instructions*: seq[Instruction] + + Instruction = object + kind*: string + entries*: seq[tuple[content: Content]] + + ContentEntryType* = enum + TimelineTimelineItem + TimelineTimelineCursor + + Content = object + case entryType*: ContentEntryType + of TimelineTimelineItem: + itemContent*: tuple[userResults: UserData] + of TimelineTimelineCursor: + value*: string + cursorType*: string + +proc renameHook*(v: var Instruction; fieldName: var string) = + if fieldName == "type": + fieldName = "kind" diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim new file mode 100644 index 0000000..dded4eb --- /dev/null +++ b/src/experimental/types/graphuser.nim @@ -0,0 +1,12 @@ +import user + +type + GraphUser* = object + data*: tuple[user: UserData] + + UserData* = object + result*: UserResult + + UserResult = object + legacy*: RawUser + restId*: string diff --git a/src/experimental/types/timeline.nim b/src/experimental/types/timeline.nim new file mode 100644 index 0000000..28239ad --- /dev/null +++ b/src/experimental/types/timeline.nim @@ -0,0 +1,23 @@ +import std/tables +import user + +type + Search* = object + globalObjects*: GlobalObjects + timeline*: Timeline + + GlobalObjects = object + users*: Table[string, RawUser] + + Timeline = object + instructions*: seq[Instructions] + + Instructions = object + addEntries*: tuple[entries: seq[Entry]] + + Entry = object + entryId*: string + content*: tuple[operation: Operation] + + Operation = object + cursor*: tuple[value, cursorType: string] diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim index e3afaf0..1c8a5c3 100644 --- a/src/experimental/types/user.nim +++ b/src/experimental/types/user.nim @@ -1,3 +1,4 @@ +import options import common type @@ -16,9 +17,11 @@ type mediaCount*: int verified*: bool protected*: bool + profileLinkColor*: string profileBannerUrl*: string profileImageUrlHttps*: string - profileLinkColor*: string + profileImageExtensions*: Option[ImageExtensions] + pinnedTweetIdsStr*: seq[string] Entities* = object url*: Urls @@ -26,3 +29,15 @@ type Urls* = object urls*: seq[Url] + + ImageExtensions = object + mediaColor*: tuple[r: Ok] + + Ok = object + ok*: Palette + + Palette = object + palette*: seq[tuple[rgb: Color]] + + Color* = object + red*, green*, blue*: int diff --git a/src/parser.nim b/src/parser.nim index 4f3cc04..ae5e505 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -26,16 +26,6 @@ proc parseUser(js: JsonNode; id=""): User = result.expandUserEntities(js) -proc parseGraphUser*(js: JsonNode; id: string): User = - if js.isNull: return - - with user, js{"data", "user", "result", "legacy"}: - result = parseUser(user, id) - - with pinned, user{"pinned_tweet_ids_str"}: - if pinned.kind == JArray and pinned.len > 0: - result.pinnedTweet = parseBiggestInt(pinned[0].getStr) - proc parseGraphList*(js: JsonNode): List = if js.isNull: return @@ -55,25 +45,6 @@ proc parseGraphList*(js: JsonNode): List = banner: list{"custom_banner_media", "media_info", "url"}.getImageStr ) -proc parseGraphListMembers*(js: JsonNode; cursor: string): Result[User] = - result = Result[User]( - beginning: cursor.len == 0, - query: Query(kind: userList) - ) - - if js.isNull: return - - let root = js{"data", "list", "members_timeline", "timeline", "instructions"} - for instruction in root: - if instruction{"type"}.getStr == "TimelineAddEntries": - for entry in instruction{"entries"}: - let content = entry{"content"} - if content{"entryType"}.getStr == "TimelineTimelineItem": - with legacy, content{"itemContent", "user_results", "result", "legacy"}: - result.content.add parseUser(legacy) - elif content{"cursorType"}.getStr == "Bottom": - result.bottom = content{"value"}.getStr - proc parsePoll(js: JsonNode): Poll = let vals = js{"binding_values"} @@ -395,26 +366,6 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod elif "bottom" in r{"entryId"}.getStr: res.bottom = r.getCursor -proc parseUsers*(js: JsonNode; after=""): Result[User] = - result = Result[User](beginning: after.len == 0) - let global = parseGlobalObjects(? js) - - let instructions = ? js{"timeline", "instructions"} - if instructions.len == 0: return - - result.parseInstructions(global, instructions) - - for e in instructions[0]{"addEntries", "entries"}: - let entry = e{"entryId"}.getStr - if "user-" in entry: - let id = entry.getId - if id in global.users: - result.content.add global.users[id] - elif "cursor-top" in entry: - result.top = e.getCursor - elif "cursor-bottom" in entry: - result.bottom = e.getCursor - proc parseTimeline*(js: JsonNode; after=""): Timeline = result = Timeline(beginning: after.len == 0) let global = parseGlobalObjects(? js) diff --git a/src/parserutils.nim b/src/parserutils.nim index 0e7c8b5..a605ea4 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -8,7 +8,7 @@ let unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})" unReplace = "$1@$2" - htRegex = re"(^|[^\w-_./?])([##$])([\w_]+)" + htRegex = re"(^|[^\w-_./?])([#$]|#)([\w_]+)" htReplace = "$1$2$3" type diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 206bb51..469157a 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -102,8 +102,9 @@ proc cache*(data: Tweet) {.async.} = proc cacheRss*(query: string; rss: Rss) {.async.} = let key = "rss:" & query pool.withAcquire(r): - dawait r.hSet(key, "rss", rss.feed) dawait r.hSet(key, "min", rss.cursor) + if rss.cursor != "suspended": + dawait r.hSet(key, "rss", compress(rss.feed)) dawait r.expire(key, rssCacheTime) template deserialize(data, T) = @@ -182,6 +183,10 @@ proc getCachedRss*(key: string): Future[Rss] {.async.} = pool.withAcquire(r): result.cursor = await r.hGet(k, "min") if result.cursor.len > 2: - result.feed = await r.hGet(k, "rss") + if result.cursor != "suspended": + let feed = await r.hGet(k, "rss") + if feed.len > 0 and feed != redisNil: + try: result.feed = uncompress feed + except: echo "Decompressing RSS failed: ", feed else: result.cursor.setLen 0 diff --git a/src/routes/rss.nim b/src/routes/rss.nim index af7312d..40aa6a7 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -1,14 +1,14 @@ # SPDX-License-Identifier: AGPL-3.0-only import asyncdispatch, strutils, tables, times, hashes, uri -import jester, supersnappy +import jester import router_utils, timeline import ../query include "../views/rss.nimf" -export times, hashes, supersnappy +export times, hashes proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} = var profile: Profile @@ -36,7 +36,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. return Rss(feed: profile.user.username, cursor: "suspended") if profile.user.fullname.len > 0: - let rss = compress renderTimelineRss(profile, cfg, multi=(names.len > 1)) + let rss = renderTimelineRss(profile, cfg, multi=(names.len > 1)) return Rss(feed: rss, cursor: profile.tweets.bottom) template respRss*(rss, page) = @@ -48,11 +48,11 @@ template respRss*(rss, page) = resp Http404, showError(page & info & "not found", cfg) elif rss.cursor.len == 9 and rss.cursor == "suspended": - resp Http404, showError(getSuspended(rss.feed), cfg) + resp Http404, showError(getSuspended(@"name"), cfg) let headers = {"Content-Type": "application/rss+xml; charset=utf-8", "Min-Id": rss.cursor} - resp Http200, headers, uncompress rss.feed + resp Http200, headers, rss.feed proc createRssRouter*(cfg: Config) = router rss: @@ -75,8 +75,7 @@ proc createRssRouter*(cfg: Config) = let tweets = await getSearch[Tweet](query, cursor) rss.cursor = tweets.bottom - rss.feed = compress renderSearchRss(tweets.content, query.text, - genQueryUrl(query), cfg) + rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) await cacheRss(key, rss) respRss(rss, "Search") @@ -157,7 +156,7 @@ proc createRssRouter*(cfg: Config) = list = await getCachedList(id=(@"id")) timeline = await getListTimeline(list.id, cursor) rss.cursor = timeline.bottom - rss.feed = compress renderListRss(timeline.content, list, cfg) + rss.feed = renderListRss(timeline.content, list, cfg) await cacheRss(key, rss) respRss(rss, "List") diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 3df67f4..a0a6e21 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -86,7 +86,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; html = renderTweetSearch(timeline, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) - var profile = await fetchProfile(after, query) + var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins) template u: untyped = profile.user if u.suspended: diff --git a/src/sass/profile/_base.scss b/src/sass/profile/_base.scss index ae6b801..b7f33e6 100644 --- a/src/sass/profile/_base.scss +++ b/src/sass/profile/_base.scss @@ -42,12 +42,16 @@ top: 50px; } -.profile-result .username { - margin: 0 !important; -} +.profile-result { + min-height: 54px; -.profile-result .tweet-header { - margin-bottom: unset; + .username { + margin: 0 !important; + } + + .tweet-header { + margin-bottom: unset; + } } @media(max-width: 700px) { diff --git a/src/views/general.nim b/src/views/general.nim index 7054fd4..82902d4 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=15") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=16") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2") if theme.len > 0: diff --git a/src/views/preferences.nim b/src/views/preferences.nim index ba1bcc0..0ce4687 100644 --- a/src/views/preferences.nim +++ b/src/views/preferences.nim @@ -35,7 +35,7 @@ macro renderPrefs*(): untyped = proc renderPreferences*(prefs: Prefs; path: string; themes: seq[string]): VNode = buildHtml(tdiv(class="overlay-panel")): fieldset(class="preferences"): - form(`method`="post", action="/saveprefs"): + form(`method`="post", action="/saveprefs", autocomplete="off"): refererField path renderPrefs() diff --git a/src/views/search.nim b/src/views/search.nim index 5f831ca..f4510d4 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -23,7 +23,7 @@ const toggles = { proc renderSearch*(): VNode = buildHtml(tdiv(class="panel-container")): tdiv(class="search-bar"): - form(`method`="get", action="/search"): + form(`method`="get", action="/search", autocomplete="off"): hiddenField("f", "users") input(`type`="text", name="q", autofocus="", placeholder="ユーザー名の入力…", dir="auto") button(`type`="submit"): icon "search" @@ -57,7 +57,8 @@ proc isPanelOpen(q: Query): bool = proc renderSearchPanel*(query: Query): VNode = let user = query.fromUser.join(",") let action = if user.len > 0: &"/{user}/search" else: "/search" - buildHtml(form(`method`="get", action=action, class="search-field")): + buildHtml(form(`method`="get", action=action, + class="search-field", autocomplete="off")): hiddenField("f", "tweets") genInput("q", "", query.text, "検索…", class="pref-inline") button(`type`="submit"): icon "search" @@ -111,7 +112,7 @@ proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string; proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode = buildHtml(tdiv(class="timeline-container")): tdiv(class="timeline-header"): - form(`method`="get", action="/search", class="search-field"): + form(`method`="get", action="/search", class="search-field", autocomplete="off"): hiddenField("f", "users") genInput("q", "", results.query.text, "ユーザー名の入力…", class="pref-inline") button(`type`="submit"): icon "search" diff --git a/src/views/timeline.nim b/src/views/timeline.nim index 3456bdb..afe35dc 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -95,7 +95,7 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string; if not results.beginning: renderNewer(results.query, parseUri(path).path) - if pinned.isSome: + if not prefs.hidePins and pinned.isSome: let tweet = get pinned renderTweet(tweet, prefs, path, showThread=tweet.hasThread)