From e4f634705b9f9a21d0c0f9413390faca1151f244 Mon Sep 17 00:00:00 2001 From: Mitarashi Date: Mon, 24 Jan 2022 21:53:59 +0200 Subject: [PATCH 01/15] Fix #521 --- src/parserutils.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parserutils.nim b/src/parserutils.nim index 0e7c8b5..d4a0aca 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -8,7 +8,7 @@ let unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})" unReplace = "$1@$2" - htRegex = re"(^|[^\w-_./?])([##$])([\w_]+)" + htRegex = re"(^|[^\w-_./?])([#$]|#)([\w_]+)" htReplace = "$1$2$3" type @@ -210,7 +210,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) = replacements.deduplicate replacements.sort(cmp) - + user.bio = orig.replacedWith(replacements, 0 .. orig.len) user.bio = user.bio.replacef(unRegex, unReplace) .replacef(htRegex, htReplace) From d56628ed2f245e043efff9e053b6a4b3b9ccbb9c Mon Sep 17 00:00:00 2001 From: Mitarashi Date: Mon, 24 Jan 2022 21:55:14 +0200 Subject: [PATCH 02/15] removed spaces --- src/parserutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parserutils.nim b/src/parserutils.nim index d4a0aca..a605ea4 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -210,7 +210,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) = replacements.deduplicate replacements.sort(cmp) - + user.bio = orig.replacedWith(replacements, 0 .. orig.len) user.bio = user.bio.replacef(unRegex, unReplace) .replacef(htRegex, htReplace) From ae7091e69d7e3fbbf56b2897ef42111d37d43abb Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 26 Jan 2022 17:24:03 +0100 Subject: [PATCH 03/15] Add experimental GraphQL user parser --- src/api.nim | 6 ++-- src/experimental/parser/graphql.nim | 8 +++++ src/experimental/parser/user.nim | 46 ++++++++++++++++------------- src/experimental/types/graphql.nim | 12 ++++++++ src/experimental/types/user.nim | 1 + src/parser.nim | 10 ------- 6 files changed, 49 insertions(+), 34 deletions(-) create mode 100644 src/experimental/parser/graphql.nim create mode 100644 src/experimental/types/graphql.nim diff --git a/src/api.nim b/src/api.nim index 503af9e..8c97493 100644 --- a/src/api.nim +++ b/src/api.nim @@ -2,14 +2,14 @@ import asyncdispatch, httpclient, uri, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser -import experimental/parser/user +import experimental/parser/[user, graphql] proc getGraphUser*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return let variables = %*{"userId": id, "withSuperFollowsUserFields": true} - js = await fetch(graphUser ? {"variables": $variables}, Api.userRestId) - result = parseGraphUser(js, id) + js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId) + result = parseGraphUser(js) proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = let diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim new file mode 100644 index 0000000..df5c40a --- /dev/null +++ b/src/experimental/parser/graphql.nim @@ -0,0 +1,8 @@ +import jsony +import ../types/graphql, user +from ../../types import User + +proc parseGraphUser*(json: string): User = + let raw = json.fromJson(GraphUser) + result = toUser raw.data.user.result.legacy + result.id = raw.data.user.result.restId diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim index 095d25c..767945b 100644 --- a/src/experimental/parser/user.nim +++ b/src/experimental/parser/user.nim @@ -37,6 +37,30 @@ proc getBanner(user: RawUser): string = if user.profileLinkColor.len > 0: return '#' & user.profileLinkColor +proc toUser*(raw: RawUser): User = + result = User( + id: raw.idStr, + username: raw.screenName, + fullname: raw.name, + location: raw.location, + bio: raw.description, + following: raw.friendsCount, + followers: raw.followersCount, + tweets: raw.statusesCount, + likes: raw.favouritesCount, + media: raw.mediaCount, + verified: raw.verified, + protected: raw.protected, + joinDate: parseTwitterDate(raw.createdAt), + banner: getBanner(raw), + userPic: getImageUrl(raw.profileImageUrlHttps).replace("_normal", "") + ) + + if raw.pinnedTweetIdsStr.len > 0: + result.pinnedTweet = parseBiggestInt(raw.pinnedTweetIdsStr[0]) + + result.expandUserEntities(raw) + proc parseUser*(json: string; username=""): User = handleErrors: case error.code @@ -44,24 +68,4 @@ proc parseUser*(json: string; username=""): User = of userNotFound: return else: echo "[error - parseUser]: ", error - let user = json.fromJson(RawUser) - - result = User( - id: user.idStr, - username: user.screenName, - fullname: user.name, - location: user.location, - bio: user.description, - following: user.friendsCount, - followers: user.followersCount, - tweets: user.statusesCount, - likes: user.favouritesCount, - media: user.mediaCount, - verified: user.verified, - protected: user.protected, - joinDate: parseTwitterDate(user.createdAt), - banner: getBanner(user), - userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "") - ) - - result.expandUserEntities(user) + result = toUser json.fromJson(RawUser) diff --git a/src/experimental/types/graphql.nim b/src/experimental/types/graphql.nim new file mode 100644 index 0000000..6697d07 --- /dev/null +++ b/src/experimental/types/graphql.nim @@ -0,0 +1,12 @@ +import user + +type + GraphUserResult* = object + legacy*: RawUser + restId*: string + + GraphUserData* = object + result*: GraphUserResult + + GraphUser* = object + data*: tuple[user: GraphUserData] diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim index e3afaf0..c048ba2 100644 --- a/src/experimental/types/user.nim +++ b/src/experimental/types/user.nim @@ -19,6 +19,7 @@ type profileBannerUrl*: string profileImageUrlHttps*: string profileLinkColor*: string + pinnedTweetIdsStr*: seq[string] Entities* = object url*: Urls diff --git a/src/parser.nim b/src/parser.nim index 4f3cc04..177eaed 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -26,16 +26,6 @@ proc parseUser(js: JsonNode; id=""): User = result.expandUserEntities(js) -proc parseGraphUser*(js: JsonNode; id: string): User = - if js.isNull: return - - with user, js{"data", "user", "result", "legacy"}: - result = parseUser(user, id) - - with pinned, user{"pinned_tweet_ids_str"}: - if pinned.kind == JArray and pinned.len > 0: - result.pinnedTweet = parseBiggestInt(pinned[0].getStr) - proc parseGraphList*(js: JsonNode): List = if js.isNull: return From a54d6aa1eb09a8bebc014e2120acb27b7aad4d0c Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 26 Jan 2022 17:57:46 +0100 Subject: [PATCH 04/15] Add experimental GraphQL list members parser --- src/api.nim | 2 +- src/experimental/parser/graphql.nim | 23 +++++++++++++-- src/experimental/types/graphlistmembers.nim | 31 +++++++++++++++++++++ src/experimental/types/graphql.nim | 12 -------- src/experimental/types/graphuser.nim | 12 ++++++++ src/parser.nim | 19 ------------- 6 files changed, 65 insertions(+), 34 deletions(-) create mode 100644 src/experimental/types/graphlistmembers.nim delete mode 100644 src/experimental/types/graphql.nim create mode 100644 src/experimental/types/graphuser.nim diff --git a/src/api.nim b/src/api.nim index 8c97493..8fdaf3d 100644 --- a/src/api.nim +++ b/src/api.nim @@ -37,7 +37,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} "withSuperFollowsTweetFields": false } url = graphListMembers ? {"variables": $variables} - result = parseGraphListMembers(await fetch(url, Api.listMembers), after) + result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after) proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index df5c40a..b00ab24 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -1,8 +1,27 @@ import jsony -import ../types/graphql, user -from ../../types import User +import user, ../types/[graphuser, graphlistmembers] +from ../../types import User, Result, Query, QueryKind proc parseGraphUser*(json: string): User = let raw = json.fromJson(GraphUser) result = toUser raw.data.user.result.legacy result.id = raw.data.user.result.restId + +proc parseGraphListMembers*(json, cursor: string): Result[User] = + result = Result[User]( + beginning: cursor.len == 0, + query: Query(kind: userList) + ) + + let raw = json.fromJson(GraphListMembers) + for instruction in raw.data.list.membersTimeline.timeline.instructions: + if instruction.kind == "TimelineAddEntries": + for entry in instruction.entries: + case entry.content.entryType + of TimelineTimelineItem: + let userResult = entry.content.itemContent.userResults.result + if userResult.restId.len > 0: + result.content.add toUser userResult.legacy + of TimelineTimelineCursor: + if entry.content.cursorType == "Bottom": + result.bottom = entry.content.value diff --git a/src/experimental/types/graphlistmembers.nim b/src/experimental/types/graphlistmembers.nim new file mode 100644 index 0000000..4cb3757 --- /dev/null +++ b/src/experimental/types/graphlistmembers.nim @@ -0,0 +1,31 @@ +import graphuser + +type + GraphListMembers* = object + data*: tuple[list: List] + + List = object + membersTimeline*: tuple[timeline: Timeline] + + Timeline = object + instructions*: seq[Instruction] + + Instruction = object + kind*: string + entries*: seq[tuple[content: Content]] + + ContentEntryType* = enum + TimelineTimelineItem + TimelineTimelineCursor + + Content = object + case entryType*: ContentEntryType + of TimelineTimelineItem: + itemContent*: tuple[userResults: UserData] + of TimelineTimelineCursor: + value*: string + cursorType*: string + +proc renameHook*(v: var Instruction; fieldName: var string) = + if fieldName == "type": + fieldName = "kind" diff --git a/src/experimental/types/graphql.nim b/src/experimental/types/graphql.nim deleted file mode 100644 index 6697d07..0000000 --- a/src/experimental/types/graphql.nim +++ /dev/null @@ -1,12 +0,0 @@ -import user - -type - GraphUserResult* = object - legacy*: RawUser - restId*: string - - GraphUserData* = object - result*: GraphUserResult - - GraphUser* = object - data*: tuple[user: GraphUserData] diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim new file mode 100644 index 0000000..dded4eb --- /dev/null +++ b/src/experimental/types/graphuser.nim @@ -0,0 +1,12 @@ +import user + +type + GraphUser* = object + data*: tuple[user: UserData] + + UserData* = object + result*: UserResult + + UserResult = object + legacy*: RawUser + restId*: string diff --git a/src/parser.nim b/src/parser.nim index 177eaed..580f449 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -45,25 +45,6 @@ proc parseGraphList*(js: JsonNode): List = banner: list{"custom_banner_media", "media_info", "url"}.getImageStr ) -proc parseGraphListMembers*(js: JsonNode; cursor: string): Result[User] = - result = Result[User]( - beginning: cursor.len == 0, - query: Query(kind: userList) - ) - - if js.isNull: return - - let root = js{"data", "list", "members_timeline", "timeline", "instructions"} - for instruction in root: - if instruction{"type"}.getStr == "TimelineAddEntries": - for entry in instruction{"entries"}: - let content = entry{"content"} - if content{"entryType"}.getStr == "TimelineTimelineItem": - with legacy, content{"itemContent", "user_results", "result", "legacy"}: - result.content.add parseUser(legacy) - elif content{"cursorType"}.getStr == "Bottom": - result.bottom = content{"value"}.getStr - proc parsePoll(js: JsonNode): Poll = let vals = js{"binding_values"} From 49a2fbb0700234ad6aec40a65ba44cc83ec0b555 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 26 Jan 2022 18:24:34 +0100 Subject: [PATCH 05/15] Support profile image color parsing in wip parser --- src/experimental/parser/user.nim | 7 ++++++- src/experimental/types/user.nim | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim index 767945b..b9aaa3c 100644 --- a/src/experimental/parser/user.nim +++ b/src/experimental/parser/user.nim @@ -1,4 +1,4 @@ -import std/[algorithm, unicode, re, strutils] +import std/[algorithm, unicode, re, strutils, strformat] import jsony import utils, slices import ../types/user as userType @@ -34,9 +34,14 @@ proc expandUserEntities(user: var User; raw: RawUser) = proc getBanner(user: RawUser): string = if user.profileBannerUrl.len > 0: return user.profileBannerUrl & "/1500x500" + if user.profileLinkColor.len > 0: return '#' & user.profileLinkColor + if user.profileImageExtensions.mediaColor.r.ok.palette.len > 0: + let color = user.profileImageExtensions.mediaColor.r.ok.palette[0].rgb + return &"#{color.red:02x}{color.green:02x}{color.blue:02x}" + proc toUser*(raw: RawUser): User = result = User( id: raw.idStr, diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim index c048ba2..430bc8e 100644 --- a/src/experimental/types/user.nim +++ b/src/experimental/types/user.nim @@ -18,6 +18,7 @@ type protected*: bool profileBannerUrl*: string profileImageUrlHttps*: string + profileImageExtensions*: ImageExtensions profileLinkColor*: string pinnedTweetIdsStr*: seq[string] @@ -27,3 +28,15 @@ type Urls* = object urls*: seq[Url] + + ImageExtensions = object + mediaColor*: tuple[r: Ok] + + Ok = object + ok*: Palette + + Palette = object + palette*: seq[tuple[rgb: Color]] + + Color* = object + red*, green*, blue*: int From 4738ec33854f46c0c23a1181fbbdac291bb9a5a3 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 26 Jan 2022 20:27:11 +0100 Subject: [PATCH 06/15] Add experimental user search parser --- src/api.nim | 5 ++++- src/experimental/parser/timeline.nim | 28 ++++++++++++++++++++++++++++ src/experimental/parser/user.nim | 10 ++++++---- src/experimental/types/timeline.nim | 23 +++++++++++++++++++++++ src/experimental/types/user.nim | 5 +++-- src/parser.nim | 20 -------------------- src/sass/profile/_base.scss | 14 +++++++++----- src/views/general.nim | 2 +- 8 files changed, 74 insertions(+), 33 deletions(-) create mode 100644 src/experimental/parser/timeline.nim create mode 100644 src/experimental/types/timeline.nim diff --git a/src/api.nim b/src/api.nim index 8fdaf3d..f421bb7 100644 --- a/src/api.nim +++ b/src/api.nim @@ -3,6 +3,7 @@ import asyncdispatch, httpclient, uri, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser import experimental/parser/[user, graphql] +import experimental/parser/timeline as timelineParser proc getGraphUser*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return @@ -85,10 +86,12 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = const searchMode = ("result_filter", "user") parse = parseUsers + fetchFunc = fetchRaw else: const searchMode = ("tweet_search_mode", "live") parse = parseTimeline + fetchFunc = fetch let q = genQueryParam(query) if q.len == 0 or q == emptyQuery: @@ -96,7 +99,7 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = let url = search ? genParams(searchParams & @[("q", q), searchMode], after) try: - result = parse(await fetch(url, Api.search), after) + result = parse(await fetchFunc(url, Api.search), after) result.query = query except InternalError: return Result[T](beginning: true, query: query) diff --git a/src/experimental/parser/timeline.nim b/src/experimental/parser/timeline.nim new file mode 100644 index 0000000..351ca85 --- /dev/null +++ b/src/experimental/parser/timeline.nim @@ -0,0 +1,28 @@ +import std/[strutils, tables] +import jsony +import user, ../types/timeline +from ../../types import Result, User + +proc getId(id: string): string {.inline.} = + let start = id.rfind("-") + if start < 0: return id + id[start + 1 ..< id.len] + +proc parseUsers*(json: string; after=""): Result[User] = + result = Result[User](beginning: after.len == 0) + + let raw = json.fromJson(Search) + if raw.timeline.instructions.len == 0: + return + + for e in raw.timeline.instructions[0].addEntries.entries: + let id = e.entryId.getId + if e.entryId.startsWith("user"): + if id in raw.globalObjects.users: + result.content.add toUser raw.globalObjects.users[id] + elif e.entryId.startsWith("cursor"): + let cursor = e.content.operation.cursor + if cursor.cursorType == "Top": + result.top = cursor.value + elif cursor.cursorType == "Bottom": + result.bottom = cursor.value diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim index b9aaa3c..dc760f0 100644 --- a/src/experimental/parser/user.nim +++ b/src/experimental/parser/user.nim @@ -1,4 +1,4 @@ -import std/[algorithm, unicode, re, strutils, strformat] +import std/[algorithm, unicode, re, strutils, strformat, options] import jsony import utils, slices import ../types/user as userType @@ -38,9 +38,11 @@ proc getBanner(user: RawUser): string = if user.profileLinkColor.len > 0: return '#' & user.profileLinkColor - if user.profileImageExtensions.mediaColor.r.ok.palette.len > 0: - let color = user.profileImageExtensions.mediaColor.r.ok.palette[0].rgb - return &"#{color.red:02x}{color.green:02x}{color.blue:02x}" + if user.profileImageExtensions.isSome: + let ext = get(user.profileImageExtensions) + if ext.mediaColor.r.ok.palette.len > 0: + let color = ext.mediaColor.r.ok.palette[0].rgb + return &"#{color.red:02x}{color.green:02x}{color.blue:02x}" proc toUser*(raw: RawUser): User = result = User( diff --git a/src/experimental/types/timeline.nim b/src/experimental/types/timeline.nim new file mode 100644 index 0000000..28239ad --- /dev/null +++ b/src/experimental/types/timeline.nim @@ -0,0 +1,23 @@ +import std/tables +import user + +type + Search* = object + globalObjects*: GlobalObjects + timeline*: Timeline + + GlobalObjects = object + users*: Table[string, RawUser] + + Timeline = object + instructions*: seq[Instructions] + + Instructions = object + addEntries*: tuple[entries: seq[Entry]] + + Entry = object + entryId*: string + content*: tuple[operation: Operation] + + Operation = object + cursor*: tuple[value, cursorType: string] diff --git a/src/experimental/types/user.nim b/src/experimental/types/user.nim index 430bc8e..1c8a5c3 100644 --- a/src/experimental/types/user.nim +++ b/src/experimental/types/user.nim @@ -1,3 +1,4 @@ +import options import common type @@ -16,10 +17,10 @@ type mediaCount*: int verified*: bool protected*: bool + profileLinkColor*: string profileBannerUrl*: string profileImageUrlHttps*: string - profileImageExtensions*: ImageExtensions - profileLinkColor*: string + profileImageExtensions*: Option[ImageExtensions] pinnedTweetIdsStr*: seq[string] Entities* = object diff --git a/src/parser.nim b/src/parser.nim index 580f449..ae5e505 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -366,26 +366,6 @@ proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNod elif "bottom" in r{"entryId"}.getStr: res.bottom = r.getCursor -proc parseUsers*(js: JsonNode; after=""): Result[User] = - result = Result[User](beginning: after.len == 0) - let global = parseGlobalObjects(? js) - - let instructions = ? js{"timeline", "instructions"} - if instructions.len == 0: return - - result.parseInstructions(global, instructions) - - for e in instructions[0]{"addEntries", "entries"}: - let entry = e{"entryId"}.getStr - if "user-" in entry: - let id = entry.getId - if id in global.users: - result.content.add global.users[id] - elif "cursor-top" in entry: - result.top = e.getCursor - elif "cursor-bottom" in entry: - result.bottom = e.getCursor - proc parseTimeline*(js: JsonNode; after=""): Timeline = result = Timeline(beginning: after.len == 0) let global = parseGlobalObjects(? js) diff --git a/src/sass/profile/_base.scss b/src/sass/profile/_base.scss index ae6b801..b7f33e6 100644 --- a/src/sass/profile/_base.scss +++ b/src/sass/profile/_base.scss @@ -42,12 +42,16 @@ top: 50px; } -.profile-result .username { - margin: 0 !important; -} +.profile-result { + min-height: 54px; -.profile-result .tweet-header { - margin-bottom: unset; + .username { + margin: 0 !important; + } + + .tweet-header { + margin-bottom: unset; + } } @media(max-width: 700px) { diff --git a/src/views/general.nim b/src/views/general.nim index 7054fd4..82902d4 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=15") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=16") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2") if theme.len > 0: From 3a076a9b4ea0f015755e8541c4f2bc49cd917f8b Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 26 Jan 2022 21:05:23 +0100 Subject: [PATCH 07/15] Add experimental parser module --- src/api.nim | 3 +-- src/experimental/parser.nim | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 src/experimental/parser.nim diff --git a/src/api.nim b/src/api.nim index f421bb7..708b72f 100644 --- a/src/api.nim +++ b/src/api.nim @@ -2,8 +2,7 @@ import asyncdispatch, httpclient, uri, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser -import experimental/parser/[user, graphql] -import experimental/parser/timeline as timelineParser +import experimental/parser as newParser proc getGraphUser*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return diff --git a/src/experimental/parser.nim b/src/experimental/parser.nim new file mode 100644 index 0000000..98ce7df --- /dev/null +++ b/src/experimental/parser.nim @@ -0,0 +1,2 @@ +import parser/[user, graphql, timeline] +export user, graphql, timeline From c95fc32e0efdd08f6223be502cbcab80fd3e2e45 Mon Sep 17 00:00:00 2001 From: Sam Erika Clotfelter Date: Wed, 26 Jan 2022 23:50:29 -0500 Subject: [PATCH 08/15] use docker cache --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 60aac63..90c4289 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,13 @@ EXPOSE 8080 RUN apk --no-cache add libsass-dev pcre -COPY . /src/nitter WORKDIR /src/nitter -RUN nimble build -y -d:danger -d:lto -d:strip \ +COPY nitter.nimble . +RUN nimble install -y -d:strip + +COPY . . +RUN nimble build -y -d:danger -d:lto \ && nimble scss \ && nimble md From bd1630c2ee9c1d428c46ccc8815e893e6d10fee2 Mon Sep 17 00:00:00 2001 From: Sam Erika Clotfelter Date: Thu, 27 Jan 2022 00:01:51 -0500 Subject: [PATCH 09/15] remove unneeded -y --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 90c4289..36f12cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ COPY nitter.nimble . RUN nimble install -y -d:strip COPY . . -RUN nimble build -y -d:danger -d:lto \ +RUN nimble build -d:danger -d:lto \ && nimble scss \ && nimble md From 091bb6813d1076c4379cf116a90385ebb19dc9d2 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 27 Jan 2022 14:36:12 +0100 Subject: [PATCH 10/15] Move RSS compression to Redis module, fix crash --- src/redis_cache.nim | 7 +++++-- src/routes/rss.nim | 13 ++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 206bb51..acd23e8 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -102,7 +102,7 @@ proc cache*(data: Tweet) {.async.} = proc cacheRss*(query: string; rss: Rss) {.async.} = let key = "rss:" & query pool.withAcquire(r): - dawait r.hSet(key, "rss", rss.feed) + dawait r.hSet(key, "rss", compress(rss.feed)) dawait r.hSet(key, "min", rss.cursor) dawait r.expire(key, rssCacheTime) @@ -182,6 +182,9 @@ proc getCachedRss*(key: string): Future[Rss] {.async.} = pool.withAcquire(r): result.cursor = await r.hGet(k, "min") if result.cursor.len > 2: - result.feed = await r.hGet(k, "rss") + let feed = await r.hGet(k, "rss") + if feed != redisNil: + try: result.feed = uncompress feed + except: echo "Decompressing RSS failed: ", feed else: result.cursor.setLen 0 diff --git a/src/routes/rss.nim b/src/routes/rss.nim index af7312d..f75f201 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -1,14 +1,14 @@ # SPDX-License-Identifier: AGPL-3.0-only import asyncdispatch, strutils, tables, times, hashes, uri -import jester, supersnappy +import jester import router_utils, timeline import ../query include "../views/rss.nimf" -export times, hashes, supersnappy +export times, hashes proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} = var profile: Profile @@ -36,7 +36,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. return Rss(feed: profile.user.username, cursor: "suspended") if profile.user.fullname.len > 0: - let rss = compress renderTimelineRss(profile, cfg, multi=(names.len > 1)) + let rss = renderTimelineRss(profile, cfg, multi=(names.len > 1)) return Rss(feed: rss, cursor: profile.tweets.bottom) template respRss*(rss, page) = @@ -52,7 +52,7 @@ template respRss*(rss, page) = let headers = {"Content-Type": "application/rss+xml; charset=utf-8", "Min-Id": rss.cursor} - resp Http200, headers, uncompress rss.feed + resp Http200, headers, rss.feed proc createRssRouter*(cfg: Config) = router rss: @@ -75,8 +75,7 @@ proc createRssRouter*(cfg: Config) = let tweets = await getSearch[Tweet](query, cursor) rss.cursor = tweets.bottom - rss.feed = compress renderSearchRss(tweets.content, query.text, - genQueryUrl(query), cfg) + rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg) await cacheRss(key, rss) respRss(rss, "Search") @@ -157,7 +156,7 @@ proc createRssRouter*(cfg: Config) = list = await getCachedList(id=(@"id")) timeline = await getListTimeline(list.id, cursor) rss.cursor = timeline.bottom - rss.feed = compress renderListRss(timeline.content, list, cfg) + rss.feed = renderListRss(timeline.content, list, cfg) await cacheRss(key, rss) respRss(rss, "List") From eadf722284e34c877aa999a7f7565e9e72e24abf Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 27 Jan 2022 14:41:40 +0100 Subject: [PATCH 11/15] Add compile-time define to skip building binary --- nitter.nimble | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nitter.nimble b/nitter.nimble index ea2436c..b15940c 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -5,7 +5,9 @@ author = "zedeus" description = "An alternative front-end for Twitter" license = "AGPL-3.0" srcDir = "src" -bin = @["nitter"] + +when not defined(depsOnly): + bin = @["nitter"] # Dependencies From f10519c41a7d9998388f32e30341a9d4312f6ab7 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 27 Jan 2022 14:46:24 +0100 Subject: [PATCH 12/15] Minor RSS behavior improvements --- src/redis_cache.nim | 12 +++++++----- src/routes/rss.nim | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/redis_cache.nim b/src/redis_cache.nim index acd23e8..469157a 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -102,8 +102,9 @@ proc cache*(data: Tweet) {.async.} = proc cacheRss*(query: string; rss: Rss) {.async.} = let key = "rss:" & query pool.withAcquire(r): - dawait r.hSet(key, "rss", compress(rss.feed)) dawait r.hSet(key, "min", rss.cursor) + if rss.cursor != "suspended": + dawait r.hSet(key, "rss", compress(rss.feed)) dawait r.expire(key, rssCacheTime) template deserialize(data, T) = @@ -182,9 +183,10 @@ proc getCachedRss*(key: string): Future[Rss] {.async.} = pool.withAcquire(r): result.cursor = await r.hGet(k, "min") if result.cursor.len > 2: - let feed = await r.hGet(k, "rss") - if feed != redisNil: - try: result.feed = uncompress feed - except: echo "Decompressing RSS failed: ", feed + if result.cursor != "suspended": + let feed = await r.hGet(k, "rss") + if feed.len > 0 and feed != redisNil: + try: result.feed = uncompress feed + except: echo "Decompressing RSS failed: ", feed else: result.cursor.setLen 0 diff --git a/src/routes/rss.nim b/src/routes/rss.nim index f75f201..40aa6a7 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -48,7 +48,7 @@ template respRss*(rss, page) = resp Http404, showError(page & info & "not found", cfg) elif rss.cursor.len == 9 and rss.cursor == "suspended": - resp Http404, showError(getSuspended(rss.feed), cfg) + resp Http404, showError(getSuspended(@"name"), cfg) let headers = {"Content-Type": "application/rss+xml; charset=utf-8", "Min-Id": rss.cursor} From 8ecac89ea00e3320fb8ccdcb7accbd73a4d9ddd1 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 30 Jan 2022 16:58:20 +0100 Subject: [PATCH 13/15] Fix hidePinned preference --- src/routes/timeline.nim | 2 +- src/views/timeline.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 3df67f4..a0a6e21 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -86,7 +86,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; html = renderTweetSearch(timeline, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) - var profile = await fetchProfile(after, query) + var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins) template u: untyped = profile.user if u.suspended: diff --git a/src/views/timeline.nim b/src/views/timeline.nim index ccf7c7b..54cad7a 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -95,7 +95,7 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string; if not results.beginning: renderNewer(results.query, parseUri(path).path) - if pinned.isSome: + if not prefs.hidePins and pinned.isSome: let tweet = get pinned renderTweet(tweet, prefs, path, showThread=tweet.hasThread) From b1a90cd52be1c67bd9fd921fa7624c9a2faf57ee Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 30 Jan 2022 17:17:59 +0100 Subject: [PATCH 14/15] Disable input autocomplete Fixes #433 --- src/views/preferences.nim | 2 +- src/views/search.nim | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/views/preferences.nim b/src/views/preferences.nim index 9d7ce8a..1787704 100644 --- a/src/views/preferences.nim +++ b/src/views/preferences.nim @@ -35,7 +35,7 @@ macro renderPrefs*(): untyped = proc renderPreferences*(prefs: Prefs; path: string; themes: seq[string]): VNode = buildHtml(tdiv(class="overlay-panel")): fieldset(class="preferences"): - form(`method`="post", action="/saveprefs"): + form(`method`="post", action="/saveprefs", autocomplete="off"): refererField path renderPrefs() diff --git a/src/views/search.nim b/src/views/search.nim index 8102412..94bbac8 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -23,9 +23,10 @@ const toggles = { proc renderSearch*(): VNode = buildHtml(tdiv(class="panel-container")): tdiv(class="search-bar"): - form(`method`="get", action="/search"): + form(`method`="get", action="/search", autocomplete="off"): hiddenField("f", "users") - input(`type`="text", name="q", autofocus="", placeholder="Enter username...", dir="auto") + input(`type`="text", name="q", autofocus="", + placeholder="Enter username...", dir="auto") button(`type`="submit"): icon "search" proc renderProfileTabs*(query: Query; username: string): VNode = @@ -57,7 +58,8 @@ proc isPanelOpen(q: Query): bool = proc renderSearchPanel*(query: Query): VNode = let user = query.fromUser.join(",") let action = if user.len > 0: &"/{user}/search" else: "/search" - buildHtml(form(`method`="get", action=action, class="search-field")): + buildHtml(form(`method`="get", action=action, + class="search-field", autocomplete="off")): hiddenField("f", "tweets") genInput("q", "", query.text, "Enter search...", class="pref-inline") button(`type`="submit"): icon "search" @@ -111,7 +113,7 @@ proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string; proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode = buildHtml(tdiv(class="timeline-container")): tdiv(class="timeline-header"): - form(`method`="get", action="/search", class="search-field"): + form(`method`="get", action="/search", class="search-field", autocomplete="off"): hiddenField("f", "users") genInput("q", "", results.query.text, "Enter username...", class="pref-inline") button(`type`="submit"): icon "search" From eedf6d07b8962521402c4aeacc6a7507d418b3d7 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 30 Jan 2022 17:28:31 +0100 Subject: [PATCH 15/15] Remove depsOnly check since Nimble supports it --- nitter.nimble | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nitter.nimble b/nitter.nimble index b15940c..ea2436c 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -5,9 +5,7 @@ author = "zedeus" description = "An alternative front-end for Twitter" license = "AGPL-3.0" srcDir = "src" - -when not defined(depsOnly): - bin = @["nitter"] +bin = @["nitter"] # Dependencies