このコミットが含まれているのは:
テクニカル諏訪子 2022-01-18 02:42:30 +09:00
コミット b0cace0a50
31個のファイルの変更432行の追加240行の削除

ファイルの表示

@ -33,6 +33,6 @@ jobs:
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
push: true
tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}

ファイルの表示

@ -7,6 +7,7 @@
# disable annoying warnings
warning("GcUnsafe2", off)
hint("XDeclaredButNotUsed", off)
hint("XCannotRaiseY", off)
hint("User", off)
const

ファイルの表示

@ -22,7 +22,7 @@ requires "redpool#f880f49"
requires "https://github.com/zedeus/redis#d0a0e6f"
requires "zippy#0.7.3"
requires "flatty#0.2.3"
requires "jsony#1.1.3"
requires "jsony#d0e69bd"
# Tasks

ファイルの表示

@ -1,70 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-only
import random, strformat, strutils, sequtils
randomize()
const rvs = [
"11.0", "40.0", "42.0", "43.0", "47.0", "50.0", "52.0", "53.0", "54.0",
"61.0", "66.0", "67.0", "69.0", "70.0"
]
proc rv(): string =
if rand(10) < 1: ""
else: "; rv:" & sample(rvs)
# OS
const enc = ["; U", "; N", "; I", ""]
proc linux(): string =
const
window = ["X11", "Wayland", "Unknown"]
arch = ["i686", "x86_64", "arm"]
distro = ["", "; Ubuntu/14.10", "; Ubuntu/16.10", "; Ubuntu/19.10",
"; Ubuntu", "; Fedora"]
sample(window) & sample(enc) & "; Linux " & sample(arch) & sample(distro)
proc windows(): string =
const
nt = ["5.1", "5.2", "6.0", "6.1", "6.2", "6.3", "6.4", "9.0", "10.0"]
arch = ["; WOW64", "; Win64; x64", "; ARM", ""]
trident = ["", "; Trident/5.0", "; Trident/6.0", "; Trident/7.0"]
"Windows " & sample(nt) & sample(enc) & sample(arch) & sample(trident)
const macs = toSeq(6..15).mapIt($it) & @["14_4", "10_1", "9_3"]
proc mac(): string =
"Macintosh; Intel Mac OS X 10_" & sample(macs) & sample(enc)
# Browser
proc presto(): string =
const p = ["2.12.388", "2.12.407", "22.9.168", "2.9.201", "2.8.131", "2.7.62",
"2.6.30", "2.5.24"]
const v = ["10.0", "11.0", "11.1", "11.5", "11.6", "12.00", "12.14", "12.16"]
&"Presto/{sample(p)} Version/{sample(v)}"
# Samples
proc product(): string =
const opera = ["Opera/9.80", "Opera/12.0"]
if rand(20) < 1: "Mozilla/5.0"
else: sample(opera)
proc os(): string =
let r = rand(10)
let os =
if r < 6: windows()
elif r < 9: linux()
else: mac()
&"({os}{rv()})"
proc browser(prod: string): string =
if "Opera" in prod: presto()
else: "like Gecko"
# Agent
proc getAgent*(): string =
let prod = product()
&"{prod} {os()} {browser(prod)}"

ファイルの表示

@ -2,6 +2,7 @@
import asyncdispatch, httpclient, uri, strutils
import packedjson
import types, query, formatters, consts, apiutils, parser
import experimental/parser/user
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
@ -32,14 +33,14 @@ proc getListMembers*(list: List; after=""): Future[Result[Profile]] {.async.} =
proc getProfile*(username: string): Future[Profile] {.async.} =
let
ps = genParams({"screen_name": username})
js = await fetch(userShow ? ps, Api.userShow)
result = parseUserShow(js, username=username)
json = await fetchRaw(userShow ? ps, Api.userShow)
result = parseUser(json, username)
proc getProfileById*(userId: string): Future[Profile] {.async.} =
let
ps = genParams({"user_id": userId})
js = await fetch(userShow ? ps, Api.userShow)
result = parseUserShow(js, id=userId)
json = await fetchRaw(userShow ? ps, Api.userShow)
result = parseUser(json)
proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
let

ファイルの表示

@ -1,7 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, times, strutils, uri
import packedjson, zippy
import httpclient, asyncdispatch, options, strutils, uri
import jsony, packedjson, zippy
import types, tokens, consts, parserutils, http_pool
import experimental/types/common
const
rlRemaining = "x-rate-limit-remaining"
@ -16,6 +17,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
result &= p
if ext:
result &= ("ext", "mediaStats")
result &= ("include_ext_alt_text", "true")
result &= ("include_ext_media_availability", "true")
if count.len > 0:
result &= ("count", count)
if cursor.len > 0:
@ -40,7 +43,14 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
"DNT": "1"
})
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
template updateToken() =
if api != Api.search and resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
token.setRateLimit(api, remaining, reset)
template fetchImpl(result, fetchBody) {.dirty.} =
once:
pool = HttpPool()
@ -48,37 +58,21 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
if token.tok.len == 0:
raise rateLimitError()
let headers = genHeaders(token)
try:
var resp: AsyncResponse
var body = pool.use(headers):
result = pool.use(genHeaders(token)):
resp = await c.get($url)
await resp.body
if body.len > 0:
if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip":
body = uncompress(body, dfGzip)
result = uncompress(result, dfGzip)
else:
echo "non-gzip body, url: ", url, ", body: ", body
echo "non-gzip body, url: ", url, ", body: ", result
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body
result = newJNull()
fetchBody
if api != Api.search and resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
token.setRateLimit(api, remaining, reset)
if result.getError notin {invalidToken, forbidden, badToken}:
release(token, used=true)
else:
echo "fetch error: ", result.getError
release(token, invalid=true)
raise rateLimitError()
release(token, used=true)
if resp.status == $Http400:
raise newException(InternalError, $url)
@ -89,3 +83,35 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
if "length" notin e.msg and "descriptor" notin e.msg:
release(token, invalid=true)
raise rateLimitError()
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
var body: string
fetchImpl body:
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body
result = newJNull()
updateToken()
let error = result.getError
if error in {invalidToken, forbidden, badToken}:
echo "fetch error: ", result.getError
release(token, invalid=true)
raise rateLimitError()
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result
result.setLen(0)
updateToken()
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors in {invalidToken, forbidden, badToken}:
echo "fetch error: ", errors
release(token, invalid=true)
raise rateLimitError()

ファイルの表示

@ -35,16 +35,13 @@ const
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "false",
"include_ext_alt_text": "true",
"include_reply_count": "1",
"tweet_mode": "extended",
"include_entities": "true",
"include_user_entities": "true",
"include_ext_media_color": "false",
"include_ext_media_availability": "true",
"send_error_codes": "true",
"simple_quoted_tweet": "true",
"ext": "mediaStats",
"include_quote_count": "true"
}.toSeq

67
src/experimental/parser/slices.nim ノーマルファイル
ファイルの表示

@ -0,0 +1,67 @@
import std/[macros, htmlgen, unicode]
import ../types/common
import ".."/../[formatters, utils]
type
ReplaceSliceKind = enum
rkRemove, rkUrl, rkHashtag, rkMention
ReplaceSlice* = object
slice: Slice[int]
kind: ReplaceSliceKind
url, display: string
proc cmp*(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
proc dedupSlices*(s: var seq[ReplaceSlice]) =
var
len = s.len
i = 0
while i < len:
var j = i + 1
while j < len:
if s[i].slice.a == s[j].slice.a:
s.del j
dec len
else:
inc j
inc i
proc extractUrls*(result: var seq[ReplaceSlice]; url: Url;
textLen: int; hideTwitter = false) =
let
link = url.expandedUrl
slice = url.indices[0] ..< url.indices[1]
if hideTwitter and slice.b.succ >= textLen and link.isTwitterUrl:
if slice.a < textLen:
result.add ReplaceSlice(kind: rkRemove, slice: slice)
else:
result.add ReplaceSlice(kind: rkUrl, url: link,
display: link.shortLink, slice: slice)
proc replacedWith*(runes: seq[Rune]; repls: openArray[ReplaceSlice];
textSlice: Slice[int]): string =
template extractLowerBound(i: int; idx): int =
if i > 0: repls[idx].slice.b.succ else: textSlice.a
result = newStringOfCap(runes.len)
for i, rep in repls:
result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
case rep.kind
of rkHashtag:
let
name = $runes[rep.slice.a.succ .. rep.slice.b]
symbol = $runes[rep.slice.a]
result.add a(symbol & name, href = "/search?q=%23" & name)
of rkMention:
result.add a($runes[rep.slice], href = rep.url, title = rep.display)
of rkUrl:
result.add a(rep.display, href = rep.url)
of rkRemove:
discard
let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
if rest.a <= rest.b:
result.add $runes[rest]

67
src/experimental/parser/user.nim ノーマルファイル
ファイルの表示

@ -0,0 +1,67 @@
import std/[algorithm, unicode, re, strutils]
import jsony
import utils, slices
import ../types/user as userType
from ../../types import Profile, Error
let
unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
unReplace = "$1<a href=\"/$2\">@$2</a>"
htRegex = re"(^|[^\w-_./?])([#$])([\w_]+)"
htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
proc expandProfileEntities(profile: var Profile; user: User) =
let
orig = profile.bio.toRunes
ent = user.entities
if ent.url.urls.len > 0:
profile.website = ent.url.urls[0].expandedUrl
var replacements = newSeq[ReplaceSlice]()
for u in ent.description.urls:
replacements.extractUrls(u, orig.high)
replacements.dedupSlices
replacements.sort(cmp)
profile.bio = orig.replacedWith(replacements, 0 .. orig.len)
.replacef(unRegex, unReplace)
.replacef(htRegex, htReplace)
proc getBanner(user: User): string =
if user.profileBannerUrl.len > 0:
return user.profileBannerUrl & "/1500x500"
if user.profileLinkColor.len > 0:
return '#' & user.profileLinkColor
proc parseUser*(json: string; username=""): Profile =
handleErrors:
case error.code
of suspended: return Profile(username: username, suspended: true)
of userNotFound: return
else: echo "[error - parseUser]: ", error
let user = json.fromJson(User)
result = Profile(
id: user.idStr,
username: user.screenName,
fullname: user.name,
location: user.location,
bio: user.description,
following: user.friendsCount,
followers: user.followersCount,
tweets: user.statusesCount,
likes: user.favouritesCount,
media: user.mediaCount,
verified: user.verified,
protected: user.protected,
joinDate: parseTwitterDate(user.createdAt),
banner: getBanner(user),
userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
)
result.expandProfileEntities(user)

22
src/experimental/parser/utils.nim ノーマルファイル
ファイルの表示

@ -0,0 +1,22 @@
# SPDX-License-Identifier: AGPL-3.0-only
import std/[sugar, strutils, times]
import ../types/common
import ../../utils as uutils
template parseTime(time: string; f: static string; flen: int): DateTime =
if time.len != flen: return
parse(time, f, utc())
proc parseIsoDate*(date: string): DateTime =
date.parseTime("yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
proc parseTwitterDate*(date: string): DateTime =
date.parseTime("ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
proc getImageUrl*(url: string): string =
url.dup(removePrefix(twimg), removePrefix(https))
template handleErrors*(body) =
if json.startsWith("{\"errors"):
for error {.inject.} in json.fromJson(Errors).errors:
body

20
src/experimental/types/common.nim ノーマルファイル
ファイルの表示

@ -0,0 +1,20 @@
from ../../types import Error
type
Url* = object
url*: string
expandedUrl*: string
displayUrl*: string
indices*: array[2, int]
ErrorObj* = object
code*: Error
message*: string
Errors* = object
errors*: seq[ErrorObj]
proc contains*(codes: set[Error]; errors: Errors): bool =
for e in errors.errors:
if e.code in codes:
return true

28
src/experimental/types/user.nim ノーマルファイル
ファイルの表示

@ -0,0 +1,28 @@
import common
type
User* = object
idStr*: string
name*: string
screenName*: string
location*: string
description*: string
entities*: Entities
createdAt*: string
followersCount*: int
friendsCount*: int
favouritesCount*: int
statusesCount*: int
mediaCount*: int
verified*: bool
protected*: bool
profileBannerUrl*: string
profileImageUrlHttps*: string
profileLinkColor*: string
Entities* = object
url*: Urls
description*: Urls
Urls* = object
urls*: seq[Url]

ファイルの表示

@ -33,23 +33,25 @@ proc getUrlPrefix*(cfg: Config): string =
if cfg.useHttps: https & cfg.hostname
else: "http://" & cfg.hostname
proc stripHtml*(text: string): string =
proc shortLink*(text: string; length=28): string =
result = text.replace(wwwRegex, "")
if result.len > length:
result = result[0 ..< length] & ""
proc stripHtml*(text: string; shorten=false): string =
var html = parseHtml(text)
for el in html.findAll("a"):
let link = el.attr("href")
if "http" in link:
if el.len == 0: continue
el[0].text = link
el[0].text =
if shorten: link.shortLink
else: link
html.innerText()
proc sanitizeXml*(text: string): string =
text.replace(illegalXmlRegex, "")
proc shortLink*(text: string; length=28): string =
result = text.replace(wwwRegex, "")
if result.len > length:
result = result[0 ..< length] & ""
proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
result = body

ファイルの表示

@ -14,11 +14,11 @@ proc parseProfile(js: JsonNode; id=""): Profile =
bio: js{"description"}.getStr,
userPic: js{"profile_image_url_https"}.getImageStr.replace("_normal", ""),
banner: js.getBanner,
following: $js{"friends_count"}.getInt,
followers: $js{"followers_count"}.getInt,
tweets: $js{"statuses_count"}.getInt,
likes: $js{"favourites_count"}.getInt,
media: $js{"media_count"}.getInt,
following: js{"friends_count"}.getInt,
followers: js{"followers_count"}.getInt,
tweets: js{"statuses_count"}.getInt,
likes: js{"favourites_count"}.getInt,
media: js{"media_count"}.getInt,
verified: js{"verified"}.getBool,
protected: js{"protected"}.getBool,
joinDate: js{"created_at"}.getTime
@ -26,21 +26,6 @@ proc parseProfile(js: JsonNode; id=""): Profile =
result.expandProfileEntities(js)
proc parseUserShow*(js: JsonNode; username=""; id=""): Profile =
if id.len > 0:
result = Profile(id: id)
else:
result = Profile(username: username)
if js.isNull: return
with error, js{"errors"}:
if error.getError == suspended:
result.suspended = true
return
result = parseProfile(js)
proc parseGraphList*(js: JsonNode): List =
if js.isNull: return

ファイルの表示

@ -1,6 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, times, macros, htmlgen, unicode, options, algorithm
import std/re
import std/[strutils, times, macros, htmlgen, options, algorithm, re]
import std/unicode except strip
import packedjson
import types, utils, formatters
@ -119,18 +119,6 @@ proc getBanner*(js: JsonNode): string =
if color.len > 0:
return '#' & color
# use primary color from profile picture color histogram
with p, js{"profile_image_extensions", "mediaColor", "r", "ok", "palette"}:
if p.len > 0:
let pal = p[0]{"rgb"}
result = "#"
result.add toHex(pal{"red"}.getInt, 2)
result.add toHex(pal{"green"}.getInt, 2)
result.add toHex(pal{"blue"}.getInt, 2)
return
return "#161616"
proc getTombstone*(js: JsonNode): string =
result = js{"tombstoneInfo", "richText", "text"}.getStr
result.removeSuffix(" Learn more")
@ -275,3 +263,4 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
replacements.sort(cmp)
tweet.text = orig.replacedWith(replacements, textSlice)
.strip(leading=false)

ファイルの表示

@ -13,6 +13,9 @@ var
rssCacheTime: int
listCacheTime*: int
template dawait(future) =
discard await future
# flatty can't serialize DateTime, so we need to define this
proc toFlatty*(s: var string, x: DateTime) =
s.toFlatty(x.toTime().toUnix())
@ -33,9 +36,9 @@ proc migrate*(key, match: string) {.async.} =
let list = await r.scan(newCursor(0), match, 100000)
r.startPipelining()
for item in list:
discard await r.del(item)
dawait r.del(item)
await r.setk(key, "true")
discard await r.flushPipeline()
dawait r.flushPipeline()
proc initRedisPool*(cfg: Config) {.async.} =
try:
@ -47,6 +50,7 @@ proc initRedisPool*(cfg: Config) {.async.} =
await migrate("snappyRss", "rss:*")
await migrate("userBuckets", "p:*")
await migrate("profileDates", "p:*")
await migrate("profileStats", "p:*")
pool.withAcquire(r):
# optimize memory usage for profile ID buckets
@ -67,7 +71,7 @@ proc get(query: string): Future[string] {.async.} =
proc setEx(key: string; time: int; data: string) {.async.} =
pool.withAcquire(r):
discard await r.setEx(key, time, data)
dawait r.setEx(key, time, data)
proc cache*(data: List) {.async.} =
await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
@ -76,29 +80,29 @@ proc cache*(data: PhotoRail; name: string) {.async.} =
await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
proc cache*(data: Profile) {.async.} =
if data.username.len == 0 or data.id.len == 0: return
if data.username.len == 0: return
let name = toLower(data.username)
pool.withAcquire(r):
r.startPipelining()
discard await r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
discard await r.setEx("i:" & data.id , baseCacheTime, data.username)
discard await r.hSet(name.pidKey, name, data.id)
discard await r.flushPipeline()
dawait r.setEx(name.profileKey, baseCacheTime, compress(toFlatty(data)))
if data.id.len > 0:
dawait r.hSet(name.pidKey, name, data.id)
dawait r.flushPipeline()
proc cacheProfileId*(username, id: string) {.async.} =
proc cacheProfileId(username, id: string) {.async.} =
if username.len == 0 or id.len == 0: return
let name = toLower(username)
pool.withAcquire(r):
discard await r.hSet(name.pidKey, name, id)
dawait r.hSet(name.pidKey, name, id)
proc cacheRss*(query: string; rss: Rss) {.async.} =
let key = "rss:" & query
pool.withAcquire(r):
r.startPipelining()
discard await r.hSet(key, "rss", rss.feed)
discard await r.hSet(key, "min", rss.cursor)
discard await r.expire(key, rssCacheTime)
discard await r.flushPipeline()
dawait r.hSet(key, "rss", rss.feed)
dawait r.hSet(key, "min", rss.cursor)
dawait r.expire(key, rssCacheTime)
dawait r.flushPipeline()
proc getProfileId*(username: string): Future[string] {.async.} =
let name = toLower(username)
@ -113,15 +117,21 @@ proc getCachedProfile*(username: string; fetch=true): Future[Profile] {.async.}
result = fromFlatty(uncompress(prof), Profile)
elif fetch:
result = await getProfile(username)
await cacheProfileId(result.username, result.id)
if result.suspended:
await cache(result)
proc getCachedProfileUsername*(userId: string): Future[string] {.async.} =
let username = await get("i:" & userId)
let
key = "i:" & userId
username = await get(key)
if username != redisNil:
result = username
else:
let profile = await getProfileById(userId)
result = profile.username
await cache(profile)
await setEx(key, baseCacheTime, result)
proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return

ファイルの表示

@ -1,9 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, strutils, options
import jester
import ".."/[types, api], ../views/embed
import jester, karax/vdom
import ".."/[types, api]
import ../views/[embed, tweet, general]
import router_utils
export api, embed
export api, embed, vdom, tweet, general, router_utils
proc createEmbedRouter*(cfg: Config) =
router embed:
@ -12,4 +14,23 @@ proc createEmbedRouter*(cfg: Config) =
if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
resp Http404
resp renderVideoEmbed(cfg, convo.tweet)
resp renderVideoEmbed(convo.tweet, cfg, request)
get "/@user/status/@id/embed":
let
convo = await getTweet(@"id")
prefs = cookiePrefs()
path = getPath()
if convo == nil or convo.tweet == nil:
resp Http404
resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
get "/embed/Tweet.html":
let id = @"id"
if id.len > 0:
redirect("/i/status/" & id & "/embed")
else:
resp Http404

ファイルの表示

@ -5,7 +5,7 @@ import asynchttpserver, asyncstreams, asyncfile, asyncnet
import jester
import router_utils
import ".."/[types, formatters, agents, utils]
import ".."/[types, formatters, utils]
export asynchttpserver, asyncstreams, asyncfile, asyncnet
export httpclient, os, strutils, asyncstreams, base64, re
@ -14,10 +14,8 @@ const
m3u8Mime* = "application/vnd.apple.mpegurl"
maxAge* = "max-age=604800"
let mediaAgent* = getAgent()
proc safeFetch*(url, agent: string): Future[string] {.async.} =
let client = newAsyncHttpClient(userAgent=agent)
proc safeFetch*(url: string): Future[string] {.async.} =
let client = newAsyncHttpClient()
try: result = await client.getContent(url)
except: discard
finally: client.close()
@ -34,7 +32,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
result = Http200
let
request = req.getNativeReq()
client = newAsyncHttpClient(userAgent=mediaAgent)
client = newAsyncHttpClient()
try:
let res = await client.get(url)
@ -116,14 +114,14 @@ proc createMediaRouter*(cfg: Config) =
var content: string
if ".vmap" in url:
let m3u8 = getM3u8Url(await safeFetch(url, mediaAgent))
let m3u8 = getM3u8Url(await safeFetch(url))
if m3u8.len > 0:
content = await safeFetch(url, mediaAgent)
content = await safeFetch(url)
else:
resp Http404
if ".m3u8" in url:
let vid = await safeFetch(url, mediaAgent)
let vid = await safeFetch(url)
content = proxifyVideo(vid, cookiePref(proxyVideos))
resp content, m3u8Mime

ファイルの表示

@ -4,12 +4,12 @@ from jester import Request, cookies
import ../views/general
import ".."/[utils, prefs, types]
export utils, prefs, types
export utils, prefs, types, uri
template savePref*(pref, value: string; req: Request; expire=false) =
if not expire or pref in cookies(req):
setCookie(pref, value, daysForward(when expire: -10 else: 360),
httpOnly=true, secure=cfg.useHttps)
httpOnly=true, secure=cfg.useHttps, sameSite=None)
template cookiePrefs*(): untyped {.dirty.} =
getPrefs(cookies(request))

ファイルの表示

@ -19,8 +19,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
names = getNames(name)
if names.len == 1:
(profile, timeline) =
await fetchSingleTimeline(after, query, skipRail=true)
(profile, timeline) = await fetchTimeline(after, query, skipRail=true)
else:
var q = query
q.fromUser = names

ファイルの表示

@ -19,8 +19,8 @@ proc getQuery*(request: Request; tab, name: string): Query =
of "search": initQuery(params(request), name=name)
else: Query(fromUser: @[name])
proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
Future[(Profile, Timeline, PhotoRail)] {.async.} =
proc fetchTimeline*(after: string; query: Query; skipRail=false):
Future[(Profile, Timeline, PhotoRail)] {.async.} =
let name = query.fromUser[0]
var
@ -30,20 +30,13 @@ proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
if profileId.len == 0:
profile = await getCachedProfile(name)
profileId = if profile.suspended: "s"
else: profile.id
if profileId.len > 0:
await cacheProfileId(profile.username, profileId)
profileId = profile.id
fetched = true
if profileId.len == 0 or profile.protected:
result[0] = profile
return
elif profileId == "s":
result[0] = Profile(username: name, suspended: true)
return
if profile.protected or profile.suspended:
return (profile, Timeline(), @[])
elif profileId.len == 0:
return (Profile(username: name), Timeline(), @[])
var rail: Future[PhotoRail]
if skipRail or profile.protected or query.kind == media:
@ -86,7 +79,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
html = renderTweetSearch(timeline, prefs, getPath())
return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
var (p, t, r) = await fetchSingleTimeline(after, query)
var (p, t, r) = await fetchTimeline(after, query)
if p.suspended: return showError(getSuspended(p.username), cfg)
if p.id.len == 0: return
@ -139,7 +132,7 @@ proc createTimelineRouter*(cfg: Config) =
timeline.beginning = true
resp $renderTweetSearch(timeline, prefs, getPath())
else:
var (_, timeline, _) = await fetchSingleTimeline(after, query, skipRail=true)
var (_, timeline, _) = await fetchTimeline(after, query, skipRail=true)
if timeline.content.len == 0: resp Http404
timeline.beginning = true
resp $renderTimelineTweets(timeline, prefs, getPath())

ファイルの表示

@ -15,23 +15,22 @@
}
.profile-banner {
padding-bottom: 4px;
margin-bottom: 4px;
background-color: var(--bg_panel);
a {
display: inherit;
line-height: 0;
display: block;
position: relative;
padding: 33.34% 0 0 0;
}
img {
width: 100%;
max-width: 100%;
position: absolute;
top: 0;
}
}
.profile-banner-color {
width: 100%;
padding-bottom: 25%;
}
.profile-tab {
padding: 0 4px 0 0;
box-sizing: border-box;

ファイルの表示

@ -35,19 +35,25 @@
}
.profile-card-avatar {
display: block;
display: inline-block;
position: relative;
width: 100%;
padding-bottom: 6px;
margin-right: 4px;
margin-bottom: 6px;
&:after {
content: '';
display: block;
margin-top: 100%;
}
img {
display: block;
box-sizing: border-box;
position: absolute;
width: 100%;
height: 100%;
margin: 0;
border: 4px solid var(--darker_grey);
background: var(--bg_color);
background: var(--bg_panel);
}
}
@ -113,8 +119,8 @@
}
.profile-card-avatar {
height: 60px;
width: unset;
width: 80px;
height: 80px;
img {
border-width: 2px;

ファイルの表示

@ -32,6 +32,7 @@
a {
position: relative;
border-radius: 5px;
background-color: var(--darker_grey);
&:before {
content: "";

ファイルの表示

@ -98,11 +98,14 @@
}
.avatar {
position: absolute;
&.round {
border-radius: 50%;
}
&.mini {
position: unset;
margin-right: 5px;
margin-top: -1px;
width: 20px;
@ -110,6 +113,24 @@
}
}
.tweet-embed {
display: flex;
flex-direction: column;
justify-content: center;
height: 100%;
background-color: var(--bg_panel);
.tweet-content {
font-size: 18px
}
.tweet-body {
display: flex;
flex-direction: column;
max-height: calc(100vh - 0.75em * 2);
}
}
.attribution {
display: flex;
pointer-events: all;

ファイルの表示

@ -2,12 +2,12 @@
import asyncdispatch, httpclient, times, sequtils, json, random
import strutils, tables
import zippy
import types, agents, consts, http_pool
import types, consts, http_pool
const
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
maxAge = 3.hours # tokens expire after 3 hours
maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
failDelay = initDuration(minutes=30)
var
@ -65,7 +65,6 @@ proc fetchToken(): Future[Token] {.async.} =
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.5",
"connection": "keep-alive",
"user-agent": getAgent(),
"authorization": auth
})

ファイルの表示

@ -48,17 +48,16 @@ type
id*: string
username*: string
fullname*: string
lowername*: string
location*: string
website*: string
bio*: string
userPic*: string
banner*: string
following*: string
followers*: string
tweets*: string
likes*: string
media*: string
following*: int
followers*: int
tweets*: int
likes*: int
media*: int
verified*: bool
protected*: bool
suspended*: bool

ファイルの表示

@ -1,18 +1,19 @@
# SPDX-License-Identifier: AGPL-3.0-only
import options
import karax/[karaxdsl, vdom]
from jester import Request
import ".."/[types, formatters]
import general, tweet
const doctype = "<!DOCTYPE html>\n"
proc renderVideoEmbed*(cfg: Config; tweet: Tweet): string =
proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
let thumb = get(tweet.video).thumb
let vidUrl = getVideoEmbed(cfg, tweet.id)
let prefs = Prefs(hlsPlayback: true)
let node = buildHtml(html(lang="en")):
renderHead(prefs, cfg, video=vidUrl, images=(@[thumb]))
renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
tdiv(class="embed-video"):
renderVideo(get(tweet.video), prefs, "")

ファイルの表示

@ -11,6 +11,9 @@ const
doctype = "<!DOCTYPE html>\n"
lp = readFile("public/lp.svg")
proc toTheme(theme: string): string =
theme.toLowerAscii.replace(" ", "_")
proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
var path = req.params.getOrDefault("referer")
if path.len == 0:
@ -33,9 +36,13 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
icon "info", title="About", href="/about"
icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
images: seq[string] = @[]; banner=""; ogTitle=""; theme="";
proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
video=""; images: seq[string] = @[]; banner=""; ogTitle="";
rss=""; canonical=""): VNode =
var theme = prefs.theme.toTheme
if "theme" in req.params:
theme = req.params["theme"].toTheme
let ogType =
if video.len > 0: "video"
elif rss.len > 0: "object"
@ -45,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
buildHtml(head):
link(rel="stylesheet", type="text/css", href="/css/style.css?v=10")
link(rel="stylesheet", type="text/css", href="/css/style.css?v=15")
link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
if theme.len > 0:
@ -118,15 +125,12 @@ proc renderHead*(prefs: Prefs; cfg: Config; titleText=""; desc=""; video="";
proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
titleText=""; desc=""; ogTitle=""; rss=""; video="";
images: seq[string] = @[]; banner=""): string =
var theme = toLowerAscii(prefs.theme).replace(" ", "_")
if "theme" in req.params:
theme = toLowerAscii(req.params["theme"]).replace(" ", "_")
let canonical = getTwitterLink(req.path, req.params)
let node = buildHtml(html(lang="en")):
renderHead(prefs, cfg, titleText, desc, video, images, banner, ogTitle,
theme, rss, canonical)
renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
rss, canonical)
body:
renderNavbar(cfg, req, rss, canonical)

ファイルの表示

@ -5,12 +5,12 @@ import karax/[karaxdsl, vdom, vstyles]
import renderutils, search
import ".."/[types, utils, formatters]
proc renderStat(num, class: string; text=""): VNode =
proc renderStat(num: int; class: string; text=""): VNode =
let t = if text.len > 0: text else: class
buildHtml(li(class=class)):
span(class="profile-stat-header"): text capitalizeAscii(t)
span(class="profile-stat-num"):
text if num.len == 0: "?" else: insertSep(num, ',')
text insertSep($num, ',')
proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
buildHtml(tdiv(class="profile-card")):
@ -78,18 +78,16 @@ proc renderPhotoRail(profile: Profile; photoRail: PhotoRail): VNode =
tdiv(class="photo-rail-grid"):
for i, photo in photoRail:
if i == 16: break
let col = if photo.color.len > 0: photo.color else: "#161616"
a(href=(&"/{profile.username}/status/{photo.tweetId}#m"),
style={backgroundColor: col}):
a(href=(&"/{profile.username}/status/{photo.tweetId}#m")):
genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
proc renderBanner(profile: Profile): VNode =
proc renderBanner(banner: string): VNode =
buildHtml():
if "#" in profile.banner:
tdiv(class="profile-banner-color", style={backgroundColor: profile.banner})
if banner.startsWith('#'):
a(style={backgroundColor: banner})
else:
a(href=getPicUrl(profile.banner), target="_blank"):
genImg(profile.banner)
a(href=getPicUrl(banner), target="_blank"):
genImg(banner)
proc renderProtected(username: string): VNode =
buildHtml(tdiv(class="timeline-container")):
@ -103,7 +101,8 @@ proc renderProfile*(profile: Profile; timeline: var Timeline;
buildHtml(tdiv(class="profile-tabs")):
if not prefs.hideBanner:
tdiv(class="profile-banner"):
renderBanner(profile)
if profile.banner.len > 0:
renderBanner(profile.banner)
let sticky = if prefs.stickyProfile: " sticky" else: ""
tdiv(class=(&"profile-tab{sticky}")):

ファイルの表示

@ -1,9 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, sequtils, strformat, options
import karax/[karaxdsl, vdom, vstyles]
from jester import Request
import renderutils
import ".."/[types, utils, formatters]
import general
proc getSmallPic(url: string): string =
result = url
@ -353,3 +355,8 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
if showThread:
a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
text "Show this thread"
proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
buildHtml(tdiv(class="tweet-embed")):
renderHead(prefs, cfg, req)
renderTweet(tweet, prefs, path, mainTweet=true)