Merge branch 'master' of https://github.com/zedeus/nitter

* Switch bearer token and endpoints, update parser

* Enable user search, disable tweet search

* Disable multi-user timelines for now

* Fix parsing of pinned tombstone

.dockerignore

@@ -1,6 +0,0 @@
-*.png
-*.md
-LICENSE
-docker-compose.yml
-Dockerfile
-tests/

.github/FUNDING.yml

@@ -1,3 +0,0 @@
-github: zedeus
-liberapay: zedeus
-patreon: nitter

.github/workflows/build-docker.yml

@@ -1,38 +0,0 @@
-name: CI/CD
-
-on:
-  push:
-    paths-ignore:
-      - "README.md"
-    branches:
-      - master
-
-jobs:
-  build-docker:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./Dockerfile
-          platforms: linux/amd64
-          push: true
-          tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}

.gitignore

@@ -3,9 +3,11 @@ nitter
 *.db
 /tests/__pycache__
 /tests/geckodriver.log
-/tests/downloaded_files/*
+/tests/downloaded_files
+/tests/latest_logs
 /tools/gencss
 /tools/rendermd
 /public/css/style.css
 /public/md/*.html
 nitter.conf
+dump.rdb

.travis.yml

@@ -1,51 +0,0 @@
-jobs:
-  include:
-    - stage: test
-      if: (NOT type IN (pull_request)) AND (branch = master)
-      dist: bionic
-      language: python
-      python:
-        - 3.6
-      services:
-        - docker
-        - xvfb
-      script:
-        - sudo apt update
-        - sudo apt install --force-yes chromium-chromedriver
-        - wget https://www.dropbox.com/s/ckuoaubd1crrj2k/Linux_x64_737173_chrome-linux.zip -O chrome.zip
-        - unzip chrome.zip
-        - cd chrome-linux
-        - sudo rm /usr/bin/google-chrome
-        - sudo ln -s chrome /usr/bin/google-chrome
-        - cd ..
-        - pip3 install --upgrade pip
-        - pip3 install -U seleniumbase pytest
-        - docker run -d -p 127.0.0.1:8080:8080/tcp $IMAGE_NAME:$TRAVIS_COMMIT
-        - sleep 10
-        - cd tests
-        - pytest --headless -n 8 --reruns 10 --reruns-delay 2
-    - stage: pr
-      if: type IN (pull_request)
-      dist: bionic
-      language: python
-      python:
-        - 3.6
-      services:
-        - docker
-        - xvfb
-      script:
-        - sudo apt update
-        - sudo apt install --force-yes chromium-chromedriver
-        - wget https://www.dropbox.com/s/ckuoaubd1crrj2k/Linux_x64_737173_chrome-linux.zip -O chrome.zip
-        - unzip chrome.zip
-        - cd chrome-linux
-        - sudo rm /usr/bin/google-chrome
-        - sudo ln -s chrome /usr/bin/google-chrome
-        - cd ..
-        - pip3 install --upgrade pip
-        - pip3 install -U seleniumbase pytest
-        - docker build -t $IMAGE_NAME:$TRAVIS_COMMIT .
-        - docker run -d -p 127.0.0.1:8080:8080/tcp $IMAGE_NAME:$TRAVIS_COMMIT
-        - sleep 10
-        - cd tests
-        - pytest --headless -n 8 --reruns 3 --reruns-delay 2

Dockerfile

@@ -1,23 +0,0 @@
-FROM nimlang/nim:1.6.2-alpine-regular as nim
-LABEL maintainer="setenforce@protonmail.com"
-EXPOSE 8080
-
-RUN apk --no-cache add libsass-dev pcre
-
-WORKDIR /src/nitter
-
-COPY nitter.nimble .
-RUN nimble install -y --depsOnly
-
-COPY . .
-RUN nimble build -d:danger -d:lto -d:strip \
-    && nimble scss \
-    && nimble md
-
-FROM alpine:latest
-WORKDIR /src/
-RUN apk --no-cache add pcre
-COPY --from=nim /src/nitter/nitter ./
-COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
-COPY --from=nim /src/nitter/public ./public
-CMD ./nitter

README.md

@@ -1,6 +1,5 @@
-# Nitter
+# Nitter (OpenBSD)
 
-[![Test Matrix](https://github.com/zedeus/nitter/workflows/CI/CD/badge.svg)](https://github.com/zedeus/nitter/actions?query=workflow%3ACI/CD)
 [![License](https://img.shields.io/github/license/zedeus/nitter?style=flat)](#license)
 
 A free and open source alternative Twitter front-end focused on privacy and
@@ -34,7 +33,7 @@ XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscW
 
 ## Resources
 
-The wiki contains 
+The wiki contains
 [a list of instances](https://github.com/zedeus/nitter/wiki/Instances) and
 [browser extensions](https://github.com/zedeus/nitter/wiki/Extensions)
 maintained by the community.
@@ -67,9 +66,10 @@ Twitter account.
 ## Installation
 
 ### Dependencies
-* libpcre
-* libsass
-* redis
+
+- libpcre
+- libsass
+- redis
 
 To compile Nitter you need a Nim installation, see
 [nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
@@ -94,84 +94,83 @@ $ cd nitter
 $ nimble build -d:release
 $ nimble scss
 $ nimble md
-$ cp nitter.example.conf nitter.conf
+$ mkdir /etc/nitter
+$ cp nitter.example.conf /etc/nitter/nitter.conf
+$ cp nitter /usr/local/bin
 ```
 
 Set your hostname, port, HMAC key, https (must be correct for cookies), and
 Redis info in `nitter.conf`. To run Redis, either run
-`redis-server --daemonize yes`, or `systemctl enable --now redis` (or
+`redis-server --daemonize yes`, or `rcctl enable redis && rcctl start redis` (or
 redis-server depending on the distro). Run Nitter by executing `./nitter` or
-using the systemd service below. You should run Nitter behind a reverse proxy
+using the rc service below. You should run Nitter behind a reverse proxy
 such as [Nginx](https://github.com/zedeus/nitter/wiki/Nginx) or
 [Apache](https://github.com/zedeus/nitter/wiki/Apache) for security and
 performance reasons.
 
-### Docker
 
-#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
+### rc.d
 
-To run Nitter with Docker, you'll need to install and run Redis separately
-before you can run the container. See below for how to also run Redis using
-Docker.
+To run Nitter via rc you can use this service file:
 
-To build and run Nitter in Docker:
-```bash
-docker build -t nitter:latest .
-docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
-```
+```sh
+#!/bin/ksh
 
-A prebuilt Docker image is provided as well:
-```bash
-docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest
-```
+daemon="/usr/local/bin/nitter"
 
-Using docker-compose to run both Nitter and Redis as different containers:
-Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
-```bash
-docker-compose up -d
-```
+. /etc/rc.d/rc.subr
 
-Note the Docker commands expect a `nitter.conf` file in the directory you run
-them.
+rc_bg=YES
 
-### systemd
-
-To run Nitter via systemd you can use this service file:
-
-```ini
-[Unit]
-Description=Nitter (An alternative Twitter front-end)
-After=syslog.target
-After=network.target
-
-[Service]
-Type=simple
-
-# set user and group
-User=nitter
-Group=nitter
-
-# configure location
-WorkingDirectory=/home/nitter/nitter
-ExecStart=/home/nitter/nitter/nitter
-
-Restart=always
-RestartSec=15
-
-[Install]
-WantedBy=multi-user.target
+rc_cmd $1
 ```
 
 Then enable and run the service:
-`systemctl enable --now nitter.service`
+`rcctl enable nitter && rcctl start nitter`
 
-### Logging
+### relayd
+```
+# $OpenBSD: relayd.conf,v 1.5 2018/05/06 20:56:55 benno Exp $
+#
+# Macros
+#
+relayd_addr="0.0.0.0"
+insrv3_addr="192.168.10.103"
 
-Nitter currently prints some errors to stdout, and there is no real logging
-implemented. If you're running Nitter with systemd, you can check stdout like
-this: `journalctl -u nitter.service` (add `--follow` to see just the last 15
-lines). If you're running the Docker image, you can do this:
-`docker logs --follow *nitter container id*`
+table <twitter> { $insrv3_addr }
+
+http protocol reverse {
+  tcp { nodelay, sack }
+  tls ciphers "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256"
+  tls keypair "owacon.moe"
+
+  match request header append "X-Forwarded-For" value "$REMOTE_ADDR"
+  match request header append "X-Forwarded-Port" value "$REMOTE_PORT"
+
+  match response header set "X-Frame-Options" value "deny"
+  match response header set "X-XSS-Protection" value "1; mode=block"
+  match response header set "X-Content-Type-Options" value "nosniff"
+  match response header set "Strict-Transport-Security" value "max-age=31536000; includeSubDomains; preload"
+  match response header set "Permissions-Policy" value "accelerometer=()"
+  match response header set "Cache-Control" value "max-age=86400"
+
+  pass request quick header "Host" value "twitter.owacon.moe" forward to <twitter>
+}
+
+relay www_tls {
+  listen on $relayd_addr port 443 tls
+  protocol reverse
+
+  forward to <twitter> port 8089 check tcp
+}
+
+relay www_http {
+  listen on $relayd_addr port 80
+  protocol reverse
+
+  forward to <twitter> port 8089 check tcp
+}
+```
 
 ## Contact
 

config.nims

@@ -1,5 +1,6 @@
 --define:ssl
 --define:useStdLib
+--threads:off
 
 # workaround httpbeast file upload bug
 --assertions:off

docker-compose.yml

@@ -1,25 +0,0 @@
-version: "3"
-
-services:
-
-  nitter:
-    image: zedeus/nitter:latest
-    container_name: nitter
-    ports:
-      - "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
-    volumes:
-      - ./nitter.conf:/src/nitter.conf:ro
-    depends_on:
-      - nitter-redis
-    restart: unless-stopped
-
-  nitter-redis:
-    image: redis:6-alpine
-    container_name: nitter-redis
-    command: redis-server --save 60 1 --loglevel warning
-    volumes:
-      - nitter-redis:/data
-    restart: unless-stopped
-
-volumes:
-  nitter-redis:

nitter.example.conf

@@ -1,21 +1,21 @@
 [Server]
+hostname = "twitter.owacon.moe"
+title = "オワコンTwitter"
 address = "0.0.0.0"
 port = 8088
-https = false  # disable to enable cookies when not using https
+https = true  # disable to enable cookies when not using https
 httpMaxConnections = 100
 staticDir = "./public"
-title = "nitter"
-hostname = "twitter.076.ne.jp"
 
 [Cache]
 listMinutes = 240  # how long to cache list info (not the tweets, so keep it high)
 rssMinutes = 10  # how long to cache rss queries
-redisHost = "localhost"  # Change to "nitter-redis" if using docker-compose
+redisHost = "localhost"
 redisPort = 6379
 redisPassword = ""
-redisConnections = 20  # connection pool size
+redisConnections = 20  # minimum open connections in pool
 redisMaxConnections = 30
-# max, new connections are opened when none are available, but if the pool size
+# new connections are opened when none are available, but if the pool size
 # goes above this, they're closed when released. don't worry about this unless
 # you receive tons of requests per second
 
@@ -23,24 +23,25 @@ redisMaxConnections = 30
 hmacKey = "secretkey"  # random key for cryptographic signing of video urls
 base64Media = false  # use base64 encoding for proxied media urls
 enableRSS = true  # set this to false to disable RSS feeds
-enableDebug = false  # enable request logs and debug endpoints
+enableDebug = false  # enable request logs and debug endpoints (/.tokens)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
 tokenCount = 10
 # minimum amount of usable tokens. tokens are used to authorize API requests,
-# but they expire after ~1 hour, and have a limit of 187 requests.
-# the limit gets reset every 15 minutes, and the pool is filled up so there's
-# always at least $tokenCount usable tokens. again, only increase this if
-# you receive major bursts all the time
+# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
+# the limits reset every 15 minutes, and the pool is filled up so there's
+# always at least `tokenCount` usable tokens. only increase this if you receive
+# major bursts all the time and don't have a rate limiting setup via e.g. nginx
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]
 theme = "Loli"
-replaceTwitter = "twitter.076.ne.jp"
-replaceYouTube = "youtube.076.ne.jp"
-replaceReddit = ""
+replaceTwitter = "twitter.owacon.moe"
+replaceYouTube = "youtube.owacon.moe"
+replaceReddit = "teddit.net"
 replaceInstagram = ""
-replaceOdysee = "odysee.076.ne.jp"
+replaceOdysee = "odysee.owacon.moe"
+replacePixiv = "pixiv.owacon.moe"
 proxyVideos = true
 hlsPlayback = false
 infiniteScroll = false

nitter.nimble

@@ -11,18 +11,18 @@ bin           = @["nitter"]
 # Dependencies
 
 requires "nim >= 1.4.8"
-requires "jester >= 0.5.0"
-requires "karax#fa4a2dc"
-requires "sass#e683aa1"
-requires "nimcrypto#a5742a9"
-requires "markdown#abdbe5e"
-requires "packedjson#d11d167"
-requires "supersnappy#2.1.1"
+requires "jester#baca3f"
+requires "karax#5cf360c"
+requires "sass#7dfdd03"
+requires "nimcrypto#4014ef9"
+requires "markdown#158efe3"
+requires "packedjson#9e6fbb6"
+requires "supersnappy#6c94198"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#0.7.3"
-requires "flatty#0.2.3"
-requires "jsony#d0e69bd"
+requires "zippy#ca5989a"
+requires "flatty#e668085"
+requires "jsony#ea811be"
 
 
 # Tasks

public/css/themes/dracula.css

@@ -0,0 +1,41 @@
+body {
+    --bg_color: #282a36;
+    --fg_color: #f8f8f2;
+    --fg_faded: #818eb6;
+    --fg_dark: var(--fg_faded);
+    --fg_nav: var(--accent);
+
+    --bg_panel: #343746;
+    --bg_elements: #292b36;
+    --bg_overlays: #44475a;
+    --bg_hover: #2f323f;
+
+    --grey: var(--fg_faded);
+    --dark_grey: #44475a;
+    --darker_grey: #3d4051;
+    --darkest_grey: #363948;
+    --border_grey: #44475a;
+
+    --accent: #bd93f9;
+    --accent_light: #caa9fa;
+    --accent_dark: var(--accent);
+    --accent_border: #ff79c696;
+
+    --play_button: #ffb86c;
+    --play_button_hover: #ffc689;
+
+    --more_replies_dots: #bd93f9;
+    --error_red: #ff5555;
+
+    --verified_blue: var(--accent);
+    --icon_text: ##F8F8F2;
+
+    --tab: #6272a4;
+    --tab_selected: var(--accent);
+
+    --profile_stat: #919cbf;
+}
+
+.search-bar > form input::placeholder{
+    color: var(--fg_faded);
+}

public/md/about.md

@@ -4,7 +4,7 @@ Nitter is a free and open source alternative Twitter front-end focused on
 privacy and performance. The source is available on GitHub at
 <https://github.com/zedeus/nitter>
 
-テクニカル諏訪子に編集したソースコードはこちらで御座います→ <https://git.076.ne.jp/TechnicalSuwako/Nitter-mod>
+テクニカル諏訪子に編集したソースコードはこちらで御座います→ <https://gitler.moe/TechnicalSuwako/Nitter-mod>
 
 * No JavaScript or ads
 * All requests go through the backend, client never talks to Twitter

public/robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+Disallow: /
+Crawl-delay: 1
+User-agent: Twitterbot
+Disallow:

src/api.nim

@@ -4,122 +4,159 @@ import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
 
-proc getGraphUser*(id: string): Future[User] {.async.} =
+proc getGraphUser*(username: string): Future[User] {.async.} =
+  if username.len == 0: return
+  let
+    variables = """{"screen_name": "$1"}""" % username
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetchRaw(graphUser ? params, Api.userScreenName)
+  result = parseGraphUser(js)
+
+proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    variables = %*{"userId": id, "withSuperFollowsUserFields": true}
-    js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
+    variables = """{"rest_id": "$1"}""" % id
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetchRaw(graphUserById ? params, Api.userRestId)
   result = parseGraphUser(js)
 
+proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = userTweetsVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    (url, apiId) = case kind
+                   of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
+                   of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
+                   of TimelineKind.media: (graphUserMedia, Api.userMedia)
+    js = await fetch(url ? params, apiId)
+  result = parseGraphTimeline(js, "user", after)
+
+proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = listTweetsVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphListTweets ? params, Api.listTweets)
+  result = parseGraphTimeline(js, "list", after).tweets
+
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
-    variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
-    url = graphListBySlug ? {"variables": $variables}
-  result = parseGraphList(await fetch(url, Api.listBySlug))
+    variables = %*{"screenName": name, "listSlug": list}
+    params = {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
 
 proc getGraphList*(id: string): Future[List] {.async.} =
   let
-    variables = %*{"listId": id, "withHighlightedLabel": false}
-    url = graphList ? {"variables": $variables}
-  result = parseGraphList(await fetch(url, Api.list))
+    variables = """{"listId": "$1"}""" % id
+    params = {"variables": variables, "features": gqlFeatures}
+  result = parseGraphList(await fetch(graphListById ? params, Api.list))
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
   if list.id.len == 0: return
-  let
+  var
     variables = %*{
       "listId": list.id,
-      "cursor": after,
-      "withSuperFollowsUserFields": false,
       "withBirdwatchPivots": false,
       "withDownvotePerspective": false,
       "withReactionsMetadata": false,
-      "withReactionsPerspective": false,
-      "withSuperFollowsTweetFields": false
+      "withReactionsPerspective": false
     }
-    url = graphListMembers ? {"variables": $variables}
+  if after.len > 0:
+    variables["cursor"] = % after
+  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
   result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
-proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
+proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
   let
-    ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
-    url = listTimeline ? ps
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+    variables = """{"rest_id": "$1"}""" % id
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphTweetResult ? params, Api.tweetResult)
+  result = parseGraphTweetResult(js)
 
-proc getUser*(username: string): Future[User] {.async.} =
-  if username.len == 0: return
-  let
-    ps = genParams({"screen_name": username})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json, username)
-
-proc getUserById*(userId: string): Future[User] {.async.} =
-  if userId.len == 0: return
-  let
-    ps = genParams({"user_id": userId})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json)
-
-proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
+proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
   if id.len == 0: return
   let
-    ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
-    url = timeline / (id & ".json") ? ps
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = tweetVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphTweet ? params, Api.tweetDetail)
+  result = parseGraphConversation(js, id)
 
-proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
-  if id.len == 0: return
-  let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
+  result = (await getGraphTweet(id, after)).replies
+  result.beginning = after.len == 0
+
+proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
+  result = await getGraphTweet(id)
+  if after.len > 0:
+    result.replies = await getReplies(id, after)
+
+proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
+  let q = genQueryParam(query)
+  if q.len == 0 or q == emptyQuery:
+    return Profile(tweets: Timeline(query: query, beginning: true))
+
+  var
+    variables = %*{
+      "rawQuery": q,
+      "count": 20,
+      "product": "Latest",
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
+    }
+  if after.len > 0:
+    variables["cursor"] = % after
+  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  result = Profile(tweets: parseGraphSearch(await fetch(url, Api.search), after))
+  result.tweets.query = query
+
+proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
+  let q = genQueryParam(query)
+  if q.len == 0 or q == emptyQuery:
+    return Timeline(query: query, beginning: true)
+
+  let url = tweetSearch ? genParams({
+    "q": q,
+    "tweet_search_mode": "live",
+    "max_id": after
+  })
+
+  result = parseTweetSearch(await fetch(url, Api.search))
+  result.query = query
+  if after.len == 0:
+    result.beginning = true
+
+proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
+  if query.text.len == 0:
+    return Result[User](query: query, beginning: true)
+
+  var url = userSearch ? {
+    "q": query.text,
+    "skip_status": "1",
+    "count": "20",
+    "page": page
+  }
+
+  result = parseUsers(await fetchRaw(url, Api.userSearch))
+  result.query = query
+  if page.len == 0:
+    result.bottom = "2"
+  elif page.allCharsInSet(Digits):
+    result.bottom = $(parseInt(page) + 1)
 
 proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
   if name.len == 0: return
   let
     ps = genParams({"screen_name": name, "trim_user": "true"},
-                   count="18", ext=false)
+                    count="18", ext=false)
     url = photoRail ? ps
   result = parsePhotoRail(await fetch(url, Api.timeline))
 
-proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
-  when T is User:
-    const
-      searchMode = ("result_filter", "user")
-      parse = parseUsers
-      fetchFunc = fetchRaw
-  else:
-    const
-      searchMode = ("tweet_search_mode", "live")
-      parse = parseTimeline
-      fetchFunc = fetch
-
-  let q = genQueryParam(query)
-  if q.len == 0 or q == emptyQuery:
-    return Result[T](beginning: true, query: query)
-
-  let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
-  try:
-    result = parse(await fetchFunc(url, Api.search), after)
-    result.query = query
-  except InternalError:
-    return Result[T](beginning: true, query: query)
-
-proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
-  let url = tweet / (id & ".json") ? genParams(cursor=after)
-  result = parseConversation(await fetch(url, Api.tweet), id)
-
-proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
-  result = (await getTweetImpl(id, after)).replies
-  result.beginning = after.len == 0
-
-proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
-  result = await getTweetImpl(id)
-  if after.len > 0:
-    result.replies = await getReplies(id, after)
-
-proc getStatus*(id: string): Future[Tweet] {.async.} =
-  let url = status / (id & ".json") ? genParams()
-  result = parseStatus(await fetch(url, Api.status))
-
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)
   try:

src/apiutils.nim

@@ -17,13 +17,13 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
     result &= p
   if ext:
     result &= ("ext", "mediaStats")
-    result &= ("include_ext_alt_text", "true")
-    result &= ("include_ext_media_availability", "true")
+    result &= ("include_ext_alt_text", "1")
+    result &= ("include_ext_media_availability", "1")
   if count.len > 0:
     result &= ("count", count)
   if cursor.len > 0:
     # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to them back into a plus
+    # so we need to turn them back into a plus
     if " " in cursor:
       result &= ("cursor", cursor.replace(" ", "+"))
     else:
@@ -44,7 +44,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
   })
 
 template updateToken() =
-  if api != Api.search and resp.headers.hasKey(rlRemaining):
+  if resp.headers.hasKey(rlRemaining):
     let
       remaining = parseInt(resp.headers[rlRemaining])
       reset = parseInt(resp.headers[rlReset])
@@ -61,12 +61,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   try:
     var resp: AsyncResponse
     pool.use(genHeaders(token)):
-      resp = await c.get($url)
-      result = await resp.body
+      template getContent =
+        resp = await c.get($url)
+        result = await resp.body
+
+      getContent()
 
       if resp.status == $Http503:
         badClient = true
-        raise newException(InternalError, result)
+        raise newException(BadClientError, "Bad client")
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -82,6 +85,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
+  except BadClientError as e:
+    release(token, used=true)
+    raise e
   except Exception as e:
     echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
     if "length" notin e.msg and "descriptor" notin e.msg:
@@ -100,7 +106,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
     updateToken()
 
     let error = result.getError
-    if error in {invalidToken, forbidden, badToken}:
+    if error in {invalidToken, badToken}:
       echo "fetch error: ", result.getError
       release(token, invalid=true)
       raise rateLimitError()
@@ -115,7 +121,7 @@ proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
 
     if result.startsWith("{\"errors"):
       let errors = result.fromJson(Errors)
-      if errors in {invalidToken, forbidden, badToken}:
+      if errors in {invalidToken, badToken}:
         echo "fetch error: ", errors
         release(token, invalid=true)
         raise rateLimitError()

src/config.nim

@@ -22,7 +22,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
 
     staticDir: cfg.get("Server", "staticDir", "./public"),
     title: cfg.get("Server", "title", "０７６ツイッター"),
-    hostname: cfg.get("Server", "hostname", "twitter.076.ne.jp"),
+    hostname: cfg.get("Server", "hostname", "twitter.owacon.moe"),
 
     # Cache
     listCacheTime: cfg.get("Cache", "listMinutes", 120),
@@ -34,8 +34,14 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     redisMaxConns: cfg.get("Cache", "redisMaxConnections", 30),
     redisPassword: cfg.get("Cache", "redisPassword", ""),
 
-    replaceYouTube: cfg.get("Preferences", "replaceYouTube", "youtube.076.ne.jp"),
-    replaceOdysee: cfg.get("Preferences", "replaceOdysee", "odysee.076.ne.jp"),
+    replaceYouTube: cfg.get("Preferences", "replaceYouTube", "youtube.owacon.moe"),
+    replaceOdysee: cfg.get("Preferences", "replaceOdysee", "odysee.owacon.moe"),
+    replacePixivA: cfg.get("Preferences", "replacePixivA", "pixiv.owacon.moe/artworks/?id="),
+    replacePixivU: cfg.get("Preferences", "replacePixivU", "pixiv.owacon.moe/users/?id="),
+    replacePixivB: cfg.get("Preferences", "replacePixivB", "pixiv.owacon.moe/users/bookmarks/?id="),
+    replacePixivI: cfg.get("Preferences", "replacePixivI", "pixiv.owacon.moe/users/illusts/?id="),
+    replacePixivM: cfg.get("Preferences", "replacePixivM", "pixiv.owacon.moe/users/manga/?id="),
+    replacePixiv: cfg.get("Preferences", "replacePixiv", "pixiv.owacon.moe"),
 
     # Config
     hmacKey: cfg.get("Config", "hmacKey", "secretkey"),

src/consts.nim

@@ -1,28 +1,29 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils
+import uri, sequtils, strutils
 
 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
 
   api = parseUri("https://api.twitter.com")
   activate* = $(api / "1.1/guest/activate.json")
 
-  userShow* = api / "1.1/users/show.json"
   photoRail* = api / "1.1/statuses/media_timeline.json"
-  status* = api / "1.1/statuses/show"
-  search* = api / "2/search/adaptive.json"
-
-  timelineApi = api / "2/timeline"
-  timeline* = timelineApi / "profile"
-  mediaTimeline* = timelineApi / "media"
-  listTimeline* = timelineApi / "list.json"
-  tweet* = timelineApi / "conversation"
+  userSearch* = api / "1.1/users/search.json"
+  tweetSearch* = api / "1.1/search/tweets.json"
 
   graphql = api / "graphql"
-  graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
-  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
-  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
-  graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
+  graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
+  graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
+  graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
+  graphTweet* = graphql / "83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2"
+  graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
+  graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
+  graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
+  graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
+  graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
+  graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
 
   timelineParams* = {
     "include_profile_interstitial_type": "0",
@@ -33,27 +34,76 @@ const
     "include_mute_edge": "0",
     "include_can_dm": "0",
     "include_can_media_tag": "1",
+    "include_ext_is_blue_verified": "1",
     "skip_status": "1",
     "cards_platform": "Web-12",
     "include_cards": "1",
-    "include_composer_source": "false",
+    "include_composer_source": "0",
     "include_reply_count": "1",
     "tweet_mode": "extended",
-    "include_entities": "true",
-    "include_user_entities": "true",
-    "include_ext_media_color": "false",
-    "send_error_codes": "true",
-    "simple_quoted_tweet": "true",
-    "include_quote_count": "true"
+    "include_entities": "1",
+    "include_user_entities": "1",
+    "include_ext_media_color": "0",
+    "send_error_codes": "1",
+    "simple_quoted_tweet": "1",
+    "include_quote_count": "1"
   }.toSeq
 
-  searchParams* = {
-    "query_source": "typed_query",
-    "pc": "1",
-    "spelling_corrections": "1"
-  }.toSeq
-  ## top: nothing
-  ## latest: "tweet_search_mode: live"
-  ## user:   "result_filter: user"
-  ## photos: "result_filter: photos"
-  ## videos: "result_filter: videos"
+  gqlFeatures* = """{
+  "android_graphql_skip_api_media_color_palette": false,
+  "blue_business_profile_image_shape_enabled": false,
+  "creator_subscriptions_subscription_count_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+  "hidden_profile_likes_enabled": false,
+  "highlights_tweets_tab_ui_enabled": false,
+  "interactive_text_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "longform_notetweets_inline_media_enabled": false,
+  "longform_notetweets_richtext_consumption_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": false,
+  "responsive_web_enhance_cards_enabled": false,
+  "responsive_web_graphql_exclude_directive_enabled": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": false,
+  "responsive_web_media_download_video_enabled": false,
+  "responsive_web_text_conversations_enabled": false,
+  "responsive_web_twitter_article_tweet_consumption_enabled": false,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+  "rweb_lists_timeline_redesign_enabled": true,
+  "spaces_2022_h2_clipping": true,
+  "spaces_2022_h2_spaces_communities": true,
+  "standardized_nudges_misinfo": false,
+  "subscriptions_verification_info_enabled": true,
+  "subscriptions_verification_info_reason_enabled": true,
+  "subscriptions_verification_info_verified_since_enabled": true,
+  "super_follow_badge_privacy_enabled": false,
+  "super_follow_exclusive_tweet_notifications_enabled": false,
+  "super_follow_tweet_api_enabled": false,
+  "super_follow_user_api_enabled": false,
+  "tweet_awards_web_tipping_enabled": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+  "tweetypie_unmention_optimization_enabled": false,
+  "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
+  "verified_phone_label_enabled": false,
+  "vibe_api_enabled": false,
+  "view_counts_everywhere_api_enabled": false
+}""".replace(" ", "").replace("\n", "")
+
+  tweetVariables* = """{
+  "focalTweetId": "$1",
+  $2
+  "includeHasBirdwatchNotes": false
+}"""
+
+  userTweetsVariables* = """{
+  "rest_id": "$1", $2
+  "count": 20
+}"""
+
+  listTweetsVariables* = """{
+  "rest_id": "$1", $2
+  "count": 20
+}"""

src/experimental/parser.nim

@@ -1,2 +1,2 @@
-import parser/[user, graphql, timeline]
-export user, graphql, timeline
+import parser/[user, graphql]
+export user, graphql

src/experimental/parser/graphql.nim

@@ -1,11 +1,20 @@
+import options
 import jsony
 import user, ../types/[graphuser, graphlistmembers]
 from ../../types import User, Result, Query, QueryKind
 
 proc parseGraphUser*(json: string): User =
+  if json.len == 0 or json[0] != '{':
+    return
+
   let raw = json.fromJson(GraphUser)
-  result = toUser raw.data.user.result.legacy
-  result.id = raw.data.user.result.restId
+
+  if raw.data.userResult.result.unavailableReason.get("") == "Suspended":
+    return User(suspended: true)
+
+  result = toUser raw.data.userResult.result.legacy
+  result.id = raw.data.userResult.result.restId
+  result.verified = result.verified or raw.data.userResult.result.isBlueVerified
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
   result = Result[User](

src/experimental/parser/timeline.nim

@@ -1,28 +0,0 @@
-import std/[strutils, tables]
-import jsony
-import user, ../types/timeline
-from ../../types import Result, User
-
-proc getId(id: string): string {.inline.} =
-  let start = id.rfind("-")
-  if start < 0: return id
-  id[start + 1 ..< id.len]
-
-proc parseUsers*(json: string; after=""): Result[User] =
-  result = Result[User](beginning: after.len == 0)
-
-  let raw = json.fromJson(Search)
-  if raw.timeline.instructions.len == 0:
-    return
-
-  for e in raw.timeline.instructions[0].addEntries.entries:
-    let id = e.entryId.getId
-    if e.entryId.startsWith("user"):
-      if id in raw.globalObjects.users:
-        result.content.add toUser raw.globalObjects.users[id]
-    elif e.entryId.startsWith("cursor"):
-      let cursor = e.content.operation.cursor
-      if cursor.cursorType == "Top":
-        result.top = cursor.value
-      elif cursor.cursorType == "Bottom":
-        result.bottom = cursor.value

src/experimental/parser/unifiedcard.nim

@@ -66,6 +66,8 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
       durationMs: videoInfo.durationMillis,
       variants: videoInfo.variants
     )
+  of model3d:
+    result.title = "Unsupported 3D model ad"
 
 proc parseUnifiedCard*(json: string): Card =
   let card = json.fromJson(UnifiedCard)
@@ -82,6 +84,10 @@ proc parseUnifiedCard*(json: string): Card =
       component.parseMedia(card, result)
     of buttonGroup:
       discard
+    of ComponentType.hidden:
+      result.kind = CardKind.hidden
+    of ComponentType.unknown:
+      echo "ERROR: Unknown component type: ", json
 
     case component.kind
     of twitterListDetails:

src/experimental/parser/user.nim

@@ -1,14 +1,14 @@
-import std/[algorithm, unicode, re, strutils, strformat, options]
+import std/[algorithm, unicode, re, strutils, strformat, options, nre]
 import jsony
 import utils, slices
 import ../types/user as userType
-from ../../types import User, Error
+from ../../types import Result, User, Error
 
 let
-  unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
+  unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
 
-  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
+  htRegex = nre.re"""(*U)(^|[^\w-_.?])([#＃$])([\w_]*+)(?!</a>|">|#)"""
   htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
 
 proc expandUserEntities(user: var User; raw: RawUser) =
@@ -29,7 +29,7 @@ proc expandUserEntities(user: var User; raw: RawUser) =
 
   user.bio = orig.replacedWith(replacements, 0 .. orig.len)
                  .replacef(unRegex, unReplace)
-                 .replacef(htRegex, htReplace)
+                 .replace(htRegex, htReplace)
 
 proc getBanner(user: RawUser): string =
   if user.profileBannerUrl.len > 0:
@@ -76,3 +76,12 @@ proc parseUser*(json: string; username=""): User =
     else: echo "[error - parseUser]: ", error
 
   result = toUser json.fromJson(RawUser)
+
+proc parseUsers*(json: string; after=""): Result[User] =
+  result = Result[User](beginning: after.len == 0)
+
+  # starting with '{' means it's an error
+  if json[0] == '[':
+    let raw = json.fromJson(seq[RawUser])
+    for user in raw:
+      result.content.add user.toUser

src/experimental/types/graphuser.nim

@@ -1,8 +1,9 @@
+import options
 import user
 
 type
   GraphUser* = object
-    data*: tuple[user: UserData]
+    data*: tuple[userResult: UserData]
 
   UserData* = object
     result*: UserResult
@@ -10,3 +11,5 @@ type
   UserResult = object
     legacy*: RawUser
     restId*: string
+    isBlueVerified*: bool
+    unavailableReason*: Option[string]

src/experimental/types/unifiedcard.nim

@@ -17,6 +17,8 @@ type
     twitterListDetails
     communityDetails
     mediaWithDetailsHorizontal
+    hidden
+    unknown
 
   Component* = object
     kind*: ComponentType
@@ -47,7 +49,7 @@ type
     vanity*: string
 
   MediaType* = enum
-    photo, video
+    photo, video, model3d
 
   MediaEntity* = object
     kind*: MediaType
@@ -70,10 +72,37 @@ type
   Text = object
     content: string
 
-  HasTypeField = Component | Destination | MediaEntity | AppStoreData
+  TypeField = Component | Destination | MediaEntity | AppStoreData
 
 converter fromText*(text: Text): string = text.content
 
-proc renameHook*(v: var HasTypeField; fieldName: var string) =
+proc renameHook*(v: var TypeField; fieldName: var string) =
   if fieldName == "type":
     fieldName = "kind"
+
+proc enumHook*(s: string; v: var ComponentType) =
+  v = case s
+      of "details": details
+      of "media": media
+      of "swipeable_media": swipeableMedia
+      of "button_group": buttonGroup
+      of "app_store_details": appStoreDetails
+      of "twitter_list_details": twitterListDetails
+      of "community_details": communityDetails
+      of "media_with_details_horizontal": mediaWithDetailsHorizontal
+      of "commerce_drop_details": hidden
+      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
+
+proc enumHook*(s: string; v: var AppType) =
+  v = case s
+      of "android_app": androidApp
+      of "iphone_app": iPhoneApp
+      of "ipad_app": iPadApp
+      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
+
+proc enumHook*(s: string; v: var MediaType) =
+  v = case s
+      of "video": video
+      of "photo": photo
+      of "model3d": model3d
+      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo

src/formatters.nim

@@ -12,9 +12,15 @@ let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
   twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
 
-  ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
+  ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
   igRegex = re"(www\.)?instagram\.com"
   odRegex = re"(www\.)?(odysee\.com|lbry\.tv|open\.lbry\.com)"
+  pxaRegex = re"(www\.)?(pixiv\.net)/artworks/"
+  pxuRegex = re"(www\.)?(pixiv\.net)/users/"
+  pxbRegex = re"(www\.)?(pixiv\.net)/users/bookmarks/"
+  pxiRegex = re"(www\.)?(pixiv\.net)/users/illusts/"
+  pxmRegex = re"(www\.)?(pixiv\.net)/users/manga/"
+  pxRegex = re"(www\.)?(pixiv\.net)"
 
   rdRegex = re"(?<![.b])((www|np|new|amp|old)\.)?reddit.com"
   rdShortRegex = re"(?<![.b])redd\.it\/"
@@ -57,8 +63,6 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
 
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
     result = result.replace(ytRegex, prefs.replaceYouTube)
-    if prefs.replaceYouTube in result:
-      result = result.replace("/c/", "/")
 
   if prefs.replaceInstagram.len > 0:
     result = result.replace(igRegex, prefs.replaceInstagram)
@@ -66,6 +70,24 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   if prefs.replaceOdysee.len > 0:
     result = result.replace(odRegex, prefs.replaceOdysee)
 
+  if prefs.replacePixivA.len > 0:
+    result = result.replace(pxaRegex, prefs.replacePixivA)
+
+  if prefs.replacePixivU.len > 0:
+    result = result.replace(pxuRegex, prefs.replacePixivU)
+
+  if prefs.replacePixivB.len > 0:
+    result = result.replace(pxbRegex, prefs.replacePixivB)
+
+  if prefs.replacePixivI.len > 0:
+    result = result.replace(pxiRegex, prefs.replacePixivI)
+
+  if prefs.replacePixivM.len > 0:
+    result = result.replace(pxmRegex, prefs.replacePixivM)
+
+  if prefs.replacePixiv.len > 0:
+    result = result.replace(pxRegex, prefs.replacePixiv)
+
   if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
     result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
     result = result.replace(cards, prefs.replaceTwitter & "/cards")
@@ -79,11 +101,8 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
     if prefs.replaceReddit in result and "/gallery/" in result:
       result = result.replace("/gallery/", "/comments/")
 
-  if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
-    result = result.replace(igRegex, prefs.replaceInstagram)
-
   if absolute.len > 0 and "href" in result:
-    result = result.replace("href=\"/", "href=\"" & absolute & "/")
+    result = result.replace("href=\"/", &"href=\"{absolute}/")
 
 proc getM3u8Url*(content: string): string =
   var matches: array[1, string]
@@ -120,10 +139,10 @@ proc pageDesc*(user: User): string =
   if user.bio.len > 0:
     stripHtml(user.bio)
   else:
-    "The latest tweets from " & user.fullname
+    user.fullname & "さんから最新のツイート"
 
 proc getJoinDate*(user: User): string =
-  user.joinDate.format("'Joined' MMMM YYYY")
+  user.joinDate.format("YYYY'年'MM'月に参加'")
 
 proc getJoinDateFull*(user: User): string =
   user.joinDate.format("h:mm tt - d MMM YYYY")

src/http_pool.nim

@@ -42,5 +42,11 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
   except ProtocolError:
     # Twitter closed the connection, retry
     body
+  except BadClientError:
+    # Twitter returned 503, we need a new client
+    pool.release(c, true)
+    badClient = false
+    c = pool.acquire(heads)
+    body
   finally:
     pool.release(c, badClient)

src/nitter.nim

@@ -15,7 +15,7 @@ import routes/[
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
 
-let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
+let configPath = getEnv("NITTER_CONF_FILE", "/etc/nitter/nitter.conf")
 let (cfg, fullCfg) = getConfig(configPath)
 
 if not cfg.enableDebug:
@@ -56,6 +56,7 @@ settings:
   port = Port(cfg.port)
   staticDir = cfg.staticDir
   bindAddr = cfg.address
+  reusePort = true
 
 routes:
   get "/":
@@ -84,20 +85,23 @@ routes:
     resp Http500, showError(
       &"An error occurred, please {link} with the URL you tried to visit.", cfg)
 
-  error RateLimitError:
+  error BadClientError:
     echo error.exc.name, ": ", error.exc.msg
+    resp Http500, showError("Network error occurred, please try again.", cfg)
+
+  error RateLimitError:
     const link = a("another instance", href = instancesUrl)
     resp Http429, showError(
       &"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
 
-  extend unsupported, ""
-  extend preferences, ""
-  extend resolver, ""
   extend rss, ""
+  extend status, ""
   extend search, ""
   extend timeline, ""
-  extend list, ""
-  extend status, ""
   extend media, ""
+  extend list, ""
+  extend preferences, ""
+  extend resolver, ""
   extend embed, ""
   extend debug, ""
+  extend unsupported, ""

src/parser.nim

@@ -4,6 +4,8 @@ import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
+proc parseGraphTweet(js: JsonNode): Tweet
+
 proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
   result = User(
@@ -19,13 +21,20 @@ proc parseUser(js: JsonNode; id=""): User =
     tweets: js{"statuses_count"}.getInt,
     likes: js{"favourites_count"}.getInt,
     media: js{"media_count"}.getInt,
-    verified: js{"verified"}.getBool,
+    verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
     protected: js{"protected"}.getBool,
     joinDate: js{"created_at"}.getTime
   )
 
   result.expandUserEntities(js)
 
+proc parseGraphUser(js: JsonNode): User =
+  let user = ? js{"user_result", "result"}
+  result = parseUser(user{"legacy"})
+
+  if "is_blue_verified" in user:
+    result.verified = user{"is_blue_verified"}.getBool()
+
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 
@@ -38,14 +47,13 @@ proc parseGraphList*(js: JsonNode): List =
   result = List(
     id: list{"id_str"}.getStr,
     name: list{"name"}.getStr,
-    username: list{"user", "legacy", "screen_name"}.getStr,
-    userId: list{"user", "rest_id"}.getStr,
+    username: list{"user_results", "result", "legacy", "screen_name"}.getStr,
+    userId: list{"user_results", "result", "rest_id"}.getStr,
     description: list{"description"}.getStr,
     members: list{"member_count"}.getInt,
-    banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
+    banner: list{"custom_banner_media", "media_info", "original_img_url"}.getImageStr
   )
 
-
 proc parsePoll(js: JsonNode): Poll =
   let vals = js{"binding_values"}
   # name format is pollNchoice_*
@@ -73,13 +81,17 @@ proc parseGif(js: JsonNode): Gif =
 proc parseVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"media_url_https"}.getImageStr,
-    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
-    available: js{"ext_media_availability", "status"}.getStr == "available",
+    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt),
+    available: true,
     title: js{"ext_alt_text"}.getStr,
     durationMs: js{"video_info", "duration_millis"}.getInt
     # playbackType: mp4
   )
 
+  with status, js{"ext_media_availability", "status"}:
+    if status.getStr.len > 0 and status.getStr.toLowerAscii != "available":
+      result.available = false
+
   with title, js{"additional_media_info", "title"}:
     result.title = title.getStr
 
@@ -87,10 +99,15 @@ proc parseVideo(js: JsonNode): Video =
     result.description = description.getStr
 
   for v in js{"video_info", "variants"}:
+    let
+      contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary"))
+      url = v{"url"}.getStr
+
     result.variants.add VideoVariant(
-      contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
+      contentType: contentType,
       bitrate: v{"bitrate"}.getInt,
-      url: v{"url"}.getStr
+      url: url,
+      resolution: if contentType == mp4: getMp4Resolution(url) else: 0
     )
 
 proc parsePromoVideo(js: JsonNode): Video =
@@ -181,7 +198,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
      result.url.len == 0 or result.url.startsWith("card://"):
     result.url = getPicUrl(result.image)
 
-proc parseTweet(js: JsonNode): Tweet =
+proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
   if js.isNull: return
   result = Tweet(
     id: js{"id_str"}.getId,
@@ -202,14 +219,28 @@ proc parseTweet(js: JsonNode): Tweet =
 
   result.expandTweetEntities(js)
 
-  if js{"is_quote_status"}.getBool:
+  # fix for pinned threads
+  if result.hasThread and result.threadId == 0:
+    result.threadId = js{"self_thread", "id_str"}.getId
+
+  if "retweeted_status" in js:
+    result.retweet = some Tweet()
+  elif js{"is_quote_status"}.getBool:
     result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
 
+  # legacy
   with rt, js{"retweeted_status_id_str"}:
     result.retweet = some Tweet(id: rt.getId)
     return
 
-  with jsCard, js{"card"}:
+  # graphql
+  with rt, js{"retweeted_status_result", "result"}:
+    # needed due to weird edgecase where the actual tweet data isn't included
+    if "legacy" in rt:
+      result.retweet = some parseGraphTweet(rt)
+      return
+
+  if jsCard.kind != JNull:
     let name = jsCard{"name"}.getStr
     if "poll" in name:
       if "image" in name:
@@ -229,11 +260,19 @@ proc parseTweet(js: JsonNode): Tweet =
       of "video":
         result.video = some(parseVideo(m))
         with user, m{"additional_media_info", "source_user"}:
-          result.attribution = some(parseUser(user))
+          if user{"id"}.getInt > 0:
+            result.attribution = some(parseUser(user))
+          else:
+            result.attribution = some(parseGraphUser(user))
       of "animated_gif":
         result.gif = some(parseGif(m))
       else: discard
 
+      with url, m{"url"}:
+        if result.text.endsWith(url.getStr):
+          result.text.removeSuffix(url.getStr)
+          result.text = result.text.strip()
+
   with jsWithheld, js{"withheld_in_countries"}:
     let withheldInCountries: seq[string] =
       if jsWithheld.kind != JArray: @[]
@@ -248,6 +287,30 @@ proc parseTweet(js: JsonNode): Tweet =
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
+proc parseLegacyTweet(js: JsonNode): Tweet =
+  result = parseTweet(js, js{"card"})
+  if not result.isNil and result.available:
+    result.user = parseUser(js{"user"})
+
+    if result.quote.isSome:
+      result.quote = some parseLegacyTweet(js{"quoted_status"})
+
+proc parseTweetSearch*(js: JsonNode): Timeline =
+  if js.kind == JNull or "statuses" notin js:
+    return Timeline(beginning: true)
+
+  for tweet in js{"statuses"}:
+    let parsed = parseLegacyTweet(tweet)
+
+    if parsed.retweet.isSome:
+      parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"})
+
+    result.content.add @[parsed]
+
+  let cursor = js{"search_metadata", "next_results"}.getStr
+  if cursor.len > 0 and "max_id" in cursor:
+    result.bottom = cursor[cursor.find("=") + 1 .. cursor.find("&q=")]
+
 proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
   let intId = if id.len > 0: parseBiggestInt(id) else: 0
   result = global.tweets.getOrDefault(id, Tweet(id: intId))
@@ -266,16 +329,6 @@ proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
     else:
       result.retweet = some Tweet()
 
-proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
-  let pin = js{"pinEntry", "entry", "entryId"}.getStr
-  if pin.len == 0: return
-
-  let id = pin.getId
-  if id notin global.tweets: return
-
-  global.tweets[id].pinned = true
-  return finalizeTweet(global, id)
-
 proc parseGlobalObjects(js: JsonNode): GlobalObjects =
   result = GlobalObjects()
   let
@@ -286,80 +339,16 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
     result.users[k] = parseUser(v, k)
 
   for k, v in tweets:
-    var tweet = parseTweet(v)
+    var tweet = parseTweet(v, v{"tweet_card"})
     if tweet.user.id in result.users:
       tweet.user = result.users[tweet.user.id]
     result.tweets[k] = tweet
 
-proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
-  result.thread = Chain()
-
-  let thread = js{"content", "item", "content", "conversationThread"}
-  with cursor, thread{"showMoreCursor"}:
-    result.thread.cursor = cursor{"value"}.getStr
-    result.thread.hasMore = true
-
-  for t in thread{"conversationComponents"}:
-    let content = t{"conversationTweetComponent", "tweet"}
-
-    if content{"displayType"}.getStr == "SelfThread":
-      result.self = true
-
-    var tweet = finalizeTweet(global, content{"id"}.getStr)
-    if not tweet.available:
-      tweet.tombstone = getTombstone(content{"tombstone"})
-    result.thread.content.add tweet
-
-proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
-  result = Conversation(replies: Result[Chain](beginning: true))
-  let global = parseGlobalObjects(? js)
-
-  let instructions = ? js{"timeline", "instructions"}
-  if instructions.len == 0:
-    return
-
-  for e in instructions[0]{"addEntries", "entries"}:
-    let entry = e{"entryId"}.getStr
-    if "tweet" in entry or "tombstone" in entry:
-      let tweet = finalizeTweet(global, e.getEntryId)
-      if $tweet.id != tweetId:
-        result.before.content.add tweet
-      else:
-        result.tweet = tweet
-    elif "conversationThread" in entry:
-      let (thread, self) = parseThread(e, global)
-      if thread.content.len > 0:
-        if self:
-          result.after = thread
-        else:
-          result.replies.content.add thread
-    elif "cursor-showMore" in entry:
-      result.replies.bottom = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.replies.bottom = e.getCursor
-
-proc parseStatus*(js: JsonNode): Tweet =
-  with e, js{"errors"}:
-    if e.getError == tweetNotFound:
-      return
-
-  result = parseTweet(js)
-  if not result.isNil:
-    result.user = parseUser(js{"user"})
-
-    with quote, js{"quoted_status"}:
-      result.quote = some parseStatus(js{"quoted_status"})
-
 proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
   if js.kind != JArray or js.len == 0:
     return
 
   for i in js:
-    when T is Tweet:
-      if res.beginning and i{"pinEntry"}.notNull:
-        with pin, parsePin(i, global):
-          res.content.add pin
-
     with r, i{"replaceEntry", "entry"}:
       if "top" in r{"entryId"}.getStr:
         res.top = r.getCursor
@@ -390,7 +379,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
       result.top = e.getCursor
     elif "cursor-bottom" in entry:
       result.bottom = e.getCursor
-    elif entry.startsWith("sq-C"):
+    elif entry.startsWith("sq-cursor"):
       with cursor, e{"content", "operation", "cursor"}:
         if cursor{"cursorType"}.getStr == "Bottom":
           result.bottom = cursor{"value"}.getStr
@@ -398,9 +387,13 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
           result.top = cursor{"value"}.getStr
 
 proc parsePhotoRail*(js: JsonNode): PhotoRail =
+  with error, js{"error"}:
+    if error.getStr == "Not authorized.":
+      return
+
   for tweet in js:
     let
-      t = parseTweet(tweet)
+      t = parseTweet(tweet, js{"tweet_card"})
       url = if t.photos.len > 0: t.photos[0]
             elif t.video.isSome: get(t.video).thumb
             elif t.gif.isSome: get(t.gif).thumb
@@ -409,3 +402,157 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
 
     if url.len == 0: continue
     result.add GalleryPhoto(url: url, tweetId: $t.id)
+
+proc parseGraphTweet(js: JsonNode): Tweet =
+  if js.kind == JNull:
+    return Tweet()
+
+  case js{"__typename"}.getStr
+  of "TweetUnavailable":
+    return Tweet()
+  of "TweetTombstone":
+    with text, js{"tombstone", "richText"}:
+      return Tweet(text: text.getTombstone)
+    with text, js{"tombstone", "text"}:
+      return Tweet(text: text.getTombstone)
+    return Tweet()
+  of "TweetPreviewDisplay":
+    return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
+  of "TweetWithVisibilityResults":
+    return parseGraphTweet(js{"tweet"})
+
+  var jsCard = copy(js{"tweet_card", "legacy"})
+  if jsCard.kind != JNull:
+    var values = newJObject()
+    for val in jsCard["binding_values"]:
+      values[val["key"].getStr] = val["value"]
+    jsCard["binding_values"] = values
+
+  result = parseTweet(js{"legacy"}, jsCard)
+  result.id = js{"rest_id"}.getId
+  result.user = parseGraphUser(js{"core"})
+
+  with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
+    result.expandNoteTweetEntities(noteTweet)
+
+  if result.quote.isSome:
+    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
+
+proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
+  let thread = js{"content", "items"}
+  for t in js{"content", "items"}:
+    let entryId = t{"entryId"}.getStr
+    if "cursor-showmore" in entryId:
+      let cursor = t{"item", "content", "value"}
+      result.thread.cursor = cursor.getStr
+      result.thread.hasMore = true
+    elif "tweet" in entryId:
+      let tweet = parseGraphTweet(t{"item", "content", "tweetResult", "result"})
+      result.thread.content.add tweet
+
+      if t{"item", "content", "tweetDisplayType"}.getStr == "SelfThread":
+        result.self = true
+
+proc parseGraphTweetResult*(js: JsonNode): Tweet =
+  with tweet, js{"data", "tweet_result", "result"}:
+    result = parseGraphTweet(tweet)
+
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
+  result = Conversation(replies: Result[Chain](beginning: true))
+
+  let instructions = ? js{"data", "timeline_response", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for e in instructions[0]{"entries"}:
+    let entryId = e{"entryId"}.getStr
+    if entryId.startsWith("tweet"):
+      with tweetResult, e{"content", "content", "tweetResult", "result"}:
+        let tweet = parseGraphTweet(tweetResult)
+
+        if not tweet.available:
+          tweet.id = parseBiggestInt(entryId.getId())
+
+        if $tweet.id == tweetId:
+          result.tweet = tweet
+        else:
+          result.before.content.add tweet
+    elif entryId.startsWith("tombstone"):
+      let id = entryId.getId()
+      let tweet = Tweet(
+        id: parseBiggestInt(id),
+        available: false,
+        text: e{"content", "content", "tombstoneInfo", "richText"}.getTombstone
+      )
+
+      if id == tweetId:
+        result.tweet = tweet
+      else:
+        result.before.content.add tweet
+    elif entryId.startsWith("conversationthread"):
+      let (thread, self) = parseGraphThread(e)
+      if self:
+        result.after = thread
+      else:
+        result.replies.content.add thread
+    elif entryId.startsWith("cursor-bottom"):
+      result.replies.bottom = e{"content", "content", "value"}.getStr
+
+proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
+  result = Profile(tweets: Timeline(beginning: after.len == 0))
+
+  let instructions =
+    if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"}
+    else: ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
+
+  if instructions.len == 0:
+    return
+
+  for i in instructions:
+    if i{"__typename"}.getStr == "TimelineAddEntries":
+      for e in i{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "content", "tweetResult", "result"}:
+            let tweet = parseGraphTweet(tweetResult)
+            if not tweet.available:
+              tweet.id = parseBiggestInt(entryId.getId())
+            result.tweets.content.add tweet
+        elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
+          let (thread, self) = parseGraphThread(e)
+          result.tweets.content.add thread.content
+        elif entryId.startsWith("cursor-bottom"):
+          result.tweets.bottom = e{"content", "value"}.getStr
+    if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry":
+      with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}:
+        let tweet = parseGraphTweet(tweetResult)
+        tweet.pinned = true
+        if not tweet.available and tweet.tombstone.len == 0:
+          let entryId = i{"entry", "entryId"}.getEntryId
+          if entryId.len > 0:
+            tweet.id = parseBiggestInt(entryId)
+        result.pinned = some tweet
+
+proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
+  result = Timeline(beginning: after.len == 0)
+
+  let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for instruction in instructions:
+    let typ = instruction{"type"}.getStr
+    if typ == "TimelineAddEntries":
+      for e in instructions[0]{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+            let tweet = parseGraphTweet(tweetResult)
+            if not tweet.available:
+              tweet.id = parseBiggestInt(entryId.getId())
+            result.content.add tweet
+        elif entryId.startsWith("cursor-bottom"):
+          result.bottom = e{"content", "value"}.getStr
+    elif typ == "TimelineReplaceEntry":
+      if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
+        result.bottom = instruction{"entry", "content", "value"}.getStr

src/parserutils.nim

@@ -28,13 +28,13 @@ template `?`*(js: JsonNode): untyped =
   if j.isNull: return
   j
 
-template `with`*(ident, value, body): untyped =
-  block:
+template with*(ident, value, body): untyped =
+  if true:
     let ident {.inject.} = value
     if ident != nil: body
 
-template `with`*(ident; value: JsonNode; body): untyped =
-  block:
+template with*(ident; value: JsonNode; body): untyped =
+  if true:
     let ident {.inject.} = value
     if value.notNull: body
 
@@ -130,9 +130,24 @@ proc getBanner*(js: JsonNode): string =
       return
 
 proc getTombstone*(js: JsonNode): string =
-  result = js{"tombstoneInfo", "richText", "text"}.getStr
+  result = js{"text"}.getStr
   result.removeSuffix(" Learn more")
 
+proc getMp4Resolution*(url: string): int =
+  # parses the height out of a URL like this one:
+  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
+  const vidSep = "/vid/"
+  let
+    vidIdx = url.find(vidSep) + vidSep.len
+    resIdx = url.find('x', vidIdx) + 1
+    res = url[resIdx ..< url.find("/", resIdx)]
+
+  try:
+    return parseInt(res)
+  except ValueError:
+    # cannot determine resolution (e.g. m3u8/non-mp4 video)
+    return 0
+
 proc extractSlice(js: JsonNode): Slice[int] =
   result = js["indices"][0].getInt ..< js["indices"][1].getInt
 
@@ -215,47 +230,37 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
   user.bio = user.bio.replacef(unRegex, unReplace)
                      .replacef(htRegex, htReplace)
 
-proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
-  let
-    orig = tweet.text.toRunes
-    textRange = js{"display_text_range"}
-    textSlice = textRange{0}.getInt .. textRange{1}.getInt
-    hasQuote = js{"is_quote_status"}.getBool
-    hasCard = tweet.card.isSome
-
-  var replyTo = ""
-  if tweet.replyId != 0:
-    with reply, js{"in_reply_to_screen_name"}:
-      tweet.reply.add reply.getStr
-      replyTo = reply.getStr
-
-  let ent = ? js{"entities"}
+proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
+                        replyTo=""; hasQuote=false) =
+  let hasCard = tweet.card.isSome
 
   var replacements = newSeq[ReplaceSlice]()
 
-  with urls, ent{"urls"}:
+  with urls, entities{"urls"}:
     for u in urls:
       let urlStr = u["url"].getStr
-      if urlStr.len == 0 or urlStr notin tweet.text:
+      if urlStr.len == 0 or urlStr notin text:
         continue
+
       replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
+
       if hasCard and u{"url"}.getStr == get(tweet.card).url:
         get(tweet.card).url = u{"expanded_url"}.getStr
 
-  with media, ent{"media"}:
+  with media, entities{"media"}:
     for m in media:
       replacements.extractUrls(m, textSlice.b, hideTwitter = true)
 
-  if "hashtags" in ent:
-    for hashtag in ent["hashtags"]:
+  if "hashtags" in entities:
+    for hashtag in entities["hashtags"]:
       replacements.extractHashtags(hashtag)
 
-  if "symbols" in ent:
-    for symbol in ent["symbols"]:
+  if "symbols" in entities:
+    for symbol in entities["symbols"]:
       replacements.extractHashtags(symbol)
 
-  if "user_mentions" in ent:
-    for mention in ent["user_mentions"]:
+  if "user_mentions" in entities:
+    for mention in entities["user_mentions"]:
       let
         name = mention{"screen_name"}.getStr
         slice = mention.extractSlice
@@ -272,5 +277,27 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   replacements.deduplicate
   replacements.sort(cmp)
 
-  tweet.text = orig.replacedWith(replacements, textSlice)
-                   .strip(leading=false)
+  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
+
+proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entities"}
+    hasQuote = js{"is_quote_status"}.getBool
+    textRange = js{"display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+
+  var replyTo = ""
+  if tweet.replyId != 0:
+    with reply, js{"in_reply_to_screen_name"}:
+      replyTo = reply.getStr
+      tweet.reply.add replyTo
+
+  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote)
+
+proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entity_set"}
+    text = js{"text"}.getStr
+    textSlice = 0..text.runeLen
+
+  tweet.expandTextEntities(entities, text, textSlice)

src/prefs_impl.nim

@@ -50,7 +50,7 @@ macro genPrefs*(prefDsl: untyped) =
     const `name`*: PrefList = toOrderedTable(`table`)
 
 genPrefs:
-  Display:
+  "表示":
     theme(select, "Nitter"):
       "テーマ"
 
@@ -58,10 +58,10 @@ genPrefs:
       "無限スクロール（JS必須）"
 
     stickyProfile(checkbox, true):
-      "Make profile sidebar stick to top"
+      "プロフィールサイドバーはページ上に付く"
 
     bidiSupport(checkbox, false):
-      "Support bidirectional text (makes clicking on tweets harder)"
+      "両方向文字対応（ツイートをクリックしにくくなる）"
 
     hideTweetStats(checkbox, false):
       "ツイートの詳細（返事、RT、いいね）を隠す"
@@ -76,9 +76,9 @@ genPrefs:
       "ツイートに返事を隠す"
 
     squareAvatars(checkbox, false):
-      "Square profile pictures"
+      "四角形プロフィールアイコン"
 
-  Media:
+  "メディア":
     mp4Playback(checkbox, true):
       "MP4再生（GIFのみ）"
 
@@ -94,26 +94,50 @@ genPrefs:
     autoplayGifs(checkbox, true):
       "自動的にGIFの再生"
 
-  "Link replacements (blank to disable)":
-    replaceTwitter(input, "twitter.076.ne.jp"):
+  "リンクの変換（無効にするには、かっこにして下さい）":
+    replaceTwitter(input, "twitter.owacon.moe"):
       "ツイッター → Nitter"
-      placeholder: "twitter.076.ne.jp"
+      placeholder: "twitter.owacon.moe"
 
-    replaceYouTube(input, "youtube.076.ne.jp"):
+    replaceYouTube(input, "youtube.owacon.moe"):
       "ユーチューブ → Invidious"
-      placeholder: "youtube.076.ne.jp"
+      placeholder: "youtube.owacon.moe"
 
-    replaceReddit(input, "reddit.076.ne.jp"):
+    replaceReddit(input, "teddit.net"):
       "Reddit → Teddit/Libreddit"
-      placeholder: "reddit.076.ne.jp"
+      placeholder: "teddit.net"
 
-    replaceInstagram(input, "instagram.076.ne.jp"):
+    replaceInstagram(input, ""):
       "インスタグラム → Bibliogram"
-      placeholder: "instagram.076.ne.jp"
+      placeholder: ""
 
-    replaceOdysee(input, "odysee.076.ne.jp"):
+    replaceOdysee(input, "odysee.owacon.moe"):
       "オディシー → Librarian"
-      placeholder: "odysee.076.ne.jp"
+      placeholder: "odysee.owacon.moe"
+
+    replacePixivA(input, "pixiv.owacon.moe/artworks/?id="):
+      "Vixipイラスト"
+      placeholder: "pixiv.owacon.moe/artworks/?id="
+
+    replacePixivU(input, "pixiv.owacon.moe/users/?id="):
+      "Vixipユーザー"
+      placeholder: "pixiv.owacon.moe/users/?id="
+
+    replacePixivB(input, "pixiv.owacon.moe/users/bookmarks/?id="):
+      "Vixipユーザーのブックマーク"
+      placeholder: "pixiv.owacon.moe/users/bookmarks/?id="
+
+    replacePixivI(input, "pixiv.owacon.moe/users/illusts/?id="):
+      "Vixipユーザーのイラスト"
+      placeholder: "pixiv.owacon.moe/users/illusts/?id="
+
+    replacePixivM(input, "pixiv.owacon.moe/users/manga/?id="):
+      "Vixipユーザーの漫画"
+      placeholder: "pixiv.owacon.moe/users/manga/?id="
+
+    replacePixiv(input, "pixiv.owacon.moe"):
+      "Pixiv → vixip"
+      placeholder: "pixiv.owacon.moe"
 
 iterator allPrefs*(): Pref =
   for k, v in prefList:

src/query.nim

@@ -93,11 +93,11 @@ proc genQueryUrl*(query: Query): string =
   if query.text.len > 0:
     params.add "q=" & encodeUrl(query.text)
   for f in query.filters:
-    params.add "f-" & f & "=on"
+    params.add &"f-{f}=on"
   for e in query.excludes:
-    params.add "e-" & e & "=on"
+    params.add &"e-{e}=on"
   for i in query.includes.filterIt(it != "nativeretweets"):
-    params.add "i-" & i & "=on"
+    params.add &"i-{i}=on"
 
   if query.since.len > 0:
     params.add "since=" & query.since

src/redis_cache.nim

@@ -118,11 +118,11 @@ proc getUserId*(username: string): Future[string] {.async.} =
   pool.withAcquire(r):
     result = await r.hGet(name.uidKey, name)
     if result == redisNil:
-      let user = await getUser(username)
+      let user = await getGraphUser(username)
       if user.suspended:
         return "suspended"
       else:
-        await cacheUserId(name, user.id)
+        await all(cacheUserId(name, user.id), cache(user))
         return user.id
 
 proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
@@ -130,8 +130,7 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
   if prof != redisNil:
     prof.deserialize(User)
   elif fetch:
-    let userId = await getUserId(username)
-    result = await getGraphUser(userId)
+    result = await getGraphUser(username)
     await cache(result)
 
 proc getCachedUsername*(userId: string): Future[string] {.async.} =
@@ -142,9 +141,11 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
   if username != redisNil:
     result = username
   else:
-    let user = await getUserById(userId)
+    let user = await getGraphUserById(userId)
     result = user.username
     await setEx(key, baseCacheTime, result)
+    if result.len > 0 and user.id.len > 0:
+      await all(cacheUserId(result, user.id), cache(user))
 
 proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
   if id == 0: return
@@ -152,8 +153,8 @@ proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
   if tweet != redisNil:
     tweet.deserialize(Tweet)
   else:
-    result = await getStatus($id)
-    if result.isNil:
+    result = await getGraphTweetResult($id)
+    if not result.isNil:
       await cache(result)
 
 proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =

src/routes/embed.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, options
+import asyncdispatch, strutils, strformat, options
 import jester, karax/vdom
 import ".."/[types, api]
 import ../views/[embed, tweet, general]
@@ -10,27 +10,27 @@ export api, embed, vdom, tweet, general, router_utils
 proc createEmbedRouter*(cfg: Config) =
   router embed:
     get "/i/videos/tweet/@id":
-      let convo = await getTweet(@"id")
-      if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
+      let tweet = await getGraphTweetResult(@"id")
+      if tweet == nil or tweet.video.isNone:
         resp Http404
 
-      resp renderVideoEmbed(convo.tweet, cfg, request)
+      resp renderVideoEmbed(tweet, cfg, request)
 
     get "/@user/status/@id/embed":
       let
-        convo = await getTweet(@"id")
+        tweet = await getGraphTweetResult(@"id")
         prefs = cookiePrefs()
         path = getPath()
 
-      if convo == nil or convo.tweet == nil:
+      if tweet == nil:
         resp Http404
 
-      resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
+      resp renderTweetEmbed(tweet, path, prefs, cfg, request)
 
     get "/embed/Tweet.html":
       let id = @"id"
 
       if id.len > 0:
-        redirect("/i/status/" & id & "/embed")
+        redirect(&"/i/status/{id}/embed")
       else:
         resp Http404

src/routes/list.nim

@@ -1,23 +1,25 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, uri
+import strutils, strformat, uri
 
 import jester
 
 import router_utils
 import ".."/[types, redis_cache, api]
 import ../views/[general, timeline, list]
-export getListTimeline, getGraphList
 
 template respList*(list, timeline, title, vnode: typed) =
   if list.id.len == 0 or list.name.len == 0:
-    resp Http404, showError("List " & @"id" & " not found", cfg)
+    resp Http404, showError(&"""List "{@"id"}" not found""", cfg)
 
   let
     html = renderList(vnode, timeline.query, list)
-    rss = "/i/lists/$1/rss" % [@"id"]
+    rss = &"""/i/lists/{@"id"}/rss"""
 
   resp renderMain(html, request, cfg, prefs, titleText=title, rss=rss, banner=list.banner)
 
+proc title*(list: List): string =
+  &"@{list.username}/{list.name}"
+
 proc createListRouter*(cfg: Config) =
   router list:
     get "/@name/lists/@slug/?":
@@ -28,24 +30,22 @@ proc createListRouter*(cfg: Config) =
         slug = decodeUrl(@"slug")
         list = await getCachedList(@"name", slug)
       if list.id.len == 0:
-        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
-      redirect("/i/lists/" & list.id)
+        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
+      redirect(&"/i/lists/{list.id}")
 
     get "/i/lists/@id/?":
       cond '.' notin @"id"
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
-        title = "@" & list.username & "/" & list.name
-        timeline = await getListTimeline(list.id, getCursor())
+        timeline = await getGraphListTweets(list.id, getCursor())
         vnode = renderTimelineTweets(timeline, prefs, request.path)
-      respList(list, timeline, title, vnode)
+      respList(list, timeline, list.title, vnode)
 
     get "/i/lists/@id/members":
       cond '.' notin @"id"
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
-        title = "@" & list.username & "/" & list.name
         members = await getGraphListMembers(list, getCursor())
-      respList(list, members, title, renderTimelineUsers(members, prefs, request.path))
+      respList(list, members, list.title, renderTimelineUsers(members, prefs, request.path))

src/routes/media.nim

@@ -88,6 +88,20 @@ proc createMediaRouter*(cfg: Config) =
     get "/pic/?":
       resp Http404
 
+    get re"^\/pic\/orig\/(enc)?\/?(.+)":
+      var url = decoded(request, 1)
+      if "twimg.com" notin url:
+        url.insert(twimg)
+      if not url.startsWith(https):
+        url.insert(https)
+      url.add("?name=orig")
+
+      let uri = parseUri(url)
+      cond isTwitterUrl(uri) == true
+
+      let code = await proxyMedia(request, url)
+      check code
+
     get re"^\/pic\/(enc)?\/?(.+)":
       var url = decoded(request, 1)
       if "twimg.com" notin url:

src/routes/rss.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, tables, times, hashes, uri
+import asyncdispatch, tables, times, hashes, uri
 
 import jester
 
@@ -10,6 +10,11 @@ include "../views/rss.nimf"
 
 export times, hashes
 
+proc redisKey*(page, name, cursor: string): string =
+  result = page & ":" & name
+  if cursor.len > 0:
+    result &= ":" & cursor
+
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
   var profile: Profile
   let
@@ -22,14 +27,12 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
   else:
     var q = query
     q.fromUser = names
-    profile = Profile(
-      tweets: await getSearch[Tweet](q, after),
-      # this is kinda dumb
-      user: User(
-        username: name,
-        fullname: names.join(" | "),
-        userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
-      )
+    profile.tweets = await getTweetSearch(q, after)
+    # this is kinda dumb
+    profile.user = User(
+      username: name,
+      fullname: names.join(" | "),
+      userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
     )
 
   if profile.user.suspended:
@@ -42,8 +45,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
   if rss.cursor.len == 0:
     let info = case page
-               of "User": " \"$1\" " % @"name"
-               of "List": " $1 " % @"id"
+               of "User": " \"" & @"name" & "\" "
+               of "List": " \"" & @"id" & "\" "
                else: " "
 
     resp Http404, showError(page & info & "not found", cfg)
@@ -67,13 +70,13 @@ proc createRssRouter*(cfg: Config) =
 
       let
         cursor = getCursor()
-        key = "search:" & $hash(genQueryUrl(query)) & ":" & cursor
+        key = redisKey("search", $hash(genQueryUrl(query)), cursor)
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "Search")
 
-      let tweets = await getSearch[Tweet](query, cursor)
+      let tweets = await getTweetSearch(query, cursor)
       rss.cursor = tweets.bottom
       rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
 
@@ -84,9 +87,8 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       let
-        cursor = getCursor()
         name = @"name"
-        key = "twitter:" & name & ":" & cursor
+        key = redisKey("twitter", name, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -101,18 +103,20 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       cond @"tab" in ["with_replies", "media", "search"]
-      let name = @"name"
-      let query =
-        case @"tab"
-        of "with_replies": getReplyQuery(name)
-        of "media": getMediaQuery(name)
-        of "search": initQuery(params(request), name=name)
-        else: Query(fromUser: @[name])
+      let
+        name = @"name"
+        tab = @"tab"
+        query =
+          case tab
+          of "with_replies": getReplyQuery(name)
+          of "media": getMediaQuery(name)
+          # of "search": initQuery(params(request), name=name)
+          else: Query(fromUser: @[name])
 
-      var key = @"tab" & ":" & @"name" & ":"
-      if @"tab" == "search":
-        key &= $hash(genQueryUrl(query)) & ":"
-      key &= getCursor()
+      let searchKey = if tab != "search": ""
+                      else: ":" & $hash(genQueryUrl(query))
+
+      let key = redisKey(tab, name & searchKey, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -143,18 +147,17 @@ proc createRssRouter*(cfg: Config) =
     get "/i/lists/@id/rss":
       cond cfg.enableRss
       let
+        id = @"id"
         cursor = getCursor()
-        key =
-          if cursor.len == 0: "lists:" & @"id"
-          else: "lists:" & @"id" & ":" & cursor
+        key = redisKey("lists", id, cursor)
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "List")
 
       let
-        list = await getCachedList(id=(@"id"))
-        timeline = await getListTimeline(list.id, cursor)
+        list = await getCachedList(id=id)
+        timeline = await getGraphListTweets(list.id, cursor)
       rss.cursor = timeline.bottom
       rss.feed = renderListRss(timeline.content, list, cfg)
 

src/routes/search.nim

@@ -14,25 +14,31 @@ export search
 proc createSearchRouter*(cfg: Config) =
   router search:
     get "/search/?":
-      if @"q".len > 500:
+      let q = @"q"
+      if q.len > 500:
         resp Http400, showError("Search input too long.", cfg)
 
       let
         prefs = cookiePrefs()
         query = initQuery(params(request))
+        title = "Search" & (if q.len > 0: " (" & q & ")" else: "")
 
       case query.kind
       of users:
-        if "," in @"q":
-          redirect("/" & @"q")
-        let users = await getSearch[User](query, getCursor())
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
+        if "," in q:
+          redirect("/" & q)
+        var users: Result[User]
+        try:
+          users = await getUserSearch(query, getCursor())
+        except InternalError:
+          users = Result[User](beginning: true, query: query)
+        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
       of tweets:
         let
-          tweets = await getSearch[Tweet](query, getCursor())
+          tweets = await getTweetSearch(query, getCursor())
           rss = "/search/rss?" & genQueryUrl(query)
         resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
-                        request, cfg, prefs, rss=rss)
+                        request, cfg, prefs, title, rss=rss)
       else:
         resp Http404, showError("Invalid search", cfg)
 
@@ -42,4 +48,4 @@ proc createSearchRouter*(cfg: Config) =
     get "/opensearch":
       let url = getUrlPrefix(cfg) & "/search?q="
       resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                    generateOpenSearchXML(cfg.title, cfg.hostname, url)
+                     generateOpenSearchXML(cfg.title, cfg.hostname, url)

src/routes/status.nim

@@ -16,17 +16,21 @@ proc createStatusRouter*(cfg: Config) =
   router status:
     get "/@name/status/@id/?":
       cond '.' notin @"name"
-      cond not @"id".any(c => not c.isDigit)
+      let id = @"id"
+
+      if id.len > 19 or id.any(c => not c.isDigit):
+        resp Http404, showError("Invalid tweet ID", cfg)
+
       let prefs = cookiePrefs()
 
       # used for the infinite scroll feature
       if @"scroll".len > 0:
-        let replies = await getReplies(@"id", getCursor())
+        let replies = await getReplies(id, getCursor())
         if replies.content.len == 0:
           resp Http404, ""
         resp $renderReplies(replies, prefs, getPath())
 
-      let conv = await getTweet(@"id", getCursor())
+      let conv = await getTweet(id, getCursor())
       if conv == nil:
         echo "nil conv"
 
@@ -72,3 +76,6 @@ proc createStatusRouter*(cfg: Config) =
 
     get "/i/web/status/@id":
       redirect("/i/status/" & @"id")
+      
+    get "/@name/thread/@id/?":
+      redirect("/$1/status/$2" % [@"name", @"id"])

src/routes/timeline.nim

@@ -45,33 +45,21 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
     after.setLen 0
 
   let
-    timeline =
-      case query.kind
-      of posts: getTimeline(userId, after)
-      of replies: getTimeline(userId, after, replies=true)
-      of media: getMediaTimeline(userId, after)
-      else: getSearch[Tweet](query, after)
-
     rail =
       skipIf(skipRail or query.kind == media, @[]):
         getCachedPhotoRail(name)
 
-    user = await getCachedUser(name)
+    user = getCachedUser(name)
 
-  var pinned: Option[Tweet]
-  if not skipPinned and user.pinnedTweet > 0 and
-     after.len == 0 and query.kind in {posts, replies}:
-    let tweet = await getCachedTweet(user.pinnedTweet)
-    if not tweet.isNil:
-      tweet.pinned = true
-      pinned = some tweet
+  result =
+    case query.kind
+    of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after)
+    of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
+    of media: await getGraphUserTweets(userId, TimelineKind.media, after)
+    else: Profile(tweets: await getTweetSearch(query, after))
 
-  result = Profile(
-    user: user,
-    pinned: pinned,
-    tweets: await timeline,
-    photoRail: await rail
-  )
+  result.user = await user
+  result.photoRail = await rail
 
   if result.user.protected or result.user.suspended:
     return
@@ -82,7 +70,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
                    rss, after: string): Future[string] {.async.} =
   if query.fromUser.len != 1:
     let
-      timeline = await getSearch[Tweet](query, after)
+      timeline = await getTweetSearch(query, after)
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
@@ -123,7 +111,7 @@ proc createTimelineRouter*(cfg: Config) =
 
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
-      cond @"name" notin ["pic", "gif", "video"]
+      cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
       cond @"tab" in ["with_replies", "media", "search", ""]
       let
         prefs = cookiePrefs()
@@ -137,7 +125,7 @@ proc createTimelineRouter*(cfg: Config) =
       # used for the infinite scroll feature
       if @"scroll".len > 0:
         if query.fromUser.len != 1:
-          var timeline = await getSearch[Tweet](query, after)
+          var timeline = (await getGraphSearch(query, after)).tweets
           if timeline.content.len == 0: resp Http404
           timeline.beginning = true
           resp $renderTweetSearch(timeline, prefs, getPath())

src/sass/profile/card.scss

@@ -73,9 +73,9 @@
         }
     }
 
-    .profile-joindate, .profile-location, profile-website {
+    .profile-joindate, .profile-location, .profile-website {
         color: var(--fg_faded);
-        margin: 2px 0;
+        margin: 1px 0;
         width: 100%;
     }
 }

src/sass/tweet/_base.scss

@@ -98,10 +98,9 @@
 }
 
 .avatar {
-    position: absolute;
-
     &.round {
         border-radius: 50%;
+        -webkit-user-select: none;
     }
     
     &.mini {
@@ -133,6 +132,10 @@
     .card-image img {
         height: auto;
     }
+    
+    .avatar {
+        position: absolute;
+    }
 }
 
 .attribution {
@@ -197,6 +200,7 @@
 
 .tweet-stats {
     margin-bottom: -3px;
+    -webkit-user-select: none;
 }
 
 .tweet-stat {
@@ -228,6 +232,7 @@
     left: 0;
     top: 0;
     position: absolute;
+    -webkit-user-select: none;
 
     &:hover {
         background-color: var(--bg_hover);

src/sass/tweet/thread.scss

@@ -23,7 +23,6 @@
     font-size: 18px;
 }
 
-
 @media(max-width: 600px) {
     .main-tweet .tweet-content {
         font-size: 16px;
@@ -111,3 +110,29 @@
     margin-left: 58px;
     padding: 7px 0;
 }
+
+.timeline-item.thread.more-replies-thread {
+    padding: 0 0.75em;
+
+    &::before {
+        top: 40px;
+        margin-bottom: 31px;
+    }
+
+    .more-replies {
+        display: flex;
+        padding-top: unset !important;
+        margin-top: 8px;
+
+        &::before {
+            display: inline-block;
+            position: relative;
+            top: -1px;
+            line-height: 0.4em;
+        }
+
+        .more-replies-text {
+            display: inline;
+        }
+    }
+}

src/sass/tweet/video.scss

@@ -3,7 +3,7 @@
 
 video {
     max-height: 100%;
-    max-width: 100%;
+    width: 100%;
 }
 
 .gallery-video {

src/tokens.nim

@@ -1,8 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, httpclient, times, sequtils, json, random
 import strutils, tables
-import zippy
-import types, consts, http_pool
+import types, consts
 
 const
   maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
@@ -11,9 +10,14 @@ const
   failDelay = initDuration(minutes=30)
 
 var
-  clientPool: HttpPool
   tokenPool: seq[Token]
   lastFailed: Time
+  enableLogging = false
+
+let headers = newHttpHeaders({"authorization": auth})
+
+template log(str) =
+  if enableLogging: echo "[tokens] ", str
 
 proc getPoolJson*(): JsonNode =
   var
@@ -37,9 +41,11 @@ proc getPoolJson*(): JsonNode =
       let
         maxReqs =
           case api
-          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
-          of Api.timeline: 187
-          else: 180
+          of Api.timeline, Api.search: 180
+          of Api.userTweets, Api.userTweetsAndReplies, Api.userRestId,
+             Api.userScreenName, Api.tweetDetail, Api.tweetResult: 500
+          of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500
+          of Api.userSearch: 900
         reqs = maxReqs - token.apis[api].remaining
 
       reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
@@ -60,25 +66,23 @@ proc fetchToken(): Future[Token] {.async.} =
   if getTime() - lastFailed < failDelay:
     raise rateLimitError()
 
-  let headers = newHttpHeaders({
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-    "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.5",
-    "connection": "keep-alive",
-    "authorization": auth
-  })
+  let client = newAsyncHttpClient(headers=headers)
 
   try:
     let
-      resp = clientPool.use(headers): await c.postContent(activate)
-      tokNode = parseJson(uncompress(resp))["guest_token"]
+      resp = await client.postContent(activate)
+      tokNode = parseJson(resp)["guest_token"]
       tok = tokNode.getStr($(tokNode.getInt))
       time = getTime()
 
     return Token(tok: tok, init: time, lastUse: time)
   except Exception as e:
-    lastFailed = getTime()
-    echo "fetching token failed: ", e.msg
+    echo "[tokens] fetching token failed: ", e.msg
+    if "Try again" notin e.msg:
+      echo "[tokens] fetching tokens paused, resuming in 30 minutes"
+      lastFailed = getTime()
+  finally:
+    client.close()
 
 proc expired(token: Token): bool =
   let time = getTime()
@@ -100,6 +104,9 @@ proc isReady(token: Token; api: Api): bool =
 proc release*(token: Token; used=false; invalid=false) =
   if token.isNil: return
   if invalid or token.expired:
+    if invalid: log "discarding invalid token"
+    elif token.expired: log "discarding expired token"
+
     let idx = tokenPool.find(token)
     if idx > -1: tokenPool.delete(idx)
   elif used:
@@ -115,6 +122,7 @@ proc getToken*(api: Api): Future[Token] {.async.} =
   if not result.isReady(api):
     release(result)
     result = await fetchToken()
+    log "added new token to pool"
     tokenPool.add result
 
   if not result.isNil:
@@ -143,10 +151,11 @@ proc poolTokens*(amount: int) {.async.} =
     except: discard
 
     if not newToken.isNil:
+      log "added new token to pool"
       tokenPool.add newToken
 
 proc initTokenPool*(cfg: Config) {.async.} =
-  clientPool = HttpPool()
+  enableLogging = cfg.enableDebug
 
   while true:
     if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens:

src/types.nim

@@ -7,17 +7,28 @@ genPrefsType()
 type
   RateLimitError* = object of CatchableError
   InternalError* = object of CatchableError
+  BadClientError* = object of CatchableError
+
+  TimelineKind* {.pure.} = enum
+    tweets
+    replies
+    media
 
   Api* {.pure.} = enum
-    userShow
+    tweetDetail
+    tweetResult
     timeline
     search
-    tweet
+    userSearch
     list
     listBySlug
     listMembers
+    listTweets
     userRestId
-    status
+    userScreenName
+    userTweets
+    userTweetsAndReplies
+    userMedia
 
   RateLimit* = object
     remaining*: int
@@ -34,17 +45,22 @@ type
     null = 0
     noUserMatches = 17
     protectedUser = 22
+    missingParams = 25
     couldntAuth = 32
     doesntExist = 34
+    invalidParam = 47
     userNotFound = 50
     suspended = 63
     rateLimited = 88
     invalidToken = 89
     listIdOrSlug = 112
     tweetNotFound = 144
+    tweetNotAuthorized = 179
     forbidden = 200
     badToken = 239
     noCsrf = 353
+    tweetUnavailable = 421
+    tweetCensored = 422
 
   User* = object
     id*: string
@@ -75,6 +91,7 @@ type
     contentType*: VideoType
     url*: string
     bitrate*: int
+    resolution*: int
 
   Video* = object
     durationMs*: int
@@ -144,6 +161,7 @@ type
     imageDirectMessage = "image_direct_message"
     audiospace = "audiospace"
     newsletterPublication = "newsletter_publication"
+    hidden
     unknown
     
   Card* = object
@@ -174,6 +192,8 @@ type
     available*: bool
     tombstone*: string
     location*: string
+    # Unused, needed for backwards compat
+    source*: string
     stats*: TweetStats
     retweet*: Option[Tweet]
     attribution*: Option[User]
@@ -185,6 +205,8 @@ type
     video*: Option[Video]
     photos*: seq[string]
 
+  Tweets* = seq[Tweet]
+
   Result*[T] = object
     content*: seq[T]
     top*, bottom*: string
@@ -192,7 +214,7 @@ type
     query*: Query
 
   Chain* = object
-    content*: seq[Tweet]
+    content*: Tweets
     hasMore*: bool
     cursor*: string
 
@@ -202,7 +224,7 @@ type
     after*: Chain
     replies*: Result[Chain]
 
-  Timeline* = Result[Tweet]
+  Timeline* = Result[Tweets]
 
   Profile* = object
     user*: User
@@ -252,6 +274,12 @@ type
     replaceTwitter*: string
     replaceYouTube*: string
     replaceOdysee*: string
+    replacePixivA*: string
+    replacePixivU*: string
+    replacePixivB*: string
+    replacePixivI*: string
+    replacePixivM*: string
+    replacePixiv*: string
     replaceReddit*: string
     replaceInstagram*: string
 
@@ -260,3 +288,6 @@ type
 
 proc contains*(thread: Chain; tweet: Tweet): bool =
   thread.content.anyIt(it.id == tweet.id)
+
+proc add*(timeline: var seq[Tweets]; tweet: Tweet) =
+  timeline.add @[tweet]

src/utils.nim

@@ -42,6 +42,12 @@ proc getPicUrl*(link: string): string =
   else:
     &"/pic/{encodeUrl(link)}"
 
+proc getOrigPicUrl*(link: string): string =
+  if base64Media:
+    &"/pic/orig/enc/{encode(link, safe=true)}"
+  else:
+    &"/pic/orig/{encodeUrl(link)}"
+
 proc filterParams*(params: Table): seq[(string, string)] =
   for p in params.pairs():
     if p[1].len > 0 and p[0] notin nitterParams:

src/views/embed.nim

@@ -11,11 +11,12 @@ const doctype = "<!DOCTYPE html>\n"
 proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
   let thumb = get(tweet.video).thumb
   let vidUrl = getVideoEmbed(cfg, tweet.id)
-  let prefs = Prefs(hlsPlayback: true)
+  let prefs = Prefs(hlsPlayback: true, mp4Playback: true)
   let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
 
-    tdiv(class="embed-video"):
-      renderVideo(get(tweet.video), prefs, "")
+    body:
+      tdiv(class="embed-video"):
+        renderVideo(get(tweet.video), prefs, "")
 
   result = doctype & $node

src/views/general.nim

@@ -90,22 +90,23 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     meta(property="og:type", content=ogType)
     meta(property="og:title", content=(if ogTitle.len > 0: ogTitle else: titleText))
     meta(property="og:description", content=stripHtml(desc))
-    meta(property="og:site_name", content="Nitter")
-    meta(property="og:locale", content="en_US")
+    meta(property="og:site_name", content="オワコンTwitter")
+    meta(property="og:locale", content="ja_JP")
+    meta(property="twitter:title", content=(if ogTitle.len > 0: ogTitle else: titleText))
+    meta(property="twitter:description", content=stripHtml(desc))
 
-    if banner.len > 0:
+    if banner.len > 0 and not banner.startsWith('#'):
       let bannerUrl = getPicUrl(banner)
       link(rel="preload", type="image/png", href=bannerUrl, `as`="image")
 
     for url in images:
-      let suffix = if "400x400" in url or url.endsWith("placeholder.png"): ""
-                   else: "?name=small"
-      let preloadUrl = getPicUrl(url & suffix)
+      let preloadUrl = if "400x400" in url: getPicUrl(url)
+                       else: getSmallPic(url)
       link(rel="preload", type="image/png", href=preloadUrl, `as`="image")
 
       let image = getUrlPrefix(cfg) & getPicUrl(url)
       meta(property="og:image", content=image)
-      meta(property="twitter:image:src", content=image)
+      meta(property="twitter:image", content=image)
 
       if rss.len > 0:
         meta(property="twitter:card", content="summary")

src/views/preferences.nim

@@ -41,7 +41,7 @@ proc renderPreferences*(prefs: Prefs; path: string; themes: seq[string]): VNode
         renderPrefs()
 
         h4(class="note"):
-          text "Preferences are stored client-side using cookies without any personal information."
+          text "設定は個人情報無しのクッキーとしてクライエントサイドで保存されます。"
 
         button(`type`="submit", class="pref-submit"):
           text "保存"

src/views/profile.nim

@@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
           span:
             let url = replaceUrls(user.website, prefs)
             icon "link"
-            a(href=url): text shortLink(url)
+            a(href=url): text url.shortLink
 
       tdiv(class="profile-joindate"):
         span(title=getJoinDateFull(user)):
@@ -68,18 +68,21 @@ proc renderPhotoRail(profile: Profile): VNode =
   buildHtml(tdiv(class="photo-rail-card")):
     tdiv(class="photo-rail-header"):
       a(href=(&"/{profile.user.username}/media")):
-        icon "picture", count & " Photos and videos"
+        icon "picture", count & "枚画像や動画"
 
     input(id="photo-rail-grid-toggle", `type`="checkbox")
     label(`for`="photo-rail-grid-toggle", class="photo-rail-header-mobile"):
-      icon "picture", count & " Photos and videos"
+      icon "picture", count & "枚画像や動画"
       icon "down"
 
     tdiv(class="photo-rail-grid"):
       for i, photo in profile.photoRail:
         if i == 16: break
+        let photoSuffix =
+          if "format" in photo.url or "placeholder" in photo.url: ""
+          else: ":thumb"
         a(href=(&"/{profile.user.username}/status/{photo.tweetId}#m")):
-          genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
+          genImg(photo.url & photoSuffix)
 
 proc renderBanner(banner: string): VNode =
   buildHtml():
@@ -105,7 +108,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
         renderBanner(profile.user.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
-    tdiv(class=(&"profile-tab{sticky}")):
+    tdiv(class=("profile-tab" & sticky)):
       renderUserCard(profile.user, prefs)
       if profile.photoRail.len > 0:
         renderPhotoRail(profile)

src/views/renderutils.nim

@@ -1,11 +1,19 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils
+import strutils, strformat
 import karax/[karaxdsl, vdom, vstyles]
 import ".."/[types, utils]
 
+const smallWebp* = "?name=small&format=webp"
+
+proc getSmallPic*(url: string): string =
+  result = url
+  if "?" notin url and not url.endsWith("placeholder.png"):
+    result &= smallWebp
+  result = getPicUrl(result)
+
 proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
   var c = "icon-" & icon
-  if class.len > 0: c = c & " " & class
+  if class.len > 0: c = &"{c} {class}"
   buildHtml(tdiv(class="icon-container")):
     if href.len > 0:
       a(class=c, title=title, href=href)
@@ -51,29 +59,23 @@ proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNod
 proc genCheckbox*(pref, label: string; state: bool): VNode =
   buildHtml(label(class="pref-group checkbox-container")):
     text label
-    if state: input(name=pref, `type`="checkbox", checked="")
-    else: input(name=pref, `type`="checkbox")
+    input(name=pref, `type`="checkbox", checked=state)
     span(class="checkbox")
 
-proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
+proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
   let p = placeholder
   buildHtml(tdiv(class=("pref-group pref-input " & class))):
     if label.len > 0:
       label(`for`=pref): text label
-    if state.len == 0:
-      input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
-    else:
-      input(name=pref, `type`="text", placeholder=p, value=state)
+    input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))
 
 proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
   buildHtml(tdiv(class="pref-group pref-input")):
     label(`for`=pref): text label
     select(name=pref):
       for opt in options:
-        if opt == state:
-          option(value=opt, selected=""): text opt
-        else:
-          option(value=opt): text opt
+        option(value=opt, selected=(opt == state)):
+          text opt
 
 proc genDate*(pref, state: string): VNode =
   buildHtml(span(class="date-input")):
@@ -85,12 +87,9 @@ proc genImg*(url: string; class=""): VNode =
     img(src=getPicUrl(url), class=class, alt="")
 
 proc getTabClass*(query: Query; tab: QueryKind): string =
-  result = "tab-item"
-  if query.kind == tab:
-    result &= " active"
+  if query.kind == tab: "tab-item active"
+  else: "tab-item"
 
 proc getAvatarClass*(prefs: Prefs): string =
-  if prefs.squareAvatars:
-    "avatar"
-  else:
-    "avatar round"
+  if prefs.squareAvatars: "avatar"
+  else: "avatar round"

src/views/rss.nimf

@@ -56,29 +56,33 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #end if
 #end proc
 #
-#proc renderRssTweets(tweets: seq[Tweet]; cfg: Config): string =
+#proc renderRssTweets(tweets: seq[Tweets]; cfg: Config; userId=""): string =
 #let urlPrefix = getUrlPrefix(cfg)
 #var links: seq[string]
-#for t in tweets:
-#  let retweet = if t.retweet.isSome: t.user.username else: ""
-#  let tweet = if retweet.len > 0: t.retweet.get else: t
-#  let link = getLink(tweet)
-#  if link in links: continue
-#  end if
-#  links.add link
-    <item>
-      <title>${getTitle(tweet, retweet)}</title>
-      <dc:creator>@${tweet.user.username}</dc:creator>
-      <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
-      <pubDate>${getRfc822Time(tweet)}</pubDate>
-      <guid>${urlPrefix & link}</guid>
-      <link>${urlPrefix & link}</link>
-    </item>
+#for thread in tweets:
+#  for tweet in thread:
+#    if userId.len > 0 and tweet.user.id != userId: continue
+#    end if
+#
+#    let retweet = if tweet.retweet.isSome: tweet.user.username else: ""
+#    let tweet = if retweet.len > 0: tweet.retweet.get else: tweet
+#    let link = getLink(tweet)
+#    if link in links: continue
+#    end if
+#    links.add link
+      <item>
+        <title>${getTitle(tweet, retweet)}</title>
+        <dc:creator>@${tweet.user.username}</dc:creator>
+        <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
+        <pubDate>${getRfc822Time(tweet)}</pubDate>
+        <guid>${urlPrefix & link}</guid>
+        <link>${urlPrefix & link}</link>
+      </item>
+#  end for
 #end for
 #end proc
 #
 #proc renderTimelineRss*(profile: Profile; cfg: Config; multi=false): string =
-#let prefs = Prefs(replaceTwitter: cfg.hostname, replaceYouTube: cfg.replaceYouTube, replaceOdysee: cfg.replaceOdysee)
 #let urlPrefix = getUrlPrefix(cfg)
 #result = ""
 #let handle = (if multi: "" else: "@") & profile.user.username
@@ -103,14 +107,13 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
       <height>128</height>
     </image>
 #if profile.tweets.content.len > 0:
-${renderRssTweets(profile.tweets.content, cfg)}
+${renderRssTweets(profile.tweets.content, cfg, userId=profile.user.id)}
 #end if
   </channel>
 </rss>
 #end proc
 #
-#proc renderListRss*(tweets: seq[Tweet]; list: List; cfg: Config): string =
-#let prefs = Prefs(replaceTwitter: cfg.hostname, replaceYouTube: cfg.replaceYouTube, replaceOdysee: cfg.replaceOdysee)
+#proc renderListRss*(tweets: seq[Tweets]; list: List; cfg: Config): string =
 #let link = &"{getUrlPrefix(cfg)}/i/lists/{list.id}"
 #result = ""
 <?xml version="1.0" encoding="UTF-8"?>
@@ -119,7 +122,7 @@ ${renderRssTweets(profile.tweets.content, cfg)}
     <atom:link href="${link}" rel="self" type="application/rss+xml" />
     <title>${xmltree.escape(list.name)} / @${list.username}</title>
     <link>${link}</link>
-    <description>${getDescription(list.name & " by @" & list.username, cfg)}</description>
+    <description>${getDescription(&"{list.name} by @{list.username}", cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
 ${renderRssTweets(tweets, cfg)}
@@ -127,8 +130,7 @@ ${renderRssTweets(tweets, cfg)}
 </rss>
 #end proc
 #
-#proc renderSearchRss*(tweets: seq[Tweet]; name, param: string; cfg: Config): string =
-#let prefs = Prefs(replaceTwitter: cfg.hostname, replaceYouTube: cfg.replaceYouTube, replaceOdysee: cfg.replaceOdysee)
+#proc renderSearchRss*(tweets: seq[Tweets]; name, param: string; cfg: Config): string =
 #let link = &"{getUrlPrefix(cfg)}/search"
 #let escName = xmltree.escape(name)
 #result = ""
@@ -138,7 +140,7 @@ ${renderRssTweets(tweets, cfg)}
     <atom:link href="${link}" rel="self" type="application/rss+xml" />
     <title>Search results for "${escName}"</title>
     <link>${link}</link>
-    <description>${getDescription("Search \"" & escName & "\"", cfg)}</description>
+    <description>${getDescription(&"Search \"{escName}\"", cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
 ${renderRssTweets(tweets, cfg)}

src/views/search.nim

@@ -62,12 +62,10 @@ proc renderSearchPanel*(query: Query): VNode =
     hiddenField("f", "tweets")
     genInput("q", "", query.text, "検索…", class="pref-inline")
     button(`type`="submit"): icon "search"
-    if isPanelOpen(query):
-      input(id="search-panel-toggle", `type`="checkbox", checked="")
-    else:
-      input(id="search-panel-toggle", `type`="checkbox")
-    label(`for`="search-panel-toggle"):
-      icon "down"
+
+    input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
+    label(`for`="search-panel-toggle"): icon "down"
+
     tdiv(class="search-panel"):
       for f in @["filter", "exclude"]:
         span(class="search-title"): text capitalize(f)
@@ -87,9 +85,9 @@ proc renderSearchPanel*(query: Query): VNode =
             genDate("until", query.until)
         tdiv:
           span(class="search-title"): text "Near"
-          genInput("near", "", query.near, placeholder="地域…")
+          genInput("near", "", query.near, "地域…", autofocus=false)
 
-proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
+proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string;
                         pinned=none(Tweet)): VNode =
   let query = results.query
   buildHtml(tdiv(class="timeline-container")):

src/views/timeline.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, sequtils, algorithm, uri, options
+import strutils, strformat, algorithm, uri, options
 import karax/[karaxdsl, vdom]
 
 import ".."/[types, query, formatters]
@@ -39,24 +39,22 @@ proc renderNoneFound(): VNode =
     h2(class="timeline-none"):
       text "何も見つけられませんでした"
 
-proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
+proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode =
   buildHtml(tdiv(class="thread-line")):
     let sortedThread = thread.sortedByIt(it.id)
     for i, tweet in sortedThread:
+      # thread has a gap, display "more replies" link
+      if i > 0 and tweet.replyId != sortedThread[i - 1].id:
+        tdiv(class="timeline-item thread more-replies-thread"):
+          tdiv(class="more-replies"):
+            a(class="more-replies-text", href=getLink(tweet)):
+              text "more replies"
+
       let show = i == thread.high and sortedThread[0].id != tweet.threadId
       let header = if tweet.pinned or tweet.retweet.isSome: "with-header " else: ""
       renderTweet(tweet, prefs, path, class=(header & "thread"),
                   index=i, last=(i == thread.high), showThread=show)
 
-proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] =
-  result = @[it]
-  if it.retweet.isSome or it.replyId in threads: return
-  for t in tweets:
-    if t.id == result[0].replyId:
-      result.insert t
-    elif t.replyId == result[0].id:
-      result.add t
-
 proc renderUser(user: User; prefs: Prefs): VNode =
   buildHtml(tdiv(class="timeline-item")):
     a(class="tweet-link", href=("/" & user.username))
@@ -89,7 +87,7 @@ proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode =
     else:
       renderNoMore()
 
-proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
+proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
                            pinned=none(Tweet)): VNode =
   buildHtml(tdiv(class="timeline")):
     if not results.beginning:
@@ -105,26 +103,26 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
       else:
         renderNoneFound()
     else:
-      var
-        threads: seq[int64]
-        retweets: seq[int64]
+      var retweets: seq[int64]
 
-      for tweet in results.content:
-        let rt = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
+      for thread in results.content:
+        if thread.len == 1:
+          let
+            tweet = thread[0]
+            retweetId = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
 
-        if tweet.id in threads or rt in retweets or tweet.id in retweets or
-           tweet.pinned and prefs.hidePins: continue
+          if retweetId in retweets or tweet.id in retweets or
+             tweet.pinned and prefs.hidePins:
+            continue
 
-        let thread = results.content.threadFilter(threads, tweet)
-        if thread.len < 2:
           var hasThread = tweet.hasThread
-          if rt != 0:
-            retweets &= rt
+          if retweetId != 0 and tweet.retweet.isSome:
+            retweets &= retweetId
             hasThread = get(tweet.retweet).hasThread
           renderTweet(tweet, prefs, path, showThread=hasThread)
         else:
           renderThread(thread, prefs, path)
-          threads &= thread.mapIt(it.id)
 
-      renderMore(results.query, results.bottom)
+      if results.bottom.len > 0:
+        renderMore(results.query, results.bottom)
       renderToTop()

src/views/tweet.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, sequtils, strformat, options
+import strutils, sequtils, strformat, options, algorithm
 import karax/[karaxdsl, vdom, vstyles]
 from jester import Request
 
@@ -7,26 +7,21 @@ import renderutils
 import ".."/[types, utils, formatters]
 import general
 
-proc getSmallPic(url: string): string =
-  result = url
-  if "?" notin url and not url.endsWith("placeholder.png"):
-    result &= ":small"
-  result = getPicUrl(result)
+const doctype = "<!DOCTYPE html>\n"
 
 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
   let url = getPicUrl(user.getUserPic("_mini"))
   buildHtml():
     img(class=(prefs.getAvatarClass & " mini"), src=url)
 
-proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
+proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode =
   buildHtml(tdiv):
-    if retweet.len > 0:
-      tdiv(class="retweet-header"):
-        span: icon "retweet", retweet & " retweeted"
-
-    if tweet.pinned:
+    if pinned:
       tdiv(class="pinned"):
         span: icon "pin", "Pinned Tweet"
+    elif retweet.len > 0:
+      tdiv(class="retweet-header"):
+        span: icon "retweet", retweet & " retweeted"
 
     tdiv(class="tweet-header"):
       a(class="tweet-avatar", href=("/" & tweet.user.username)):
@@ -57,19 +52,21 @@ proc renderAlbum(tweet: Tweet): VNode =
           tdiv(class="attachment image"):
             let
               named = "name=" in photo
-              orig = if named: photo else: photo & "?name=orig"
-              small = if named: photo else: photo & "?name=small"
-            a(href=getPicUrl(orig), class="still-image", target="_blank"):
+              small = if named: photo else: photo & smallWebp
+            a(href=getOrigPicUrl(photo), class="still-image", target="_blank"):
               genImg(small)
 
-proc isPlaybackEnabled(prefs: Prefs; video: Video): bool =
-  case video.playbackType
+proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
+  case playbackType
   of mp4: prefs.mp4Playback
   of m3u8, vmap: prefs.hlsPlayback
 
-proc renderVideoDisabled(video: Video; path: string): VNode =
+proc hasMp4Url(video: Video): bool =
+  video.variants.anyIt(it.contentType == mp4)
+
+proc renderVideoDisabled(playbackType: VideoType; path: string): VNode =
   buildHtml(tdiv(class="video-overlay")):
-    case video.playbackType
+    case playbackType
     of mp4:
       p: text "mp4 playback disabled in preferences"
     of m3u8, vmap:
@@ -84,9 +81,11 @@ proc renderVideoUnavailable(video: Video): VNode =
       p: text "This media is unavailable"
 
 proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
-  let container =
-    if video.description.len > 0 or video.title.len > 0: " card-container"
-    else: ""
+  let
+    container = if video.description.len == 0 and video.title.len == 0: ""
+                else: " card-container"
+    playbackType = if not prefs.proxyVideos and video.hasMp4Url: mp4
+                   else: video.playbackType
 
   buildHtml(tdiv(class="attachments card")):
     tdiv(class="gallery-video" & container):
@@ -95,22 +94,21 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
         if not video.available:
           img(src=thumb)
           renderVideoUnavailable(video)
-        elif not prefs.isPlaybackEnabled(video):
+        elif not prefs.isPlaybackEnabled(playbackType):
           img(src=thumb)
-          renderVideoDisabled(video, path)
+          renderVideoDisabled(playbackType, path)
         else:
-          let vid = video.variants.filterIt(it.contentType == video.playbackType)
-          let source = getVidUrl(vid[0].url)
-          case video.playbackType
+          let
+            vars = video.variants.filterIt(it.contentType == playbackType)
+            vidUrl = vars.sortedByIt(it.resolution)[^1].url
+            source = if prefs.proxyVideos: getVidUrl(vidUrl)
+                     else: vidUrl
+          case playbackType
           of mp4:
-            if prefs.muteVideos:
-              video(poster=thumb, controls="", muted=""):
-                source(src=source, `type`="video/mp4")
-            else:
-              video(poster=thumb, controls=""):
-                source(src=source, `type`="video/mp4")
+            video(poster=thumb, controls="", muted=prefs.muteVideos):
+              source(src=source, `type`="video/mp4")
           of m3u8, vmap:
-            video(poster=thumb, data-url=source, data-autoload="false")
+            video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
             verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
             tdiv(class="overlay-circle"): span(class="overlay-triangle")
             verbatim "</div>"
@@ -124,14 +122,9 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
   buildHtml(tdiv(class="attachments media-gif")):
     tdiv(class="gallery-gif", style={maxHeight: "unset"}):
       tdiv(class="attachment"):
-        let thumb = getSmallPic(gif.thumb)
-        let url = getPicUrl(gif.url)
-        if prefs.autoplayGifs:
-          video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""):
-            source(src=url, `type`="video/mp4")
-        else:
-          video(class="gif", poster=thumb, controls="", muted="", loop=""):
-            source(src=url, `type`="video/mp4")
+        video(class="gif", poster=getSmallPic(gif.thumb), autoplay=prefs.autoplayGifs,
+              controls="", muted="", loop=""):
+          source(src=getPicUrl(gif.url), `type`="video/mp4")
 
 proc renderPoll(poll: Poll): VNode =
   buildHtml(tdiv(class="poll")):
@@ -146,7 +139,7 @@ proc renderPoll(poll: Poll): VNode =
         span(class="poll-choice-value"): text percStr
         span(class="poll-choice-option"): text poll.options[i]
     span(class="poll-info"):
-      text insertSep($poll.votes, ',') & " votes • " & poll.status
+      text &"{insertSep($poll.votes, ',')} votes • {poll.status}"
 
 proc renderCardImage(card: Card): VNode =
   buildHtml(tdiv(class="card-image-container")):
@@ -296,7 +289,10 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.quote.isSome:
         renderQuote(tweet.quote.get(), prefs, path)
 
-  let fullTweet = tweet
+  let
+    fullTweet = tweet
+    pinned = tweet.pinned
+
   var retweet: string
   var tweet = fullTweet
   if tweet.retweet.isSome:
@@ -309,7 +305,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
 
     tdiv(class="tweet-body"):
       var views = ""
-      renderHeader(tweet, retweet, prefs)
+      renderHeader(tweet, retweet, pinned, prefs)
 
       if not afterTweet and index == 0 and tweet.reply.len > 0 and
          (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username):
@@ -325,7 +321,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.attribution.isSome:
         renderAttribution(tweet.attribution.get(), prefs)
 
-      if tweet.card.isSome:
+      if tweet.card.isSome and tweet.card.get().kind != hidden:
         renderCard(tweet.card.get(), prefs, path)
 
       if tweet.photos.len > 0:
@@ -344,7 +340,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderQuote(tweet.quote.get(), prefs, path)
 
       if mainTweet:
-        p(class="tweet-published"): text getTime(tweet)
+        p(class="tweet-published"): text &"{getTime(tweet)}"
 
       if tweet.mediaTags.len > 0:
         renderMediaTags(tweet.mediaTags)
@@ -356,7 +352,12 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
 
-proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
-  buildHtml(tdiv(class="tweet-embed")):
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): string =
+  let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req)
-    renderTweet(tweet, prefs, path, mainTweet=true)
+
+    body:
+      tdiv(class="tweet-embed"):
+        renderTweet(tweet, prefs, path, mainTweet=true)
+
+  result = doctype & $node

tests/requirements.txt

@@ -0,0 +1 @@
+seleniumbase

tests/test_card.py

@@ -3,11 +3,6 @@ from parameterized import parameterized
 
 
 card = [
-    ['Thom_Wolf/status/1122466524860702729',
-     'pytorch/fairseq',
-     'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - pytorch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
-     'github.com', True],
-
     ['nim_lang/status/1136652293510717440',
      'Version 0.20.0 released',
      'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!',
@@ -21,14 +16,19 @@ card = [
     ['FluentAI/status/1116417904831029248',
      'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
      'One of the only ways to improve Alexa is to have human beings check it for errors',
-     'theverge.com', True]
+     'theverge.com', True],
+
+    ['nim_lang/status/1082989146040340480',
+     'Nim in 2018: A short recap',
+     'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
+     'nim-lang.org', True]
 ]
 
 no_thumb = [
-    ['Bountysource/status/1141879700639215617',
-     'Post a bounty on kivy/plyer!',
-     'Automation and Screen Reader Support',
-     'bountysource.com'],
+    ['Thom_Wolf/status/1122466524860702729',
+     'facebookresearch/fairseq',
+     'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
+     'github.com'],
 
     ['brent_p/status/1088857328680488961',
      'Hts Nim Sugar',
@@ -38,17 +38,7 @@ no_thumb = [
     ['voidtarget/status/1133028231672582145',
      'sinkingsugar/nimqt-example',
      'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.',
-     'github.com'],
-
-    ['mobile_test/status/490378953744318464',
-     'Nantasket Beach',
-     'Explore this photo titled Nantasket Beach by Ben Sandofsky (@sandofsky) on 500px',
-     '500px.com'],
-
-    ['nim_lang/status/1082989146040340480',
-     'Nim in 2018: A short recap',
-     'Posted in r/programming by u/miran1',
-     'reddit.com']
+     'github.com']
 ]
 
 playable = [
@@ -63,17 +53,6 @@ playable = [
      'youtube.com']
 ]
 
-# promo = [
-    # ['BangOlufsen/status/1145698701517754368',
-    #  'Upgrade your journey', '',
-    #  'www.bang-olufsen.com'],
-
-    # ['BangOlufsen/status/1154934429900406784',
-    #  'Learn more about Beosound Shape', '',
-    #  'www.bang-olufsen.com']
-# ]
-
-
 class CardTest(BaseTestCase):
     @parameterized.expand(card)
     def test_card(self, tweet, title, description, destination, large):
@@ -81,7 +60,7 @@ class CardTest(BaseTestCase):
         c = Card(Conversation.main + " ")
         self.assert_text(title, c.title)
         self.assert_text(destination, c.destination)
-        self.assertIn('_img', self.get_image_url(c.image + ' img'))
+        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
         if len(description) > 0:
             self.assert_text(description, c.description)
         if large:
@@ -104,17 +83,7 @@ class CardTest(BaseTestCase):
         c = Card(Conversation.main + " ")
         self.assert_text(title, c.title)
         self.assert_text(destination, c.destination)
-        self.assertIn('_img', self.get_image_url(c.image + ' img'))
+        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
         self.assert_element_visible('.card-overlay')
         if len(description) > 0:
             self.assert_text(description, c.description)
-
-    # @parameterized.expand(promo)
-    # def test_card_promo(self, tweet, title, description, destination):
-    #     self.open_nitter(tweet)
-    #     c = Card(Conversation.main + " ")
-    #     self.assert_text(title, c.title)
-    #     self.assert_text(destination, c.destination)
-    #     self.assert_element_visible('.video-overlay')
-    #     if len(description) > 0:
-    #         self.assert_text(description, c.description)

tests/test_profile.py

@@ -4,8 +4,8 @@ from parameterized import parameterized
 profiles = [
         ['mobile_test', 'Test account',
          'Test Account. test test Testing username with @mobile_test_2 and a #hashtag',
-         'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '100'],
-        ['mobile_test_2', 'mobile test 2', '', '', '', 'Joined January 2011', '13']
+         'San Francisco, CA', 'example.com/foobar', '2009年10月に参加', '100'],
+        ['mobile_test_2', 'mobile test 2', '', '', '', '2011年1月に参加', '13']
 ]
 
 verified = [['jack'], ['elonmusk']]
@@ -17,11 +17,6 @@ protected = [
 
 invalid = [['thisprofiledoesntexist'], ['%']]
 
-banner_color = [
-    ['nim_lang', '22, 25, 32'],
-    ['rustlang', '35, 31, 32']
-]
-
 banner_image = [
     ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
 ]
@@ -39,7 +34,7 @@ class ProfileTest(BaseTestCase):
             (location, Profile.location),
             (website, Profile.website),
             (joinDate, Profile.joinDate),
-            (mediaCount + " Photos and videos", Profile.mediaCount)
+            (mediaCount + "枚画像や動画", Profile.mediaCount)
         ]
 
         for text, selector in tests:
@@ -71,14 +66,8 @@ class ProfileTest(BaseTestCase):
         self.assert_text(f'User "{username}" not found')
 
     def test_suspended(self):
-        self.open_nitter('user')
-        self.assert_text('User "user" has been suspended')
-
-    @parameterized.expand(banner_color)
-    def test_banner_color(self, username, color):
-        self.open_nitter(username)
-        banner = self.find_element(Profile.banner + ' a')
-        self.assertIn(color, banner.value_of_css_property('background-color'))
+        self.open_nitter('suspendme')
+        self.assert_text('User "suspendme" has been suspended')
 
     @parameterized.expand(banner_image)
     def test_banner_image(self, username, url):

tests/test_quote.py

@@ -3,14 +3,18 @@ from parameterized import parameterized
 
 text = [
     ['elonmusk/status/1138136540096319488',
-     'Tesla Owners Online', '@Model3Owners',
+     'TREV PAGE', '@Model3Owners',
      """As of March 58.4% of new car sales in Norway are electric. 
 
 What are we doing wrong? reuters.com/article/us-norwa…"""],
 
-    ['nim_lang/status/924694255364341760',
-     'Hacker News', '@newsycombinator',
-     'Why Rust fails hard at scientific computing andre-ratsimbazafy.com/why-r…']
+    ['nim_lang/status/1491461266849808397#m',
+     'Nim language', '@nim_lang',
+     """What's better than Nim 1.6.0?
+
+Nim 1.6.2 :)
+
+nim-lang.org/blog/2021/12/17…"""]
 ]
 
 image = [

tests/test_search.py

@@ -2,8 +2,8 @@ from base import BaseTestCase
 from parameterized import parameterized
 
 
-class SearchTest(BaseTestCase):
-    @parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
-    def test_username_search(self, username):
-        self.search_username(username)
-        self.assert_text(f'{username}')
+#class SearchTest(BaseTestCase):
+    #@parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
+    #def test_username_search(self, username):
+        #self.search_username(username)
+        #self.assert_text(f'{username}')

tests/test_tweet.py

@@ -16,7 +16,7 @@ timeline = [
 ]
 
 status = [
-    [20, 'jack⚡️', 'jack', '21 Mar 2006', 'just setting up my twttr'],
+    [20, 'jack', 'jack', '21 Mar 2006', 'just setting up my twttr'],
     [134849778302464000, 'The Twoffice', 'TheTwoffice', '11 Nov 2011', 'test'],
     [105685475985080322, 'The Twoffice', 'TheTwoffice', '22 Aug 2011', 'regular tweet'],
     [572593440719912960, 'Test account', 'mobile_test', '3 Mar 2015', 'testing test']
@@ -71,12 +71,12 @@ emoji = [
 
 retweet = [
     [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
-    [3, 'mobile_test_8', 'mobile test 8', 'jack⚡️', '@jack', 'twttr']
+    [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
 ]
 
-reply = [
-    ['mobile_test/with_replies', 15]
-]
+# reply = [
+#     ['mobile_test/with_replies', 15]
+# ]
 
 
 class TweetTest(BaseTestCase):
@@ -137,8 +137,8 @@ class TweetTest(BaseTestCase):
         self.open_nitter(tweet)
         self.assert_text('Tweet not found', '.error-panel')
 
-    @parameterized.expand(reply)
-    def test_thread(self, tweet, num):
-        self.open_nitter(tweet)
-        thread = self.find_element(f'.timeline > div:nth-child({num})')
-        self.assertIn(thread.get_attribute('class'), 'thread-line')
+    # @parameterized.expand(reply)
+    # def test_thread(self, tweet, num):
+    #     self.open_nitter(tweet)
+    #     thread = self.find_element(f'.timeline > div:nth-child({num})')
+    #     self.assertIn(thread.get_attribute('class'), 'thread-line')