Merge branch 'master' of https://github.com/zedeus/nitter

.gitignore

@@ -3,9 +3,11 @@ nitter
 *.db
 /tests/__pycache__
 /tests/geckodriver.log
-/tests/downloaded_files/*
+/tests/downloaded_files
+/tests/latest_logs
 /tools/gencss
 /tools/rendermd
 /public/css/style.css
 /public/md/*.html
 nitter.conf
+dump.rdb

README.md

@@ -34,7 +34,7 @@ XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscW
 
 ## Resources
 
-The wiki contains 
+The wiki contains
 [a list of instances](https://github.com/zedeus/nitter/wiki/Instances) and
 [browser extensions](https://github.com/zedeus/nitter/wiki/Extensions)
 maintained by the community.
@@ -67,9 +67,10 @@ Twitter account.
 ## Installation
 
 ### Dependencies
-* libpcre
-* libsass
-* redis
+
+- libpcre
+- libsass
+- redis
 
 To compile Nitter you need a Nim installation, see
 [nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
@@ -115,18 +116,21 @@ before you can run the container. See below for how to also run Redis using
 Docker.
 
 To build and run Nitter in Docker:
+
 ```bash
 docker build -t nitter:latest .
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
 ```
 
 A prebuilt Docker image is provided as well:
+
 ```bash
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest
 ```
 
 Using docker-compose to run both Nitter and Redis as different containers:
 Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
+
 ```bash
 docker-compose up -d
 ```

nitter.nimble

@@ -12,15 +12,15 @@ bin           = @["nitter"]
 
 requires "nim >= 1.4.8"
 requires "jester >= 0.5.0"
-requires "karax#fa4a2dc"
+requires "karax#5498909"
 requires "sass#e683aa1"
 requires "nimcrypto#a5742a9"
-requires "markdown#abdbe5e"
+requires "markdown#a661c26"
 requires "packedjson#d11d167"
 requires "supersnappy#2.1.1"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#0.7.3"
+requires "zippy#0.9.11"
 requires "flatty#0.2.3"
 requires "jsony#d0e69bd"
 

public/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /
+User-agent: Twitterbot
+Disallow:

src/formatters.nim

@@ -67,11 +67,11 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
     result = result.replace(odRegex, prefs.replaceOdysee)
 
   if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
-    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
+    result = result.replace(tco, &"{https}{prefs.replaceTwitter}/t.co")
     result = result.replace(cards, prefs.replaceTwitter & "/cards")
     result = result.replace(twRegex, prefs.replaceTwitter)
     result = result.replacef(twLinkRegex, a(
-      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
+      prefs.replaceTwitter & "$2", href = &"{https}{prefs.replaceTwitter}$1"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
     result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
@@ -83,7 +83,7 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
     result = result.replace(igRegex, prefs.replaceInstagram)
 
   if absolute.len > 0 and "href" in result:
-    result = result.replace("href=\"/", "href=\"" & absolute & "/")
+    result = result.replace("href=\"/", &"href=\"{absolute}/")
 
 proc getM3u8Url*(content: string): string =
   var matches: array[1, string]

src/nitter.nim

@@ -85,7 +85,6 @@ routes:
       &"An error occurred, please {link} with the URL you tried to visit.", cfg)
 
   error RateLimitError:
-    echo error.exc.name, ": ", error.exc.msg
     const link = a("another instance", href = instancesUrl)
     resp Http429, showError(
       &"Instance has been rate limited.<br>Use {link} or try again later.", cfg)

src/parser.nim

@@ -87,10 +87,15 @@ proc parseVideo(js: JsonNode): Video =
     result.description = description.getStr
 
   for v in js{"video_info", "variants"}:
+    let
+      contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary"))
+      url = v{"url"}.getStr
+
     result.variants.add VideoVariant(
-      contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
+      contentType: contentType,
       bitrate: v{"bitrate"}.getInt,
-      url: v{"url"}.getStr
+      url: url,
+      resolution: if contentType == mp4: getMp4Resolution(url) else: 0
     )
 
 proc parsePromoVideo(js: JsonNode): Video =

src/parserutils.nim

@@ -137,6 +137,21 @@ proc getSource*(js: JsonNode): string =
   let src = js{"source"}.getStr
   result = src.substr(src.find('>') + 1, src.rfind('<') - 1)
 
+proc getMp4Resolution*(url: string): int =
+  # parses the height out of a URL like this one:
+  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
+  const vidSep = "/vid/"
+  let
+    vidIdx = url.find(vidSep) + vidSep.len
+    resIdx = url.find('x', vidIdx) + 1
+    res = url[resIdx ..< url.find("/", resIdx)]
+
+  try:
+    return parseInt(res)
+  except ValueError:
+    # cannot determine resolution (e.g. m3u8/non-mp4 video)
+    return 0
+
 proc extractSlice(js: JsonNode): Slice[int] =
   result = js["indices"][0].getInt ..< js["indices"][1].getInt
 

src/query.nim

@@ -93,11 +93,11 @@ proc genQueryUrl*(query: Query): string =
   if query.text.len > 0:
     params.add "q=" & encodeUrl(query.text)
   for f in query.filters:
-    params.add "f-" & f & "=on"
+    params.add &"f-{f}=on"
   for e in query.excludes:
-    params.add "e-" & e & "=on"
+    params.add &"e-{e}=on"
   for i in query.includes.filterIt(it != "nativeretweets"):
-    params.add "i-" & i & "=on"
+    params.add &"i-{i}=on"
 
   if query.since.len > 0:
     params.add "since=" & query.since

src/routes/embed.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, options
+import asyncdispatch, strutils, strformat, options
 import jester, karax/vdom
 import ".."/[types, api]
 import ../views/[embed, tweet, general]
@@ -31,6 +31,6 @@ proc createEmbedRouter*(cfg: Config) =
       let id = @"id"
 
       if id.len > 0:
-        redirect("/i/status/" & id & "/embed")
+        redirect(&"/i/status/{id}/embed")
       else:
         resp Http404

src/routes/list.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, uri
+import strutils, strformat, uri
 
 import jester
 
@@ -10,14 +10,17 @@ export getListTimeline, getGraphList
 
 template respList*(list, timeline, title, vnode: typed) =
   if list.id.len == 0 or list.name.len == 0:
-    resp Http404, showError("List " & @"id" & " not found", cfg)
+    resp Http404, showError(&"""List "{@"id"}" not found""", cfg)
 
   let
     html = renderList(vnode, timeline.query, list)
-    rss = "/i/lists/$1/rss" % [@"id"]
+    rss = &"""/i/lists/{@"id"}/rss"""
 
   resp renderMain(html, request, cfg, prefs, titleText=title, rss=rss, banner=list.banner)
 
+proc title*(list: List): string =
+  &"@{list.username}/{list.name}"
+
 proc createListRouter*(cfg: Config) =
   router list:
     get "/@name/lists/@slug/?":
@@ -28,24 +31,22 @@ proc createListRouter*(cfg: Config) =
         slug = decodeUrl(@"slug")
         list = await getCachedList(@"name", slug)
       if list.id.len == 0:
-        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
-      redirect("/i/lists/" & list.id)
+        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
+      redirect(&"/i/lists/{list.id}")
 
     get "/i/lists/@id/?":
       cond '.' notin @"id"
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
-        title = "@" & list.username & "/" & list.name
         timeline = await getListTimeline(list.id, getCursor())
         vnode = renderTimelineTweets(timeline, prefs, request.path)
-      respList(list, timeline, title, vnode)
+      respList(list, timeline, list.title, vnode)
 
     get "/i/lists/@id/members":
       cond '.' notin @"id"
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
-        title = "@" & list.username & "/" & list.name
         members = await getGraphListMembers(list, getCursor())
-      respList(list, members, title, renderTimelineUsers(members, prefs, request.path))
+      respList(list, members, list.title, renderTimelineUsers(members, prefs, request.path))

src/routes/media.nim

@@ -88,6 +88,20 @@ proc createMediaRouter*(cfg: Config) =
     get "/pic/?":
       resp Http404
 
+    get re"^\/pic\/orig\/(enc)?\/?(.+)":
+      var url = decoded(request, 1)
+      if "twimg.com" notin url:
+        url.insert(twimg)
+      if not url.startsWith(https):
+        url.insert(https)
+      url.add("?name=orig")
+
+      let uri = parseUri(url)
+      cond isTwitterUrl(uri) == true
+
+      let code = await proxyMedia(request, url)
+      check code
+
     get re"^\/pic\/(enc)?\/?(.+)":
       var url = decoded(request, 1)
       if "twimg.com" notin url:

src/routes/rss.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, tables, times, hashes, uri
+import asyncdispatch, strutils, strformat, tables, times, hashes, uri
 
 import jester
 
@@ -42,8 +42,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
   if rss.cursor.len == 0:
     let info = case page
-               of "User": " \"$1\" " % @"name"
-               of "List": " $1 " % @"id"
+               of "User": &""" "{@"name"}" """
+               of "List": &""" "{@"id"}" """
                else: " "
 
     resp Http404, showError(page & info & "not found", cfg)
@@ -67,7 +67,7 @@ proc createRssRouter*(cfg: Config) =
 
       let
         cursor = getCursor()
-        key = "search:" & $hash(genQueryUrl(query)) & ":" & cursor
+        key = &"search:{hash(genQueryUrl(query))}:cursor"
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -86,7 +86,7 @@ proc createRssRouter*(cfg: Config) =
       let
         cursor = getCursor()
         name = @"name"
-        key = "twitter:" & name & ":" & cursor
+        key = &"twitter:{name}:{cursor}"
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -109,7 +109,7 @@ proc createRssRouter*(cfg: Config) =
         of "search": initQuery(params(request), name=name)
         else: Query(fromUser: @[name])
 
-      var key = @"tab" & ":" & @"name" & ":"
+      var key = &"""{@"tab"}:{@"name"}:"""
       if @"tab" == "search":
         key &= $hash(genQueryUrl(query)) & ":"
       key &= getCursor()
@@ -132,11 +132,11 @@ proc createRssRouter*(cfg: Config) =
         cursor = getCursor()
 
       if list.id.len == 0:
-        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
+        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
 
-      let url = "/i/lists/" & list.id & "/rss"
+      let url = &"/i/lists/{list.id}/rss"
       if cursor.len > 0:
-        redirect(url & "?cursor=" & encodeUrl(cursor, false))
+        redirect(&"{url}?cursor={encodeUrl(cursor, false)}")
       else:
         redirect(url)
 
@@ -146,7 +146,7 @@ proc createRssRouter*(cfg: Config) =
         cursor = getCursor()
         key =
           if cursor.len == 0: "lists:" & @"id"
-          else: "lists:" & @"id" & ":" & cursor
+          else: &"""lists:{@"id"}:{cursor}"""
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:

src/routes/search.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, uri
+import strutils, strformat, uri
 
 import jester
 
@@ -37,7 +37,7 @@ proc createSearchRouter*(cfg: Config) =
         resp Http404, showError("Invalid search", cfg)
 
     get "/hashtag/@hash":
-      redirect("/search?q=" & encodeUrl("#" & @"hash"))
+      redirect(&"""/search?q={encodeUrl("#" & @"hash")}""")
 
     get "/opensearch":
       let url = getUrlPrefix(cfg) & "/search?q="

src/routes/timeline.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, sequtils, uri, options, times
+import asyncdispatch, strutils, strformat, sequtils, uri, options, times
 import jester, karax/vdom
 
 import router_utils
@@ -102,7 +102,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
 template respTimeline*(timeline: typed) =
   let t = timeline
   if t.len == 0:
-    resp Http404, showError("User \"" & @"name" & "\" not found", cfg)
+    resp Http404, showError(&"""User "{@"name"}" not found""", cfg)
   resp t
 
 template respUserId*() =

src/tokens.nim

@@ -14,6 +14,10 @@ var
   clientPool: HttpPool
   tokenPool: seq[Token]
   lastFailed: Time
+  enableLogging = false
+
+template log(str) =
+  if enableLogging: echo "[tokens] ", str
 
 proc getPoolJson*(): JsonNode =
   var
@@ -77,8 +81,10 @@ proc fetchToken(): Future[Token] {.async.} =
 
     return Token(tok: tok, init: time, lastUse: time)
   except Exception as e:
-    lastFailed = getTime()
-    echo "fetching token failed: ", e.msg
+    echo "[tokens] fetching token failed: ", e.msg
+    if "Try again" notin e.msg:
+      echo "[tokens] fetching tokens paused, resuming in 30 minutes"
+      lastFailed = getTime()
 
 proc expired(token: Token): bool =
   let time = getTime()
@@ -100,6 +106,9 @@ proc isReady(token: Token; api: Api): bool =
 proc release*(token: Token; used=false; invalid=false) =
   if token.isNil: return
   if invalid or token.expired:
+    if invalid: log "discarding invalid token"
+    elif token.expired: log "discarding expired token"
+
     let idx = tokenPool.find(token)
     if idx > -1: tokenPool.delete(idx)
   elif used:
@@ -115,6 +124,7 @@ proc getToken*(api: Api): Future[Token] {.async.} =
   if not result.isReady(api):
     release(result)
     result = await fetchToken()
+    log "added new token to pool"
     tokenPool.add result
 
   if not result.isNil:
@@ -143,10 +153,12 @@ proc poolTokens*(amount: int) {.async.} =
     except: discard
 
     if not newToken.isNil:
+      log "added new token to pool"
       tokenPool.add newToken
 
 proc initTokenPool*(cfg: Config) {.async.} =
   clientPool = HttpPool()
+  enableLogging = cfg.enableDebug
 
   while true:
     if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens:

src/types.nim

@@ -75,6 +75,7 @@ type
     contentType*: VideoType
     url*: string
     bitrate*: int
+    resolution*: int
 
   Video* = object
     durationMs*: int

src/utils.nim

@@ -42,6 +42,12 @@ proc getPicUrl*(link: string): string =
   else:
     &"/pic/{encodeUrl(link)}"
 
+proc getOrigPicUrl*(link: string): string =
+  if base64Media:
+    &"/pic/orig/enc/{encode(link, safe=true)}"
+  else:
+    &"/pic/orig/{encodeUrl(link)}"
+
 proc filterParams*(params: Table): seq[(string, string)] =
   for p in params.pairs():
     if p[1].len > 0 and p[0] notin nitterParams:

src/views/general.nim

@@ -81,7 +81,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
 
     title:
       if titleText.len > 0:
-        text titleText & " | " & cfg.title
+        text &"{titleText}|{cfg.title}"
       else:
         text cfg.title
 

src/views/renderutils.nim

@@ -1,11 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils
+import strutils, strformat
 import karax/[karaxdsl, vdom, vstyles]
 import ".."/[types, utils]
 
 proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
   var c = "icon-" & icon
-  if class.len > 0: c = c & " " & class
+  if class.len > 0: c = &"{c} {class}"
   buildHtml(tdiv(class="icon-container")):
     if href.len > 0:
       a(class=c, title=title, href=href)

src/views/rss.nimf

@@ -119,7 +119,7 @@ ${renderRssTweets(profile.tweets.content, cfg)}
     <atom:link href="${link}" rel="self" type="application/rss+xml" />
     <title>${xmltree.escape(list.name)} / @${list.username}</title>
     <link>${link}</link>
-    <description>${getDescription(list.name & " by @" & list.username, cfg)}</description>
+    <description>${getDescription(&"{list.name} by @{list.username}", cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
 ${renderRssTweets(tweets, cfg)}
@@ -138,7 +138,7 @@ ${renderRssTweets(tweets, cfg)}
     <atom:link href="${link}" rel="self" type="application/rss+xml" />
     <title>Search results for "${escName}"</title>
     <link>${link}</link>
-    <description>${getDescription("Search \"" & escName & "\"", cfg)}</description>
+    <description>${getDescription(&"Search \"{escName}\"", cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
 ${renderRssTweets(tweets, cfg)}

src/views/tweet.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, sequtils, strformat, options
+import strutils, sequtils, strformat, options, algorithm
 import karax/[karaxdsl, vdom, vstyles]
 from jester import Request
 
@@ -10,7 +10,7 @@ import general
 proc getSmallPic(url: string): string =
   result = url
   if "?" notin url and not url.endsWith("placeholder.png"):
-    result &= ":small"
+    result &= "?name=small"
   result = getPicUrl(result)
 
 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
@@ -57,19 +57,22 @@ proc renderAlbum(tweet: Tweet): VNode =
           tdiv(class="attachment image"):
             let
               named = "name=" in photo
-              orig = if named: photo else: photo & "?name=orig"
+              orig = photo
               small = if named: photo else: photo & "?name=small"
-            a(href=getPicUrl(orig), class="still-image", target="_blank"):
+            a(href=getOrigPicUrl(orig), class="still-image", target="_blank"):
               genImg(small)
 
-proc isPlaybackEnabled(prefs: Prefs; video: Video): bool =
-  case video.playbackType
+proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
+  case playbackType
   of mp4: prefs.mp4Playback
   of m3u8, vmap: prefs.hlsPlayback
 
-proc renderVideoDisabled(video: Video; path: string): VNode =
+proc hasMp4Url(video: Video): bool =
+  video.variants.anyIt(it.contentType == mp4)
+
+proc renderVideoDisabled(playbackType: VideoType; path: string): VNode =
   buildHtml(tdiv(class="video-overlay")):
-    case video.playbackType
+    case playbackType
     of mp4:
       p: text "mp4 playback disabled in preferences"
     of m3u8, vmap:
@@ -84,9 +87,11 @@ proc renderVideoUnavailable(video: Video): VNode =
       p: text "This media is unavailable"
 
 proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
-  let container =
-    if video.description.len > 0 or video.title.len > 0: " card-container"
-    else: ""
+  let
+    container = if video.description.len == 0 and video.title.len == 0: ""
+                else: " card-container"
+    playbackType = if not prefs.proxyVideos and video.hasMp4Url: mp4
+                   else: video.playbackType
 
   buildHtml(tdiv(class="attachments card")):
     tdiv(class="gallery-video" & container):
@@ -95,13 +100,16 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
         if not video.available:
           img(src=thumb)
           renderVideoUnavailable(video)
-        elif not prefs.isPlaybackEnabled(video):
+        elif not prefs.isPlaybackEnabled(playbackType):
           img(src=thumb)
-          renderVideoDisabled(video, path)
+          renderVideoDisabled(playbackType, path)
         else:
-          let vid = video.variants.filterIt(it.contentType == video.playbackType)
-          let source = getVidUrl(vid[0].url)
-          case video.playbackType
+          let
+            vars = video.variants.filterIt(it.contentType == playbackType)
+            vidUrl = vars.sortedByIt(it.resolution)[^1].url
+            source = if prefs.proxyVideos: getVidUrl(vidUrl)
+                     else: vidUrl
+          case playbackType
           of mp4:
             if prefs.muteVideos:
               video(poster=thumb, controls="", muted=""):
@@ -146,7 +154,7 @@ proc renderPoll(poll: Poll): VNode =
         span(class="poll-choice-value"): text percStr
         span(class="poll-choice-option"): text poll.options[i]
     span(class="poll-info"):
-      text insertSep($poll.votes, ',') & " votes • " & poll.status
+      text &"{insertSep($poll.votes, ',')} votes • {poll.status}"
 
 proc renderCardImage(card: Card): VNode =
   buildHtml(tdiv(class="card-image-container")):

tests/test_card.py

@@ -4,8 +4,8 @@ from parameterized import parameterized
 
 card = [
     ['Thom_Wolf/status/1122466524860702729',
-     'pytorch/fairseq',
-     'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - pytorch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
+     'facebookresearch/fairseq',
+     'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
      'github.com', True],
 
     ['nim_lang/status/1136652293510717440',

tests/test_quote.py

@@ -8,9 +8,13 @@ text = [
 
 What are we doing wrong? reuters.com/article/us-norwa…"""],
 
-    ['nim_lang/status/924694255364341760',
-     'Hacker News', '@newsycombinator',
-     'Why Rust fails hard at scientific computing andre-ratsimbazafy.com/why-r…']
+    ['nim_lang/status/1491461266849808397#m',
+     'Nim language', '@nim_lang',
+     """What's better than Nim 1.6.0?
+
+Nim 1.6.2 :)
+
+nim-lang.org/blog/2021/12/17…"""]
 ]
 
 image = [

tests/test_tweet.py

@@ -16,7 +16,7 @@ timeline = [
 ]
 
 status = [
-    [20, 'jack⚡️', 'jack', '21 Mar 2006', 'just setting up my twttr'],
+    [20, 'jack', 'jack', '21 Mar 2006', 'just setting up my twttr'],
     [134849778302464000, 'The Twoffice', 'TheTwoffice', '11 Nov 2011', 'test'],
     [105685475985080322, 'The Twoffice', 'TheTwoffice', '22 Aug 2011', 'regular tweet'],
     [572593440719912960, 'Test account', 'mobile_test', '3 Mar 2015', 'testing test']
@@ -71,7 +71,7 @@ emoji = [
 
 retweet = [
     [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
-    [3, 'mobile_test_8', 'mobile test 8', 'jack⚡️', '@jack', 'twttr']
+    [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
 ]
 
 reply = [