Switch to packedjson to try lowering memory usage

このコミットが含まれているのは:
Zed 2020-06-02 16:22:44 +02:00
コミット fb591e43b8
5個のファイルの変更37行の追加33行の削除

ファイルの表示

@ -11,10 +11,8 @@ bin = @["nitter"]
# Dependencies # Dependencies
requires "nim >= 1.2.0" requires "nim >= 1.2.0"
requires "norm#head"
requires "jester#head" requires "jester#head"
requires "regex#2d96bab" requires "regex#2d96bab"
requires "q >= 0.0.7"
requires "nimcrypto >= 0.4.11" requires "nimcrypto >= 0.4.11"
requires "karax >= 1.1.2" requires "karax >= 1.1.2"
requires "sass" requires "sass"
@ -22,6 +20,7 @@ requires "markdown#head"
requires "https://github.com/zedeus/redis#head" requires "https://github.com/zedeus/redis#head"
requires "redpool#head" requires "redpool#head"
requires "msgpack4nim >= 0.3.1" requires "msgpack4nim >= 0.3.1"
requires "packedjson"
# Tasks # Tasks

ファイルの表示

@ -1,4 +1,5 @@
import asyncdispatch, httpclient, uri, strutils, json import asyncdispatch, httpclient, uri, strutils
import packedjson
import types, query, formatters, consts, apiutils, parser import types, query, formatters, consts, apiutils, parser
proc getGraphProfile*(username: string): Future[Profile] {.async.} = proc getGraphProfile*(username: string): Future[Profile] {.async.} =

ファイルの表示

@ -1,5 +1,6 @@
import httpclient, asyncdispatch, options, times, strutils, json, uri import httpclient, asyncdispatch, options, times, strutils, uri
import types, agents, tokens, consts import packedjson
import types, agents, tokens, consts, parserutils
proc genParams*(pars: openarray[(string, string)] = @[]; proc genParams*(pars: openarray[(string, string)] = @[];
cursor=""): seq[(string, string)] = cursor=""): seq[(string, string)] =
@ -41,19 +42,20 @@ proc fetch*(url: Uri; retried=false; oldApi=false): Future[JsonNode] {.async.} =
if resp.status != $Http200: if resp.status != $Http200:
if "Bad guest token" in body: if "Bad guest token" in body:
return keepToken = false
return newJNull()
elif not body.startsWith('{'): elif not body.startsWith('{'):
echo resp.status, " ", body echo resp.status, " ", body
return return newJNull()
result = parseJson(body) result = parseJson(body)
if result{"errors"} != nil and result.getError == forbidden: if result{"errors"}.notNull and result.getError == forbidden:
keepToken = false keepToken = false
echo "bad token" echo "bad token"
except: except:
echo "error: ", url echo "error: ", url
result = nil result = newJNull()
finally: finally:
if keepToken: if keepToken:
token.release() token.release()

ファイルの表示

@ -1,8 +1,9 @@
import json, strutils, options, tables, times, math import strutils, options, tables, times, math
import packedjson
import types, parserutils import types, parserutils
proc parseProfile(js: JsonNode; id=""): Profile = proc parseProfile(js: JsonNode; id=""): Profile =
if js == nil: return if js.isNull: return
result = Profile( result = Profile(
id: if id.len > 0: id else: js{"id_str"}.getStr, id: if id.len > 0: id else: js{"id_str"}.getStr,
username: js{"screen_name"}.getStr, username: js{"screen_name"}.getStr,
@ -24,7 +25,7 @@ proc parseProfile(js: JsonNode; id=""): Profile =
result.expandProfileEntities(js) result.expandProfileEntities(js)
proc parseUserShow*(js: JsonNode; username: string): Profile = proc parseUserShow*(js: JsonNode; username: string): Profile =
if js == nil: return if js.isNull: return
with error, js{"errors"}: with error, js{"errors"}:
result = Profile(username: username) result = Profile(username: username)
if error.getError == suspended: if error.getError == suspended:
@ -34,7 +35,7 @@ proc parseUserShow*(js: JsonNode; username: string): Profile =
result = parseProfile(js) result = parseProfile(js)
proc parseGraphProfile*(js: JsonNode; username: string): Profile = proc parseGraphProfile*(js: JsonNode; username: string): Profile =
if js == nil: return if js.isNull: return
with error, js{"errors"}: with error, js{"errors"}:
result = Profile(username: username) result = Profile(username: username)
if error.getError == suspended: if error.getError == suspended:
@ -46,12 +47,12 @@ proc parseGraphProfile*(js: JsonNode; username: string): Profile =
parseProfile(user, id) parseProfile(user, id)
proc parseGraphList*(js: JsonNode): List = proc parseGraphList*(js: JsonNode): List =
if js == nil: return if js.isNull: return
var list = js{"data", "user_by_screen_name", "list"} var list = js{"data", "user_by_screen_name", "list"}
if list == nil: if list.isNull:
list = js{"data", "list"} list = js{"data", "list"}
if list == nil: if list.isNull:
return return
result = List( result = List(
@ -70,7 +71,7 @@ proc parseListMembers*(js: JsonNode; cursor: string): Result[Profile] =
query: Query(kind: userList) query: Query(kind: userList)
) )
if js == nil: return if js.isNull: return
result.top = js{"previous_cursor_str"}.getStr result.top = js{"previous_cursor_str"}.getStr
result.bottom = js{"next_cursor_str"}.getStr result.bottom = js{"next_cursor_str"}.getStr
@ -116,7 +117,7 @@ proc parseVideo(js: JsonNode): Video =
for v in js{"video_info", "variants"}: for v in js{"video_info", "variants"}:
result.variants.add VideoVariant( result.variants.add VideoVariant(
videoType: v{"content_type"}.to(VideoType), videoType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
bitrate: v{"bitrate"}.getInt, bitrate: v{"bitrate"}.getInt,
url: v{"url"}.getStr url: v{"url"}.getStr
) )
@ -192,14 +193,14 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
break break
proc parseTweet(js: JsonNode): Tweet = proc parseTweet(js: JsonNode): Tweet =
if js == nil: return if js.isNull: return
result = Tweet( result = Tweet(
id: js{"id_str"}.getId, id: js{"id_str"}.getId,
threadId: js{"conversation_id_str"}.getId, threadId: js{"conversation_id_str"}.getId,
replyId: js{"in_reply_to_status_id_str"}.getId, replyId: js{"in_reply_to_status_id_str"}.getId,
text: js{"full_text"}.getStr, text: js{"full_text"}.getStr,
time: js{"created_at"}.getTime, time: js{"created_at"}.getTime,
hasThread: js{"self_thread"} != nil, hasThread: js{"self_thread"}.notNull,
available: true, available: true,
profile: Profile(id: js{"user_id_str"}.getStr), profile: Profile(id: js{"user_id_str"}.getStr),
stats: TweetStats( stats: TweetStats(
@ -354,7 +355,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
if instructions.len == 0: return if instructions.len == 0: return
for i in instructions: for i in instructions:
if result.beginning and i{"pinEntry"} != nil: if result.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global): with pin, parsePin(i, global):
result.content.add pin result.content.add pin
else: else:

ファイルの表示

@ -1,5 +1,5 @@
import json, strutils, times, macros, htmlgen, uri, unicode, options import strutils, times, macros, htmlgen, uri, unicode, options
import regex import regex, packedjson
import types, utils, formatters import types, utils, formatters
const const
@ -11,9 +11,12 @@ const
let localTimezone = local() let localTimezone = local()
template isNull*(js: JsonNode): bool = js.kind == JNull
template notNull*(js: JsonNode): bool = js.kind != JNull
template `?`*(js: JsonNode): untyped = template `?`*(js: JsonNode): untyped =
let j = js let j = js
if j == nil: return if j.isNull: return
else: j else: j
template `with`*(ident, value, body): untyped = template `with`*(ident, value, body): untyped =
@ -24,8 +27,7 @@ template `with`*(ident, value, body): untyped =
template `with`*(ident; value: JsonNode; body): untyped = template `with`*(ident; value: JsonNode; body): untyped =
block: block:
let ident {.inject.} = value let ident {.inject.} = value
if ident != nil and ident.kind != JNull: if value.notNull: body
body
template getCursor*(js: JsonNode): string = template getCursor*(js: JsonNode): string =
js{"content", "operation", "cursor", "value"}.getStr js{"content", "operation", "cursor", "value"}.getStr
@ -50,7 +52,6 @@ proc getId*(id: string): string {.inline.} =
id[start + 1 ..< id.len] id[start + 1 ..< id.len]
proc getId*(js: JsonNode): int64 {.inline.} = proc getId*(js: JsonNode): int64 {.inline.} =
if js == nil: return
case js.kind case js.kind
of JString: return parseBiggestInt(js.getStr("0")) of JString: return parseBiggestInt(js.getStr("0"))
of JInt: return js.getBiggestInt() of JInt: return js.getBiggestInt()
@ -117,10 +118,9 @@ template getSlice(text: string; slice: seq[int]): string =
text.runeSubStr(slice[0], slice[1] - slice[0]) text.runeSubStr(slice[0], slice[1] - slice[0])
proc getSlice(text: string; js: JsonNode): string = proc getSlice(text: string; js: JsonNode): string =
if js == nil or js.kind != JArray or js.len < 2 or if js.kind != JArray or js.len < 2 or js[0].kind != JInt: return text
js[0].kind != JInt: return text
let slice = js.to(seq[int]) let slice = @[js{0}.getInt, js{1}.getInt]
text.getSlice(slice) text.getSlice(slice)
proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) = proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) =
@ -130,9 +130,9 @@ proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) =
let let
url = js{"expanded_url"}.getStr url = js{"expanded_url"}.getStr
slice = js{"indices"}.to(seq[int]) slice = js{"indices"}[1].getInt
if hideTwitter and slice[1] >= tLen and url.isTwitterUrl: if hideTwitter and slice >= tLen and url.isTwitterUrl:
text = text.replace(u, "") text = text.replace(u, "")
text.removeSuffix(' ') text.removeSuffix(' ')
text.removeSuffix('\n') text.removeSuffix('\n')
@ -178,7 +178,8 @@ proc expandProfileEntities*(profile: var Profile; js: JsonNode) =
proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
let let
orig = tweet.text orig = tweet.text
slice = js{"display_text_range"}.to(seq[int]) textRange = js{"display_text_range"}
slice = @[textRange{0}.getInt, textRange{1}.getInt]
hasQuote = js{"is_quote_status"}.getBool hasQuote = js{"is_quote_status"}.getBool
hasCard = tweet.card.isSome hasCard = tweet.card.isSome