Merge branch 'dev'

anonstream/__init__.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2022 n9k <https://git.076.ne.jp/ninya9k>
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+import json
 from collections import OrderedDict
 
 from quart_compress import Compress
@@ -8,6 +9,7 @@ from quart_compress import Compress
 from anonstream.config import update_flask_from_toml
 from anonstream.quart import Quart
 from anonstream.utils.captcha import create_captcha_factory, create_captcha_signer
+from anonstream.utils.chat import schema_to_emotes
 from anonstream.utils.user import generate_blank_allowedness
 
 __version__ = '1.4.0'
@@ -45,6 +47,10 @@ def create_app(toml_config):
     app.failures = OrderedDict() # access captcha failures
     app.allowedness = generate_blank_allowedness()
 
+    with open(app.config['EMOTE_SCHEMA']) as fp:
+      schema = json.load(fp)
+      app.emotes = schema_to_emotes(schema)
+
     # State for tasks
     app.users_update_buffer = set()
     app.stream_title = None

anonstream/chat.py

@@ -8,7 +8,7 @@ from quart import current_app, escape
 
 from anonstream.broadcast import broadcast, broadcast_users_update
 from anonstream.events import notify_event_sockets
-from anonstream.helpers.chat import generate_nonce_hash, get_scrollback
+from anonstream.helpers.chat import generate_nonce_hash, get_scrollback, insert_emotes
 from anonstream.utils.chat import get_message_for_websocket, get_approx_linespan
 
 CONFIG = current_app.config
@@ -93,7 +93,7 @@ def add_chat_message(user, nonce, comment, ignore_empty=False):
         else:
             seq = last_message['seq'] + 1
     dt = datetime.utcfromtimestamp(timestamp)
-    markup = escape(comment)
+    markup = insert_emotes(escape(comment))
     message = {
         'id': message_id,
         'seq': seq,

anonstream/config.py

@@ -39,6 +39,7 @@ def toml_to_flask_sections(config):
         toml_to_flask_section_flood,
         toml_to_flask_section_captcha,
         toml_to_flask_section_nojs,
+        toml_to_flask_section_emote,
     )
     for toml_to_flask_section in TOML_TO_FLASK_SECTIONS:
         yield toml_to_flask_section(config)
@@ -164,3 +165,9 @@ def toml_to_flask_section_nojs(config):
         'NOJS_REFRESH_USERS': round(cfg['refresh_users']),
         'NOJS_TIMEOUT_CHAT': round(cfg['timeout_chat']),
     }
+
+def toml_to_flask_section_emote(config):
+  cfg = config['emote']
+  return {
+    'EMOTE_SCHEMA': cfg['schema'],
+  }

anonstream/helpers/chat.py

@@ -3,9 +3,10 @@
 
 import hashlib
 
-from quart import current_app
+from quart import current_app, escape, Markup
 
 CONFIG = current_app.config
+EMOTES = current_app.emotes
 
 def generate_nonce_hash(nonce):
     parts = CONFIG['SECRET_KEY'] + b'nonce-hash\0' + nonce.encode()
@@ -16,3 +17,25 @@ def get_scrollback(messages):
     if len(messages) < n:
         return messages
     return list(messages)[-n:]
+
+def insert_emotes(markup):
+    assert isinstance(markup, Markup)
+    for name, regex, _position, _size in EMOTES:
+        emote_markup = (
+            f'<span class="emote" data-emote="{escape(name)}" '
+            f'title="{escape(name)}">{escape(name)}</span>'
+        )
+        markup = regex.sub(emote_markup, markup)
+    return Markup(markup)
+
+def get_emotes_for_websocket():
+    return {
+        name: {
+            'x': position[0],
+            'y': position[1],
+            'width': size[0],
+            'height': size[1],
+        }
+        for name, _regex, position, size in EMOTES
+    }
+    return tuple(EMOTES.values())

anonstream/routes/core.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 import math
+import re
 
 from quart import current_app, request, render_template, abort, make_response, redirect, url_for, send_from_directory
 from werkzeug.exceptions import Forbidden, NotFound, TooManyRequests
@@ -11,7 +12,7 @@ from anonstream.captcha import get_captcha_image, get_random_captcha_digest
 from anonstream.segments import segments, StopSendingSegments
 from anonstream.stream import is_online, get_stream_uptime
 from anonstream.user import watching, create_eyes, renew_eyes, EyesException, RatelimitedEyes, TooManyEyes, ensure_allowedness, Blacklisted, SecretClub
-from anonstream.routes.wrappers import with_user_from, auth_required, clean_cache_headers, generate_and_add_user
+from anonstream.routes.wrappers import with_user_from, auth_required, generate_and_add_user, clean_cache_headers, etag_conditional
 from anonstream.helpers.captcha import check_captcha_digest, Answer
 from anonstream.utils.security import generate_csp
 from anonstream.utils.user import identifying_string
@@ -136,6 +137,10 @@ async def access(timestamp, user_or_token):
 
 @current_app.route('/static/<filename>')
 @with_user_from(request)
+@etag_conditional
 @clean_cache_headers
 async def static(timestamp, user, filename):
-    return await send_from_directory(STATIC_DIRECTORY, filename)
+    response = await send_from_directory(STATIC_DIRECTORY, filename)
+    if filename in {'style.css', 'anonstream.js'}:
+        response.headers['Cache-Control'] = 'no-cache'
+    return response

anonstream/routes/nojs.py

@@ -10,12 +10,13 @@ from anonstream.user import add_state, pop_state, try_change_appearance, update_
 from anonstream.routes.wrappers import with_user_from, render_template_with_etag
 from anonstream.helpers.chat import get_scrollback
 from anonstream.helpers.user import get_default_name
-from anonstream.utils.chat import generate_nonce
+from anonstream.utils.chat import generate_nonce, escape_css_string, get_emotehash
 from anonstream.utils.security import generate_csp
 from anonstream.utils.user import concatenate_for_notice
 
 CONFIG = current_app.config
 USERS_BY_TOKEN = current_app.users_by_token
+EMOTES = current_app.emotes
 
 @current_app.route('/stream.html')
 @with_user_from(request)
@@ -53,8 +54,11 @@ async def nojs_chat_messages(timestamp, user):
         refresh=CONFIG['NOJS_REFRESH_MESSAGES'],
         user=user,
         users_by_token=USERS_BY_TOKEN,
+        emotes=EMOTES,
+        emotehash=get_emotehash(tuple(EMOTES)),
         messages=get_scrollback(current_app.messages),
         timeout=CONFIG['NOJS_TIMEOUT_CHAT'],
+        escape_css_string=escape_css_string,
         get_default_name=get_default_name,
     )
 

anonstream/routes/wrappers.py

@@ -213,6 +213,21 @@ def clean_cache_headers(f):
 
     return wrapper
 
+def etag_conditional(f):
+    @wraps(f)
+    async def wrapper(*args, **kwargs):
+        response = await f(*args, **kwargs)
+        etag = response.headers.get('ETag')
+        if etag is not None:
+            if match := re.fullmatch(r'"(?P<tag>.+)"', etag):
+                tag = match.group('tag')
+                if tag in request.if_none_match:
+                    return '', 304, {'ETag': etag}
+
+        return response
+
+    return wrapper
+
 def assert_allowedness(timestamp, user):
     try:
         ensure_allowedness(user, timestamp=timestamp)

anonstream/static/anonstream.js

@@ -84,6 +84,12 @@ const insert_jsmarkup = () => {
     style_tripcode_colors.nonce = CSP;
     document.head.insertAdjacentElement("beforeend", style_tripcode_colors);
   }
+  if (document.getElementById("style-emote") === null) {
+    const style_emote = document.createElement("style");
+    style_emote.id = "style-emote";
+    style_emote.nonce = CSP;
+    document.head.insertAdjacentElement("beforeend", style_emote);
+  }
   if (document.getElementById("stream__video") === null) {
     const parent = document.getElementById("stream");
     parent.insertAdjacentHTML("beforeend", jsmarkup_stream_video);
@@ -134,6 +140,7 @@ insert_jsmarkup();
 const stylesheet_color = document.styleSheets[1];
 const stylesheet_tripcode_display = document.styleSheets[2];
 const stylesheet_tripcode_colors = document.styleSheets[3];
+const stylesheet_emote = document.styleSheets[4];
 
 /* override chat form notice button */
 const chat_form = document.getElementById("chat-form_js");
@@ -275,6 +282,57 @@ const delete_chat_messages = (seqs) => {
   }
 }
 
+const hexdigest = async (string, bytelength) => {
+  uint8array = new TextEncoder().encode(string);
+  arraybuffer = await crypto.subtle.digest('sha-256', uint8array);
+  array = Array.from(new Uint8Array(arraybuffer).slice(0, bytelength));
+  hex = array.map(b => b.toString(16).padStart(2, '0')).join('');
+  return hex
+}
+const escape_css_string = (string) => {
+  /* https://drafts.csswg.org/cssom/#common-serializing-idioms */
+  const result = [];
+  for (const char of string) {
+    if (char === '\0') {
+      result.push('\ufffd');
+    } else if (char < '\u0020' || char == '\u007f') {
+      result.push(`\\${char.charCodeAt().toString(16)}`);
+    } else if (char == '"' || char == '\\') {
+      result.push(`\\${char}`);
+    } else {
+      result.push(char);
+    }
+  }
+  return result.join('');
+}
+const update_emotes = async (emotes) => {
+  const rules = [];
+  for (const key of Object.keys(emotes)) {
+    const emote = emotes[key];
+    rules.push(
+      `[data-emote="${escape_css_string(key)}"] { background-position: ${-emote.x}px ${-emote.y}px; width: ${emote.width}px; height: ${emote.height}px; }`
+    );
+  }
+  rules.sort();
+  const emotehash = await hexdigest(rules.toString(), 6);
+  const emotehash_rule = `.emote { background-image: url("/static/emotes.png?coords=${escape_css_string(encodeURIComponent(emotehash))}"); }`;
+
+  const rules_set = new Set([emotehash_rule, ...rules]);
+  const to_delete = [];
+  for (let index = 0; index < stylesheet_emote.cssRules.length; index++) {
+    const css_rule = stylesheet_emote.cssRules[index];
+    if (!rules_set.delete(css_rule.cssText)) {
+      to_delete.push(index);
+    }
+  }
+  for (const rule of rules_set) {
+    stylesheet_emote.insertRule(rule);
+  }
+  for (const index of to_delete.reverse()) {
+    stylesheet_emote.deleteRule(index + rules_set.size);
+  }
+}
+
 let users = {};
 let stats = null;
 let stats_received = null;
@@ -594,7 +652,7 @@ const show_offline_screen = () => {
   stream.dataset.offline = "";
 }
 
-const on_websocket_message = (event) => {
+const on_websocket_message = async (event) => {
   //console.log("websocket message", event);
   const receipt = JSON.parse(event.data);
   switch (receipt.type) {
@@ -631,7 +689,7 @@ const on_websocket_message = (event) => {
       // chat form submit button
       chat_form_submit.disabled = false;
 
-      // remove messages the server isn't acknowledging the existance of
+      // remove messages the server isn't acknowledging the existence of
       const seqs = new Set(receipt.messages.map((message) => {return message.seq;}));
       const to_delete = [];
       for (const chat_message of chat_messages.children) {
@@ -676,6 +734,9 @@ const on_websocket_message = (event) => {
       chat_appearance_form_name.setAttribute("placeholder", default_name[user.broadcaster]);
       chat_appearance_form_color.setAttribute("value", user.color);
 
+      // emote coordinates
+      await update_emotes(receipt.emotes);
+
       // insert new messages
       const last = chat_messages.children.length == 0 ? null : chat_messages.children[chat_messages.children.length - 1];
       const last_seq = last === null ? null : parseInt(last.dataset.seq);

anonstream/static/style.css

@@ -263,6 +263,12 @@ noscript {
   overflow-wrap: anywhere;
   line-height: 1.3125;
 }
+.emote {
+  display: inline-block;
+  font-size: 0;
+  vertical-align: middle;
+  cursor: default;
+}
 .tripcode {
   padding: 0 5px;
   border-radius: 7px;

anonstream/templates/nojs_chat_messages.html

@@ -8,9 +8,9 @@
 <head>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <meta http-equiv="content-security-policy" content="default-src 'none'; style-src 'nonce-{{ csp }}';">
-  <meta http-equiv="refresh" content="{{ refresh }}">
-  <meta http-equiv="refresh" content="{{ refresh + 1 }}; url={{ url_for('nojs_chat_messages_redirect', token=user.token) }}">
+  <meta http-equiv="content-security-policy" content="default-src 'none'; style-src 'nonce-{{ csp }}'; img-src 'self';">
+  <!--<meta http-equiv="refresh" content="{{ refresh }}">-->
+  <meta http-equiv="refresh" content="{{ refresh }}; url={{ url_for('nojs_chat_messages_redirect', token=user.token) }}">
   <style nonce="{{ csp }}">
     html {
       height: 100%;
@@ -133,6 +133,13 @@
       overflow-wrap: anywhere;
       line-height: 1.3125;
     }
+    .emote {
+      background-image: url("{{ escape_css_string(url_for('static', filename='emotes.png', coords=emotehash)) | safe }}");
+      display: inline-block;
+      font-size: 0;
+      vertical-align: middle;
+      cursor: default;
+    }
     .tripcode {
       padding: 0 5px;
       border-radius: 7px;
@@ -154,6 +161,14 @@
         {% endif %}
       {% endwith %}
     {% endfor %}
+
+    {% for name, _regex, (x, y), (width, height) in emotes %}
+      [data-emote="{{ escape_css_string(name) | safe }}"] {
+        background-position: {{ -x }}px {{ -y }}px;
+        width: {{ width }}px;
+        height: {{ height }}px;
+      }
+    {% endfor %}
   </style>
 </head>
 <body>

anonstream/utils/chat.py

@@ -4,7 +4,11 @@
 import base64
 import hashlib
 import math
+import re
 import secrets
+from functools import lru_cache
+
+from quart import escape
 
 class NonceReuse(Exception):
     pass
@@ -26,3 +30,46 @@ def get_approx_linespan(text):
     linespan = sum(map(height, text.splitlines()))
     linespan = linespan if linespan > 0 else 1
     return linespan
+
+def schema_to_emotes(schema):
+    emotes = []
+    for name, coords in schema.items():
+        assert not re.search(r'\s', name), \
+            'whitespace is not allowed in emote names'
+        name_markup = escape(name)
+        regex = re.compile(
+            r'(?:^|(?<=\s|\W))%s(?:$|(?=\s|\W))' % re.escape(name_markup)
+        )
+        position, size = tuple(coords['position']), tuple(coords['size'])
+        emotes.append((name, regex, position, size))
+    return emotes
+
+def escape_css_string(string):
+    '''
+    https://drafts.csswg.org/cssom/#common-serializing-idioms
+    '''
+    result = []
+    for char in string:
+        if char == '\0':
+            result.append('\ufffd')
+        elif char < '\u0020' or char == '\u007f':
+            result.append(f'\\{ord(char):x}')
+        elif char == '"' or char == '\\':
+            result.append(f'\\{char}')
+        else:
+            result.append(char)
+    return ''.join(result)
+
+@lru_cache(maxsize=1)
+def get_emotehash(emotes):
+    rules = []
+    for name, _regex, (x, y), (width, height) in sorted(emotes):
+        rule = (
+            f'[data-emote="{escape_css_string(name)}"] '
+            f'{{ background-position: {-x}px {-y}px; '
+            f'width: {width}px; height: {height}px; }}'
+        )
+        rules.append(rule.encode())
+    plaintext = b','.join(rules)
+    digest = hashlib.sha256(plaintext).digest()
+    return digest[:6].hex()

anonstream/websocket.py

@@ -11,6 +11,7 @@ from anonstream.captcha import get_random_captcha_digest_for
 from anonstream.chat import get_all_messages_for_websocket, add_chat_message, Rejected
 from anonstream.user import get_all_users_for_websocket, see, reading, verify, deverify, BadCaptcha, try_change_appearance, ensure_allowedness, AllowednessException
 from anonstream.wrappers import with_timestamp, get_timestamp
+from anonstream.helpers.chat import get_emotes_for_websocket
 from anonstream.utils.chat import generate_nonce
 from anonstream.utils.user import identifying_string
 from anonstream.utils.websocket import parse_websocket_data, Malformed, WS
@@ -36,6 +37,7 @@ async def websocket_outbound(queue, user):
         'scrollback': CONFIG['MAX_CHAT_SCROLLBACK'],
         'digest': get_random_captcha_digest_for(user),
         'pingpong': CONFIG['TASK_BROADCAST_PING'],
+        'emotes': get_emotes_for_websocket(),
     })
     while True:
         payload = await queue.get()

config.toml

@@ -62,7 +62,7 @@ max_name_length = 24
 min_name_contrast = 3.0
 background_color = "#232327"
 max_tripcode_password_length = 1024
-legacy_tripcode_algorithm = false
+legacy_tripcode_algorithm = true
 force_captcha_every = 40
 
 [flood.messages]
@@ -89,3 +89,6 @@ refresh_messages = 4.0
 refresh_info = 6.0
 refresh_users = 6.0
 timeout_chat = 30.0
+
+[emote]
+schema = "emotes.json"

emotes.json

@@ -0,0 +1 @@
+{}