Use farside.link for frontend alternatives in results (#560)

* Integrate Farside into Whoogle When instances are ratelimited (when a captcha is returned instead of the user's search results) the user can now hop to a new instance via Farside, a new backend service that redirects users to working instances of a particular frontend. In this case, it presents a user with a Farside link to a new Whoogle (or Searx) instance instead, so that the user can resume their search. For the generated Farside->Whoogle link, the generated link includes the user's current Whoogle configuration settings as URL params, to ensure a more seamless transition between instances. This doesn't translate to the Farside->Searx link, but potentially could with some changes. * Expand conversion of config<->url params Config settings can now be translated to and from URL params using a predetermined set of "safe" keys (i.e. config settings that easily translate to URL params). * Allow jumping instances via Farside when ratelimited When instances are ratelimited (when a captcha is returned instead of the user's search results) the user can now hop to a new instance via Farside, a new backend service that redirects users to working instances of a particular frontend. In this case, it presents a user with a Farside link to a new Whoogle (or Searx) instance instead, so that the user can resume their search. For the generated Farside->Whoogle link, the generated link includes the user's current Whoogle configuration settings as URL params, to ensure a more seamless transition between instances. This doesn't translate to the Farside->Searx link, but potentially could with some changes. Closes #554 Closes #559
2021-12-08 17:27:33 -07:00 · 2021-12-08 17:27:33 -07:00 · 634d179568
--- a/12
+++ b/12
@ -47,18 +47,18 @@ ENV HTTPS_ONLY=$use_https
 ARG whoogle_port=5000
 ENV EXPOSE_PORT=$whoogle_port

-ARG twitter_alt='nitter.net'
+ARG twitter_alt='farside.link/nitter'
 ENV WHOOGLE_ALT_TW=$twitter_alt
-ARG youtube_alt='invidious.snopyta.org'
+ARG youtube_alt='farside.link/invidious'
 ENV WHOOGLE_ALT_YT=$youtube_alt
-ARG instagram_alt='bibliogram.art/u'
+ARG instagram_alt='farside.link/bibliogram'
 ENV WHOOGLE_ALT_IG=$instagram_alt
-ARG reddit_alt='libredd.it'
+ARG reddit_alt='farside.link/libreddit'
 ENV WHOOGLE_ALT_RD=$reddit_alt
+ARG medium_alt='farside.link/scribe'
+ENV WHOOGLE_ALT_MD=$medium_alt
 ARG translate_alt='lingva.ml'
 ENV WHOOGLE_ALT_TL=$translate_alt
-ARG medium_alt='scribe.rip'
-ENV WHOOGLE_ALT_MD=$medium_alt

 WORKDIR /whoogle

--- a/README.md
+++ b/README.md
@ -197,12 +197,12 @@ Description=Whoogle
 # Site alternative configurations, uncomment to enable
 # Note: If not set, the feature will still be available
 # with default values.
-#Environment=WHOOGLE_ALT_TW=nitter.net
-#Environment=WHOOGLE_ALT_YT=invidious.snopyta.org
-#Environment=WHOOGLE_ALT_IG=bibliogram.art/u
-#Environment=WHOOGLE_ALT_RD=libredd.it
+#Environment=WHOOGLE_ALT_TW=farside.link/nitter
+#Environment=WHOOGLE_ALT_YT=farside.link/invidious
+#Environment=WHOOGLE_ALT_IG=farside.link/bibliogram/u
+#Environment=WHOOGLE_ALT_RD=farside.link/libreddit
+#Environment=WHOOGLE_ALT_MD=farside.link/scribe
 #Environment=WHOOGLE_ALT_TL=lingva.ml
-#Environment=WHOOGLE_ALT_MD=scribe.rip
 # Load values from dotenv only
 #Environment=WHOOGLE_DOTENV=1
 Type=simple
--- a/app.json
+++ b/app.json
@ -47,22 +47,27 @@
    },
    "WHOOGLE_ALT_TW": {
      "description": "The site to use as a replacement for twitter.com when site alternatives are enabled in the config.",
-      "value": "nitter.net",
+      "value": "farside.link/nitter",
      "required": false
    },
    "WHOOGLE_ALT_YT": {
      "description": "The site to use as a replacement for youtube.com when site alternatives are enabled in the config.",
-      "value": "invidious.snopyta.org",
+      "value": "farside.link/invidious",
      "required": false
    },
    "WHOOGLE_ALT_IG": {
      "description": "The site to use as a replacement for instagram.com when site alternatives are enabled in the config.",
-      "value": "bibliogram.art/u",
+      "value": "farside.link/bibliogram/u",
      "required": false
    },
    "WHOOGLE_ALT_RD": {
      "description": "The site to use as a replacement for reddit.com when site alternatives are enabled in the config.",
-      "value": "libredd.it",
+      "value": "farside.link/libreddit",
+      "required": false
+    },
+    "WHOOGLE_ALT_MD": {
+      "description": "The site to use as a replacement for medium.com when site alternatives are enabled in the config.",
+      "value": "farside.link/scribe",
      "required": false
    },
    "WHOOGLE_ALT_TL": {
@ -70,11 +75,6 @@
      "value": "lingva.ml",
      "required": false
    },
-    "WHOOGLE_ALT_MD": {
-      "description": "The site to use as a replacement for medium.com when site alternatives are enabled in the config.",
-      "value": "scribe.rip",
-      "required": false
-    },
    "WHOOGLE_MINIMAL": {
        "description": "Remove everything except basic result cards from all search queries (set to 1 or leave blank)",
        "value": "",
--- a/app/models/config.py
+++ b/app/models/config.py
@ -17,7 +17,7 @@ class Config:
        self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
        self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
        self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
-        self.ctry = os.getenv('WHOOGLE_CONFIG_COUNTRY', 'US')
+        self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', 'US')
        self.theme = os.getenv('WHOOGLE_CONFIG_THEME', 'system')
        self.safe = read_config_bool('WHOOGLE_CONFIG_SAFE')
        self.dark = read_config_bool('WHOOGLE_CONFIG_DARK')  # deprecated
@ -33,9 +33,13 @@ class Config:
        self.safe_keys = [
            'lang_search',
            'lang_interface',
-            'ctry',
-            'dark',
-            'theme'
+            'country',
+            'theme',
+            'alts',
+            'new_tab',
+            'view_image',
+            'block',
+            'safe'
        ]

        # Skip setting custom config if there isn't one
@ -105,5 +109,26 @@ class Config:
        for param_key in params.keys():
            if not self.is_safe_key(param_key):
                continue
-            self[param_key] = params.get(param_key)
+            param_val = params.get(param_key)
+
+            if param_val == 'off':
+                param_val = False
+            elif param_val.isdigit():
+                param_val = int(param_val)
+
+            self[param_key] = param_val
        return self
+
+    def to_params(self) -> str:
+        """Generates a set of safe params for using in Whoogle URLs
+
+        Returns:
+            str -- a set of URL parameters
+        """
+        param_str = ''
+        for safe_key in self.safe_keys:
+            if not self[safe_key]:
+                continue
+            param_str = param_str + f'&{safe_key}={self[safe_key]}'
+
+        return param_str
--- a/app/request.py
+++ b/app/request.py
@ -120,7 +120,7 @@ def gen_query(query, args, config) -> str:
    if 'chips' in args:
        param_dict['chips'] = '&chips=' + args.get('chips')

-    param_dict['gl'] = ('&gl=' + config.ctry) if config.ctry else ''
+    param_dict['gl'] = ('&gl=' + config.country) if config.country else ''
    param_dict['hl'] = '&hl=' + (
        config.lang_interface.replace('lang_', '')
        if config.lang_interface else ''
--- a/app/routes.py
+++ b/app/routes.py
@ -310,7 +310,16 @@ def search():
    translate_to = localization_lang.replace('lang_', '')

    # Return 503 if temporarily blocked by captcha
-    resp_code = 503 if has_captcha(str(response)) else 200
+    if has_captcha(str(response)):
+        return render_template(
+            'error.html',
+            blocked=True,
+            error_message=translation['ratelimit'],
+            translation=translation,
+            farside='https://farside.link',
+            config=g.user_config,
+            query=urlparse.unquote(query),
+            params=g.user_config.to_params()), 503
    response = bold_search_terms(response, query)

    # Feature to display IP address
@ -351,7 +360,7 @@ def search():
            search_type=search_util.search_type,
            mobile=g.user_request.mobile)
                       if 'isch' not in
-                          search_util.search_type else '')), resp_code
+                          search_util.search_type else '')), 200


@app.route(f'/{Endpoint.config}', methods=['GET', 'POST', 'PUT'])
--- a/app/static/css/dark-theme.css
+++ b/app/static/css/dark-theme.css
@ -138,10 +138,14 @@ select {
    color: var(--whoogle-dark-contrast-text) !important;
 }

-#gh-link {
+.link {
    color: var(--whoogle-dark-contrast-text);
 }

+.link-color {
+    color: var(--whoogle-dark-result-url) !important;
+}
+
 .autocomplete-items {
    border: 1px solid var(--whoogle-dark-element-bg);
 }
--- a/app/static/css/error.css
+++ b/app/static/css/error.css
@ -0,0 +1,9 @@
+html {
+    font-size: 1.3rem;
+}
+
+@media (max-width: 1000px) {
+    html {
+        font-size: 3rem;
+    }
+}
--- a/app/static/css/light-theme.css
+++ b/app/static/css/light-theme.css
@ -125,10 +125,14 @@ input {
    color: var(--whoogle-contrast-text);
 }

-#gh-link {
+.link {
    color: var(--whoogle-element-bg);
 }

+.link-color {
+    color: var(--whoogle-result-url) !important;
+}
+
 .autocomplete-items {
    border: 1px solid var(--whoogle-element-bg);
 }
--- a/app/static/css/main.css
+++ b/app/static/css/main.css
@ -177,3 +177,10 @@ details summary {
    padding: 10px;
    font-weight: bold;
 }
+
+/* Mobile styles */
+@media (max-width: 1000px) {
+    select {
+        width: 100%;
+    }
+}
--- a/app/static/settings/translations.json
+++ b/app/static/settings/translations.json
@ -33,7 +33,9 @@
        "translate": "translate",
        "light": "light",
        "dark": "dark",
-        "system": "system"
+        "system": "system",
+        "ratelimit": "Instance has been ratelimited",
+        "continue-search": "Continue your search with "
    },
    "lang_nl": {
        "search": "Zoeken",
@ -69,7 +71,9 @@
        "translate": "vertalen",
        "light": "helder",
        "dark": "donker",
-        "system": "systeeminstellingen"
+        "system": "systeeminstellingen",
+        "ratelimit": "Instantie is beperkt in snelheid",
+        "continue-search": "Ga verder met zoeken met "
    },
    "lang_de": {
        "search": "Suchen",
@ -105,7 +109,9 @@
        "translate": "Übersetzen",
        "light": "hell",
        "dark": "dunkel",
-        "system": "Systemeinstellung"
+        "system": "Systemeinstellung",
+        "ratelimit": "Instanz wurde ratenbegrenzt",
+        "continue-search": "Setzen Sie Ihre Suche fort mit "
    },
    "lang_es": {
        "search": "Buscar",
@ -141,7 +147,9 @@
        "translate": "traducir",
        "light": "brillante",
        "dark": "oscuro",
-        "system": "configuración del sistema"
+        "system": "configuración del sistema",
+        "ratelimit": "La instancia ha sido ratelimited",
+        "continue-search": "Continúe su búsqueda con "
    },
    "lang_it": {
        "search": "Cerca",
@ -177,7 +185,9 @@
        "translate": "tradurre",
        "light": "luminoso",
        "dark": "notte",
-        "system": "impostazioni di sistema"
+        "system": "impostazioni di sistema",
+        "ratelimit": "L'istanza è stata limitata alla velocità",
+        "continue-search": "Continua la tua ricerca con "
    },
    "lang_pt": {
        "search": "Pesquisar",
@ -213,7 +223,9 @@
        "translate": "traduzir",
        "light": "brilhante",
        "dark": "escuro",
-        "system": "configuração de sistema"
+        "system": "configuração de sistema",
+        "ratelimit": "A instância foi limitada pela taxa",
+        "continue-search": "Continue sua pesquisa com "
    },
    "lang_ru": {
        "search": "Поиск",
@ -249,7 +261,9 @@
        "translate": "перевести",
        "light": "светлое",
        "dark": "темное",
-        "system": "системное"
+        "system": "системное",
+        "ratelimit": "Число экземпляров ограничено",
+        "continue-search": "Продолжайте поиск с "
    },
    "lang_zh-CN": {
        "search": "搜索",
@ -285,7 +299,9 @@
        "translate": "翻译",
        "light": "明亮的",
        "dark": "黑暗的",
-        "system": "系统设置"
+        "system": "系统设置",
+        "ratelimit": "实例已被限速",
+        "continue-search": "继续搜索 "
    },
    "lang_si": {
        "search": "සොයන්න",
@ -321,7 +337,9 @@
        "translate": "පරිවර්තනය කරන්න",
        "light": "දීප්තිමත්",
        "dark": "අඳුරු",
-        "system": "පද්ධතිය"
+        "system": "පද්ධතිය",
+        "ratelimit": "උදාහරණය අනුපාත කර ඇත",
+        "continue-search": "සමඟ ඔබේ සෙවීම දිගටම කරගෙන යන්න"
    },
    "lang_fr": {
        "search": "Chercher",
@ -357,7 +375,9 @@
        "translate": "Traduire",
        "light": "clair",
        "dark": "sombre",
-        "system": "système"
+        "system": "système",
+        "ratelimit": "Le débit de l'instance a été limité",
+        "continue-search": "Continuez votre recherche avec "
    },
    "lang_fa": {
        "search": "جستجو",
@ -393,7 +413,9 @@
        "translate": "ترجمه",
        "light": "روشن",
        "dark": "تیره",
-        "system": "سیستم"
+        "system": "سیستم",
+        "ratelimit": "نمونه با نرخ محدود شده است",
+        "continue-search": "جستجوی خود را با "
    },
    "lang_cs": {
        "search": "Hledat",
@ -429,7 +451,9 @@
        "translate": "Přeložit",
        "light": "Světlý",
        "dark": "Tmavý",
-        "system": "Systémový"
+        "system": "Systémový",
+        "ratelimit": "Instance byla omezena sazbou",
+        "continue-search": "Pokračujte ve vyhledávání pomocí "
    },
    "lang_zh-TW": {
        "search": "搜尋",
@ -465,7 +489,9 @@
        "translate": "翻譯",
        "light": "明亮的",
        "dark": "黑暗的",
-        "system": "依系統"
+        "system": "依系統",
+        "ratelimit": "實例已被限速",
+        "continue-search": "繼續搜索 "
    },
    "lang_bg": {
        "search": "Търсене",
@ -501,7 +527,9 @@
        "translate": "превод",
        "light": "светла",
        "dark": "тъмна",
-        "system": "системна"
+        "system": "системна",
+        "ratelimit": "Екземплярът е с ограничена скорост",
+        "continue-search": "Продължете търсенето си с "
    },
    "lang_hi": {
        "search": "खोज",
@ -537,9 +565,11 @@
        "translate": "अनुवाद करना",
        "light": "रोशनी",
        "dark": "अंधेरा",
-        "system": "प्रणाली"
+        "system": "प्रणाली",
+        "ratelimit": "इंस्टेंस को सीमित कर दिया गया है",
+        "continue-search": "के साथ अपनी खोज जारी रखें "
    },
-        "lang_ja": {
+    "lang_ja": {
        "search": "検索",
        "config": "設定",
        "config-country": "国を設定する",
@ -573,6 +603,8 @@
        "translate": "翻訳",
        "light": "ライト",
        "dark": "ダーク",
-        "system": "自動"
-    } 
+        "system": "自動",
+        "ratelimit": "インスタンスはレート制限されています",
+        "continue-search": "で検索を続ける "
+    }
 }
--- a/app/templates/error.html
+++ b/app/templates/error.html
@ -1,6 +1,40 @@
-<h1>Error</h1>
-<hr>
-<p>
-    Error: "{{ error_message|safe }}"
-</p>
-<a href="/">Return Home</a>
+{% if config.theme %}
+    {% if config.theme == 'system' %}
+        <style>
+            @import "{{ cb_url('light-theme.css') }}" screen;
+            @import "{{ cb_url('dark-theme.css') }}" screen and (prefers-color-scheme: dark);
+        </style>
+    {% else %}
+        <link rel="stylesheet" href="{{ cb_url(config.theme + '-theme.css') }}"/>
+    {% endif %}
+{% else %}
+    <link rel="stylesheet" href="{{ cb_url(('dark' if config.dark else 'light') + '-theme.css') }}"/>
+{% endif %}
+<link rel="stylesheet" href="{{ cb_url('main.css') }}">
+<link rel="stylesheet" href="{{ cb_url('error.css') }}">
+<style>{{ config.style }}</style>
+<div>
+    <h1>Error</h1>
+    <p>
+        {{ error_message|safe }}
+    </p>
+    <hr>
+    <p>
+    {% if blocked is defined %}
+        <h4>{{ translation['continue-search'] }} <a class="link" href="https://github.com/benbusby/farside">Farside</a>!</h4>
+        Whoogle:
+        <br>
+        <a class="link-color" href="{{farside}}/whoogle/search?q={{query}}{{params}}">
+            {{farside}}/whoogle/search?q={{query}}{{params}}
+        </a>
+        <br><br>
+        Searx:
+        <br>
+        <a class="link-color" href="{{farside}}/searx/search?q={{query}}">
+            {{farside}}/searx/search?q={{query}}
+        </a>
+        <hr>
+    {% endif %}
+    </p>
+    <a class="link" href="home">Return Home</a>
+</div>
--- a/app/templates/footer.html
+++ b/app/templates/footer.html
@ -1,9 +1,9 @@
 <footer>
    <p class="footer">
        Whoogle Search v{{ version_number }} ||
-        <a id="gh-link" href="https://github.com/benbusby/whoogle-search">{{ translation['github-link'] }}</a>
+        <a class="link" href="https://github.com/benbusby/whoogle-search">{{ translation['github-link'] }}</a>
        {% if newest_version %}
             || <span class="update_available">Update Available 🟢</span>
        {% endif %}
    </p>
-</footer>
+</footer>
--- a/app/templates/index.html
+++ b/app/templates/index.html
@ -86,15 +86,15 @@
            <div class="config-fields">
                <form id="config-form" action="config" method="post">
                    <div class="config-options">
-                        <div class="config-div config-div-ctry">
-                            <label for="config-ctry">{{ translation['config-country'] }}: </label>
-                            <select name="ctry" id="config-ctry">
-                                {% for ctry in countries %}
-                                    <option value="{{ ctry.value }}"
-                                            {% if ctry.value in config.ctry %}
+                        <div class="config-div config-div-country">
+                            <label for="config-country">{{ translation['config-country'] }}: </label>
+                            <select name="country" id="config-country">
+                                {% for country in countries %}
+                                    <option value="{{ country.value }}"
+                                            {% if country.value in config.country %}
                                            selected
                                            {% endif %}>
-                                        {{ ctry.name }}
+                                        {{ country.name }}
                                    </option>
                                {% endfor %}
                            </select>
--- a/app/utils/results.py
+++ b/app/utils/results.py
@ -24,14 +24,14 @@ BLACKLIST = [
 ]

 SITE_ALTS = {
-    'twitter.com': os.getenv('WHOOGLE_ALT_TW', 'nitter.net'),
-    'youtube.com': os.getenv('WHOOGLE_ALT_YT', 'invidious.snopyta.org'),
-    'instagram.com': os.getenv('WHOOGLE_ALT_IG', 'bibliogram.art/u'),
-    'reddit.com': os.getenv('WHOOGLE_ALT_RD', 'libredd.it'),
+    'twitter.com': os.getenv('WHOOGLE_ALT_TW', 'farside.link/nitter'),
+    'youtube.com': os.getenv('WHOOGLE_ALT_YT', 'farside.link/invidious'),
+    'instagram.com': os.getenv('WHOOGLE_ALT_IG', 'farside.link/bibliogram/u'),
+    'reddit.com': os.getenv('WHOOGLE_ALT_RD', 'farside.link/libreddit'),
    **dict.fromkeys([
        'medium.com',
        'levelup.gitconnected.com'
-    ], os.getenv('WHOOGLE_ALT_MD', 'scribe.rip'))
+    ], os.getenv('WHOOGLE_ALT_MD', 'farside.link/scribe'))
 }


--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -31,13 +31,13 @@ services:
      #- WHOOGLE_PROXY_LOC=<proxy host/ip>
      # Site alternative configurations, uncomment to enable
      # Note: If not set, the feature will still be available
-      # with default values. 
-      #- WHOOGLE_ALT_TW=nitter.net
-      #- WHOOGLE_ALT_YT=invidious.snopyta.org
-      #- WHOOGLE_ALT_IG=bibliogram.art/u
-      #- WHOOGLE_ALT_RD=libredd.it
+      # with default values.
+      #- WHOOGLE_ALT_TW=farside.link/nitter
+      #- WHOOGLE_ALT_YT=farside.link/invidious
+      #- WHOOGLE_ALT_IG=farside.link/bibliogram/u
+      #- WHOOGLE_ALT_RD=farside.link/libreddit
+      #- WHOOGLE_ALT_MD=farside.link/scribe
      #- WHOOGLE_ALT_TL=lingva.ml
-      #- WHOOGLE_ALT_MD=scribe.rip
    #env_file: # Alternatively, load variables from whoogle.env
      #- whoogle.env
    ports:
--- a/test/conftest.py
+++ b/test/conftest.py
@ -9,7 +9,7 @@ demo_config = {
    'nojs': str(random.getrandbits(1)),
    'lang_interface': random.choice(app.config['LANGUAGES'])['value'],
    'lang_search': random.choice(app.config['LANGUAGES'])['value'],
-    'ctry': random.choice(app.config['COUNTRIES'])['value']
+    'country': random.choice(app.config['COUNTRIES'])['value']
 }


--- a/test/test_results.py
+++ b/test/test_results.py
@ -86,20 +86,6 @@ def test_block_results(client):
        assert 'pinterest.com' not in urlparse(link['href']).netloc


-# TODO: Unit test the site alt method instead -- the results returned
-# are too unreliable for this test in particular.
-# def test_site_alts(client):
-    # rv = client.post('/search', data=dict(q='twitter official account'))
-    # assert b'twitter.com/Twitter' in rv.data
-
-    # client.post('/config', data=dict(alts=True))
-    # assert json.loads(client.get('/config').data)['alts']
-
-    # rv = client.post('/search', data=dict(q='twitter official account'))
-    # assert b'twitter.com/Twitter' not in rv.data
-    # assert b'nitter.net/Twitter' in rv.data
-
-
 def test_recent_results(client):
    times = {
        'past year': 365,
--- a/whoogle.template.env
+++ b/whoogle.template.env
@ -7,12 +7,12 @@
 #  - docker-compose: Uncomment the env_file option
 #  - docker: Add "--env-file ./whoogle.env" to your build command

-#WHOOGLE_ALT_TW=nitter.net
-#WHOOGLE_ALT_YT=invidious.snopyta.org
-#WHOOGLE_ALT_IG=bibliogram.art/u
-#WHOOGLE_ALT_RD=libredd.it
+#WHOOGLE_ALT_TW=farside.link/nitter
+#WHOOGLE_ALT_YT=farside.link/invidious
+#WHOOGLE_ALT_IG=farside.link/bibliogram/u
+#WHOOGLE_ALT_RD=farside.link/libreddit
+#WHOOGLE_ALT_MD=farside.link/scribe
 #WHOOGLE_ALT_TL=lingva.ml
-#WHOOGLE_ALT_MD=scribe.rip
 #WHOOGLE_USER=""
 #WHOOGLE_PASS=""
 #WHOOGLE_PROXY_USER=""
@ -56,7 +56,7 @@
 #WHOOGLE_CONFIG_NEW_TAB=1

 # Enable View Image option
-#WHOOGLE_CONFIG_VIEW_IMAGE=1 
+#WHOOGLE_CONFIG_VIEW_IMAGE=1

 # Search using GET requests only (exposes query in logs)
 #WHOOGLE_CONFIG_GET_ONLY=1