Separated SearX and SearXNG https://github.com/libredirect/browser_extension/issues/700

.github/workflows/main.yml

@@ -21,7 +21,7 @@ jobs:
       - name: install python packages
         run: |
           python -m pip install --upgrade pip
-          pip install requests colorama
+          pip install requests colorama pyyaml
       
       - name: run script
         run: python instances.py

instances.py

@@ -8,6 +8,7 @@ from urllib.parse import urlparse
 import re
 from colorama import Fore, Style
 import socket
+import yaml
 
 mightyList = {}
 networks = {}
@@ -335,42 +336,47 @@ def linvgatranslate():
         'lingva', 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', None, False)
 
 
-def searx_searxng():
+def searxng():
     r = requests.get(
         'https://searx.space/data/instances.json')
     rJson = json.loads(r.text)
-    searxList = {}
-    searxList['clearnet'] = []
-    searxList['tor'] = []
-    searxList['i2p'] = []
-    searxList['loki'] = []
     searxngList = {}
     searxngList['clearnet'] = []
     searxngList['tor'] = []
     searxngList['i2p'] = []
     searxngList['loki'] = []
-    for item in rJson['instances']:
-        if re.search(torRegex, item[:-1]):
-            if (rJson['instances'][item].get('generator') == 'searxng'):
-                searxngList['tor'].append(item[:-1])
-            else:
-                searxList['tor'].append(item[:-1])
-        elif re.search(i2pRegex, item[:-1]):
-            if (rJson['instances'][item].get('generator') == 'searxng'):
-                searxngList['i2p'].append(item[:-1])
-            else:
-                searxList['i2p'].append(item[:-1])
-        else:
-            if (rJson['instances'][item].get('generator') == 'searxng'):
-                searxngList['clearnet'].append(item[:-1])
-            else:
-                searxList['clearnet'].append(item[:-1])
+    
+
+    for item in rJson['instances']:
+        if re.search(torRegex, item[:-1] and rJson['instances'][item].get('generator') == 'searxng'):
+            searxngList['tor'].append(item[:-1])
+        elif re.search(i2pRegex, item[:-1] and rJson['instances'][item].get('generator') == 'searxng'):
+            searxngList['i2p'].append(item[:-1])
+        elif rJson['instances'][item].get('generator') == 'searxng':
+            searxngList['clearnet'].append(item[:-1])
 
-    mightyList['searx'] = searxList
     mightyList['searxng'] = searxngList
     print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearX, SearXNG')
 
 
+def searx():
+    searxList = {}
+    searxList['clearnet'] = []
+    searxList['tor'] = []
+    searxList['i2p'] = []
+    searxList['loki'] = []
+    r = requests.get(
+        'https://raw.githubusercontent.com/searx/searx-instances/master/searxinstances/instances.yml')
+    data = yaml.safe_load(r.text)
+    for key in data:
+        searxList['clearnet'].append(key)
+        if 'additional_urls' in data[key]:
+            for additional_url in data[key]['additional_urls']:
+                if data[key]['additional_urls'][additional_url] == "Hidden Service":
+                    searxList['tor'].append(additional_url)
+    mightyList['searx'] = searxList
+
+
 def whoogle():
     fetchRegexList('whoogle', 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md',
                    r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ")
@@ -501,6 +507,7 @@ def gothub():
         r"\| \[.*\]\((https:\/{2}.*?)\).*\|.*(?!No|Yes)"
     )
 
+
 def mikuInvidious():
     fetchFromFile('mikuInvidious')
 
@@ -517,6 +524,8 @@ wolfreeAlpha_url_list = [
     "https://uqq.gitlab.io"
 ]
 wolfreeAlpha_url_list_i = 0
+
+
 def wolfreeAlpha(i):
     global wolfreeAlpha_url_list_i
     frontend = 'wolfreeAlpha'
@@ -561,7 +570,8 @@ libremdb()
 simplytranslate()
 linvgatranslate()
 libreTranslate()
-searx_searxng()
+searxng()
+searx()
 whoogle()
 librex()
 rimgo()
@@ -585,7 +595,6 @@ mikuInvidious()
 wolfreeAlpha(wolfreeAlpha_url_list_i)
 
 
-
 mightyList = filterLastSlash(mightyList)
 mightyList = idnaEncode(mightyList)