diff --git a/README.md b/README.md new file mode 100644 index 0000000..9dfbb01 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +The automated instances list for the mighty LibRedirect project! + +Just run: +```bash +$ python3 instances.py +``` \ No newline at end of file diff --git a/blacklist.json b/blacklist.json new file mode 100644 index 0000000..43738c1 --- /dev/null +++ b/blacklist.json @@ -0,0 +1,79 @@ +{ + "cloudflare": [ + "https://invidious.kavin.rocks", + "https://invidious.epicsite.xyz", + "https://yt.oelrichsgarcia.de", + "https://piped.video", + "https://piped.moomoo.me", + "https://piped.syncpundit.io", + "https://piped.mha.fi", + "https://piped.garudalinux.org", + "https://proxitok.marcopisco.com", + "https://proxitok.lunar.icu", + "https://send.q1q.wtf", + "https://transferer.seinetransfert.fr", + "https://nitter.domain.glass", + "https://birdsite.xanny.family", + "https://notabird.site", + "https://fuckthesacklers.network", + "https://nitter.winscloud.net", + "https://twitter.dr460nf1r3.org", + "https://nitter.garudalinux.org", + "https://nitter.catalyst.sx", + "https://nitter.rawbit.ninja", + "https://nitter.privacytools.io", + "https://nitter.sneed.network", + "https://n.sneed.network", + "https://nitter.twei.space", + "https://nitter.d420.de", + "https://nitter.marcopisco.com", + "https://nitter.lunar.icu", + "https://twitter.moe.ngo", + "https://nitter.freedit.eu", + "https://libreddit.eu.org", + "https://libreddit.domain.glass", + "https://r.nf", + "https://reddi.tk", + "https://r.walkx.fyi", + "https://reddit.dr460nf1r3.org", + "https://libreddit.cachyos.org", + "https://libreddit.freedit.eu", + "https://libreddit.garudalinux.org", + "https://www.futureddit.gq", + "https://lr.aeong.one", + "https://libreddit.lunar.icu", + "https://teddit.domain.glass", + "https://teddit.httpjames.space", + "https://teddit.garudalinux.org", + "https://tedd.it", + "https://teddit.privacytools.io", + "https://teddit.rawbit.ninja", + "https://scribe.rawbit.ninja", + "https://quetre.marcopisco.com", + "https://quetre.lunar.icu", + "https://libremdb.lunar.icu", + "https://lingva.garudalinux.org", + "https://translate.dr460nf1r3.org", + "https://libretranslate.com", + "https://search.garudalinux.org", + "https://search.dr460nf1r3.org", + "https://whoogle.lunar.icu", + "https://i.actionsack.com", + "https://rimgo.marcopisco.com", + "https://beatbump.ml", + "https://bin.klowdee.host", + "https://privatebin.io", + "https://bonus01.hwb0307.com", + "https://0.klowdee.host", + "https://sendpass.ru", + "https://0a0bin.klowdee.host", + "https://0.0g.gg", + "https://privload.com", + "https://andoc.yateam.cc", + "https://p.darklab.sh", + "https://luckvintage.com" + ], + "authenticate": [ + "https://nitter:nitter@nitter.nixnet.services" + ] +} \ No newline at end of file diff --git a/data.json b/data.json new file mode 100644 index 0000000..9194ac0 --- /dev/null +++ b/data.json @@ -0,0 +1,927 @@ +{ + "invidious": { + "clearnet": [ + "https://yewtu.be", + "https://vid.puffyan.us", + "https://inv.riverside.rocks", + "https://invidious.kavin.rocks", + "https://watch.thekitty.zone", + "https://y.com.sb", + "https://invidious.nerdvpn.de", + "https://yt.artemislena.eu", + "https://invidious.tiekoetter.com", + "https://invidious.flokinet.to", + "https://inv.vern.cc", + "https://inv.odyssey346.dev", + "https://invidious.snopyta.org", + "https://invidious.baczek.me", + "https://invidious.sethforprivacy.com", + "https://yt.funami.tech", + "https://invidious.drivet.xyz", + "https://invidious.silur.me", + "https://vid.priv.au", + "https://iv.ggtyler.dev", + "https://invidious.epicsite.xyz", + "https://yt.oelrichsgarcia.de", + "https://invidious.slipfox.xyz", + "https://invidious.dhusch.de", + "https://invidious.weblibre.org", + "https://invidious.esmailelbob.xyz", + "https://iv.melmac.space", + "https://invidious.privacydev.net", + "https://inv.bp.projectsegfau.lt", + "https://invidious.lidarshield.cloud", + "https://invidious.namazso.eu" + ], + "tor": [ + "http://w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd.onion", + "http://kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad.onion", + "http://grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad.onion", + "http://osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd.onion", + "http://u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad.onion", + "http://euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd.onion", + "http://invidious.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://inv.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd.onion", + "http://ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid.onion", + "http://iv.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd.onion", + "http://invidious.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion", + "http://c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid.onion" + ], + "i2p": [ + "http://verni6dr4qxjgjumnvesxerh5rvhv6oy5ddeibaqy5d7tgbiiyfa.b32.i2p" + ], + "loki": [] + }, + "piped": { + "clearnet": [ + "https://piped.video", + "https://piped.moomoo.me", + "https://piped.syncpundit.io", + "https://piped.mha.fi", + "https://watch.whatever.social", + "https://piped.garudalinux.org", + "https://yt.jae.fi", + "https://il.ax", + "https://piped.esmailelbob.xyz", + "https://piped.projectsegfau.lt", + "https://piped.privacydev.net", + "https://piped.palveluntarjoaja.eu", + "https://piped.smnz.de", + "https://piped.adminforge.de", + "https://piped.qdi.fi", + "https://piped.hostux.net", + "https://pd.vern.cc", + "https://piped.chauvet.pro" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "pipedMaterial": { + "clearnet": [ + "https://piped-material.xn--17b.net", + "https://piped-material.ftp.sh" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "cloudtube": { + "clearnet": [ + "https://tube.cadence.moe", + "https://tube.boritsch.de" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "proxiTok": { + "clearnet": [ + "https://proxitok.pabloferreiro.es", + "https://proxitok.pussthecat.org", + "https://tok.habedieeh.re", + "https://proxitok.esmailelbob.xyz", + "https://proxitok.privacydev.net", + "https://proxitok.odyssey346.dev", + "https://tok.artemislena.eu", + "https://tok.adminforge.de", + "https://proxitok.manasiwibi.com", + "https://tik.hostux.net", + "https://tt.vern.cc", + "https://proxitok.mha.fi", + "https://proxitok.pufe.org", + "https://proxitok.marcopisco.com", + "https://cringe.whatever.social", + "https://proxitok.lunar.icu" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "send": { + "clearnet": [ + "https://send.vis.ee", + "https://send.zcyph.cc", + "https://send.q1q.wtf", + "https://send.ephemeral.land", + "https://send.mni.li", + "https://send.monks.tools", + "https://send.boblorange.net", + "https://send.aurorabilisim.com", + "https://send.datahoarder.dev", + "https://fileupload.ggc-project.de", + "https://drop.chapril.org", + "https://send.jeugdhulp.be", + "https://files.psu.ru", + "https://send.portailpro.net", + "https://transfer.acted.org", + "https://send.epsilonrecords.nl", + "https://transfer.fm", + "https://send.datenpost.app", + "https://transferer.seinetransfert.fr" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "nitter": { + "clearnet": [ + "https://nitter.net", + "https://nitter.lacontrevoie.fr", + "https://nitter.pussthecat.org", + "https://nitter:nitter@nitter.nixnet.services", + "https://nitter.fdn.fr", + "https://nitter.1d4.us", + "https://nitter.kavin.rocks", + "https://nitter.unixfox.eu", + "https://nitter.domain.glass", + "https://nitter.namazso.eu", + "https://birdsite.xanny.family", + "https://nitter.hu", + "https://nitter.moomoo.me", + "https://bird.trom.tf", + "https://nitter.it", + "https://twitter.censors.us", + "https://nitter.grimneko.de", + "https://twitter.076.ne.jp", + "https://nitter.fly.dev", + "https://notabird.site", + "https://nitter.weiler.rocks", + "https://nitter.sethforprivacy.com", + "https://nitter.cutelab.space", + "https://nitter.nl", + "https://nitter.mint.lgbt", + "https://nitter.bus-hit.me", + "https://fuckthesacklers.network", + "https://nitter.esmailelbob.xyz", + "https://tw.artemislena.eu", + "https://nitter.winscloud.net", + "https://nitter.tiekoetter.com", + "https://nitter.spaceint.fr", + "https://nitter.privacy.com.de", + "https://nitter.poast.org", + "https://nitter.bird.froth.zone", + "https://nitter.dcs0.hu", + "https://twitter.dr460nf1r3.org", + "https://nitter.garudalinux.org", + "https://twitter.beparanoid.de", + "https://nitter.cz", + "https://nitter.privacydev.net", + "https://nitter.evil.site", + "https://tweet.lambda.dance", + "https://nitter.kylrth.com", + "https://nitter.foss.wtf", + "https://nitter.priv.pw", + "https://nitter.tokhmi.xyz", + "https://nitter.catalyst.sx", + "https://unofficialbird.com", + "https://nitter.projectsegfau.lt", + "https://singapore.unofficialbird.com", + "https://nitter.fprivacy.com", + "https://canada.unofficialbird.com", + "https://india.unofficialbird.com", + "https://nederland.unofficialbird.com", + "https://uk.unofficialbird.com", + "https://n.l5.ca", + "https://nitter.slipfox.xyz", + "https://nitter.soopy.moe", + "https://nitter.qwik.space", + "https://read.whatever.social", + "https://nitter.rawbit.ninja", + "https://nt.vern.cc", + "https://ntr.odyssey346.dev", + "https://nitter.ir", + "https://nitter.privacytools.io", + "https://nitter.sneed.network", + "https://n.sneed.network", + "https://nitter.manasiwibi.com", + "https://nitter.smnz.de", + "https://nitter.twei.space", + "https://nitter.inpt.fr", + "https://nitter.d420.de", + "https://nitter.caioalonso.com", + "https://nitter.at", + "https://nitter.drivet.xyz", + "https://nitter.pw", + "https://nitter.nicfab.eu", + "https://bird.habedieeh.re", + "https://nitter.hostux.net", + "https://nitter.adminforge.de", + "https://nitter.platypush.tech", + "https://nitter.mask.sh", + "https://nitter.pufe.org", + "https://nitter.us.projectsegfau.lt", + "https://nitter.marcopisco.com", + "https://nitter.arcticfoxes.net", + "https://t.com.sb", + "https://nitter.selfhostable.net", + "https://nitter.kling.gg", + "https://twt.funami.tech", + "http://nitter.ktachibana.party", + "https://nitter.riverside.rocks", + "https://bird.wilbvr.me", + "https://nitter.girlboss.ceo", + "https://nitter.lunar.icu", + "https://twitter.moe.ngo", + "https://nitter.freedit.eu" + ], + "tor": [ + "http://nitter7bryz3jv7e3uekphigvmoyoem4al3fynerxkj22dmoxoq553qd.onion", + "http://npf37k3mtzwxreiw52ccs5ay4e6qt2fkcs2ndieurdyn2cuzzsfyfvid.onion", + "http://26oq3gioiwcmfojub37nz5gzbkdiqp7fue5kvye7d4txv4ny6fb4wwid.onion", + "http://vfaomgh4jxphpbdfizkm5gbtjahmei234giqj4facbwhrfjtcldauqad.onion", + "http://iwgu3cv7ywf3gssed5iqtavmrlszgsxazkmwwnt4h2kdait75thdyrqd.onion", + "http://erpnncl5nhyji3c32dcfmztujtl3xaddqb457jsbkulq24zqq7ifdgad.onion", + "http://ckzuw5misyahmg7j5t5xwwuj3bwy62jfolxyux4brfflramzsvvd3syd.onion", + "http://jebqj47jgxleaiosfcxfibx2xdahjettuydlxbg64azd4khsxv6kawid.onion", + "http://nttr2iupbb6fazdpr2rgbooon2tzbbsvvkagkgkwohhodjzj43stxhad.onion", + "http://nitraeju2mipeziu2wtcrqsxg7h62v5y4eqgwi75uprynkj74gevvuqd.onion", + "http://nitter.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://ibsboeui2im5o7dxnik3s5yghufumgy5abevtij5nbizequfpu4qi4ad.onion", + "http://ec5nvbycpfa5k6ro77blxgkyrzbkv7uy6r5cngcbkadtjj2733nm3uyd.onion", + "http://twitter.ap3jtgssd7qn4vfoa62gi2ijddoblslbeuw5dan3xt7myvqpura5i4qd.onion", + "http://twitter.prnoid54e44a4bduq5due64jkk7wcnkxcp5kv3juncm7veptjcqudgyd.onion", + "http://nitter.micohauwkjbyw5meacrb4ipicwvwg4xtzl7y7viv53kig2mdcsvwkyyd.onion", + "http://fbdi323aj3t6hdx4jtkuwxrnafzskquciowu2yu3jizfssaq35aow6ad.onion", + "http://nitterqdyumlovt7tjqpdjrluitgmtpa53qq3idlpgoe4kxo7gs3xvad.onion", + "http://nitter.privpw3tndpkw6pnp3g727zfgfdzbu3k6a7chv226s3xymv2p4eiuqyd.onion", + "http://aaaaaxx74hbzeibageieowryhsqvimbdiitfkfe47tvgawkwkjh3coqd.onion", + "http://wiio4sgs4247brk7hj6qck2jxnvldwfdbguigc5ivpxrsegnliyfvuqd.onion", + "http://qwikxx2erhx6qrymued6ox2qkf2yeogjwypqvzoif4fqkljixasr6oid.onion", + "http://4g47cxugkohbweao2x66nnxxfoe3k7gdfzxej537nhdbwr522sbjxeqd.onion", + "http://nt.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://tw.lpoaj7z2zkajuhgnlltpeqh3zyq7wk2iyeggqaduhgxhyajtdt2j7wad.onion", + "http://r2eqimhkvxboaltbdsectoo3hkf476pyemsdykclexzajbfx5v6ojlyd.onion", + "http://li7snkj6oituazbkr5clmilccwwumhd2dntbhttxomy4dfakeeoar4qd.onion", + "http://a5xyidyppowvblric6k6nixgf2eqwnb7zzeaarj7slqbv7tb6ip5t3ad.onion", + "http://bird.habeehrhadazsw3izbrbilqajalfyqqln54mrja3iwpqxgcuxnus7eid.onion", + "http://codeine3hsqnnkb3dsu6ft4tunlomr3lmuml5hcoqmfkgiqfv2brdqqd.onion", + "http://nitter.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion", + "http://jdadsqkjereqronxzovgvd453snh5jsvo7mukfwt2feth353nobqotad.onion", + "http://oawb4ub2x2jehgg7k7ll4uoay34re5nbbnwnztpphdec6wkcxgarwbad.onion" + ], + "i2p": [ + "http://axd6uavsstsrvstva4mzlzh4ct76rc6zdug3nxdgeitrzczhzf4q.b32.i2p", + "http://u6ikd6zndl3c4dsdq4mmujpntgeevdk5qzkfb57r4tnfeccrn2qa.b32.i2p", + "http://gseczlzmiv23p5vhsktyd7whquq2uy3c5fgkmdohh453qp3daoua.b32.i2p", + "http://tm4rwkeysv3zz3q5yacyr4rlmca2c4etkdobfvuqzt6vsfsu4weq.b32.i2p", + "http://vernzdedoxuflrrxc4vbatbkpjh4k22ecgiqgimdiif62onhagva.b32.i2p", + "http://pxrhfmieeeqdu5pi4duscycsxqefqphaxgvvrvdtc3ckqn2mcora.b32.i2p" + ], + "loki": [ + "http://nitter.priv.loki" + ] + }, + "bibliogram": { + "clearnet": [ + "https://bibliogram.1d4.us", + "https://bibliogram.froth.zone", + "https://ig.tokhmi.xyz", + "https://ig.beparanoid.de", + "https://bibliogram.priv.pw" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "libreddit": { + "clearnet": [ + "https://libreddit.spike.codes", + "https://safereddit.com", + "https://libreddit.kavin.rocks", + "https://reddit.invak.id", + "https://libreddit.eu.org", + "https://reddit.simo.sh", + "https://lr.riverside.rocks", + "https://libreddit.strongthany.cc", + "https://libreddit.domain.glass", + "https://r.nf", + "https://reddit.stuehieyr.com", + "https://lr.mint.lgbt", + "https://libreddit.drivet.xyz", + "https://libreddit.pussthecat.org", + "https://libreddit.northboot.xyz", + "https://libreddit.totaldarkness.net", + "https://libreddit.esmailelbob.xyz", + "https://lr.vern.cc", + "https://libreddit.nl", + "https://reddi.tk", + "https://r.walkx.fyi", + "https://libreddit.kylrth.com", + "https://libreddit.tiekoetter.com", + "https://reddit.rtrace.io", + "https://libreddit.privacydev.net", + "https://r.ahwx.org", + "https://bob.fr.to", + "https://libreddit.dcs0.hu", + "https://reddit.dr460nf1r3.org", + "https://rd.jae.su", + "https://libreddit.foss.wtf", + "https://l.opnxng.com", + "https://libreddit.cachyos.org", + "https://rd.funami.tech", + "https://libreddit.projectsegfau.lt", + "https://lr.slipfox.xyz", + "https://libreddit.oxymagnesium.com", + "https://reddit.utsav2.dev", + "https://libreddit.freedit.eu", + "https://libreddit.mha.fi", + "https://libreddit.garudalinux.org", + "https://libreddit.mask.sh", + "https://www.futureddit.gq", + "https://lr.4201337.xyz", + "https://lr.odyssey346.dev", + "https://reddit.selfhostable.net", + "https://lr.artemislena.eu", + "https://discuss.whatever.social", + "https://libreddit.pufe.org", + "https://lr.aeong.one", + "https://reddit.smnz.de", + "https://libreddit.bus-hit.me", + "https://reddit.leptons.xyz", + "https://libreddit.lunar.icu" + ], + "tor": [ + "http://ojwp2gtj7dq7scd7gnbac6wp53tklgsicteabrnx2pr7zai64wriiaad.onion", + "http://ecue64ybzvn6vjzl37kcsnwt4ycmbsyf74nbttyg7rkc3t3qwnj7mcyd.onion", + "http://libreddit.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://ojwp2gtj7dq7scd7gnbac6wp53tklgsicteabrnx2pr7zai64wriiaad.onion", + "http://lr.lpoaj7z2zkajuhgnlltpeqh3zyq7wk2iyeggqaduhgxhyajtdt2j7wad.onion", + "http://libreddit.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion" + ], + "i2p": [], + "loki": [] + }, + "teddit": { + "clearnet": [ + "https://teddit.net", + "https://teddit.ggc-project.de", + "https://teddit.zaggy.nl", + "https://teddit.namazso.eu", + "https://teddit.tinfoil-hat.net", + "https://teddit.domain.glass", + "https://snoo.ioens.is", + "https://teddit.httpjames.space", + "https://incogsnoo.com", + "https://teddit.pussthecat.org", + "https://reddit.lol", + "https://teddit.sethforprivacy.com", + "https://teddit.totaldarkness.net", + "https://teddit.adminforge.de", + "https://teddit.bus-hit.me", + "https://teddit.froth.zone", + "https://rdt.trom.tf", + "https://teddit.encrypted-data.xyz", + "https://i.opnxng.com", + "https://teddit.tokhmi.xyz", + "https://teddit.garudalinux.org", + "https://tedd.it", + "https://teddit.privacytools.io", + "https://td.vern.cc", + "https://teddit.rawbit.ninja", + "https://teddit.artemislena.eu", + "https://teddit.manasiwibi.com", + "https://teddit.hostux.net" + ], + "tor": [ + "http://snoo.ioensistjs7wd746zluwixvojbbkxhr37lepdvwtdfeav673o64iflqd.onion", + "http://ibarajztopxnuhabfu7fg6gbudynxofbnmvis3ltj6lfx47b6fhrd5qd.onion", + "http://tedditfyn6idalzso5wam5qd3kdtxoljjhbrbbx34q2xkcisvshuytad.onion", + "http://dawtyi5e2cfyfmoht4izmczi42aa2zwh6wi34zwvc6rzf2acpxhrcrad.onion", + "http://qtpvyiaqhmwccxwzsqubd23xhmmrt75tdyw35kp43w4hvamsgl3x27ad.onion", + "http://jnuonmf2n36sfdmyksqqqyab3w63cq4kx24olyjleh5z6zzfvyt7uqqd.onion", + "http://td.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://yqu4yj5lju7bmlwpzpmltb5gsu6cw7nnbcxxx4iqemwa56nxjiggf4qd.onion", + "http://teddit.lpoaj7z2zkajuhgnlltpeqh3zyq7wk2iyeggqaduhgxhyajtdt2j7wad.onion", + "http://24fympskbrdgbf4afuvhqwwl2tv3y2vwxg5t2ktozd4j5b3fob5ntzyd.onion" + ], + "i2p": [ + "http://xugoqcf2pftm76vbznx4xuhrzyb5b6zwpizpnw2hysexjdn5l2tq.b32.i2p", + "http://teddit.i2p", + "http://vzeiwzi7ogwl3ijrfek4fbtwhvamxcpyqoc3s4vcgnhlp54s5clq.b32.i2p", + "http://verncco2oaxjikammz4pi7umzp673cme6zuemx7yeeewspwrw3va.b32.i2p" + ], + "loki": [] + }, + "scribe": { + "clearnet": [ + "https://scribe.rip", + "https://scribe.nixnet.services", + "https://scribe.citizen4.eu", + "https://scribe.bus-hit.me", + "https://scribe.froth.zone", + "https://scribe.esmailelbob.xyz", + "https://scribe.privacydev.net", + "https://scribe.rawbit.ninja", + "https://sc.vern.cc" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "quetre": { + "clearnet": [ + "https://quetre.iket.me", + "https://qr.vern.cc", + "https://quetre.pussthecat.org", + "https://quetre.tokhmi.xyz", + "https://quetre.projectsegfau.lt", + "https://quetre.esmailelbob.xyz", + "https://quetre.odyssey346.dev", + "https://quetre.privacydev.net", + "https://ask.habedieeh.re", + "https://quetre.marcopisco.com", + "https://quetre.blackdrgn.nl", + "https://quetre.pufe.org", + "https://quetre.lunar.icu", + "https://que.wilbvr.me" + ], + "tor": [ + "http://quetre.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://qr.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://ask.habeehrhadazsw3izbrbilqajalfyqqln54mrja3iwpqxgcuxnus7eid.onion", + "http://quetre.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion" + ], + "i2p": [ + "http://vernnflenvsqccuanaun7yydnmturi4jkyxlyzhn6ultpje66c3q.b32.i2p" + ], + "loki": [] + }, + "libremdb": { + "clearnet": [ + "https://libremdb.iket.me", + "https://libremdb.pussthecat.org", + "https://libremdbeu.herokuapp.com", + "https://lmdb.tokhmi.xyz", + "https://libremdb.esmailelbob.xyz", + "https://ld.vern.cc", + "https://binge.whatever.social", + "https://libremdb.lunar.icu" + ], + "tor": [ + "http://libremdb.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://ld.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" + ], + "i2p": [ + "http://vernz3ubrntql4wrgyrssd6u3qzi36zrhz2agbo6vibzbs5olk2q.b32.i2p" + ], + "loki": [] + }, + "simplyTranslate": { + "clearnet": [ + "https://simplytranslate.org", + "https://st.tokhmi.xyz", + "https://translate.josias.dev", + "https://translate.namazso.eu", + "https://translate.riverside.rocks", + "https://simplytranslate.manerakai.com", + "https://translate.bus-hit.me", + "https://simplytranslate.pussthecat.org", + "https://translate.northboot.xyz", + "https://translate.tiekoetter.com", + "https://simplytranslate.esmailelbob.xyz", + "https://tl.vern.cc", + "https://translate.slipfox.xyz", + "https://st.privacydev.net", + "https://translate.priv.pw", + "https://st.odyssey346.dev" + ], + "tor": [ + "http://fyng2tsmzmvxmojzbbwmfnsn2lrcyftf4cw6rk5j2v2huliazud3fjid.onion", + "http://xxtbwyb5z5bdvy2f6l2yquu5qilgkjeewno4qfknvb3lkg3nmoklitid.onion", + "http://translate.prnoid54e44a4bduq5due64jkk7wcnkxcp5kv3juncm7veptjcqudgyd.onion", + "http://simplytranslate.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://tl.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://st.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion" + ], + "i2p": [ + "http://kmnkkp4v4ublygtnyq46oacy3qw75nrxxniriuaersumw7bfsm2a.b32.i2p", + "http://verna7avzgd4qqal7k2onjzwxcceqby2gwvya2a2frdswb7z2k4q.b32.i2p" + ], + "loki": [ + "http://translate.priv.loki" + ] + }, + "lingva": { + "clearnet": [ + "https://lingva.ml", + "https://translate.igna.wtf", + "https://translate.plausibility.cloud", + "https://translate.projectsegfau.lt", + "https://lingva.garudalinux.org", + "https://translate.dr460nf1r3.org", + "https://translate.jae.fi" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "libreTranslate": { + "clearnet": [ + "https://libretranslate.com", + "https://libretranslate.de", + "https://translate.argosopentech.com", + "https://translate.api.skitzen.com", + "https://translate.fortytwo-it.com", + "https://translate.terraprint.co", + "https://lt.vern.cc" + ], + "tor": [ + "http://lt.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" + ], + "i2p": [ + "http://vernf45n7mxwqnp5riaax7p67pwcl7wcefdcnqqvim7ckdx4264a.b32.i2p" + ], + "loki": [] + }, + "searx": { + "clearnet": [ + "https://jsearch.pw", + "https://search.kvj.ovh", + "https://search.stinpriza.org", + "https://search.trom.tf", + "https://searx.bissisoft.com", + "https://searx.divided-by-zero.eu", + "https://searx.dresden.network", + "https://searx.fossencdi.org", + "https://searx.gnu.style", + "https://searx.mastodontech.de", + "https://searx.mxchange.org", + "https://searx.nakhan.net", + "https://searx.nixnet.services", + "https://searx.roflcopter.fr", + "https://searx.ru", + "https://searx.semipvt.com", + "https://searx.sp-codes.de", + "https://searx.stuehieyr.com", + "https://searx.tyil.nl", + "https://searx.webheberg.info", + "https://searx.win", + "https://searx.xyz", + "https://searx.zapashcanon.fr", + "https://sh0.it", + "https://spot.murena.io", + "https://suche.tromdienste.de", + "https://sx.catgirl.cloud" + ], + "tor": [ + "http://3afisqjw2rxm6z7mmstyt5rx75qfqrgxnkzftknbp2vhipr2nrmrjdyd.onion", + "http://z5vawdol25vrmorm4yydmohsd4u6rdoj2sylvoi3e3nqvxkvpqul7bqd.onion", + "http://zbuc3bbzbfdqqo2x46repx2ddajbha6fpsjeeptjhhhhzji3zopxdqyd.onion", + "http://f4qfqajszpx5b7itzxt6mb7kj4ktpgbdq7lq6xaiqyqx6a7de3epptad.onion", + "http://searx.micohauwkjbyw5meacrb4ipicwvwg4xtzl7y7viv53kig2mdcsvwkyyd.onion" + ], + "i2p": [ + "http://ransack.i2p", + "http://mqamk4cfykdvhw5kjez2gnvse56gmnqxn7vkvvbuor4k4j2lbbnq.b32.i2p" + ], + "loki": [] + }, + "searxng": { + "clearnet": [ + "https://baresearch.org", + "https://coppedge.info", + "https://darmarit.org/searx", + "https://dynabyte.ca", + "https://etsi.me", + "https://gsch.eu", + "https://intelwinds.com", + "https://jackgoss.xyz", + "https://northboot.xyz", + "https://notsearch.uk", + "https://ooglester.com", + "https://opnxng.com", + "https://paulgo.io", + "https://priv.au", + "https://privatesearch.dev", + "https://privatus.live", + "https://s.frlt.one", + "https://s.trung.fun", + "https://s.zhaocloud.net", + "https://search.0relay.com", + "https://search.0xgingi.com", + "https://search.affusio.com", + "https://search.bus-hit.me", + "https://search.charleseroop.com", + "https://search.chemicals-in-the-water.eu", + "https://search.cronobox.one", + "https://search.disroot.org", + "https://search.drivet.xyz", + "https://search.gcomm.ch", + "https://search.kiwitalk.de", + "https://search.leptons.xyz", + "https://search.mdosch.de", + "https://search.mpx.wtf", + "https://search.neet.works", + "https://search.ononoki.org", + "https://search.privacyguides.net", + "https://search.projectsegfau.lt", + "https://search.rabbit-company.com", + "https://search.rhscz.eu", + "https://search.rowie.at", + "https://search.sapti.me", + "https://search.serginho.dev", + "https://search.smnz.de", + "https://search.suenram.us", + "https://search.unlocked.link", + "https://search.us.projectsegfau.lt", + "https://search.uspersec.com", + "https://search.zzls.xyz", + "https://searx.baczek.me", + "https://searx.be", + "https://searx.becomesovran.com", + "https://searx.catfluori.de", + "https://searx.chocoflan.net", + "https://searx.cthd.icu", + "https://searx.delicta.pp.ua", + "https://searx.ericaftereric.top", + "https://searx.esmailelbob.xyz", + "https://searx.fi", + "https://searx.fmac.xyz", + "https://searx.gnous.eu", + "https://searx.juancord.xyz", + "https://searx.mha.fi", + "https://searx.namejeff.xyz", + "https://searx.netzspielplatz.de", + "https://searx.oakleycord.dev", + "https://searx.orion-hub.fr", + "https://searx.priv.pw", + "https://searx.prvcy.eu", + "https://searx.rasp.fr", + "https://searx.sev.monster", + "https://searx.si", + "https://searx.slipfox.xyz/searx", + "https://searx.tiekoetter.com", + "https://searx.tuxcloud.net", + "https://searx.work", + "https://searxng.au", + "https://searxng.bandarawela.net", + "https://searxng.fingerbib.xyz", + "https://searxng.nicfab.eu", + "https://srx.cosmohub.io", + "https://swag.pw", + "https://sx.fukt.lol", + "https://www.gruble.de", + "https://www.jabber-germany.de/searx", + "https://xo.wtf" + ], + "tor": [ + "http://w5rl6wsd7mzj4bdkbuqvzidet5osdsm5jhg2f7nvfidakfq5exda5wid.onion", + "http://4n53nafyi77iplnbrpmxnp3x4exbswwxigujaxy3b37fvr7bvlopxeyd.onion", + "http://privateoz3u5utrimal2edr56j3r5caakektxxgixigdkycuxigvquid.onion", + "http://searxyqdegtqksicnwpxa24ipnb2ylmp6iutnnwtazbhup4dit4eeyqd.onion", + "http://searchvrzxoqa4k76duqh4wtzpr553hfl53qpytclxf7zci53lwklfyd.onion", + "http://gbat2pbpg7ys3fi3pbp64667tt5x66mg45xok35bxdw7v55brm7a27yd.onion", + "http://b2c7fvbhnfvwnl6oh2tydhzfx4i37kmmrycq42heqbbe7wovq3uzenid.onion", + "http://searxdr3pqz4nydgnqocsia2xbywptxbkympa2emn7zlgggrir4bkfad.onion", + "http://searx.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://searxfilowxokbogygrigir4wqxfxqzuxofxgdon7dg6rsii4yxzytyd.onion", + "http://searx.privpw3tndpkw6pnp3g727zfgfdzbu3k6a7chv226s3xymv2p4eiuqyd.onion", + "http://rq2w52kyrif3xpfihkgjnhqm3a5aqhoikpv72z3drpjglfzc2wr5z4yd.onion", + "http://fub6vgedgeadlu3ctskrpkcqjruh76tckwtj5swfhyblgml2tzgzckqd.onion/searx", + "http://searx3aolosaf3urwnhpynlhuokqsgz47si4pzz5hvb7uuzyjncl2tid.onion", + "https://lgmekfnpealdqhnctyg3p5fuelpdwnbqbwnl2ickpdoodzatzgz27ryd.onion" + ], + "i2p": [], + "loki": [] + }, + "whoogle": { + "clearnet": [ + "https://search.albony.xyz", + "https://search.garudalinux.org", + "https://search.dr460nf1r3.org", + "https://s.tokhmi.xyz", + "https://search.sethforprivacy.com", + "https://whoogle.dcs0.hu", + "https://whoogle.esmailelbob.xyz", + "https://gowogle.voring.me", + "https://whoogle.privacydev.net", + "https://wg.vern.cc", + "https://www.indexia.gq", + "https://whoogle.lunar.icu" + ], + "tor": [ + "http://whoglqjdkgt2an4tdepberwqz3hk7tjo4kqgdnuj77rt7nshw2xqhqad.onion", + "http://nuifgsnbb2mcyza74o7illtqmuaqbwu4flam3cdmsrnudwcmkqur37qd.onion", + "http://whoogle.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://whoogle.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion" + ], + "i2p": [ + "http://verneks7rfjptpz5fpii7n7nrxilsidi2qxepeuuf66c3tsf4nhq.b32.i2p" + ], + "loki": [] + }, + "librex": { + "clearnet": [ + "https://librex.beparanoid.de", + "https://lx.vern.cc", + "https://lx.sesu.cc", + "https://librex.mikata.ru", + "https://librex.terryiscool160.xyz", + "https://search.davidovski.xyz", + "https://search.funami.tech", + "https://search.madreyk.xyz", + "https://search.pabloferreiro.es", + "https://buscar.weblibre.org", + "https://search.ahwx.org", + "https://librex.pufe.org", + "https://search.milivojevic.in.rs", + "https://librex.ratakor.com", + "https://search.tildevarsh.in", + "https://librex.yogeshlamichhane.com.np", + "https://librex.myroware.eu" + ], + "tor": [ + "http://librex.2356uhnbpv5nk3bni5bv6jg2cd6lgj664kwx3lhyelstpttpyv4kk2qd.onion", + "http://lx.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://lx.swxoebbpeqiiixyhbuh3vbw53pdrmtbiaj2sqveol6kkn5rpapfi4aad.onion", + "http://f7ssz7l3biu4fugwctfpcx4txg5yq4gqhrt473ledsuc3ivtd3omniid.onion", + "http://librex.n53wt4ivvfdfaqkwldgdzfsubszukie2an6auja6x2wp3e3oa7v2gqyd.onion" + ], + "i2p": [ + "http://vernziqfqvweijfaacmwazohgpdo2bt2ib2jlupt2pwwu27bhgxq.b32.i2p", + "http://37bw4xelfgivsih5s4s5cozkxkhgoeqx5bjfjnrnmv5utdtk45sa.b32.i2p" + ], + "loki": [] + }, + "rimgo": { + "clearnet": [ + "https://i.bcow.xyz", + "https://rimgo.pussthecat.org", + "https://rimgo.totaldarkness.net", + "https://rimgo.bus-hit.me", + "https://rimgo.esmailelbob.xyz", + "https://i.actionsack.com", + "https://rimgo.privacydev.net", + "https://imgur.artemislena.eu", + "https://rimgo.vern.cc", + "https://rimgo.hostux.net", + "https://ri.zzls.xyz", + "https://rimgo.marcopisco.com" + ], + "tor": [ + "http://rimgo.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://rimgo.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" + ], + "i2p": [], + "loki": [] + }, + "librarian": { + "clearnet": [ + "https://lbry.bcow.xyz", + "https://odysee.076.ne.jp", + "https://librarian.pussthecat.org", + "https://lbry.projectsegfau.lt", + "https://librarian.esmailelbob.xyz", + "https://lbry.vern.cc", + "https://lbry.slipfox.xyz", + "https://lbry.mywire.org" + ], + "tor": [ + "http://librarian.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://lbry.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion", + "http://5znbzx2xcymhddzekfjib3isgqq4ilcyxa2bsq6vqmnvbtgu4f776lqd.onion", + "http://zjmls4urccc5wxc64ilbv4oqarjab6pbanvmhd2qqt6p7fgqrjjnryqd.onion", + "http://bxewpsswttslepw27w2hhxhlizwm7l7y54x3jw5cfrb64hb6lgc557ad.onion" + ], + "i2p": [], + "loki": [] + }, + "beatbump": { + "clearnet": [ + "https://beatbump.ml", + "https://beatbump.esmailelbob.xyz" + ], + "tor": [ + "http://beatbump.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion" + ], + "i2p": [], + "loki": [] + }, + "hyperpipe": { + "clearnet": [ + "https://hyperpipe.surge.sh", + "https://hyperpipe.esmailelbob.xyz", + "https://listen.whatever.social", + "https://music.adminforge.de" + ], + "tor": [ + "http://hyperpipe.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion" + ], + "i2p": [], + "loki": [] + }, + "facil": { + "clearnet": [ + "https://facilmap.org" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "osm": { + "clearnet": [ + "https://www.openstreetmap.org" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "simpleertube": { + "clearnet": [ + "https://tube.simple-web.org", + "https://simpleertube.esmailelbob.xyz", + "https://stube.tokhmi.xyz", + "https://st.vern.cc" + ], + "tor": [ + "http://simpleertube.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion", + "http://st.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion" + ], + "i2p": [ + "http://verntblehhel4dx6ntyxkj4mryvic3s3yptzxewzwnldq3esi4qq.b32.i2p" + ], + "loki": [] + }, + "breezeWiki": { + "clearnet": [ + "https://breezewiki.com", + "https://breezewiki.pussthecat.org", + "https://bw.odyssey346.dev", + "https://bw.vern.cc", + "https://breezewiki.esmailelbob.xyz", + "https://bw.artemislena.eu" + ], + "tor": [], + "i2p": [], + "loki": [] + }, + "privateBin": { + "clearnet": [ + "https://bin.bus-hit.me", + "https://0.jaegers.net", + "https://privatebin.net", + "https://paste.0xfc.de", + "https://bin.klowdee.host", + "https://paste.ononoki.org", + "https://paste.aryu.de", + "https://privatebin.io", + "https://cpaste.org", + "https://pastebin.hot-chilli.net", + "https://bonus01.hwb0307.com", + "https://paste.momobako.com", + "https://0.klowdee.host", + "https://sendpass.ru", + "https://0a0bin.klowdee.host", + "https://0.0g.gg", + "https://paste.hostux.net", + "https://pastebin.grey.pw", + "https://paste.ottertelecom.com", + "https://privatebin.at", + "https://snip.dssr.ch", + "https://bin.foss.wtf", + "https://paste.tildeverse.org", + "https://secrets.true.nl", + "https://secrets.cebador.fr", + "https://pastebin.linqhost.nl", + "https://paste.itefix.net", + "https://paste.tuxcloud.net", + "https://privatebin.rockhost.se", + "https://bin.0xfc.de", + "https://paste.07f.de", + "https://notebin.de", + "https://privload.com", + "https://privatebin.wsmm.de", + "https://vip.ci", + "https://andoc.yateam.cc", + "https://pb.vern.cc", + "https://paste.techbag.app", + "https://p.darklab.sh", + "https://luckvintage.com" + ], + "tor": [], + "i2p": [], + "loki": [] + } +} \ No newline at end of file diff --git a/fixed/beatbump.json b/fixed/beatbump.json new file mode 100644 index 0000000..c5c7812 --- /dev/null +++ b/fixed/beatbump.json @@ -0,0 +1,6 @@ +{ + "clearnet": ["https://beatbump.ml", "https://beatbump.esmailelbob.xyz"], + "tor": ["http://beatbump.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion"], + "i2p": [], + "loki": [] +} diff --git a/fixed/bibliogram.json b/fixed/bibliogram.json new file mode 100644 index 0000000..b08aa7a --- /dev/null +++ b/fixed/bibliogram.json @@ -0,0 +1,6 @@ +{ + "clearnet": ["https://bibliogram.1d4.us", "https://bibliogram.froth.zone", "https://ig.tokhmi.xyz", "https://ig.beparanoid.de", "https://bibliogram.priv.pw"], + "tor": [], + "i2p": [], + "loki": [] +} diff --git a/fixed/cloudtube.json b/fixed/cloudtube.json new file mode 100644 index 0000000..6372158 --- /dev/null +++ b/fixed/cloudtube.json @@ -0,0 +1,6 @@ +{ + "clearnet": ["https://tube.cadence.moe", "https://tube.boritsch.de"], + "tor": [], + "i2p": [], + "loki": [] +} diff --git a/fixed/facil.json b/fixed/facil.json new file mode 100644 index 0000000..6c89eb1 --- /dev/null +++ b/fixed/facil.json @@ -0,0 +1,6 @@ +{ + "clearnet": ["https://facilmap.org"], + "tor": [], + "i2p": [], + "loki": [] +} diff --git a/fixed/osm.json b/fixed/osm.json new file mode 100644 index 0000000..715fbad --- /dev/null +++ b/fixed/osm.json @@ -0,0 +1,8 @@ +{ + "clearnet": [ + "https://www.openstreetmap.org" + ], + "tor": [], + "i2p": [], + "loki": [] +} \ No newline at end of file diff --git a/instances.py b/instances.py new file mode 100755 index 0000000..e7d8467 --- /dev/null +++ b/instances.py @@ -0,0 +1,518 @@ +#!/usr/bin/python3 + +import traceback +import logging +import requests +import json +from urllib.parse import urlparse +import re +from colorama import Fore, Style +import socket + +mightyList = {} +networks = {} + +startRegex = r"https?:\/{2}(?:[^\s\/]+\.)*" +endRegex = "(?:\/[^\s\/]+)*\/?" +torRegex = startRegex + "onion" + endRegex +i2pRegex = startRegex + "i2p" + endRegex +lokiRegex = startRegex + "loki" + endRegex +authRegex = r"https?:\/{2}\S+:\S+@(?:[^\s\/]+\.)*[a-zA-Z0-9]+" + endRegex + +# 2.0 because Libredirect is currently on version 2.x.x +headers = {'User-Agent': 'Libredirect-instance-fetcher/2.0'} + +with open('networks.json', 'rt') as tmp: + networks = json.load(tmp) + + +def filterLastSlash(urlList): + tmp = {} + for frontend in urlList: + tmp[frontend] = {} + for network in urlList[frontend]: + tmp[frontend][network] = [] + for url in urlList[frontend][network]: + if url.endswith('/'): + tmp[frontend][network].append(url[:-1]) + print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url) + else: + tmp[frontend][network].append(url) + return tmp + + +def idnaEncode(urlList): + tmp = {} + for frontend in urlList: + tmp[frontend] = {} + for network in urlList[frontend]: + tmp[frontend][network] = [] + for url in urlList[frontend][network]: + try: + encodedUrl = url.encode("idna").decode("utf8") + tmp[frontend][network].append(encodedUrl) + if (encodedUrl != url): + print(Fore.YELLOW + "Fixed " + Style.RESET_ALL + url) + except Exception: + tmp[frontend][network].append(url) + return tmp + + +def ip2bin(ip): return "".join( + map( + str, + [ + "{0:08b}".format(int(x)) for x in ip.split(".") + ] + ) +) + + +def get_cloudflare_ips(): + r = requests.get('https://www.cloudflare.com/ips-v4') + return r.text.split('\n') + + +cloudflare_ips = get_cloudflare_ips() + + +def is_cloudflare(url): + instance_ip = None + try: + instance_ip = socket.gethostbyname(urlparse(url).hostname) + if instance_ip is None: + return False + except Exception: + return False + instance_bin = ip2bin(instance_ip) + + for cloudflare_ip_mask in cloudflare_ips: + cloudflare_ip = cloudflare_ip_mask.split('/')[0] + cloudflare_bin = ip2bin(cloudflare_ip) + + mask = int(cloudflare_ip_mask.split('/')[1]) + cloudflare_bin_masked = cloudflare_bin[:mask] + instance_bin_masked = instance_bin[:mask] + + if cloudflare_bin_masked == instance_bin_masked: + print(url + ' is behind ' + Fore.RED + + 'cloudflare' + Style.RESET_ALL) + return True + return False + + +def is_authenticate(url): + try: + if re.match(authRegex, url): + print(url + ' requires ' + Fore.RED + + 'authentication' + Style.RESET_ALL) + return True + r = requests.get(url, timeout=5, headers=headers) + if 'www-authenticate' in r.headers: + print(url + ' requires ' + Fore.RED + + 'authentication' + Style.RESET_ALL) + return True + except Exception: + return False + return False + + +def fetchCache(frontend, name): + try: + with open('./data.json') as file: + mightyList[frontend] = json.load(file)[frontend] + print(Fore.YELLOW + 'Failed' + Style.RESET_ALL + ' to fetch ' + name) + except Exception: + print(Fore.RED + 'Failed' + Style.RESET_ALL + ' to get cached ' + name) + + +def fetchFromFile(frontend, name): + with open('./fixed/' + frontend + '.json') as file: + mightyList[frontend] = json.load(file) + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + + +def fetchJsonList(frontend, name, url, urlItem, jsonObject): + try: + r = requests.get(url, headers=headers) + rJson = json.loads(r.text) + if jsonObject: + rJson = rJson['instances'] + _list = {} + for network in networks: + _list[network] = [] + if type(urlItem) == dict: + for item in rJson: + for network in networks: + if urlItem[network] is not None: + if urlItem[network] in item and item[urlItem[network]] is not None: + if item[urlItem[network]].strip() != '': + _list[network].append(item[urlItem[network]]) + else: + for item in rJson: + tmpItem = item + if urlItem is not None: + tmpItem = item[urlItem] + if tmpItem.strip() == '': + continue + elif re.search(torRegex, tmpItem): + _list['tor'].append(tmpItem) + elif re.search(i2pRegex, tmpItem): + _list['i2p'].append(tmpItem) + elif re.search(lokiRegex, tmpItem): + _list['loki'].append(tmpItem) + else: + _list['clearnet'].append(tmpItem) + + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) + + +def fetchRegexList(frontend, name, url, regex): + try: + r = requests.get(url, headers=headers) + _list = {} + for network in networks: + _list[network] = [] + + tmp = re.findall(regex, r.text) + + for item in tmp: + if item.strip() == "": + continue + elif re.search(torRegex, item): + _list['tor'].append(item) + elif re.search(i2pRegex, item): + _list['i2p'].append(item) + elif re.search(lokiRegex, item): + _list['loki'].append(item) + else: + _list['clearnet'].append(item) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) + + +def fetchTextList(frontend, name, url, prepend): + try: + _list = {} + for network in networks: + _list[network] = [] + + if type(url) == dict: + for network in networks: + if url[network] is not None: + r = requests.get(url[network], headers=headers) + tmp = r.text.strip().split('\n') + for item in tmp: + item = prepend[network] + item + _list[network].append(item) + else: + r = requests.get(url, headers=headers) + tmp = r.text.strip().split('\n') + + for item in tmp: + item = prepend + item + if re.search(torRegex, item): + _list['tor'].append(item) + elif re.search(i2pRegex, item): + _list['i2p'].append(item) + elif re.search(lokiRegex, item): + _list['loki'].append(item) + else: + _list['clearnet'].append(item) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) + + +def invidious(): + name = 'Invidious' + frontend = 'invidious' + url = 'https://api.invidious.io/instances.json' + try: + _list = {} + _list['clearnet'] = [] + _list['tor'] = [] + _list['i2p'] = [] + _list['loki'] = [] + r = requests.get(url, headers=headers) + rJson = json.loads(r.text) + for instance in rJson: + if instance[1]['type'] == 'https': + _list['clearnet'].append(instance[1]['uri']) + elif instance[1]['type'] == 'onion': + _list['tor'].append(instance[1]['uri']) + elif instance[1]['type'] == 'i2p': + _list['i2p'].append(instance[1]['uri']) + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) + + +def piped(): + frontend = 'piped' + name = 'Piped' + try: + _list = {} + _list['clearnet'] = [] + _list['tor'] = [] + _list['i2p'] = [] + _list['loki'] = [] + r = requests.get( + 'https://raw.githubusercontent.com/wiki/TeamPiped/Piped/Instances.md', headers=headers) + + tmp = re.findall( + r'(?:[^\s\/]+\.)+[a-zA-Z]+ (?:\(Official\) )?\| (https:\/{2}(?:[^\s\/]+\.)+[a-zA-Z]+) \| ', r.text) + for item in tmp: + try: + url = requests.get(item, timeout=5, headers=headers).url + if url.strip("/") == item: + continue + else: + _list['clearnet'].append(url) + except Exception: + logging.error(traceback.format_exc()) + continue + mightyList[frontend] = _list + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + name) + except Exception: + fetchCache(frontend, name) + logging.error(traceback.format_exc()) + + +def pipedMaterial(): + fetchRegexList('pipedMaterial', 'Piped-Material', 'https://raw.githubusercontent.com/mmjee/Piped-Material/master/README.md', + r"\| (https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]*) +\| Production") + + +def cloudtube(): + fetchFromFile('cloudtube', 'Cloudtube') + + +def proxitok(): + fetchRegexList('proxiTok', 'ProxiTok', 'https://raw.githubusercontent.com/wiki/pablouser1/ProxiTok/Public-instances.md', + r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)(?: \(Official\))? +\|(?:(?: [A-Z]*.*\|.*\|)|(?:$))") + + +def send(): + fetchRegexList('send', 'Send', 'https://gitlab.com/timvisee/send-instances/-/raw/master/README.md', + r"- ([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}") + + +def nitter(): + fetchRegexList('nitter', 'Nitter', 'https://raw.githubusercontent.com/wiki/zedeus/nitter/Instances.md', + r"(?:(?:\| )|(?:- ))\[(?:(?:\S+\.)+[a-zA-Z0-9]+)\/?\]\((https?:\/{2}(?:\S+\.)+[a-zA-Z0-9]+)\/?\)(?:(?: (?:\((?:\S+ ?\S*)\) )? *\| [^❌]{1,4} +\|(?:(?:\n)|(?: ❌)|(?: ✅)|(?: ❓)|(?: \[)))|(?:\n))") + + +def bibliogram(): + fetchFromFile('bibliogram', 'Bibliogram') + + +def libreddit(): + fetchJsonList('libreddit', 'Libreddit', 'https://github.com/libreddit/libreddit-instances/raw/master/instances.json', + {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, True) + + +def teddit(): + fetchJsonList('teddit', 'Teddit', 'https://codeberg.org/teddit/teddit/raw/branch/main/instances.json', + {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) + + +def scribe(): + fetchJsonList('scribe', 'Scribe', + 'https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json', None, False) + + +def quetre(): + fetchRegexList('quetre', 'Quetre', 'https://raw.githubusercontent.com/zyachel/quetre/main/README.md', + r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") + + +def libremdb(): + fetchRegexList('libremdb', 'libremdb', 'https://raw.githubusercontent.com/zyachel/libremdb/main/README.md', + r"\| \[.*\]\(([-a-zA-Z0-9@:%_\+.~#?&//=]{2,}\.[a-z0-9]{2,}\b(?:\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?)\)*\|*[A-Z]{0,}.*\|.*\|") + + +def simpleertube(): + fetchTextList('simpleertube', 'SimpleerTube', {'clearnet': 'https://simple-web.org/instances/simpleertube', 'tor': 'https://simple-web.org/instances/simpleertube_onion', + 'i2p': 'https://simple-web.org/instances/simpleertube_i2p', 'loki': None}, {'clearnet': 'https://', 'tor': 'http://', 'i2p': 'http://', 'loki': 'http://'}) + + +def simplytranslate(): + fetchTextList('simplyTranslate', 'SimplyTranslate', {'clearnet': 'https://simple-web.org/instances/simplytranslate', 'tor': 'https://simple-web.org/instances/simplytranslate_onion', + 'i2p': 'https://simple-web.org/instances/simplytranslate_i2p', 'loki': 'https://simple-web.org/instances/simplytranslate_loki'}, {'clearnet': 'https://', 'tor': 'http://', 'i2p': 'http://', 'loki': 'http://'}) + + +def linvgatranslate(): + fetchJsonList('lingva', 'LingvaTranslate', + 'https://raw.githubusercontent.com/TheDavidDelta/lingva-translate/main/instances.json', None, False) + + +def searx_searxng(): + r = requests.get( + 'https://searx.space/data/instances.json', headers=headers) + rJson = json.loads(r.text) + searxList = {} + searxList['clearnet'] = [] + searxList['tor'] = [] + searxList['i2p'] = [] + searxList['loki'] = [] + searxngList = {} + searxngList['clearnet'] = [] + searxngList['tor'] = [] + searxngList['i2p'] = [] + searxngList['loki'] = [] + for item in rJson['instances']: + if re.search(torRegex, item[:-1]): + if (rJson['instances'][item].get('generator') == 'searxng'): + searxngList['tor'].append(item[:-1]) + else: + searxList['tor'].append(item[:-1]) + elif re.search(i2pRegex, item[:-1]): + if (rJson['instances'][item].get('generator') == 'searxng'): + searxngList['i2p'].append(item[:-1]) + else: + searxList['i2p'].append(item[:-1]) + else: + if (rJson['instances'][item].get('generator') == 'searxng'): + searxngList['clearnet'].append(item[:-1]) + else: + searxList['clearnet'].append(item[:-1]) + + mightyList['searx'] = searxList + mightyList['searxng'] = searxngList + print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'SearX, SearXNG') + + +def whoogle(): + fetchRegexList('whoogle', 'Whoogle', 'https://raw.githubusercontent.com/benbusby/whoogle-search/main/README.md', + r"\| \[https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)*(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\) \| ") + + +def librex(): + fetchJsonList('librex', 'LibreX', 'https://raw.githubusercontent.com/hnhx/librex/main/instances.json', + {'clearnet': 'clearnet', 'tor': 'tor', 'i2p': 'i2p', 'loki': None}, True) + + +def rimgo(): + fetchJsonList('rimgo', 'rimgo', 'https://codeberg.org/video-prize-ranch/rimgo/raw/branch/main/instances.json', + {'clearnet': 'url', 'tor': 'onion', 'i2p': 'i2p', 'loki': None}, False) + + +def librarian(): + fetchJsonList('librarian', 'Librarian', + 'https://codeberg.org/librarian/librarian/raw/branch/main/instances.json', 'url', True) + + +def beatbump(): + fetchFromFile('beatbump', 'Beatbump') + + +def hyperpipe(): + fetchJsonList('hyperpipe', 'Hyperpipe', + 'https://codeberg.org/Hyperpipe/pages/raw/branch/main/api/frontend.json', 'url', False) + + +def facil(): + fetchFromFile('facil', 'FacilMap') + + +def osm(): + fetchFromFile('osm', 'OpenStreetMap') + + +def libreTranslate(): + fetchRegexList('libreTranslate', 'LibreTranslate', 'https://raw.githubusercontent.com/LibreTranslate/LibreTranslate/main/README.md', + r"\[(?:[^\s\/]+\.)+[a-zA-Z0-9]+\]\((https?:\/{2}(?:[^\s\/]+\.)+[a-zA-Z0-9]+)\/?\)\|") + + +def breezeWiki(): + fetchJsonList('breezeWiki', 'BreezeWiki', + 'https://docs.breezewiki.com/files/instances.json', 'instance', False) + + +def privateBin(): + fetchJsonList('privateBin', 'PrivateBin', + 'https://privatebin.info/directory/api?top=100&https_redirect=true&min_rating=A&csp_header=true&min_uptime=100&attachments=true', 'url', False) + + +def isValid(url): # This code is contributed by avanitrachhadiya2155 + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) + except Exception: + return False + + +invidious() +piped() +pipedMaterial() +cloudtube() +proxitok() +send() +nitter() +bibliogram() +libreddit() +teddit() +scribe() +quetre() +libremdb() +simplytranslate() +linvgatranslate() +libreTranslate() +searx_searxng() +whoogle() +librex() +rimgo() +librarian() +beatbump() +hyperpipe() +facil() +osm() +simpleertube() +breezeWiki() +privateBin() +mightyList = filterLastSlash(mightyList) +mightyList = idnaEncode(mightyList) + +cloudflare = [] +authenticate = [] +for k1, v1 in mightyList.items(): + if type(mightyList[k1]) is dict: + for k2, v2 in mightyList[k1].items(): + for instance in mightyList[k1][k2]: + if (not isValid(instance)): + mightyList[k1][k2].remove(instance) + print("removed " + instance) + else: + if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_cloudflare(instance): + cloudflare.append(instance) + if not instance.endswith('.onion') and not instance.endswith('.i2p') and not instance.endswith('.loki') and is_authenticate(instance): + authenticate.append(instance) +blacklist = { + 'cloudflare': cloudflare, + 'authenticate': authenticate, +} + +# Writing to file +json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) +with open('./data.json', 'w') as outfile: + outfile.write(json_object) +print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json') + +json_object = json.dumps(blacklist, ensure_ascii=False, indent=2) +with open('./blacklist.json', 'w') as outfile: + outfile.write(json_object) +print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/blacklist.json') + +# print(json_object) diff --git a/networks.json b/networks.json new file mode 100644 index 0000000..8a3546b --- /dev/null +++ b/networks.json @@ -0,0 +1,18 @@ +{ + "clearnet": { + "tld": "org", + "name": "Clearnet" + }, + "tor": { + "tld": "onion", + "name": "Tor" + }, + "i2p": { + "tld": "i2p", + "name": "I2P" + }, + "loki": { + "tld": "loki", + "name": "Lokinet" + } +} \ No newline at end of file