From 0b6ffbd9a6291d21e0d9dd39b065350670a58668 Mon Sep 17 00:00:00 2001 From: Indrawan I Date: Sat, 11 Feb 2023 00:17:45 +0700 Subject: [PATCH] feat: simulating nhentai request on cloudflare enabled (#17) * fix apidocs definitions * add nhentaiStrategy method * add MaybeError types * add simulateCookie to janda class * test case * env example * push release --- .env.schema | 30 ++++-- README.md | 18 ++-- package.json | 10 +- src/JandaPress.ts | 77 +++++++++++-- src/controller/3hentai/3hentaiGet.ts | 2 +- src/controller/3hentai/3hentaiRandom.ts | 2 +- src/controller/3hentai/3hentaiSearch.ts | 2 +- src/controller/asmhentai/asmhentaiGet.ts | 2 +- src/controller/asmhentai/asmhentaiRandom.ts | 2 +- src/controller/asmhentai/asmhentaiSearch.ts | 2 +- src/controller/hentai2read/hentai2readGet.ts | 2 +- .../hentai2read/hentai2readSearch.ts | 2 +- src/controller/hentaifox/hentaifoxGet.ts | 2 +- src/controller/hentaifox/hentaifoxRandom.ts | 2 +- src/controller/hentaifox/hentaifoxSearch.ts | 2 +- src/controller/nhentai/nhentaiGet.ts | 20 ++-- src/controller/nhentai/nhentaiRandom.ts | 21 ++-- src/controller/nhentai/nhentaiRelated.ts | 20 ++-- src/controller/nhentai/nhentaiSearch.ts | 20 ++-- src/controller/pururin/pururinGet.ts | 2 +- src/controller/pururin/pururinRandom.ts | 2 +- src/controller/pururin/pururinSearch.ts | 2 +- .../simply-hentai/simply-hentaiGet.ts | 2 +- src/index.ts | 4 +- src/interfaces.ts | 4 + src/scraper/nhentai/nhentaiGetController.ts | 13 ++- .../nhentai/nhentaiRelatedController.ts | 1 + .../nhentai/nhentaiSearchController.ts | 6 +- src/utils/modifier.ts | 101 ++++++++++++++++-- test/nhentaiCookietest.ts | 10 +- 30 files changed, 253 insertions(+), 132 deletions(-) diff --git a/.env.schema b/.env.schema index 3e40038..d390a38 100644 --- a/.env.schema +++ b/.env.schema @@ -1,14 +1,22 @@ -# cloudflare stuff for testing nhentai: https://github.com/Zekfad/nhentai-api/issues/25#issuecomment-1141360074 -# you can skip this as it's no longer needed -CF_COOKIE= -CF= +# railway, fly.dev, heroku, vercel or any free service, NHENTAI_IP_ORIGIN should be true +RAILWAY = sinkaroid -# your username if it's on railway.app -RAILWAY=sinkaroid +# default port +PORT = 3000 -# jandapress config -PORT=3000 -REDIS_URL=redis://default:somenicepassword@redis-666.c10.us-east-6-6.ec666.cloud.redislabs.com:1337 +# backend storage, default is redis with this format +REDIS_URL = redis://default:somenicepassword@redis-666.c10.us-east-6-6.ec666.cloud.redislabs.com:1337 -# ttl for cache in a hour -EXPIRE_CACHE=1 +# ttl expire cache (in X hour) +EXPIRE_CACHE = 1 + +# nhentai strategy +# default is true which is assign to request on origin IP instead of nhentai.net with cloudflare protection +# if you have paid instance like vps you need chromium or firefox installed and set NHENTAI_IP_ORIGIN to false +NHENTAI_IP_ORIGIN = true + +# you must set COOKIE if NHENTAI_IP_ORIGIN is false, read the jandapress docs +COOKIE = "cf_clearance=l7RsUjiZ3LHAZZKcM7BcCylwD2agwPDU7l9zkg8MzPo-1676044652-0-250" + +# you must set USER_AGENT if NHENTAI_IP_ORIGIN is false, read the jandapress docs +USER_AGENT = "jandapress/1.0.5 Node.js/16.9.1" diff --git a/README.md b/README.md index 572b433..8570d66 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,11 @@ The motivation of this project is to bring you an actionable data related doujin - [Installation](#installation) - [Docker](#docker) - [Manual](#manual) + - [Nhentai guide](#limitations) - [Running tests](#running-tests) - [Playground](https://sinkaroid.github.io/jandapress) - [Routing](#playground) - [Status response](#status-response) - - [Limitations](#limitations) - [CLosing remarks](https://github.com/sinkaroid/jandapress/blob/master/CLOSING_REMARKS.md) - [Alternative links](https://github.com/sinkaroid/jandapress/blob/master/CLOSING_REMARKS.md#alternative-links) - [Pronunciation](#Pronunciation) @@ -83,6 +83,9 @@ REDIS_URL=redis://default:somenicepassword@someredishost:1337 ## the database ur EXPIRE_CACHE=1 ## a hour ``` +## Nhentai guide +Nhentai was cloudflare protection enabled, If IP and our thoughts against them? You should implement a proxy. Check [`cookie branch`](https://github.com/sinkaroid/jandapress/tree/cookie), take a look this workaround [Zekfad/nhentai-api/issues/25#issuecomment-1141360074](https://github.com/Zekfad/nhentai-api/issues/25#issuecomment-1141360074) + ### Docker docker pull ghcr.io/sinkaroid/jandapress:latest @@ -101,12 +104,6 @@ EXPIRE_CACHE=1 ## a hour ## Running tests -Jandapress depends on -- [express](https://github.com/expressjs/express) web api framework -- [keyv](https://github.com/jaredwray/keyv) key-value storage with support for multiple backends -- [cheerio](https://cheerio.js.org/) for parsing html -- [cors](https://github.com/expressjs/cors) middleware for enabling CORS -- [rate-limit](https://github.com/nfriedly/express-rate-limit) rate-limiting middleware for express ### Start the production server `npm run start:prod` @@ -216,11 +213,8 @@ The missing piece of 3hentai.net - https://sinkaroid.github.io/jandapress/#api-3 ## Status response HTTP/1.1 200 OK - HTTP/1.1 200 (cached) - HTTP/1.1 500 (bad parameters) - -## Limitations -Nhentai was cloudflare protection enabled, If IP and our thoughts against them? You should implement a proxy. Check [`cookie branch`](https://github.com/sinkaroid/jandapress/tree/cookie), take a look this workaround [Zekfad/nhentai-api/issues/25#issuecomment-1141360074](https://github.com/Zekfad/nhentai-api/issues/25#issuecomment-1141360074) + HTTP/1.1 400 Bad Request + HTTP/1.1 500 Fail to get data ## Frequently asked questions **Q: The website response is slow** diff --git a/package.json b/package.json index 6ccaf47..ac0c2c5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jandapress", - "version": "2.0.4-dev", + "version": "2.1.1-alpha", "description": "RESTful and experimental API for the Doujinshi, Pressing the whole nhentai, pururin, hentaifox, and more.. where the official one is lack.", "main": "build/src/index.js", "scripts": { @@ -25,7 +25,7 @@ }, "apidoc": { "title": "Jandapress API Documentation", - "url" : "https://janda.mod.land", + "url": "https://janda.mod.land", "sampleUrl": "https://janda.mod.land", "name": "Jandapress" }, @@ -44,10 +44,12 @@ "express": "^4.18.1", "express-rate-limit": "^6.4.0", "express-slow-down": "^1.4.0", + "http-cookie-agent": "^5.0.2", "keyv": "^4.5.2", "phin": "^3.6.1", "pino": "^8.7.0", - "pino-pretty": "^9.1.1" + "pino-pretty": "^9.1.1", + "tough-cookie": "^4.1.2" }, "devDependencies": { "@types/cors": "^2.8.12", @@ -69,4 +71,4 @@ "engines": { "node": ">=14" } -} \ No newline at end of file +} diff --git a/src/JandaPress.ts b/src/JandaPress.ts index 55fa8ce..494b1ee 100644 --- a/src/JandaPress.ts +++ b/src/JandaPress.ts @@ -1,17 +1,80 @@ import p from "phin"; import Keyv from "keyv"; -import dotenv from "dotenv"; -dotenv.config(); +import { CookieJar } from "tough-cookie"; +import { HttpsCookieAgent } from "http-cookie-agent/http"; const keyv = new Keyv(process.env.REDIS_URL); +const strategy = process.env.NHENTAI_IP_ORIGIN || "true"; keyv.on("error", err => console.log("Connection Error", err)); const ttl = 1000 * 60 * 60 * Number(process.env.EXPIRE_CACHE); +const jar = new CookieJar(); +jar.setCookie(process.env.COOKIE || "", "https://nhentai.net/"); + class JandaPress { url: string; + useragent: string; constructor() { this.url = ""; + this.useragent = "jandapress/1.0.5 Node.js/16.9.1"; + } + + async simulateCookie(target: string, parseJson = false): Promise { + if (!parseJson) { + const res = await p({ + url: target, + followRedirects: true, + core: { + agent: new HttpsCookieAgent({ cookies: { jar, }, }), + }, + headers: { + "User-Agent": process.env.USER_AGENT || "", + }, + }); + + return res; + } else { + const res = await p({ + url: target, + parse: "json", + core: { + agent: new HttpsCookieAgent({ cookies: { jar, }, }), + }, + headers: { + "User-Agent": process.env.USER_AGENT || "", + }, + }); + + return res.body; + } + + + } + + /** + * Simulating nhentai request if origin api is not available + * You'll need [tough-cookie](https://www.npmjs.com/package/tough-cookie) and [http-cookie-agent](https://www.npmjs.com/package/http-cookie-agent) to make this work + * @param target url to fetch + * @returns Promise + * @throws Error + */ + async simulateNhentaiRequest(target: string): Promise { + if (strategy === "true") { + const res = await p({ + url: target, + parse: "json" + }); + return res.body; + } else { + try { + const res = await this.simulateCookie(target, true); + return res; + } catch (err) { + const e = err as Error; + throw new Error(e.message); + } + } } /** @@ -42,7 +105,7 @@ class JandaPress { * @param url url to fetch * @returns Buffer */ - async fetchJson(url: string) { + async fetchJson(url: string): Promise { const cached = await keyv.get(url); if (cached) { @@ -50,9 +113,9 @@ class JandaPress { return cached; } else { console.log("Fetching from source"); - const res = await p({ url: url, parse: "json" }); - await keyv.set(url, res.body, ttl); - return res.body; + const res = await this.simulateNhentaiRequest(url); + await keyv.set(url, res, ttl); + return res; } } @@ -66,9 +129,7 @@ class JandaPress { rss: `${Math.round(rss * 100) / 100} MB`, heap: `${Math.round(heap * 100) / 100}/${Math.round(heaptotal * 100) / 100} MB` }; - } - } export default JandaPress; diff --git a/src/controller/3hentai/3hentaiGet.ts b/src/controller/3hentai/3hentaiGet.ts index 93e1ce9..eec1032 100644 --- a/src/controller/3hentai/3hentaiGet.ts +++ b/src/controller/3hentai/3hentaiGet.ts @@ -20,7 +20,7 @@ export async function get3hentai(req: Request, res: Response, next: NextFunction * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/3hentai/get?book=123 diff --git a/src/controller/3hentai/3hentaiRandom.ts b/src/controller/3hentai/3hentaiRandom.ts index 618c508..7f5da8c 100644 --- a/src/controller/3hentai/3hentaiRandom.ts +++ b/src/controller/3hentai/3hentaiRandom.ts @@ -13,7 +13,7 @@ export async function random3hentai(req: Request, res: Response, next: NextFunct * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/3hentai/random diff --git a/src/controller/3hentai/3hentaiSearch.ts b/src/controller/3hentai/3hentaiSearch.ts index 942a53f..c9897bd 100644 --- a/src/controller/3hentai/3hentaiSearch.ts +++ b/src/controller/3hentai/3hentaiSearch.ts @@ -23,7 +23,7 @@ export async function search3hentai(req: Request, res: Response, next: NextFunct * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/3hentai/search?key=yuri diff --git a/src/controller/asmhentai/asmhentaiGet.ts b/src/controller/asmhentai/asmhentaiGet.ts index d9a935c..5e2cf27 100644 --- a/src/controller/asmhentai/asmhentaiGet.ts +++ b/src/controller/asmhentai/asmhentaiGet.ts @@ -20,7 +20,7 @@ export async function getAsmhentai(req: Request, res: Response, next: NextFuncti * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/asmhentai/get?book=123 diff --git a/src/controller/asmhentai/asmhentaiRandom.ts b/src/controller/asmhentai/asmhentaiRandom.ts index 17dc6a0..99a043f 100644 --- a/src/controller/asmhentai/asmhentaiRandom.ts +++ b/src/controller/asmhentai/asmhentaiRandom.ts @@ -13,7 +13,7 @@ export async function randomAsmhentai(req: Request, res: Response, next: NextFun * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/asmhentai/random diff --git a/src/controller/asmhentai/asmhentaiSearch.ts b/src/controller/asmhentai/asmhentaiSearch.ts index 9d289f0..d547026 100644 --- a/src/controller/asmhentai/asmhentaiSearch.ts +++ b/src/controller/asmhentai/asmhentaiSearch.ts @@ -20,7 +20,7 @@ export async function searchAsmhentai(req: Request, res: Response, next: NextFun * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/asmhentai/search?key=yuri diff --git a/src/controller/hentai2read/hentai2readGet.ts b/src/controller/hentai2read/hentai2readGet.ts index bb2eb68..4e5673e 100644 --- a/src/controller/hentai2read/hentai2readGet.ts +++ b/src/controller/hentai2read/hentai2readGet.ts @@ -19,7 +19,7 @@ export async function getHentai2read(req: Request, res: Response) { * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/hentai2read/get?book=butabako_shotaone_matome_fgo_hen/1 diff --git a/src/controller/hentai2read/hentai2readSearch.ts b/src/controller/hentai2read/hentai2readSearch.ts index 858cbee..4f583a7 100644 --- a/src/controller/hentai2read/hentai2readSearch.ts +++ b/src/controller/hentai2read/hentai2readSearch.ts @@ -17,7 +17,7 @@ export async function searchHentai2read(req: Request, res: Response, next: NextF * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/hentai2read/search?key=yuri diff --git a/src/controller/hentaifox/hentaifoxGet.ts b/src/controller/hentaifox/hentaifoxGet.ts index 4481204..1f8f88a 100644 --- a/src/controller/hentaifox/hentaifoxGet.ts +++ b/src/controller/hentaifox/hentaifoxGet.ts @@ -20,7 +20,7 @@ export async function getHentaifox(req: Request, res: Response, next: NextFuncti * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/hentaifox/get?book=123 diff --git a/src/controller/hentaifox/hentaifoxRandom.ts b/src/controller/hentaifox/hentaifoxRandom.ts index 5f19756..a56cf6c 100644 --- a/src/controller/hentaifox/hentaifoxRandom.ts +++ b/src/controller/hentaifox/hentaifoxRandom.ts @@ -13,7 +13,7 @@ export async function randomHentaifox(req: Request, res: Response, next: NextFun * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/hentaifox/random diff --git a/src/controller/hentaifox/hentaifoxSearch.ts b/src/controller/hentaifox/hentaifoxSearch.ts index 0101c12..90a5b4c 100644 --- a/src/controller/hentaifox/hentaifoxSearch.ts +++ b/src/controller/hentaifox/hentaifoxSearch.ts @@ -17,7 +17,7 @@ export async function searchHentaifox(req: Request, res: Response, next: NextFun * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/hentaifox/search?key=yuri diff --git a/src/controller/nhentai/nhentaiGet.ts b/src/controller/nhentai/nhentaiGet.ts index bea21ae..c5bf52d 100644 --- a/src/controller/nhentai/nhentaiGet.ts +++ b/src/controller/nhentai/nhentaiGet.ts @@ -1,7 +1,6 @@ import { scrapeContent } from "../../scraper/nhentai/nhentaiGetController"; -import c from "../../utils/options"; import { logger } from "../../utils/logger"; -import { mock, isNumeric } from "../../utils/modifier"; +import { nhentaiStrategy, isNumeric, maybeError } from "../../utils/modifier"; import { Request, Response } from "express"; export async function getNhentai(req: Request, res: Response) { @@ -10,10 +9,6 @@ export async function getNhentai(req: Request, res: Response) { if (!book) throw Error("Parameter book is required"); if (!isNumeric(book)) throw Error("Parameter book must be number"); - let actualAPI; - if (!await mock(c.NHENTAI)) actualAPI = c.NHENTAI_IP_3; - else actualAPI = c.NHENTAI; - /** * @api {get} /nhentai/get?book=:book Get nhentai * @apiName Get nhentai @@ -24,7 +19,7 @@ export async function getNhentai(req: Request, res: Response) { * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/nhentai/get?book=123 @@ -43,7 +38,7 @@ export async function getNhentai(req: Request, res: Response) { * print(await resp.json()) */ - const url = `${actualAPI}/api/gallery/${book}`; + const url = `${nhentaiStrategy()}/api/gallery/${book}`; const data = await scrapeContent(url); logger.info({ path: req.path, @@ -53,11 +48,8 @@ export async function getNhentai(req: Request, res: Response) { useragent: req.get("User-Agent") }); return res.json(data); - } catch (err: any) { - const e = { - "success": false, - "message": err.message - }; - res.json(e); + } catch (err) { + const e = err as Error; + res.status(400).json(maybeError(false, e.message)); } } diff --git a/src/controller/nhentai/nhentaiRandom.ts b/src/controller/nhentai/nhentaiRandom.ts index 5d37453..4f6c519 100644 --- a/src/controller/nhentai/nhentaiRandom.ts +++ b/src/controller/nhentai/nhentaiRandom.ts @@ -1,16 +1,10 @@ import { scrapeContent } from "../../scraper/nhentai/nhentaiGetController"; -import c from "../../utils/options"; import { logger } from "../../utils/logger"; -import { mock } from "../../utils/modifier"; -import { getIdRandomNhentai } from "../../utils/modifier"; -import { Request, Response, NextFunction } from "express"; +import { nhentaiStrategy, getIdRandomNhentai, maybeError } from "../../utils/modifier"; +import { Request, Response } from "express"; -export async function randomNhentai(req: Request, res: Response, next: NextFunction) { +export async function randomNhentai(req: Request, res: Response) { try { - let actualAPI; - if (!await mock(c.NHENTAI)) actualAPI = c.NHENTAI_IP_3; - else actualAPI = c.NHENTAI; - const id = await getIdRandomNhentai(); /** @@ -21,7 +15,7 @@ export async function randomNhentai(req: Request, res: Response, next: NextFunct * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/nhentai/random @@ -41,7 +35,7 @@ export async function randomNhentai(req: Request, res: Response, next: NextFunct * */ - const url = `${actualAPI}/api/gallery/${id}`; + const url = `${nhentaiStrategy()}/api/gallery/${id}`; const data = await scrapeContent(url, true); logger.info({ path: req.path, @@ -51,7 +45,8 @@ export async function randomNhentai(req: Request, res: Response, next: NextFunct useragent: req.get("User-Agent") }); return res.json(data); - } catch (err: any) { - next(Error(err.message)); + } catch (err) { + const e = err as Error; + res.status(400).json(maybeError(false, `Error Try again: ${e.message}`)); } } diff --git a/src/controller/nhentai/nhentaiRelated.ts b/src/controller/nhentai/nhentaiRelated.ts index e8721ac..5479dd9 100644 --- a/src/controller/nhentai/nhentaiRelated.ts +++ b/src/controller/nhentai/nhentaiRelated.ts @@ -1,7 +1,6 @@ import { scrapeContent } from "../../scraper/nhentai/nhentaiRelatedController"; -import c from "../../utils/options"; import { logger } from "../../utils/logger"; -import { mock, isNumeric } from "../../utils/modifier"; +import { nhentaiStrategy, isNumeric, maybeError } from "../../utils/modifier"; import { Request, Response } from "express"; export async function relatedNhentai(req: Request, res: Response) { @@ -10,10 +9,6 @@ export async function relatedNhentai(req: Request, res: Response) { if (!book) throw Error("Parameter book is required"); if (!isNumeric(book)) throw Error("Value must be number"); - let actualAPI; - if (!await mock(c.NHENTAI)) actualAPI = c.NHENTAI_IP_3; - else actualAPI = c.NHENTAI; - /** * @api {get} /nhentai/related?book=:book Get related nhentai * @apiName Get related nhentai @@ -24,7 +19,7 @@ export async function relatedNhentai(req: Request, res: Response) { * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/nhentai/related?book=123 @@ -43,7 +38,7 @@ export async function relatedNhentai(req: Request, res: Response) { * print(await resp.json()) */ - const url = `${actualAPI}/api/gallery/${book}/related`; + const url = `${nhentaiStrategy()}/api/gallery/${book}/related`; const data = await scrapeContent(url); logger.info({ path: req.path, @@ -53,11 +48,8 @@ export async function relatedNhentai(req: Request, res: Response) { useragent: req.get("User-Agent") }); return res.json(data); - } catch (err: any) { - const e = { - "success": false, - "message": err.message - }; - res.json(e); + } catch (err) { + const e = err as Error; + res.status(400).json(maybeError(false, e.message)); } } diff --git a/src/controller/nhentai/nhentaiSearch.ts b/src/controller/nhentai/nhentaiSearch.ts index 9c884be..6317cba 100644 --- a/src/controller/nhentai/nhentaiSearch.ts +++ b/src/controller/nhentai/nhentaiSearch.ts @@ -1,7 +1,6 @@ import { scrapeContent } from "../../scraper/nhentai/nhentaiSearchController"; -import c from "../../utils/options"; import { logger } from "../../utils/logger"; -import { mock } from "../../utils/modifier"; +import { nhentaiStrategy, maybeError } from "../../utils/modifier"; const sorting = ["popular-today", "popular-week", "popular"]; import { Request, Response } from "express"; @@ -13,10 +12,6 @@ export async function searchNhentai(req: Request, res: Response) { if (!key) throw Error("Parameter key is required"); if (!sorting.includes(sort)) throw Error("Invalid sort: " + sorting.join(", ")); - let actualAPI; - if (!await mock(c.NHENTAI)) actualAPI = c.NHENTAI_IP_3; - else actualAPI = c.NHENTAI; - /** * @api {get} /nhentai/search Search nhentai * @apiName Search nhentai @@ -28,7 +23,7 @@ export async function searchNhentai(req: Request, res: Response) { * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/nhentai/search?key=yuri @@ -48,7 +43,7 @@ export async function searchNhentai(req: Request, res: Response) { * print(await resp.json()) */ - const url = `${actualAPI}/api/galleries/search?query=${key}&sort=${sort}&page=${page}`; + const url = `${nhentaiStrategy()}/api/galleries/search?query=${key}&sort=${sort}&page=${page}`; const data = await scrapeContent(url); logger.info({ path: req.path, @@ -58,11 +53,8 @@ export async function searchNhentai(req: Request, res: Response) { useragent: req.get("User-Agent") }); return res.json(data); - } catch (err: any) { - const e = { - "success": false, - "message": err.message - }; - res.json(e); + } catch (err) { + const e = err as Error; + res.status(400).json(maybeError(false, e.message)); } } \ No newline at end of file diff --git a/src/controller/pururin/pururinGet.ts b/src/controller/pururin/pururinGet.ts index 405bb38..e290f68 100644 --- a/src/controller/pururin/pururinGet.ts +++ b/src/controller/pururin/pururinGet.ts @@ -20,7 +20,7 @@ export async function getPururin(req: Request, res: Response, next: NextFunction * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/pururin/get?book=123 diff --git a/src/controller/pururin/pururinRandom.ts b/src/controller/pururin/pururinRandom.ts index 6b6c800..7327829 100644 --- a/src/controller/pururin/pururinRandom.ts +++ b/src/controller/pururin/pururinRandom.ts @@ -16,7 +16,7 @@ export async function randomPururin(req: Request, res: Response, next: NextFunct * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/pururin/random diff --git a/src/controller/pururin/pururinSearch.ts b/src/controller/pururin/pururinSearch.ts index b6a2b03..29299e1 100644 --- a/src/controller/pururin/pururinSearch.ts +++ b/src/controller/pururin/pururinSearch.ts @@ -23,7 +23,7 @@ export async function searchPururin(req: Request, res: Response, next: NextFunct * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/pururin/search?key=yuri diff --git a/src/controller/simply-hentai/simply-hentaiGet.ts b/src/controller/simply-hentai/simply-hentaiGet.ts index a7c977f..4ba9bbf 100644 --- a/src/controller/simply-hentai/simply-hentaiGet.ts +++ b/src/controller/simply-hentai/simply-hentaiGet.ts @@ -22,7 +22,7 @@ export async function getSimplyhentai(req: Request, res: Response) { * * @apiSuccessExample {json} Success-Response: * HTTP/1.1 200 OK - * HTTP/1.1 200 (cached) + * HTTP/1.1 400 Bad Request * * @apiExample {curl} curl * curl -i https://janda.mod.land/simply-hentai/get?book=fate-grand-order/fgo-sanbunkatsuhou/all-pages diff --git a/src/index.ts b/src/index.ts index 0d29f99..b260d54 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ +import "dotenv/config"; import JandaPress from "./JandaPress"; import express from "express"; import { Request, Response, NextFunction } from "express"; @@ -6,11 +7,10 @@ import { slow, limiter } from "./utils/limit-options"; import { logger } from "./utils/logger"; import { isNumeric } from "./utils/modifier"; import * as pkg from "../package.json"; -import dotenv from "dotenv"; const janda = new JandaPress(); const app = express(); -dotenv.config(); + app.get("/", slow, limiter, (req, res) => { res.send({ diff --git a/src/interfaces.ts b/src/interfaces.ts index 8073f06..282eac1 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -34,3 +34,7 @@ interface T { count: number; } +export interface MaybeError { + message: string; +} + diff --git a/src/scraper/nhentai/nhentaiGetController.ts b/src/scraper/nhentai/nhentaiGetController.ts index 825ca96..1a036f3 100644 --- a/src/scraper/nhentai/nhentaiGetController.ts +++ b/src/scraper/nhentai/nhentaiGetController.ts @@ -1,4 +1,3 @@ -import p from "phin"; import JandaPress from "../../JandaPress"; import c from "../../utils/options"; import { getDate, timeAgo } from "../../utils/modifier"; @@ -32,10 +31,8 @@ const janda = new JandaPress(); export async function scrapeContent(url: string, random = false) { try { let res, raw; - if (random) res = await p({ url: url, parse: "json" }), - raw = res.body as Nhentai; - else res = await janda.fetchJson(url), - raw = res as Nhentai; + if (random) res = await janda.simulateNhentaiRequest(url), raw = res as Nhentai; + else res = await janda.fetchJson(url), raw = res as Nhentai; const GALLERY = "https://i.nhentai.net/galleries"; const imagesRaw = raw.images.pages; @@ -93,11 +90,13 @@ export async function scrapeContent(url: string, random = false) { }; const data = { + success: true, data: objectData, source: `${c.NHENTAI}/g/${raw.id}`, }; return data; - } catch (err: any) { - throw Error(err.message); + } catch (err) { + const e = err as Error; + throw Error(e.message); } } \ No newline at end of file diff --git a/src/scraper/nhentai/nhentaiRelatedController.ts b/src/scraper/nhentai/nhentaiRelatedController.ts index 294c321..fd0cd4d 100644 --- a/src/scraper/nhentai/nhentaiRelatedController.ts +++ b/src/scraper/nhentai/nhentaiRelatedController.ts @@ -42,6 +42,7 @@ export async function scrapeContent(url: string) { } const data = { + success: true, data: content, source: url.replace(c.NHENTAI_IP, c.NHENTAI), }; diff --git a/src/scraper/nhentai/nhentaiSearchController.ts b/src/scraper/nhentai/nhentaiSearchController.ts index f878808..2e4aae8 100644 --- a/src/scraper/nhentai/nhentaiSearchController.ts +++ b/src/scraper/nhentai/nhentaiSearchController.ts @@ -56,6 +56,7 @@ export async function scrapeContent(url: string) { } const data = { + success: true, data: content, page: Number(url.split("&page=")[1]), sort: url.split("&sort=")[1].split("&")[0], @@ -63,7 +64,8 @@ export async function scrapeContent(url: string) { }; return data; - } catch (err: any) { - throw Error(err.message); + } catch (err) { + const e = err as Error; + throw Error(e.message); } } \ No newline at end of file diff --git a/src/utils/modifier.ts b/src/utils/modifier.ts index 6d72d94..3886b55 100644 --- a/src/utils/modifier.ts +++ b/src/utils/modifier.ts @@ -1,33 +1,70 @@ +import JandaPress from "../JandaPress"; import p from "phin"; import { load } from "cheerio"; import c from "./options"; +const janda = new JandaPress(); +/** + * Get Pururin info and replace + * @param value + * @returns string + */ function getPururinInfo(value: string) { return value.replace(/\n/g, " ").replace(/\s\s+/g, " ").trim(); } +/** + * Get Pururin page count + * @param value + * @returns number + */ function getPururinPageCount(value: string) { const data = value.replace(/\n/g, " ").replace(/\s\s+/g, " ").trim().split(", ").pop(); return Number(data?.split(" ")[0]); } +/** + * Get Pururin language + * @param value + * @returns string + */ function getPururinLanguage(value: string) { return value.split(",").reverse()[1].trim(); } +/** + * Parse url + * @param url + * @returns string + */ function getUrl(url: string) { return url.replace(/^\/\//, "https://"); } +/** + * Parse id + * @param url + * @returns string + */ function getId(url: string) { return url.replace(/^https?:\/\/[^\\/]+/, "").replace(/\/$/, ""); } +/** + * Parse alphabet only + * @param input + * @returns string + */ function removeNonNumeric(input: string) { return input.replace(/[^0-9]/g, ""); } +/** + * Parse date format on nhentai + * @param date + * @returns string + */ function getDate(date: Date) { return date.toLocaleDateString("en-US", { year: "numeric", @@ -36,6 +73,11 @@ function getDate(date: Date) { }); } +/** + * Fancy time ago format + * @param input + * @returns string + */ function timeAgo(input: Date) { const date = new Date(input); const formatter: any = new Intl.RelativeTimeFormat("en"); @@ -57,6 +99,11 @@ function timeAgo(input: Date) { } } +/** + * Check nhentai status + * @param url + * @returns boolean + */ async function mock(url: string) { const site = await p({ url: url }); if (site.statusCode === 200) { @@ -68,11 +115,20 @@ async function mock(url: string) { } } -export const isNumeric = (val: string) : boolean => { +/** + * Check if string is numeric + * @param val + * @returns boolean + */ +export const isNumeric = (val: string): boolean => { return !isNaN(Number(val)); }; -export async function getIdRandomPururin (): Promise { +/** + * Simulate random on pururin + * @returns Promise + */ +export async function getIdRandomPururin(): Promise { const randomNumber = Math.floor(Math.random() * 500) + 1; const raw = await p(`${c.PURURIN}/browse/random?page=${randomNumber}`); const $ = load(raw.body); @@ -82,13 +138,14 @@ export async function getIdRandomPururin (): Promise { return parseInt(randomgallery); } -export async function getIdRandomNhentai (): Promise { - if (await mock(c.NHENTAI)) { - const res: any = await p({ - url: `${c.NHENTAI}/random`, - followRedirects: true, - }); - +/** + * Simulate random on nhentai + * @returns Promise + */ +export async function getIdRandomNhentai(): Promise { + if (process.env.NHENTAI_IP_ORIGIN === "false") { + const res: any = await janda.simulateCookie(`${c.NHENTAI}/random`); + const getId = res.socket._httpMessage.path; return parseInt(getId.replace(/^\/g\/([0-9]+)\/?$/, "$1")); } else { @@ -98,6 +155,28 @@ export async function getIdRandomNhentai (): Promise { } } +/** + * Error handler + * @param success + * @param message + * @returns object + */ +export function maybeError(success: boolean, message: string) { + return { success, message }; +} -export { getPururinInfo, getPururinPageCount, getUrl, getId, getDate, timeAgo, - mock, getPururinLanguage, removeNonNumeric }; \ No newline at end of file +/** + * Get nhentai strategy from origin api or simulating the request cookie + * @returns string + */ +export function nhentaiStrategy() { + let strategy: string; + if (process.env.NHENTAI_IP_ORIGIN === "true" || process.env.NHENTAI_IP_ORIGIN === undefined) strategy = c.NHENTAI_IP_3; + else strategy = c.NHENTAI; + return strategy; +} + +export { + getPururinInfo, getPururinPageCount, getUrl, getId, getDate, timeAgo, + mock, getPururinLanguage, removeNonNumeric +}; \ No newline at end of file diff --git a/test/nhentaiCookietest.ts b/test/nhentaiCookietest.ts index 956881d..75de8fa 100644 --- a/test/nhentaiCookietest.ts +++ b/test/nhentaiCookietest.ts @@ -5,20 +5,20 @@ import * as dotenv from "dotenv"; dotenv.config(); const jar = new CookieJar(); -jar.setCookie(process.env.CF_COOKIE || "", "https://nhentai.net/"); +jar.setCookie(process.env.COOKIE || "", "https://nhentai.net/"); async function test() { const res = await p({ - url: "https://nhentai.net/api/galleries/search?query=futa", + url: "https://nhentai.net/api/gallery/1", core: { agent: new HttpsCookieAgent({ cookies: { jar, }, }), }, "headers": { - "User-Agent": "jandapress/1.0.5 Node.js/16.9.1" // nhentai-api-client/3.4.3 Node.js/16.9.1 + "User-Agent": process.env.USER_AGENT || "jandapress/1.0.5 Node.js/16.9.1", }, }); - //check status + console.log(res.statusCode); } -test(); \ No newline at end of file +test().catch(console.error); \ No newline at end of file