feat: simulating nhentai request on cloudflare enabled (#17)

* fix apidocs definitions

* add nhentaiStrategy method

* add MaybeError types

* add simulateCookie to janda class

* test case

* env example

* push release

.env.schema

@@ -1,14 +1,22 @@
-# cloudflare stuff for testing nhentai: https://github.com/Zekfad/nhentai-api/issues/25#issuecomment-1141360074
+# railway, fly.dev, heroku, vercel or any free service, NHENTAI_IP_ORIGIN should be true
-# you can skip this as it's no longer needed
+RAILWAY = sinkaroid
-CF_COOKIE=
-CF=
 
-# your username if it's on railway.app
+# default port
-RAILWAY=sinkaroid
+PORT = 3000
 
-# jandapress config
+# backend storage, default is redis with this format
-PORT=3000
+REDIS_URL = redis://default:somenicepassword@redis-666.c10.us-east-6-6.ec666.cloud.redislabs.com:1337
-REDIS_URL=redis://default:somenicepassword@redis-666.c10.us-east-6-6.ec666.cloud.redislabs.com:1337
 
-# ttl for cache in a hour
+# ttl expire cache (in X hour)
-EXPIRE_CACHE=1
+EXPIRE_CACHE = 1
+
+# nhentai strategy
+# default is true which is assign to request on origin IP instead of nhentai.net with cloudflare protection
+# if you have paid instance like vps you need chromium or firefox installed and set NHENTAI_IP_ORIGIN to false
+NHENTAI_IP_ORIGIN = true
+
+# you must set COOKIE if NHENTAI_IP_ORIGIN is false, read the jandapress docs 
+COOKIE = "cf_clearance=l7RsUjiZ3LHAZZKcM7BcCylwD2agwPDU7l9zkg8MzPo-1676044652-0-250"
+
+# you must set USER_AGENT if NHENTAI_IP_ORIGIN is false, read the jandapress docs
+USER_AGENT = "jandapress/1.0.5 Node.js/16.9.1"

README.md

@@ -28,11 +28,11 @@ The motivation of this project is to bring you an actionable data related doujin
     - [Installation](#installation)
       - [Docker](#docker)
       - [Manual](#manual)
+      - [Nhentai guide](#limitations)
     - [Running tests](#running-tests)
   - [Playground](https://sinkaroid.github.io/jandapress)
     - [Routing](#playground)
     - [Status response](#status-response)
-  - [Limitations](#limitations)
   - [CLosing remarks](https://github.com/sinkaroid/jandapress/blob/master/CLOSING_REMARKS.md)
     - [Alternative links](https://github.com/sinkaroid/jandapress/blob/master/CLOSING_REMARKS.md#alternative-links)
   - [Pronunciation](#Pronunciation)
@@ -83,6 +83,9 @@ REDIS_URL=redis://default:somenicepassword@someredishost:1337 ## the database ur
 EXPIRE_CACHE=1 ## a hour
 ```
 
+## Nhentai guide
+Nhentai was cloudflare protection enabled, If IP and our thoughts against them? You should implement a proxy. Check [`cookie branch`](https://github.com/sinkaroid/jandapress/tree/cookie), take a look this workaround [Zekfad/nhentai-api/issues/25#issuecomment-1141360074](https://github.com/Zekfad/nhentai-api/issues/25#issuecomment-1141360074)
+
 ### Docker
 
     docker pull ghcr.io/sinkaroid/jandapress:latest
@@ -101,12 +104,6 @@ EXPIRE_CACHE=1 ## a hour
 
 
 ## Running tests
-Jandapress depends on
-- [express](https://github.com/expressjs/express) web api framework
-- [keyv](https://github.com/jaredwray/keyv) key-value storage with support for multiple backends
-- [cheerio](https://cheerio.js.org/) for parsing html
-- [cors](https://github.com/expressjs/cors) middleware for enabling CORS
-- [rate-limit](https://github.com/nfriedly/express-rate-limit) rate-limiting middleware for express
 
 ### Start the production server
 `npm run start:prod`
@@ -216,11 +213,8 @@ The missing piece of 3hentai.net - https://sinkaroid.github.io/jandapress/#api-3
 ## Status response
 
     HTTP/1.1 200 OK
-    HTTP/1.1 200 (cached)
+    HTTP/1.1 400 Bad Request
-    HTTP/1.1 500 (bad parameters)
+    HTTP/1.1 500 Fail to get data
-
-## Limitations
-Nhentai was cloudflare protection enabled, If IP and our thoughts against them? You should implement a proxy. Check [`cookie branch`](https://github.com/sinkaroid/jandapress/tree/cookie), take a look this workaround [Zekfad/nhentai-api/issues/25#issuecomment-1141360074](https://github.com/Zekfad/nhentai-api/issues/25#issuecomment-1141360074)
 
 ## Frequently asked questions 
 **Q: The website response is slow**  

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jandapress",
-  "version": "2.0.4-dev",
+  "version": "2.1.1-alpha",
   "description": "RESTful and experimental API for the Doujinshi, Pressing the whole nhentai, pururin, hentaifox, and more.. where the official one is lack.",
   "main": "build/src/index.js",
   "scripts": {
@@ -25,7 +25,7 @@
   },
   "apidoc": {
     "title": "Jandapress API Documentation",
-    "url" : "https://janda.mod.land",
+    "url": "https://janda.mod.land",
     "sampleUrl": "https://janda.mod.land",
     "name": "Jandapress"
   },
@@ -44,10 +44,12 @@
     "express": "^4.18.1",
     "express-rate-limit": "^6.4.0",
     "express-slow-down": "^1.4.0",
+    "http-cookie-agent": "^5.0.2",
     "keyv": "^4.5.2",
     "phin": "^3.6.1",
     "pino": "^8.7.0",
-    "pino-pretty": "^9.1.1"
+    "pino-pretty": "^9.1.1",
+    "tough-cookie": "^4.1.2"
   },
   "devDependencies": {
     "@types/cors": "^2.8.12",
@@ -69,4 +71,4 @@
   "engines": {
     "node": ">=14"
   }
-}
+}

src/JandaPress.ts

@@ -1,17 +1,80 @@
 import p from "phin";
 import Keyv from "keyv";
-import dotenv from "dotenv";
+import { CookieJar } from "tough-cookie";
-dotenv.config();
+import { HttpsCookieAgent } from "http-cookie-agent/http";
 
 const keyv = new Keyv(process.env.REDIS_URL);
+const strategy = process.env.NHENTAI_IP_ORIGIN || "true";
 
 keyv.on("error", err => console.log("Connection Error", err));
 const ttl = 1000 * 60 * 60 * Number(process.env.EXPIRE_CACHE);
 
+const jar = new CookieJar();
+jar.setCookie(process.env.COOKIE || "", "https://nhentai.net/");
+
 class JandaPress {
   url: string;
+  useragent: string;
   constructor() {
     this.url = "";
+    this.useragent = "jandapress/1.0.5 Node.js/16.9.1";
+  }
+
+  async simulateCookie(target: string, parseJson = false): Promise<p.IResponse | unknown> {
+    if (!parseJson) {
+      const res = await p({
+        url: target,
+        followRedirects: true,
+        core: {
+          agent: new HttpsCookieAgent({ cookies: { jar, }, }),
+        },
+        headers: {
+          "User-Agent": process.env.USER_AGENT || "",
+        },
+      });
+
+      return res;
+    } else {
+      const res = await p({
+        url: target,
+        parse: "json",
+        core: {
+          agent: new HttpsCookieAgent({ cookies: { jar, }, }),
+        },
+        headers: {
+          "User-Agent": process.env.USER_AGENT || "",
+        },
+      });
+
+      return res.body;
+    }
+
+
+  }
+
+  /**
+   * Simulating nhentai request if origin api is not available
+   * You'll need [tough-cookie](https://www.npmjs.com/package/tough-cookie) and [http-cookie-agent](https://www.npmjs.com/package/http-cookie-agent) to make this work
+   * @param target url to fetch
+   * @returns Promise<unknown>
+   * @throws Error
+   */
+  async simulateNhentaiRequest(target: string): Promise<unknown> {
+    if (strategy === "true") {
+      const res = await p({
+        url: target,
+        parse: "json"
+      });
+      return res.body;
+    } else {
+      try {
+        const res = await this.simulateCookie(target, true);
+        return res;
+      } catch (err) {
+        const e = err as Error;
+        throw new Error(e.message);
+      }
+    }
   }
 
   /**
@@ -42,7 +105,7 @@ class JandaPress {
      * @param url url to fetch
      * @returns Buffer
      */
-  async fetchJson(url: string) {
+  async fetchJson(url: string): Promise<unknown> {
     const cached = await keyv.get(url);
 
     if (cached) {
@@ -50,9 +113,9 @@ class JandaPress {
       return cached;
     } else {
       console.log("Fetching from source");
-      const res = await p({ url: url, parse: "json" });
+      const res = await this.simulateNhentaiRequest(url);
-      await keyv.set(url, res.body, ttl);
+      await keyv.set(url, res, ttl);
-      return res.body;
+      return res;
     }
   }
 
@@ -66,9 +129,7 @@ class JandaPress {
       rss: `${Math.round(rss * 100) / 100} MB`,
       heap: `${Math.round(heap * 100) / 100}/${Math.round(heaptotal * 100) / 100} MB`
     };
-
   }
-
 }
 
 export default JandaPress;

src/controller/3hentai/3hentaiGet.ts

@@ -20,7 +20,7 @@ export async function get3hentai(req: Request, res: Response, next: NextFunction
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/3hentai/get?book=123

src/controller/3hentai/3hentaiRandom.ts

@@ -13,7 +13,7 @@ export async function random3hentai(req: Request, res: Response, next: NextFunct
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/3hentai/random

src/controller/3hentai/3hentaiSearch.ts

@@ -23,7 +23,7 @@ export async function search3hentai(req: Request, res: Response, next: NextFunct
      * 
      * @apiSuccessExample {json} Success-Response:
      *    HTTP/1.1 200 OK
-     *    HTTP/1.1 200 (cached)
+     *    HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/3hentai/search?key=yuri

src/controller/asmhentai/asmhentaiGet.ts

@@ -20,7 +20,7 @@ export async function getAsmhentai(req: Request, res: Response, next: NextFuncti
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/asmhentai/get?book=123

src/controller/asmhentai/asmhentaiRandom.ts

@@ -13,7 +13,7 @@ export async function randomAsmhentai(req: Request, res: Response, next: NextFun
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/asmhentai/random

src/controller/asmhentai/asmhentaiSearch.ts

@@ -20,7 +20,7 @@ export async function searchAsmhentai(req: Request, res: Response, next: NextFun
      * 
      * @apiSuccessExample {json} Success-Response:
      *    HTTP/1.1 200 OK
-     *    HTTP/1.1 200 (cached)
+     *    HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/asmhentai/search?key=yuri

src/controller/hentai2read/hentai2readGet.ts

@@ -19,7 +19,7 @@ export async function getHentai2read(req: Request, res: Response) {
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/hentai2read/get?book=butabako_shotaone_matome_fgo_hen/1

src/controller/hentai2read/hentai2readSearch.ts

@@ -17,7 +17,7 @@ export async function searchHentai2read(req: Request, res: Response, next: NextF
      * 
      * @apiSuccessExample {json} Success-Response:
      *    HTTP/1.1 200 OK
-     *    HTTP/1.1 200 (cached)
+     *    HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/hentai2read/search?key=yuri

src/controller/hentaifox/hentaifoxGet.ts

@@ -20,7 +20,7 @@ export async function getHentaifox(req: Request, res: Response, next: NextFuncti
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/hentaifox/get?book=123

src/controller/hentaifox/hentaifoxRandom.ts

@@ -13,7 +13,7 @@ export async function randomHentaifox(req: Request, res: Response, next: NextFun
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/hentaifox/random

src/controller/hentaifox/hentaifoxSearch.ts

@@ -17,7 +17,7 @@ export async function searchHentaifox(req: Request, res: Response, next: NextFun
      * 
      * @apiSuccessExample {json} Success-Response:
      *    HTTP/1.1 200 OK
-     *    HTTP/1.1 200 (cached)
+     *    HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/hentaifox/search?key=yuri

src/controller/nhentai/nhentaiGet.ts

@@ -1,7 +1,6 @@
 import { scrapeContent } from "../../scraper/nhentai/nhentaiGetController";
-import c from "../../utils/options";
 import { logger } from "../../utils/logger";
-import { mock, isNumeric } from "../../utils/modifier";
+import { nhentaiStrategy, isNumeric, maybeError } from "../../utils/modifier";
 import { Request, Response } from "express";
 
 export async function getNhentai(req: Request, res: Response) {
@@ -10,10 +9,6 @@ export async function getNhentai(req: Request, res: Response) {
     if (!book) throw Error("Parameter book is required");
     if (!isNumeric(book)) throw Error("Parameter book must be number");
 
-    let actualAPI;
-    if (!await mock(c.NHENTAI)) actualAPI = c.NHENTAI_IP_3;
-    else actualAPI = c.NHENTAI;
-
     /**
      * @api {get} /nhentai/get?book=:book Get nhentai
      * @apiName Get nhentai
@@ -24,7 +19,7 @@ export async function getNhentai(req: Request, res: Response) {
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/nhentai/get?book=123
@@ -43,7 +38,7 @@ export async function getNhentai(req: Request, res: Response) {
      *    print(await resp.json())
      */
     
-    const url = `${actualAPI}/api/gallery/${book}`;
+    const url = `${nhentaiStrategy()}/api/gallery/${book}`;
     const data = await scrapeContent(url);
     logger.info({
       path: req.path,
@@ -53,11 +48,8 @@ export async function getNhentai(req: Request, res: Response) {
       useragent: req.get("User-Agent")
     });
     return res.json(data);
-  } catch (err: any) {
+  } catch (err) {
-    const e = {
+    const e = err as Error;
-      "success": false,
+    res.status(400).json(maybeError(false, e.message));
-      "message": err.message
-    };
-    res.json(e);
   }
 }

src/controller/nhentai/nhentaiRandom.ts

@@ -1,16 +1,10 @@
 import { scrapeContent } from "../../scraper/nhentai/nhentaiGetController";
-import c from "../../utils/options";
 import { logger } from "../../utils/logger";
-import { mock } from "../../utils/modifier";
+import { nhentaiStrategy, getIdRandomNhentai, maybeError } from "../../utils/modifier";
-import { getIdRandomNhentai } from "../../utils/modifier";
+import { Request, Response } from "express";
-import { Request, Response, NextFunction } from "express";
 
-export async function randomNhentai(req: Request, res: Response, next: NextFunction) {
+export async function randomNhentai(req: Request, res: Response) {
   try {
-    let actualAPI;
-    if (!await mock(c.NHENTAI)) actualAPI = c.NHENTAI_IP_3;
-    else actualAPI = c.NHENTAI;
-
     const id = await getIdRandomNhentai();
 
     /**
@@ -21,7 +15,7 @@ export async function randomNhentai(req: Request, res: Response, next: NextFunct
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/nhentai/random
@@ -41,7 +35,7 @@ export async function randomNhentai(req: Request, res: Response, next: NextFunct
      * 
      */
 
-    const url = `${actualAPI}/api/gallery/${id}`;
+    const url = `${nhentaiStrategy()}/api/gallery/${id}`;
     const data = await scrapeContent(url, true);
     logger.info({
       path: req.path,
@@ -51,7 +45,8 @@ export async function randomNhentai(req: Request, res: Response, next: NextFunct
       useragent: req.get("User-Agent")
     });
     return res.json(data);
-  } catch (err: any) {
+  } catch (err) {
-    next(Error(err.message));
+    const e = err as Error;
+    res.status(400).json(maybeError(false, `Error Try again: ${e.message}`));
   }
 }

src/controller/nhentai/nhentaiRelated.ts

@@ -1,7 +1,6 @@
 import { scrapeContent } from "../../scraper/nhentai/nhentaiRelatedController";
-import c from "../../utils/options";
 import { logger } from "../../utils/logger";
-import { mock, isNumeric } from "../../utils/modifier";
+import { nhentaiStrategy, isNumeric, maybeError } from "../../utils/modifier";
 import { Request, Response } from "express";
 
 export async function relatedNhentai(req: Request, res: Response) {
@@ -10,10 +9,6 @@ export async function relatedNhentai(req: Request, res: Response) {
     if (!book) throw Error("Parameter book is required");
     if (!isNumeric(book)) throw Error("Value must be number");
 
-    let actualAPI;
-    if (!await mock(c.NHENTAI)) actualAPI = c.NHENTAI_IP_3;
-    else actualAPI = c.NHENTAI;
-
     /**
      * @api {get} /nhentai/related?book=:book Get related nhentai
      * @apiName Get related nhentai
@@ -24,7 +19,7 @@ export async function relatedNhentai(req: Request, res: Response) {
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/nhentai/related?book=123
@@ -43,7 +38,7 @@ export async function relatedNhentai(req: Request, res: Response) {
      *    print(await resp.json())
      */
     
-    const url = `${actualAPI}/api/gallery/${book}/related`;
+    const url = `${nhentaiStrategy()}/api/gallery/${book}/related`;
     const data = await scrapeContent(url);
     logger.info({
       path: req.path,
@@ -53,11 +48,8 @@ export async function relatedNhentai(req: Request, res: Response) {
       useragent: req.get("User-Agent")
     });
     return res.json(data);
-  } catch (err: any) {
+  } catch (err) {
-    const e = {
+    const e = err as Error;
-      "success": false,
+    res.status(400).json(maybeError(false, e.message));
-      "message": err.message
-    };
-    res.json(e);
   }
 }

src/controller/nhentai/nhentaiSearch.ts

@@ -1,7 +1,6 @@
 import { scrapeContent } from "../../scraper/nhentai/nhentaiSearchController";
-import c from "../../utils/options";
 import { logger } from "../../utils/logger";
-import { mock } from "../../utils/modifier";
+import { nhentaiStrategy, maybeError } from "../../utils/modifier";
 const sorting = ["popular-today", "popular-week", "popular"];
 import { Request, Response } from "express";
 
@@ -13,10 +12,6 @@ export async function searchNhentai(req: Request, res: Response) {
     if (!key) throw Error("Parameter key is required");
     if (!sorting.includes(sort)) throw Error("Invalid sort: " + sorting.join(", "));
 
-    let actualAPI;
-    if (!await mock(c.NHENTAI)) actualAPI = c.NHENTAI_IP_3;
-    else actualAPI = c.NHENTAI;
-
     /**
      * @api {get} /nhentai/search Search nhentai
      * @apiName Search nhentai
@@ -28,7 +23,7 @@ export async function searchNhentai(req: Request, res: Response) {
      * 
      * @apiSuccessExample {json} Success-Response:
      *    HTTP/1.1 200 OK
-     *    HTTP/1.1 200 (cached)
+     *    HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/nhentai/search?key=yuri
@@ -48,7 +43,7 @@ export async function searchNhentai(req: Request, res: Response) {
      *    print(await resp.json())
      */
 
-    const url = `${actualAPI}/api/galleries/search?query=${key}&sort=${sort}&page=${page}`;
+    const url = `${nhentaiStrategy()}/api/galleries/search?query=${key}&sort=${sort}&page=${page}`;
     const data = await scrapeContent(url);
     logger.info({
       path: req.path,
@@ -58,11 +53,8 @@ export async function searchNhentai(req: Request, res: Response) {
       useragent: req.get("User-Agent")
     });
     return res.json(data);
-  } catch (err: any) {
+  } catch (err) {
-    const e = {
+    const e = err as Error;
-      "success": false,
+    res.status(400).json(maybeError(false, e.message));
-      "message": err.message
-    };
-    res.json(e);
   }
 }

src/controller/pururin/pururinGet.ts

@@ -20,7 +20,7 @@ export async function getPururin(req: Request, res: Response, next: NextFunction
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/pururin/get?book=123

src/controller/pururin/pururinRandom.ts

@@ -16,7 +16,7 @@ export async function randomPururin(req: Request, res: Response, next: NextFunct
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/pururin/random

src/controller/pururin/pururinSearch.ts

@@ -23,7 +23,7 @@ export async function searchPururin(req: Request, res: Response, next: NextFunct
      * 
      * @apiSuccessExample {json} Success-Response:
      *    HTTP/1.1 200 OK
-     *    HTTP/1.1 200 (cached)
+     *    HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/pururin/search?key=yuri

src/controller/simply-hentai/simply-hentaiGet.ts

@@ -22,7 +22,7 @@ export async function getSimplyhentai(req: Request, res: Response) {
      * 
      * @apiSuccessExample {json} Success-Response:
      *   HTTP/1.1 200 OK
-     *   HTTP/1.1 200 (cached)
+     *   HTTP/1.1 400 Bad Request
      * 
      * @apiExample {curl} curl
      * curl -i https://janda.mod.land/simply-hentai/get?book=fate-grand-order/fgo-sanbunkatsuhou/all-pages

src/index.ts

@@ -1,3 +1,4 @@
+import "dotenv/config";
 import JandaPress from "./JandaPress";
 import express from "express";
 import { Request, Response, NextFunction } from "express";
@@ -6,11 +7,10 @@ import { slow, limiter } from "./utils/limit-options";
 import { logger } from "./utils/logger";
 import { isNumeric } from "./utils/modifier";
 import * as pkg from "../package.json";
-import dotenv from "dotenv";
 
 const janda = new JandaPress();
 const app = express();
-dotenv.config();
+
 
 app.get("/", slow, limiter, (req, res) => {
   res.send({

src/interfaces.ts

@@ -34,3 +34,7 @@ interface T {
     count: number;
 }
 
+export interface MaybeError {
+    message: string;
+}
+

src/scraper/nhentai/nhentaiGetController.ts

@@ -1,4 +1,3 @@
-import p from "phin";
 import JandaPress from "../../JandaPress";
 import c from "../../utils/options";
 import { getDate, timeAgo } from "../../utils/modifier";
@@ -32,10 +31,8 @@ const janda = new JandaPress();
 export async function scrapeContent(url: string, random = false) {
   try {
     let res, raw;
-    if (random) res = await p({ url: url, parse: "json" }),
+    if (random) res = await janda.simulateNhentaiRequest(url), raw = res as Nhentai;
-    raw = res.body as Nhentai;
+    else res = await janda.fetchJson(url), raw = res as Nhentai;
-    else res = await janda.fetchJson(url),
-    raw = res as Nhentai;
 
     const GALLERY = "https://i.nhentai.net/galleries";
     const imagesRaw = raw.images.pages;
@@ -93,11 +90,13 @@ export async function scrapeContent(url: string, random = false) {
     };
 
     const data = {
+      success: true,
       data: objectData,
       source: `${c.NHENTAI}/g/${raw.id}`,
     };
     return data;
-  } catch (err: any) {
+  } catch (err) {
-    throw Error(err.message);
+    const e = err as Error;
+    throw Error(e.message);
   }
 }

src/scraper/nhentai/nhentaiRelatedController.ts

@@ -42,6 +42,7 @@ export async function scrapeContent(url: string) {
     }
 
     const data = {
+      success: true,
       data: content,
       source: url.replace(c.NHENTAI_IP, c.NHENTAI),
     };

src/scraper/nhentai/nhentaiSearchController.ts

@@ -56,6 +56,7 @@ export async function scrapeContent(url: string) {
     }
     
     const data = {
+      success: true,
       data: content,
       page: Number(url.split("&page=")[1]),
       sort: url.split("&sort=")[1].split("&")[0],
@@ -63,7 +64,8 @@ export async function scrapeContent(url: string) {
     };
     return data;
     
-  } catch (err: any) {
+  } catch (err) {
-    throw Error(err.message);
+    const e = err as Error;
+    throw Error(e.message);
   }
 }

src/utils/modifier.ts

@@ -1,33 +1,70 @@
+import JandaPress from "../JandaPress";
 import p from "phin";
 import { load } from "cheerio";
 import c from "./options";
 
+const janda = new JandaPress();
 
+/**
+ * Get Pururin info and replace
+ * @param value 
+ * @returns string
+ */
 function getPururinInfo(value: string) {
   return value.replace(/\n/g, " ").replace(/\s\s+/g, " ").trim();
 }
 
+/**
+ * Get Pururin page count
+ * @param value
+ * @returns number
+ */
 function getPururinPageCount(value: string) {
   const data = value.replace(/\n/g, " ").replace(/\s\s+/g, " ").trim().split(", ").pop();
   return Number(data?.split(" ")[0]);
 }
 
+/**
+ * Get Pururin language
+ * @param value
+ * @returns string
+ */
 function getPururinLanguage(value: string) {
   return value.split(",").reverse()[1].trim();
 }
 
+/**
+ * Parse url
+ * @param url
+ * @returns string
+ */
 function getUrl(url: string) {
   return url.replace(/^\/\//, "https://");
 }
 
+/**
+ * Parse id
+ * @param url
+ * @returns string
+ */
 function getId(url: string) {
   return url.replace(/^https?:\/\/[^\\/]+/, "").replace(/\/$/, "");
 }
 
+/**
+ * Parse alphabet only
+ * @param input
+ * @returns string
+ */
 function removeNonNumeric(input: string) {
   return input.replace(/[^0-9]/g, "");
 }
 
+/**
+ * Parse date format on nhentai
+ * @param date
+ * @returns string
+ */
 function getDate(date: Date) {
   return date.toLocaleDateString("en-US", {
     year: "numeric",
@@ -36,6 +73,11 @@ function getDate(date: Date) {
   });
 }
 
+/**
+ * Fancy time ago format
+ * @param input 
+ * @returns string
+ */
 function timeAgo(input: Date) {
   const date = new Date(input);
   const formatter: any = new Intl.RelativeTimeFormat("en");
@@ -57,6 +99,11 @@ function timeAgo(input: Date) {
   }
 }
 
+/**
+ * Check nhentai status
+ * @param url
+ * @returns boolean
+ */
 async function mock(url: string) {
   const site = await p({ url: url });
   if (site.statusCode === 200) {
@@ -68,11 +115,20 @@ async function mock(url: string) {
   }
 }
 
-export const isNumeric = (val: string) : boolean => {
+/** 
+ * Check if string is numeric
+ * @param val
+ * @returns boolean
+ */
+export const isNumeric = (val: string): boolean => {
   return !isNaN(Number(val));
 };
 
-export async function getIdRandomPururin (): Promise<number> {
+/** 
+ * Simulate random on pururin
+ * @returns Promise<number>
+ */
+export async function getIdRandomPururin(): Promise<number> {
   const randomNumber = Math.floor(Math.random() * 500) + 1;
   const raw = await p(`${c.PURURIN}/browse/random?page=${randomNumber}`);
   const $ = load(raw.body);
@@ -82,13 +138,14 @@ export async function getIdRandomPururin (): Promise<number> {
   return parseInt(randomgallery);
 }
 
-export async function getIdRandomNhentai (): Promise<number> {
+/**
-  if (await mock(c.NHENTAI)) {
+ * Simulate random on nhentai
-    const res: any = await p({
+ * @returns Promise<number>
-      url: `${c.NHENTAI}/random`,
+ */
-      followRedirects: true,
+export async function getIdRandomNhentai(): Promise<number> {
-    });
+  if (process.env.NHENTAI_IP_ORIGIN === "false") {
-    
+    const res: any = await janda.simulateCookie(`${c.NHENTAI}/random`);
+
     const getId = res.socket._httpMessage.path;
     return parseInt(getId.replace(/^\/g\/([0-9]+)\/?$/, "$1"));
   } else {
@@ -98,6 +155,28 @@ export async function getIdRandomNhentai (): Promise<number> {
   }
 }
 
+/**
+ * Error handler
+ * @param success
+ * @param message
+ * @returns object
+ */
+export function maybeError(success: boolean, message: string) {
+  return { success, message };
+}
 
-export { getPururinInfo, getPururinPageCount, getUrl, getId, getDate, timeAgo, 
+/**
-  mock, getPururinLanguage, removeNonNumeric };
+ * Get nhentai strategy from origin api or simulating the request cookie
+ * @returns string
+ */
+export function nhentaiStrategy() {
+  let strategy: string;
+  if (process.env.NHENTAI_IP_ORIGIN === "true" || process.env.NHENTAI_IP_ORIGIN === undefined) strategy = c.NHENTAI_IP_3;
+  else strategy = c.NHENTAI;
+  return strategy;
+}
+
+export {
+  getPururinInfo, getPururinPageCount, getUrl, getId, getDate, timeAgo,
+  mock, getPururinLanguage, removeNonNumeric
+};

test/nhentaiCookietest.ts

@@ -5,20 +5,20 @@ import * as dotenv from "dotenv";
 
 dotenv.config();
 const jar = new CookieJar();
-jar.setCookie(process.env.CF_COOKIE || "", "https://nhentai.net/");
+jar.setCookie(process.env.COOKIE || "", "https://nhentai.net/");
 
 async function test() {
   const res = await p({
-    url: "https://nhentai.net/api/galleries/search?query=futa",
+    url: "https://nhentai.net/api/gallery/1",
     core: {
       agent: new HttpsCookieAgent({ cookies: { jar, }, }),
     },
     "headers": {
-      "User-Agent": "jandapress/1.0.5 Node.js/16.9.1" // nhentai-api-client/3.4.3 Node.js/16.9.1
+      "User-Agent": process.env.USER_AGENT || "jandapress/1.0.5 Node.js/16.9.1",
     },
   });
-  //check status
+  
   console.log(res.statusCode);
 }
 
-test();
+test().catch(console.error);