feat: add nhentai.to support & improve tests (#24)

* tdd

* remove charset

* adjust client

* remake options

* add reverse proxy

* apply endpoint

* update docs

* add each controllers

* add each scraper

* pre release
このコミットが含まれているのは:
Indrawan I 2023-05-16 15:04:12 +07:00 committed by GitHub
コミット b3ee3cdcd6
この署名に対応する既知のキーがデータベースに存在しません
GPGキーID: 4AEE18F83AFDEB23
15個のファイルの変更488行の追加12行の削除

30
.github/workflows/nhentaito.yml vendored ノーマルファイル
ファイルの表示

@ -0,0 +1,30 @@
name: Nhentaito test
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [14.x]
steps:
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Install dependencies
run: npm install
- name: Build
run: npm run build
- name: Nhentaito test
run: npm run test:nhentaito

ファイルの表示

@ -44,7 +44,7 @@ The motivation of this project is to bring you an actionable data related doujin
You enjoy consume doujin sites to build web applications. There are a lot sites that have effort especially pururin, simply-hentai and etc, not official api available nor public resource that can be used for everyone. Instead making lot of abstraction and enumerating them manually, You can rely on jandapress to make less of pain. The current state is FREE to use, meant all anonymous usage is allowed no aunthentication required and CORS was enabled.
## The solution
<a href="https://github.com/sinkaroid/jandapress/wiki/Routing"><img src="https://cdn.discordapp.com/attachments/952117487166705747/1025602331456307230/jandaflow2.png" width="550"></a>
<a href="https://github.com/sinkaroid/jandapress/wiki/Routing"><img src="https://cdn.discordapp.com/attachments/1082449595033997434/1107863120275320852/jandapressflow_1.png" width="800"></a>
## Features
@ -66,6 +66,7 @@ You enjoy consume doujin sites to build web applications. There are a lot sites
| `simply-hentai` | [![Simply-hentai](https://github.com/sinkaroid/jandapress/workflows/Simply-hentai%20test/badge.svg)](https://github.com/sinkaroid/jandapress/actions/workflows/simply-hentai.yml) | ✅ | ❌ | ❌ |
| `asmhentai` | [![Asmhentai](https://github.com/sinkaroid/jandapress/workflows/Asmhentai%20test/badge.svg)](https://github.com/sinkaroid/jandapress/actions/workflows/asmhentai.yml) | ✅ | ✅ | ✅ |
| `3hentai` | [![Asmhentai](https://github.com/sinkaroid/jandapress/workflows/3hentai%20test/badge.svg)](https://github.com/sinkaroid/jandapress/actions/workflows/3hentai.yml) | ✅ | ✅ | ✅ |
| `nhentai.to` | [![Nhentaito](https://github.com/sinkaroid/jandapress/workflows/Nhentaito%20test/badge.svg)](https://github.com/sinkaroid/jandapress/actions/workflows/nhentaito.yml) | ✅ | ✅ | ✅ |
## Prerequisites
<table>
@ -267,6 +268,22 @@ The missing piece of 3hentai.net - https://sinkaroid.github.io/jandapress/#api-3
- https://janda.sinkaroid.org/3hentai/get?book=608979
- https://janda.sinkaroid.org/3hentai/search?key=futanari&page=2&sort=popular-7d
### Nhentai.to
The missing piece of nhentai.to - https://sinkaroid.github.io/jandapress/#api-nhentaito
- `/nhentaito`: nhentaito api
- **get**, takes parameters : `book`
- **search**, takes parameters : `key`, `?page`
- **related**, takes parameters : `book`
- **random**
- <u>sort parameters on search</u>
- None
- Example
- https://janda.sinkaroid.org/nhentaito/get?book=272
- https://janda.sinkaroid.org/nhentaito/search?key=futanari
- https://janda.sinkaroid.org/nhentaito/search?key=futanari&page=2
- https://janda.sinkaroid.org/nhentaito/related?book=272
## Status response
`"success": true,` or `"success": false,`

ファイルの表示

@ -1,6 +1,6 @@
{
"name": "jandapress",
"version": "2.1.7-alpha",
"version": "3.8.0-alpha",
"description": "RESTful and experimental API for the Doujinshi, Pressing the whole nhentai, pururin, hentaifox, and more.. where the official one is lack.",
"main": "build/src/index.js",
"scripts": {
@ -12,6 +12,7 @@
"start:dev": "ts-node-dev src/index.ts",
"lint": "npx eslint . --ext .ts",
"lint:fix": "npx eslint . --fix",
"test:mock": "ts-node test/mock.ts",
"build:freshdoc": "rimraf docs && rimraf template && rimraf theme.zip",
"build:template": " npm run build:freshdoc && curl https://codeload.github.com/ScathachGrip/apidocjs-theme/zip/refs/tags/v9 -o theme.zip && unzip theme.zip && mv apidocjs-theme-9 template",
"build:apidoc": "npm run build:template && npx apidoc -i src -o ./docs -t ./template/template-marine",
@ -21,7 +22,8 @@
"test:asmhentai": "npx start-server-and-test 3000 \"curl -v http://localhost:3000/asmhentai/get?book=308830 | jq '.'\"",
"test:hentai2read": "npx start-server-and-test 3000 \"curl -v http://localhost:3000/hentai2read/get?book=butabako_shotaone_matome_fgo_hen/1 | jq '.'\"",
"test:simply-hentai": "npx start-server-and-test 3000 \"curl -v http://localhost:3000/simply-hentai/get?book=fate-grand-order/fgo-sanbunkatsuhou/all-pages | jq '.'\"",
"test:3hentai": "npx start-server-and-test 3000 \"curl -v http://localhost:3000/3hentai/get?book=608979 | jq '.'\""
"test:3hentai": "npx start-server-and-test 3000 \"curl -v http://localhost:3000/3hentai/get?book=608979 | jq '.'\"",
"test:nhentaito": "npx start-server-and-test 3000 \"curl -v http://localhost:3000/nhentaito/get?book=272 | jq '.'\""
},
"apidoc": {
"title": "Jandapress API Documentation",

ファイルの表示

@ -15,7 +15,7 @@ class JandaPress {
useragent: string;
constructor() {
this.url = "";
this.useragent = "jandapress/1.0.5 Node.js/16.9.1";
this.useragent = process.env.USER_AGENT || "jandapress/1.0.5 Node.js/16.9.1";
}
async simulateCookie(target: string, parseJson = false): Promise<p.IResponse | unknown> {
@ -30,7 +30,7 @@ class JandaPress {
agent: new HttpsCookieAgent({ cookies: { jar, }, }),
},
headers: {
"User-Agent": process.env.USER_AGENT || "",
"User-Agent": this.useragent,
},
});
@ -43,7 +43,7 @@ class JandaPress {
agent: new HttpsCookieAgent({ cookies: { jar, }, }),
},
headers: {
"User-Agent": process.env.USER_AGENT || "",
"User-Agent": this.useragent,
},
});
@ -91,11 +91,22 @@ class JandaPress {
return cached;
} else if (url.includes("/random")) {
console.log("Random should not be cached");
const res = await p({ url: url, followRedirects: true });
const res = await p({
url: url,
headers: {
"User-Agent": this.useragent,
},
followRedirects: true });
return res.body;
} else {
console.log("Fetching from source");
const res = await p({ url: url, followRedirects: true });
const res = await p({
url: url,
headers: {
"User-Agent": this.useragent,
},
followRedirects: true
});
await keyv.set(url, res.body, ttl);
return res.body;
}

56
src/controller/nhentaito/nhentaiToGet.ts ノーマルファイル
ファイルの表示

@ -0,0 +1,56 @@
import { scrapeContent } from "../../scraper/nhentaito/nhentaiToGetController";
import { logger } from "../../utils/logger";
import { isNumeric, maybeError } from "../../utils/modifier";
import c from "../../utils/options";
import { Request, Response } from "express";
export async function getNhentaiTo(req: Request, res: Response) {
try {
const book = req.query.book as string;
if (!book) throw Error("Parameter book is required");
if (!isNumeric(book)) throw Error("Parameter book must be number");
/**
* @api {get} /nhentaito/get?book=:book Get nhentai.to
* @apiName Get nhentai.to
* @apiGroup nhentai.to
* @apiDescription Get a doujinshi on nhentai.to based on id
*
* @apiParam {Number} book Book ID
*
* @apiSuccessExample {json} Success-Response:
* HTTP/1.1 200 OK
* HTTP/1.1 400 Bad Request
*
* @apiExample {curl} curl
* curl -i https://janda.sinkaroid.org/nhentaito/get?book=272
*
* @apiExample {js} JS/TS
* import axios from "axios"
*
* axios.get("https://janda.sinkaroid.org/nhentaito/get?book=272")
* .then(res => console.log(res.data))
* .catch(err => console.error(err))
*
* @apiExample {python} Python
* import aiohttp
* async with aiohttp.ClientSession() as session:
* async with session.get("https://janda.sinkaroid.org/nhentaito/get?book=272") as resp:
* print(await resp.json())
*/
const url = `${c.NHENTAI_TO}/g/${book}`;
const data = await scrapeContent(url);
logger.info({
path: req.path,
query: req.query,
method: req.method,
ip: req.ip,
useragent: req.get("User-Agent")
});
return res.json(data);
} catch (err) {
const e = err as Error;
res.status(400).json(maybeError(false, e.message));
}
}

52
src/controller/nhentaito/nhentaiToRandom.ts ノーマルファイル
ファイルの表示

@ -0,0 +1,52 @@
import { scrapeContent } from "../../scraper/nhentaito/nhentaiToGetController";
import { logger } from "../../utils/logger";
import { maybeError } from "../../utils/modifier";
import c from "../../utils/options";
import { Request, Response } from "express";
export async function randomNhentaiTo(req: Request, res: Response) {
try {
/**
* @api {get} /nhentaito/random Random nhentai.to
* @apiName Random nhentai.to
* @apiGroup nhentai.to
* @apiDescription Gets random doujinshi on nhentai.to
*
* @apiSuccessExample {json} Success-Response:
* HTTP/1.1 200 OK
* HTTP/1.1 400 Bad Request
*
* @apiExample {curl} curl
* curl -i https://janda.sinkaroid.org/nhentaito/random
*
* @apiExample {js} JS/TS
* import axios from "axios"
*
* axios.get("https://janda.sinkaroid.org/nhentaito/random")
* .then(res => console.log(res.data))
* .catch(err => console.error(err))
*
* @apiExample {python} Python
* import aiohttp
* async with aiohttp.ClientSession() as session:
* async with session.get("https://janda.sinkaroid.org/nhentaito/random") as resp:
* print(await resp.json())
*
*/
const url = `${c.NHENTAI_TO}/random/`;
const data = await scrapeContent(url, true);
logger.info({
path: req.path,
query: req.query,
method: req.method,
ip: req.ip,
useragent: req.get("User-Agent")
});
return res.json(data);
} catch (err) {
const e = err as Error;
res.status(400).json(maybeError(false, `Error Try again: ${e.message}`));
}
}

56
src/controller/nhentaito/nhentaiToRelated.ts ノーマルファイル
ファイルの表示

@ -0,0 +1,56 @@
import { scrapeContent } from "../../scraper/nhentaito/nhentaiToSearchController";
import { logger } from "../../utils/logger";
import { isNumeric, maybeError } from "../../utils/modifier";
import c from "../../utils/options";
import { Request, Response } from "express";
export async function relatedNhentaiTo(req: Request, res: Response) {
try {
const book = req.query.book as string;
if (!book) throw Error("Parameter book is required");
if (!isNumeric(book)) throw Error("Parameter book must be number");
/**
* @api {get} /nhentaito/related?book=:book Get related nhentai.to
* @apiName Get related nhentai.to
* @apiGroup nhentai.to
* @apiDescription Get a related doujinshi on nhentai.to based on id
*
* @apiParam {Number} book Book ID
*
* @apiSuccessExample {json} Success-Response:
* HTTP/1.1 200 OK
* HTTP/1.1 400 Bad Request
*
* @apiExample {curl} curl
* curl -i https://janda.sinkaroid.org/nhentaito/related?book=272
*
* @apiExample {js} JS/TS
* import axios from "axios"
*
* axios.get("https://janda.sinkaroid.org/nhentaito/related?book=272")
* .then(res => console.log(res.data))
* .catch(err => console.error(err))
*
* @apiExample {python} Python
* import aiohttp
* async with aiohttp.ClientSession() as session:
* async with session.get("https://janda.sinkaroid.org/nhentaito/related?book=272") as resp:
* print(await resp.json())
*/
const url = `${c.NHENTAI_TO}/g/${book}`;
const data = await scrapeContent(url, true);
logger.info({
path: req.path,
query: req.query,
method: req.method,
ip: req.ip,
useragent: req.get("User-Agent")
});
return res.json(data);
} catch (err) {
const e = err as Error;
res.status(400).json(maybeError(false, e.message));
}
}

57
src/controller/nhentaito/nhentaiToSearch.ts ノーマルファイル
ファイルの表示

@ -0,0 +1,57 @@
import { scrapeContent } from "../../scraper/nhentaito/nhentaiToSearchController";
import { logger } from "../../utils/logger";
import { maybeError } from "../../utils/modifier";
import c from "../../utils/options";
import { Request, Response } from "express";
export async function searchNhentaiTo(req: Request, res: Response) {
try {
const key = req.query.key || "";
const page = req.query.page || 1;
if (!key) throw Error("Parameter key is required");
/**
* @api {get} /nhentaito/search Search nhentai.to
* @apiName Search nhentai.to
* @apiGroup nhentai.to
* @apiDescription Search doujinshi on nhentai.to
* @apiParam {String} key Keyword to search
* @apiParam {Number} [page=1] Page number
*
* @apiSuccessExample {json} Success-Response:
* HTTP/1.1 200 OK
* HTTP/1.1 400 Bad Request
*
* @apiExample {curl} curl
* curl -i https://janda.sinkaroid.org/nhentaito/search?key=yuri
* curl -i https://janda.sinkaroid.org/nhentaito/search?key=yuri&page=2
*
* @apiExample {js} JS/TS
* import axios from "axios"
*
* axios.get("https://janda.sinkaroid.org/nhentaito/search?key=yuri")
* .then(res => console.log(res.data))
* .catch(err => console.error(err))
*
* @apiExample {python} Python
* import aiohttp
* async with aiohttp.ClientSession() as session:
* async with session.get("https://janda.sinkaroid.org/nhentaito/search?key=yuri") as resp:
* print(await resp.json())
*/
const url = `${c.NHENTAI_TO}/search?q=${key}&page=${page}`;
const data = await scrapeContent(url);
logger.info({
path: req.path,
query: req.query,
method: req.method,
ip: req.ip,
useragent: req.get("User-Agent")
});
return res.json(data);
} catch (err) {
const e = err as Error;
res.status(400).json(maybeError(false, e.message));
}
}

ファイルの表示

@ -2,26 +2,45 @@ import { Router } from "express";
import cors from "cors";
import { slow, limiter } from "../utils/limit-options";
// hentaifox
import { searchHentaifox } from "../controller/hentaifox/hentaifoxSearch";
import { getHentaifox } from "../controller/hentaifox/hentaifoxGet";
import { getPururin } from "../controller/pururin/pururinGet";
import { randomHentaifox } from "../controller/hentaifox/hentaifoxRandom";
// pururin
import { getPururin } from "../controller/pururin/pururinGet";
import { searchPururin } from "../controller/pururin/pururinSearch";
import { randomPururin } from "../controller/pururin/pururinRandom";
// hentai2read
import { searchHentai2read } from "../controller/hentai2read/hentai2readSearch";
import { getHentai2read } from "../controller/hentai2read/hentai2readGet";
// simply-hentai
import { getSimplyhentai } from "../controller/simply-hentai/simply-hentaiGet";
// nhentai
import { getNhentai } from "../controller/nhentai/nhentaiGet";
import { searchNhentai } from "../controller/nhentai/nhentaiSearch";
import { relatedNhentai } from "../controller/nhentai/nhentaiRelated";
import { randomNhentai } from "../controller/nhentai/nhentaiRandom";
// asmhentai
import { getAsmhentai } from "../controller/asmhentai/asmhentaiGet";
import { searchAsmhentai } from "../controller/asmhentai/asmhentaiSearch";
import { randomAsmhentai } from "../controller/asmhentai/asmhentaiRandom";
// 3hentai
import { get3hentai } from "../controller/3hentai/3hentaiGet";
import { search3hentai } from "../controller/3hentai/3hentaiSearch";
import { random3hentai } from "../controller/3hentai/3hentaiRandom";
// nhentaito
import { getNhentaiTo } from "../controller/nhentaito/nhentaiToGet";
import { randomNhentaiTo } from "../controller/nhentaito/nhentaiToRandom";
import { searchNhentaiTo } from "../controller/nhentaito/nhentaiToSearch";
import { relatedNhentaiTo } from "../controller/nhentaito/nhentaiToRelated";
function scrapeRoutes() {
const router = Router();
@ -44,6 +63,10 @@ function scrapeRoutes() {
router.get("/3hentai/get", cors(), slow, limiter, get3hentai);
router.get("/3hentai/search", cors(), slow, limiter, search3hentai);
router.get("/3hentai/random", cors(), slow, limiter, random3hentai);
router.get("/nhentaito/get", cors(), slow, limiter, getNhentaiTo);
router.get("/nhentaito/random", cors(), slow, limiter, randomNhentaiTo);
router.get("/nhentaito/search", cors(), slow, limiter, searchNhentaiTo);
router.get("/nhentaito/related", cors(), slow, limiter, relatedNhentaiTo);
return router;
}

ファイルの表示

@ -0,0 +1,75 @@
import { load } from "cheerio";
import p from "phin";
import JandaPress from "../../JandaPress";
import c from "../../utils/options";
import prox from "../../utils/reverseprox";
interface IGetNhentaiTo {
title: string;
id: number;
tags: string[];
total: number;
image: string[];
}
interface IData {
success: boolean;
data: object;
source: string;
}
const janda = new JandaPress();
export async function scrapeContent(url: string, random = false) {
try {
let res, raw;
if (random) res = await p({
url: url,
headers: {
"User-Agent": process.env.USER_AGENT || "jandapress/1.0.5 Node.js/16.9.1"
},
followRedirects: true
}), raw = res.body;
else res = await janda.fetchBody(url), raw = res;
const $ = load(raw);
const title: string = $("div#info-block div#info h1").text();
if (!title) throw Error("Not found");
const tags: string[] = $("span.tags span.name").map((i, abc) => {
return $(abc).text();
}).get();
// const cover = $("div#cover img").attr("src") || "";
const total: number = parseInt(tags.pop()?.split(" ")[0] || "0");
const id: number = parseInt($("div#cover a").attr("href")?.split("/g/")[1] || "0");
const thumbnail = $("a.gallerythumb img").map((i, abc) => {
return $(abc).attr("data-src");
}).get();
const proxy = thumbnail
.map((img: string) => img?.replace(prox.NHENTAI_TO, prox.NHENTAI_TO_SOLVER));
const image = [];
for (let i = 0; i < total; i++) {
image.push(`${proxy[i]?.replace("t.", ".")}`);
}
const objectData: IGetNhentaiTo = {
title,
id,
tags,
total,
image
};
const data: IData = {
success: true,
data: objectData,
source: `${c.NHENTAI_TO}/g/${id}?re=janda`
};
return data;
} catch (err) {
const e = err as Error;
throw Error(e.message);
}
}

ファイルの表示

@ -0,0 +1,71 @@
import { load } from "cheerio";
import JandaPress from "../../JandaPress";
import prox from "../../utils/reverseprox";
interface ISearchNhentaiTo {
title: string;
id: number;
cover: string;
}
interface IData {
success: boolean;
data: object;
page: number;
source: string;
}
const janda = new JandaPress();
export async function scrapeContent(url: string, isRelated = false) {
try {
const res = await janda.fetchBody(url);
const $ = load(res);
let mode;
if (!isRelated) mode = "div.gallery";
//in in container index-container
else mode = "div.container.index-container";
const dataRaw = $(mode);
const title = dataRaw.find("div.caption").map((i, el) => {
return $(el).text();
}).get();
const id = dataRaw.find("a").map((i, el) => {
return $(el).attr("href")?.split("/")[2];
}).get();
const cover = dataRaw.find("img").map((i, el) => {
return $(el).attr("src")?.replace(prox.NHENTAI_TO, prox.NHENTAI_TO_SOLVER) || $(el).attr("data-src")?.replace(prox.NHENTAI_TO, prox.NHENTAI_TO_SOLVER);
}).get();
let looping;
if (!isRelated) looping = dataRaw;
else looping = title;
const objectData = [];
for (let i = 0; i < looping.length; i++) {
const searchResults: ISearchNhentaiTo = {
title: title[i],
id: Number(id[i]),
cover: cover[i],
};
objectData.push(searchResults);
}
if (objectData.length === 0) throw Error("No result found");
const data: IData = {
success: true,
data: objectData,
page: Number(url.split("page=")[1]),
source: url
};
return data;
} catch (err) {
const e = err as Error;
throw Error(e.message);
}
}

ファイルの表示

@ -10,5 +10,6 @@ export default {
NHENTAI_IP_3: "http://138.2.77.198:3002",
NHENTAI_IP_4: "http://129.150.63.211:3002",
ASMHENTAI: "https://asmhentai.com",
THREEHENTAI: "http://3hentai.net"
};
THREEHENTAI: "http://3hentai.net",
NHENTAI_TO: "https://nhentai.to"
};

5
src/utils/reverseprox.ts ノーマルファイル
ファイルの表示

@ -0,0 +1,5 @@
export default {
NHENTAI_TO: "cdn.dogehls.xyz",
NHENTAI_TO_SOLVER: "amber.merahputih.moe"
};

21
test/mock.ts ノーマルファイル
ファイルの表示

@ -0,0 +1,21 @@
import p from "phin";
import { load } from "cheerio";
import { name, version } from "../package.json";
const url = "https://nhentai.to/g/272";
async function test() {
const res = await p({
url: url,
"headers": {
"User-Agent": `${name}/${version} Node.js/16.9.1`,
},
});
const $ = load(res.body);
const title = $("title").text();
console.log(title);
console.log(res.statusCode);
}
test().catch(console.error);

ファイルの表示

@ -13,7 +13,6 @@
"paths": {},
"typeRoots": ["./node_modules/@types"],
"inlineSourceMap": true,
"charset": "UTF-8",
"downlevelIteration": true,
"newLine": "lf",
"strict": true,