bagood
/
gpt4free-ts
ミラー元 https://github.com/xiangsx/gpt4free-ts.git
1
0
フォーク 0
コード イシュー パッケージ プロジェクト リリース Wiki アクティビティ

feat(forefront): add account page pool

このコミットが含まれているのは:
xiang 2023-06-05 22:11:03 +08:00
親 fdbe20b60c
コミット 5181869218
5個のファイルの変更、113行の追加、184行の削除
統計情報を表示 Patchファイルをダウンロード Diffファイルをダウンロード すべてのファイルを展開 すべてのファイルを折り畳む

2
index.ts
ファイルの表示

@ -3,7 +3,6 @@ import Router from 'koa-router'
import bodyParser from 'koa-bodyparser';
import {ChatModelFactory, Model} from "./model";
import dotenv from 'dotenv';
import {freeBrowserPool} from "./pool/puppeteer";
dotenv.config();
@ -65,7 +64,6 @@ router.get('/ask/stream', async (ctx) => {
app.use(router.routes());
(async () => {
await freeBrowserPool.init(+(process.env.POOL_SIZE || 2), process.env.DEBUG === '1');
const server = app.listen(3000, () => {
console.log("Now listening: 127.0.0.1:3000");
});

113
model/forefront/index.ts
ファイルの表示

@ -1,19 +1,22 @@
import {Chat, ChatOptions, Request, Response, ResponseStream} from "../base";
import {Page} from "puppeteer";
import {FreeBrowser, freeBrowserPool} from "../../pool/puppeteer";
import {Browser, Page} from "puppeteer";
import {BrowserPool} from "../../pool/puppeteer";
import {CreateEmail, TempEmailType, TempMailMessage} from "../../utils/emailFactory";
import {CreateTlsProxy} from "../../utils/proxyAgent";
import {PassThrough} from "stream";
type PageData = {
gpt4times: number;
}
export class Forefrontnew extends Chat {
private browser: FreeBrowser | undefined;
private page: Page | undefined = undefined;
private url: string = 'https://chat.forefront.ai/';
private writing: NodeJS.Timer | undefined = undefined;
private msgSize: number = 0;
private pagePool: BrowserPool<PageData>;
constructor(options?: ChatOptions) {
super(options);
this.pagePool = new BrowserPool<PageData>(+(process.env.POOL_SIZE||2), this.init.bind(this));
}
public async ask(req: Request): Promise<Response> {
@ -33,7 +36,6 @@ export class Forefrontnew extends Chat {
private async tryValidate(validateURL: string, triedTimes: number) {
if (triedTimes === 3) {
await this.remove();
throw new Error('validate failed');
}
triedTimes += 1;
@ -45,28 +47,21 @@ export class Forefrontnew extends Chat {
}
}
private async remove() {
await freeBrowserPool.remove(this.browser?.id || "");
this.browser = undefined;
this.page = undefined;
this.msgSize = 0;
}
private async init(browser: Browser): Promise<Page> {
const [page] = await browser.pages();
await page.goto("https://accounts.forefront.ai/sign-up");
await page.setViewport({width: 1920, height: 1080});
await page.waitForSelector('#emailAddress-field');
await page.click('#emailAddress-field')
private async init(): Promise<Page> {
this.browser = freeBrowserPool.getRandom();
this.page = await this.browser.getPage("https://accounts.forefront.ai/sign-up");
await this.page.setViewport({width: 1920, height: 1080});
await this.page.waitForSelector('#emailAddress-field');
await this.page.click('#emailAddress-field')
await this.page.waitForSelector('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary')
await this.page.click('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary')
await page.waitForSelector('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary')
await page.click('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary')
const emailBox = CreateEmail(TempEmailType.TempEmail44)
const emailAddress = await emailBox.getMailAddress();
// 将文本键入焦点元素
await this.page.keyboard.type(emailAddress, {delay: 10});
await this.page.keyboard.press('Enter');
await page.keyboard.type(emailAddress, {delay: 10});
await page.keyboard.press('Enter');
const msgs = (await emailBox.waitMails()) as TempMailMessage[]
let validateURL: string | undefined;
@ -81,63 +76,57 @@ export class Forefrontnew extends Chat {
}
await this.tryValidate(validateURL, 0);
console.log('register successfully');
await this.page.waitForSelector('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)', {timeout: 10000})
await this.page.click('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)')
await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})
await page.waitForSelector('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)', {timeout: 10000})
await page.click('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)')
await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})
await this.page.waitForTimeout(2000);
await this.page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium', {timeout: 100000});
await this.page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium');
await this.page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
await this.page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
await page.waitForTimeout(2000);
await page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium', {timeout: 100000});
await page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium');
await page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
await page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
await this.page.waitForSelector('.px-4 > .flex > .grid > .h-9 > .grow')
await this.page.click('.px-4 > .flex > .grid > .h-9 > .grow')
await page.waitForSelector('.px-4 > .flex > .grid > .h-9 > .grow')
await page.click('.px-4 > .flex > .grid > .h-9 > .grow')
await this.page.waitForSelector('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
await this.page.click('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
return this.page;
await page.waitForSelector('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
await page.click('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
return page;
}
public async askStream(req: Request): Promise<ResponseStream> {
if (this.writing) {
const [page, data = {gpt4times: 0} as PageData, done, destroy] = this.pagePool.get();
if (!page) {
const pt = new PassThrough();
pt.write('Other conversation');
pt.write('please wait init.....about 1 min');
pt.end();
return {text: pt}
}
if (this.msgSize === 5) {
await this.remove();
}
this.msgSize++;
if (!this.browser || !this.page) {
this.page = await this.init();
}
try {
console.log('try find text input');
await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})
await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})
} catch (e) {
console.error(e);
}
console.log('try to find input');
await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {
await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {
timeout: 10000,
visible: true
})
console.log('found');
await this.page.click('.relative > .flex > .w-full > .text-th-primary-dark > div')
await this.page.focus('.relative > .flex > .w-full > .text-th-primary-dark > div')
await this.page.keyboard.type(req.prompt, {delay: 10});
await this.page.keyboard.press('Enter');
await this.page.waitForSelector('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div');
await page.click('.relative > .flex > .w-full > .text-th-primary-dark > div')
await page.focus('.relative > .flex > .w-full > .text-th-primary-dark > div')
await page.keyboard.type(req.prompt, {delay: 10});
await page.keyboard.press('Enter');
await page.waitForSelector('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div');
// find markdown list container
const mdList = await this.page.$('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div');
const mdList = await page.$('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div');
const md = mdList;
// get latest markdown id
let id: number = this.msgSize * 4;
let id: number = 4;
const selector = `div > .w-full:nth-child(${id}) > .flex > .flex > .post-markdown`;
await this.page.waitForSelector(selector);
const result = await this.page.$(selector)
await page.waitForSelector(selector);
const result = await page.$(selector)
// get latest markdown text
let oldText = '';
const pt = new PassThrough();
@ -155,11 +144,11 @@ export class Forefrontnew extends Chat {
pt.write(text.slice(oldText.length - text.length));
oldText = text;
}, 100)
if (!this.page) {
if (!page) {
return;
}
try {
await this.page.waitForSelector(`.w-full:nth-child(${id}) > .flex > .flex > .flex > .opacity-100`);
await page.waitForSelector(`.w-full > .flex > .flex > .flex > .opacity-100`);
const text: any = await result?.evaluate(el => {
return el.textContent;
});
@ -168,6 +157,14 @@ export class Forefrontnew extends Chat {
}
} finally {
pt.end();
await page.waitForSelector('.flex:nth-child(1) > div:nth-child(2) > .relative > .flex > .cursor-pointer')
await page.click('.flex:nth-child(1) > div:nth-child(2) > .relative > .flex > .cursor-pointer')
data.gpt4times += 1;
if (data.gpt4times >= 5) {
destroy();
} else {
done(data);
}
clearInterval(itl);
}
})().then();

3
model/index.ts
ファイルの表示

@ -1,7 +1,6 @@
import {Chat, ChatOptions} from "./base";
import {You} from "./you";
import {AiDream} from "./aidream";
import {Phind} from "./phind";
import {Forefrontnew} from "./forefront";
import {Mcbbs} from "./mcbbs";
@ -10,7 +9,6 @@ export enum Model {
You = 'you',
Forefront = 'forefront',
AiDream = 'aidream',
Phind = 'phind',
Mcbbs = 'mcbbs',
}
@ -29,7 +27,6 @@ export class ChatModelFactory {
this.modelMap.set(Model.You, new You(this.options))
this.modelMap.set(Model.Forefront, new Forefrontnew(this.options))
this.modelMap.set(Model.AiDream, new AiDream(this.options))
this.modelMap.set(Model.Phind, new Phind(this.options))
this.modelMap.set(Model.Mcbbs, new Mcbbs(this.options))
}

34
model/phind/index.ts
ファイルの表示

@ -1,34 +0,0 @@
import {Chat, ChatOptions, Request, Response, ResponseStream} from "../base";
import {Page} from "puppeteer";
import {FreeBrowser, freeBrowserPool} from "../../pool/puppeteer";
import {Stream} from "stream";
export class Phind extends Chat {
private browser: FreeBrowser | undefined;
private page: Page | undefined = undefined;
constructor(options?: ChatOptions) {
super(options);
}
public async ask(req: Request): Promise<Response> {
return Promise.resolve({text: ''});
}
public async askStream(req: Request): Promise<ResponseStream> {
if (!this.browser) {
this.browser = freeBrowserPool.getRandom();
}
if (!this.page) {
this.page = await this.browser.getPage('https://phind.com');
await this.page.setViewport({width: 1920, height: 1080})
// await this.page.waitForNavigation();
}
// await this.page.waitForSelector('.col-md-10 > .container-xl > .mb-3 > .input-group > .form-control')
// await this.page.click('.col-md-10 > .container-xl > .mb-3 > .input-group > .form-control')
// todo complete
return Promise.resolve({text: new Stream()});
}
}

145
pool/puppeteer.ts
ファイルの表示

@ -1,103 +1,74 @@
import puppeteer, {Browser, Page, PuppeteerLaunchOptions} from "puppeteer";
import fs from 'fs';
import path from "path";
import run from "node:test";
import {v4} from "uuid";
const runPath = path.join(__dirname, 'run');
export class FreeBrowser {
private browser: Browser | undefined = undefined;
private readonly options: PuppeteerLaunchOptions | undefined;
private urls: Set<string> = new Set<string>();
private pages: Record<string, Page> = {};
public readonly id: string;
constructor(id: string, options?: PuppeteerLaunchOptions) {
this.options = {
// userDataDir: path.join(runPath, id),
...options
};
this.id = id;
}
public async init() {
this.browser = await puppeteer.launch(this.options)
}
public async getPage(url: string): Promise<Page> {
if (!this.browser) {
throw new Error('Browser must init first')
}
if (this.pages[url]) {
return this.pages[url];
}
const page = await this.browser.newPage();
await page.goto(url)
this.pages[url] = page;
return page;
}
public async close() {
this.browser?.close();
}
export interface PageInfo<T> {
id: number;
ready: boolean;
page?: Page;
data?: T;
}
type PrepareFunc<T> = (browser: Browser) => Promise<Page>
class FreeBrowserPool {
private size: number = 0;
private readonly pool: FreeBrowser[];
private debug: boolean = false;
export class BrowserPool<T> {
private readonly pool: PageInfo<T>[] = [];
private readonly size: number;
private readonly prepare: PrepareFunc<T>
constructor() {
this.pool = [];
constructor(size: number, prepare: PrepareFunc<T>,) {
this.size = size
this.prepare = prepare;
this.init();
}
public async init(size: number, debug: boolean) {
this.debug = debug;
console.log(`browser pool init size:${size}`)
if (!fs.existsSync(runPath)) {
fs.mkdirSync(runPath);
}
this.size = size;
for (let i = 0; i < size; i++) {
this.pool.push(await this.newBrowser());
}
}
public getRandom(): FreeBrowser {
return this.pool[Math.floor(Math.random() * this.pool.length)]
}
private async newBrowser(): Promise<FreeBrowser> {
const options: PuppeteerLaunchOptions = {
headless: this.debug ? false : 'new',
args: ['--no-sandbox']
};
if (process.env.http_proxy) {
options.args?.push(`--proxy-server=${process.env.http_proxy}`);
}
const browser = new FreeBrowser(v4(), options);
await browser.init();
return browser;
}
public async remove(id: string) {
let removed = false;
this.pool.filter(item => {
if (item.id === id) {
item.close();
removed = true;
return false;
init() {
for (let i = 0; i < this.size; i++) {
const info: PageInfo<T> = {
id: i,
ready: false,
}
this.initOne().then(page => {
info.page = page;
info.ready = true;
}).catch(e=>{
console.error(e);
})
this.pool.push(info)
}
}
async initOne(): Promise<Page> {
const options: PuppeteerLaunchOptions = {
headless: process.env.DEBUG === "1" ? false : 'new',
args: ['--no-sandbox'],
};
const browser = await puppeteer.launch(options)
return this.prepare(browser)
}
//@ts-ignore
get(): [page: Page | undefined, data: T | undefined, done: (data: T) => void, destroy: () => void] {
for (const item of this.pool) {
if (item.ready) {
item.ready = false;
return [
item.page,
item.data,
(data: T) => {
item.ready = true
item.data = data;
},
() => {
this.initOne().then((page) => {
item.page = page
item.ready = true;
})
}
]
}
return true;
})
if (removed) {
this.pool.push(await this.newBrowser());
}
}
}
export const freeBrowserPool = new FreeBrowserPool();