feat(forefront): add account page pool
このコミットが含まれているのは:
コミット
5181869218
2
index.ts
2
index.ts
|
@ -3,7 +3,6 @@ import Router from 'koa-router'
|
||||||
import bodyParser from 'koa-bodyparser';
|
import bodyParser from 'koa-bodyparser';
|
||||||
import {ChatModelFactory, Model} from "./model";
|
import {ChatModelFactory, Model} from "./model";
|
||||||
import dotenv from 'dotenv';
|
import dotenv from 'dotenv';
|
||||||
import {freeBrowserPool} from "./pool/puppeteer";
|
|
||||||
|
|
||||||
dotenv.config();
|
dotenv.config();
|
||||||
|
|
||||||
|
@ -65,7 +64,6 @@ router.get('/ask/stream', async (ctx) => {
|
||||||
app.use(router.routes());
|
app.use(router.routes());
|
||||||
|
|
||||||
(async () => {
|
(async () => {
|
||||||
await freeBrowserPool.init(+(process.env.POOL_SIZE || 2), process.env.DEBUG === '1');
|
|
||||||
const server = app.listen(3000, () => {
|
const server = app.listen(3000, () => {
|
||||||
console.log("Now listening: 127.0.0.1:3000");
|
console.log("Now listening: 127.0.0.1:3000");
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,19 +1,22 @@
|
||||||
import {Chat, ChatOptions, Request, Response, ResponseStream} from "../base";
|
import {Chat, ChatOptions, Request, Response, ResponseStream} from "../base";
|
||||||
import {Page} from "puppeteer";
|
import {Browser, Page} from "puppeteer";
|
||||||
import {FreeBrowser, freeBrowserPool} from "../../pool/puppeteer";
|
import {BrowserPool} from "../../pool/puppeteer";
|
||||||
import {CreateEmail, TempEmailType, TempMailMessage} from "../../utils/emailFactory";
|
import {CreateEmail, TempEmailType, TempMailMessage} from "../../utils/emailFactory";
|
||||||
import {CreateTlsProxy} from "../../utils/proxyAgent";
|
import {CreateTlsProxy} from "../../utils/proxyAgent";
|
||||||
import {PassThrough} from "stream";
|
import {PassThrough} from "stream";
|
||||||
|
|
||||||
|
type PageData = {
|
||||||
|
gpt4times: number;
|
||||||
|
}
|
||||||
|
|
||||||
export class Forefrontnew extends Chat {
|
export class Forefrontnew extends Chat {
|
||||||
private browser: FreeBrowser | undefined;
|
|
||||||
private page: Page | undefined = undefined;
|
private page: Page | undefined = undefined;
|
||||||
private url: string = 'https://chat.forefront.ai/';
|
|
||||||
private writing: NodeJS.Timer | undefined = undefined;
|
|
||||||
private msgSize: number = 0;
|
private msgSize: number = 0;
|
||||||
|
private pagePool: BrowserPool<PageData>;
|
||||||
|
|
||||||
constructor(options?: ChatOptions) {
|
constructor(options?: ChatOptions) {
|
||||||
super(options);
|
super(options);
|
||||||
|
this.pagePool = new BrowserPool<PageData>(+(process.env.POOL_SIZE||2), this.init.bind(this));
|
||||||
}
|
}
|
||||||
|
|
||||||
public async ask(req: Request): Promise<Response> {
|
public async ask(req: Request): Promise<Response> {
|
||||||
|
@ -33,7 +36,6 @@ export class Forefrontnew extends Chat {
|
||||||
|
|
||||||
private async tryValidate(validateURL: string, triedTimes: number) {
|
private async tryValidate(validateURL: string, triedTimes: number) {
|
||||||
if (triedTimes === 3) {
|
if (triedTimes === 3) {
|
||||||
await this.remove();
|
|
||||||
throw new Error('validate failed');
|
throw new Error('validate failed');
|
||||||
}
|
}
|
||||||
triedTimes += 1;
|
triedTimes += 1;
|
||||||
|
@ -45,28 +47,21 @@ export class Forefrontnew extends Chat {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async remove() {
|
private async init(browser: Browser): Promise<Page> {
|
||||||
await freeBrowserPool.remove(this.browser?.id || "");
|
const [page] = await browser.pages();
|
||||||
this.browser = undefined;
|
await page.goto("https://accounts.forefront.ai/sign-up");
|
||||||
this.page = undefined;
|
await page.setViewport({width: 1920, height: 1080});
|
||||||
this.msgSize = 0;
|
await page.waitForSelector('#emailAddress-field');
|
||||||
}
|
await page.click('#emailAddress-field')
|
||||||
|
|
||||||
private async init(): Promise<Page> {
|
await page.waitForSelector('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary')
|
||||||
this.browser = freeBrowserPool.getRandom();
|
await page.click('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary')
|
||||||
this.page = await this.browser.getPage("https://accounts.forefront.ai/sign-up");
|
|
||||||
await this.page.setViewport({width: 1920, height: 1080});
|
|
||||||
await this.page.waitForSelector('#emailAddress-field');
|
|
||||||
await this.page.click('#emailAddress-field')
|
|
||||||
|
|
||||||
await this.page.waitForSelector('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary')
|
|
||||||
await this.page.click('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary')
|
|
||||||
|
|
||||||
const emailBox = CreateEmail(TempEmailType.TempEmail44)
|
const emailBox = CreateEmail(TempEmailType.TempEmail44)
|
||||||
const emailAddress = await emailBox.getMailAddress();
|
const emailAddress = await emailBox.getMailAddress();
|
||||||
// 将文本键入焦点元素
|
// 将文本键入焦点元素
|
||||||
await this.page.keyboard.type(emailAddress, {delay: 10});
|
await page.keyboard.type(emailAddress, {delay: 10});
|
||||||
await this.page.keyboard.press('Enter');
|
await page.keyboard.press('Enter');
|
||||||
|
|
||||||
const msgs = (await emailBox.waitMails()) as TempMailMessage[]
|
const msgs = (await emailBox.waitMails()) as TempMailMessage[]
|
||||||
let validateURL: string | undefined;
|
let validateURL: string | undefined;
|
||||||
|
@ -81,63 +76,57 @@ export class Forefrontnew extends Chat {
|
||||||
}
|
}
|
||||||
await this.tryValidate(validateURL, 0);
|
await this.tryValidate(validateURL, 0);
|
||||||
console.log('register successfully');
|
console.log('register successfully');
|
||||||
await this.page.waitForSelector('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)', {timeout: 10000})
|
await page.waitForSelector('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)', {timeout: 10000})
|
||||||
await this.page.click('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)')
|
await page.click('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)')
|
||||||
await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})
|
await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})
|
||||||
|
|
||||||
await this.page.waitForTimeout(2000);
|
await page.waitForTimeout(2000);
|
||||||
await this.page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium', {timeout: 100000});
|
await page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium', {timeout: 100000});
|
||||||
await this.page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium');
|
await page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium');
|
||||||
await this.page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
|
await page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
|
||||||
await this.page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
|
await page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium')
|
||||||
|
|
||||||
await this.page.waitForSelector('.px-4 > .flex > .grid > .h-9 > .grow')
|
await page.waitForSelector('.px-4 > .flex > .grid > .h-9 > .grow')
|
||||||
await this.page.click('.px-4 > .flex > .grid > .h-9 > .grow')
|
await page.click('.px-4 > .flex > .grid > .h-9 > .grow')
|
||||||
|
|
||||||
await this.page.waitForSelector('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
|
await page.waitForSelector('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
|
||||||
await this.page.click('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
|
await page.click('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)')
|
||||||
return this.page;
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async askStream(req: Request): Promise<ResponseStream> {
|
public async askStream(req: Request): Promise<ResponseStream> {
|
||||||
if (this.writing) {
|
const [page, data = {gpt4times: 0} as PageData, done, destroy] = this.pagePool.get();
|
||||||
|
if (!page) {
|
||||||
const pt = new PassThrough();
|
const pt = new PassThrough();
|
||||||
pt.write('Other conversation');
|
pt.write('please wait init.....about 1 min');
|
||||||
pt.end();
|
pt.end();
|
||||||
return {text: pt}
|
return {text: pt}
|
||||||
}
|
}
|
||||||
if (this.msgSize === 5) {
|
|
||||||
await this.remove();
|
|
||||||
}
|
|
||||||
this.msgSize++;
|
|
||||||
if (!this.browser || !this.page) {
|
|
||||||
this.page = await this.init();
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
console.log('try find text input');
|
console.log('try find text input');
|
||||||
await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})
|
await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000})
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(e);
|
console.error(e);
|
||||||
}
|
}
|
||||||
console.log('try to find input');
|
console.log('try to find input');
|
||||||
await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {
|
await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {
|
||||||
timeout: 10000,
|
timeout: 10000,
|
||||||
visible: true
|
visible: true
|
||||||
})
|
})
|
||||||
console.log('found');
|
console.log('found');
|
||||||
await this.page.click('.relative > .flex > .w-full > .text-th-primary-dark > div')
|
await page.click('.relative > .flex > .w-full > .text-th-primary-dark > div')
|
||||||
await this.page.focus('.relative > .flex > .w-full > .text-th-primary-dark > div')
|
await page.focus('.relative > .flex > .w-full > .text-th-primary-dark > div')
|
||||||
await this.page.keyboard.type(req.prompt, {delay: 10});
|
await page.keyboard.type(req.prompt, {delay: 10});
|
||||||
await this.page.keyboard.press('Enter');
|
await page.keyboard.press('Enter');
|
||||||
await this.page.waitForSelector('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div');
|
await page.waitForSelector('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div');
|
||||||
// find markdown list container
|
// find markdown list container
|
||||||
const mdList = await this.page.$('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div');
|
const mdList = await page.$('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div');
|
||||||
const md = mdList;
|
const md = mdList;
|
||||||
// get latest markdown id
|
// get latest markdown id
|
||||||
let id: number = this.msgSize * 4;
|
let id: number = 4;
|
||||||
const selector = `div > .w-full:nth-child(${id}) > .flex > .flex > .post-markdown`;
|
const selector = `div > .w-full:nth-child(${id}) > .flex > .flex > .post-markdown`;
|
||||||
await this.page.waitForSelector(selector);
|
await page.waitForSelector(selector);
|
||||||
const result = await this.page.$(selector)
|
const result = await page.$(selector)
|
||||||
// get latest markdown text
|
// get latest markdown text
|
||||||
let oldText = '';
|
let oldText = '';
|
||||||
const pt = new PassThrough();
|
const pt = new PassThrough();
|
||||||
|
@ -155,11 +144,11 @@ export class Forefrontnew extends Chat {
|
||||||
pt.write(text.slice(oldText.length - text.length));
|
pt.write(text.slice(oldText.length - text.length));
|
||||||
oldText = text;
|
oldText = text;
|
||||||
}, 100)
|
}, 100)
|
||||||
if (!this.page) {
|
if (!page) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
await this.page.waitForSelector(`.w-full:nth-child(${id}) > .flex > .flex > .flex > .opacity-100`);
|
await page.waitForSelector(`.w-full > .flex > .flex > .flex > .opacity-100`);
|
||||||
const text: any = await result?.evaluate(el => {
|
const text: any = await result?.evaluate(el => {
|
||||||
return el.textContent;
|
return el.textContent;
|
||||||
});
|
});
|
||||||
|
@ -168,6 +157,14 @@ export class Forefrontnew extends Chat {
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
pt.end();
|
pt.end();
|
||||||
|
await page.waitForSelector('.flex:nth-child(1) > div:nth-child(2) > .relative > .flex > .cursor-pointer')
|
||||||
|
await page.click('.flex:nth-child(1) > div:nth-child(2) > .relative > .flex > .cursor-pointer')
|
||||||
|
data.gpt4times += 1;
|
||||||
|
if (data.gpt4times >= 5) {
|
||||||
|
destroy();
|
||||||
|
} else {
|
||||||
|
done(data);
|
||||||
|
}
|
||||||
clearInterval(itl);
|
clearInterval(itl);
|
||||||
}
|
}
|
||||||
})().then();
|
})().then();
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
import {Chat, ChatOptions} from "./base";
|
import {Chat, ChatOptions} from "./base";
|
||||||
import {You} from "./you";
|
import {You} from "./you";
|
||||||
import {AiDream} from "./aidream";
|
import {AiDream} from "./aidream";
|
||||||
import {Phind} from "./phind";
|
|
||||||
import {Forefrontnew} from "./forefront";
|
import {Forefrontnew} from "./forefront";
|
||||||
import {Mcbbs} from "./mcbbs";
|
import {Mcbbs} from "./mcbbs";
|
||||||
|
|
||||||
|
@ -10,7 +9,6 @@ export enum Model {
|
||||||
You = 'you',
|
You = 'you',
|
||||||
Forefront = 'forefront',
|
Forefront = 'forefront',
|
||||||
AiDream = 'aidream',
|
AiDream = 'aidream',
|
||||||
Phind = 'phind',
|
|
||||||
Mcbbs = 'mcbbs',
|
Mcbbs = 'mcbbs',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -29,7 +27,6 @@ export class ChatModelFactory {
|
||||||
this.modelMap.set(Model.You, new You(this.options))
|
this.modelMap.set(Model.You, new You(this.options))
|
||||||
this.modelMap.set(Model.Forefront, new Forefrontnew(this.options))
|
this.modelMap.set(Model.Forefront, new Forefrontnew(this.options))
|
||||||
this.modelMap.set(Model.AiDream, new AiDream(this.options))
|
this.modelMap.set(Model.AiDream, new AiDream(this.options))
|
||||||
this.modelMap.set(Model.Phind, new Phind(this.options))
|
|
||||||
this.modelMap.set(Model.Mcbbs, new Mcbbs(this.options))
|
this.modelMap.set(Model.Mcbbs, new Mcbbs(this.options))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,34 +0,0 @@
|
||||||
import {Chat, ChatOptions, Request, Response, ResponseStream} from "../base";
|
|
||||||
import {Page} from "puppeteer";
|
|
||||||
import {FreeBrowser, freeBrowserPool} from "../../pool/puppeteer";
|
|
||||||
import {Stream} from "stream";
|
|
||||||
|
|
||||||
export class Phind extends Chat {
|
|
||||||
private browser: FreeBrowser | undefined;
|
|
||||||
private page: Page | undefined = undefined;
|
|
||||||
|
|
||||||
constructor(options?: ChatOptions) {
|
|
||||||
super(options);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async ask(req: Request): Promise<Response> {
|
|
||||||
return Promise.resolve({text: ''});
|
|
||||||
}
|
|
||||||
|
|
||||||
public async askStream(req: Request): Promise<ResponseStream> {
|
|
||||||
if (!this.browser) {
|
|
||||||
this.browser = freeBrowserPool.getRandom();
|
|
||||||
}
|
|
||||||
if (!this.page) {
|
|
||||||
this.page = await this.browser.getPage('https://phind.com');
|
|
||||||
await this.page.setViewport({width: 1920, height: 1080})
|
|
||||||
// await this.page.waitForNavigation();
|
|
||||||
}
|
|
||||||
|
|
||||||
// await this.page.waitForSelector('.col-md-10 > .container-xl > .mb-3 > .input-group > .form-control')
|
|
||||||
// await this.page.click('.col-md-10 > .container-xl > .mb-3 > .input-group > .form-control')
|
|
||||||
// todo complete
|
|
||||||
return Promise.resolve({text: new Stream()});
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,103 +1,74 @@
|
||||||
import puppeteer, {Browser, Page, PuppeteerLaunchOptions} from "puppeteer";
|
import puppeteer, {Browser, Page, PuppeteerLaunchOptions} from "puppeteer";
|
||||||
import fs from 'fs';
|
|
||||||
import path from "path";
|
import path from "path";
|
||||||
import run from "node:test";
|
import run from "node:test";
|
||||||
import {v4} from "uuid";
|
|
||||||
|
|
||||||
const runPath = path.join(__dirname, 'run');
|
const runPath = path.join(__dirname, 'run');
|
||||||
|
|
||||||
export class FreeBrowser {
|
export interface PageInfo<T> {
|
||||||
private browser: Browser | undefined = undefined;
|
id: number;
|
||||||
private readonly options: PuppeteerLaunchOptions | undefined;
|
ready: boolean;
|
||||||
private urls: Set<string> = new Set<string>();
|
page?: Page;
|
||||||
private pages: Record<string, Page> = {};
|
data?: T;
|
||||||
public readonly id: string;
|
|
||||||
|
|
||||||
constructor(id: string, options?: PuppeteerLaunchOptions) {
|
|
||||||
this.options = {
|
|
||||||
// userDataDir: path.join(runPath, id),
|
|
||||||
...options
|
|
||||||
};
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async init() {
|
|
||||||
this.browser = await puppeteer.launch(this.options)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public async getPage(url: string): Promise<Page> {
|
|
||||||
if (!this.browser) {
|
|
||||||
throw new Error('Browser must init first')
|
|
||||||
}
|
|
||||||
if (this.pages[url]) {
|
|
||||||
return this.pages[url];
|
|
||||||
}
|
|
||||||
const page = await this.browser.newPage();
|
|
||||||
await page.goto(url)
|
|
||||||
|
|
||||||
this.pages[url] = page;
|
|
||||||
return page;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async close() {
|
|
||||||
this.browser?.close();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type PrepareFunc<T> = (browser: Browser) => Promise<Page>
|
||||||
|
|
||||||
class FreeBrowserPool {
|
export class BrowserPool<T> {
|
||||||
private size: number = 0;
|
private readonly pool: PageInfo<T>[] = [];
|
||||||
private readonly pool: FreeBrowser[];
|
private readonly size: number;
|
||||||
private debug: boolean = false;
|
private readonly prepare: PrepareFunc<T>
|
||||||
|
|
||||||
constructor() {
|
constructor(size: number, prepare: PrepareFunc<T>,) {
|
||||||
this.pool = [];
|
this.size = size
|
||||||
|
this.prepare = prepare;
|
||||||
|
this.init();
|
||||||
}
|
}
|
||||||
|
|
||||||
public async init(size: number, debug: boolean) {
|
init() {
|
||||||
this.debug = debug;
|
for (let i = 0; i < this.size; i++) {
|
||||||
console.log(`browser pool init size:${size}`)
|
const info: PageInfo<T> = {
|
||||||
if (!fs.existsSync(runPath)) {
|
id: i,
|
||||||
fs.mkdirSync(runPath);
|
ready: false,
|
||||||
}
|
}
|
||||||
this.size = size;
|
this.initOne().then(page => {
|
||||||
for (let i = 0; i < size; i++) {
|
info.page = page;
|
||||||
this.pool.push(await this.newBrowser());
|
info.ready = true;
|
||||||
}
|
}).catch(e=>{
|
||||||
}
|
console.error(e);
|
||||||
|
})
|
||||||
public getRandom(): FreeBrowser {
|
this.pool.push(info)
|
||||||
return this.pool[Math.floor(Math.random() * this.pool.length)]
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async newBrowser(): Promise<FreeBrowser> {
|
async initOne(): Promise<Page> {
|
||||||
const options: PuppeteerLaunchOptions = {
|
const options: PuppeteerLaunchOptions = {
|
||||||
headless: this.debug ? false : 'new',
|
headless: process.env.DEBUG === "1" ? false : 'new',
|
||||||
args: ['--no-sandbox']
|
args: ['--no-sandbox'],
|
||||||
};
|
};
|
||||||
if (process.env.http_proxy) {
|
const browser = await puppeteer.launch(options)
|
||||||
options.args?.push(`--proxy-server=${process.env.http_proxy}`);
|
return this.prepare(browser)
|
||||||
}
|
}
|
||||||
const browser = new FreeBrowser(v4(), options);
|
|
||||||
await browser.init();
|
//@ts-ignore
|
||||||
return browser;
|
get(): [page: Page | undefined, data: T | undefined, done: (data: T) => void, destroy: () => void] {
|
||||||
}
|
for (const item of this.pool) {
|
||||||
|
if (item.ready) {
|
||||||
public async remove(id: string) {
|
item.ready = false;
|
||||||
let removed = false;
|
return [
|
||||||
this.pool.filter(item => {
|
item.page,
|
||||||
if (item.id === id) {
|
item.data,
|
||||||
item.close();
|
(data: T) => {
|
||||||
removed = true;
|
item.ready = true
|
||||||
return false;
|
item.data = data;
|
||||||
|
},
|
||||||
|
() => {
|
||||||
|
this.initOne().then((page) => {
|
||||||
|
item.page = page
|
||||||
|
item.ready = true;
|
||||||
|
})
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
return true;
|
|
||||||
})
|
|
||||||
if (removed) {
|
|
||||||
this.pool.push(await this.newBrowser());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const freeBrowserPool = new FreeBrowserPool();
|
|
||||||
|
|
読み込み中…
新しいイシューから参照