diff --git a/index.ts b/index.ts index 310338c..b1a34bc 100644 --- a/index.ts +++ b/index.ts @@ -3,7 +3,6 @@ import Router from 'koa-router' import bodyParser from 'koa-bodyparser'; import {ChatModelFactory, Model} from "./model"; import dotenv from 'dotenv'; -import {freeBrowserPool} from "./pool/puppeteer"; dotenv.config(); @@ -65,7 +64,6 @@ router.get('/ask/stream', async (ctx) => { app.use(router.routes()); (async () => { - await freeBrowserPool.init(+(process.env.POOL_SIZE || 2), process.env.DEBUG === '1'); const server = app.listen(3000, () => { console.log("Now listening: 127.0.0.1:3000"); }); diff --git a/model/forefront/index.ts b/model/forefront/index.ts index f3d0827..cda2920 100644 --- a/model/forefront/index.ts +++ b/model/forefront/index.ts @@ -1,19 +1,22 @@ import {Chat, ChatOptions, Request, Response, ResponseStream} from "../base"; -import {Page} from "puppeteer"; -import {FreeBrowser, freeBrowserPool} from "../../pool/puppeteer"; +import {Browser, Page} from "puppeteer"; +import {BrowserPool} from "../../pool/puppeteer"; import {CreateEmail, TempEmailType, TempMailMessage} from "../../utils/emailFactory"; import {CreateTlsProxy} from "../../utils/proxyAgent"; import {PassThrough} from "stream"; +type PageData = { + gpt4times: number; +} + export class Forefrontnew extends Chat { - private browser: FreeBrowser | undefined; private page: Page | undefined = undefined; - private url: string = 'https://chat.forefront.ai/'; - private writing: NodeJS.Timer | undefined = undefined; private msgSize: number = 0; + private pagePool: BrowserPool; constructor(options?: ChatOptions) { super(options); + this.pagePool = new BrowserPool(+(process.env.POOL_SIZE||2), this.init.bind(this)); } public async ask(req: Request): Promise { @@ -33,7 +36,6 @@ export class Forefrontnew extends Chat { private async tryValidate(validateURL: string, triedTimes: number) { if (triedTimes === 3) { - await this.remove(); throw new Error('validate failed'); } triedTimes += 1; @@ -45,28 +47,21 @@ export class Forefrontnew extends Chat { } } - private async remove() { - await freeBrowserPool.remove(this.browser?.id || ""); - this.browser = undefined; - this.page = undefined; - this.msgSize = 0; - } + private async init(browser: Browser): Promise { + const [page] = await browser.pages(); + await page.goto("https://accounts.forefront.ai/sign-up"); + await page.setViewport({width: 1920, height: 1080}); + await page.waitForSelector('#emailAddress-field'); + await page.click('#emailAddress-field') - private async init(): Promise { - this.browser = freeBrowserPool.getRandom(); - this.page = await this.browser.getPage("https://accounts.forefront.ai/sign-up"); - await this.page.setViewport({width: 1920, height: 1080}); - await this.page.waitForSelector('#emailAddress-field'); - await this.page.click('#emailAddress-field') - - await this.page.waitForSelector('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary') - await this.page.click('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary') + await page.waitForSelector('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary') + await page.click('.cl-rootBox > .cl-card > .cl-main > .cl-form > .cl-formButtonPrimary') const emailBox = CreateEmail(TempEmailType.TempEmail44) const emailAddress = await emailBox.getMailAddress(); // 将文本键入焦点元素 - await this.page.keyboard.type(emailAddress, {delay: 10}); - await this.page.keyboard.press('Enter'); + await page.keyboard.type(emailAddress, {delay: 10}); + await page.keyboard.press('Enter'); const msgs = (await emailBox.waitMails()) as TempMailMessage[] let validateURL: string | undefined; @@ -81,63 +76,57 @@ export class Forefrontnew extends Chat { } await this.tryValidate(validateURL, 0); console.log('register successfully'); - await this.page.waitForSelector('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)', {timeout: 10000}) - await this.page.click('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)') - await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000}) + await page.waitForSelector('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)', {timeout: 10000}) + await page.click('.flex > .modal > .modal-box > .flex > .px-3:nth-child(1)') + await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000}) - await this.page.waitForTimeout(2000); - await this.page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium', {timeout: 100000}); - await this.page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium'); - await this.page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium') - await this.page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium') + await page.waitForTimeout(2000); + await page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium', {timeout: 100000}); + await page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium'); + await page.waitForSelector('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium') + await page.click('.absolute > .shadow > .w-full:nth-child(2) > .flex > .font-medium') - await this.page.waitForSelector('.px-4 > .flex > .grid > .h-9 > .grow') - await this.page.click('.px-4 > .flex > .grid > .h-9 > .grow') + await page.waitForSelector('.px-4 > .flex > .grid > .h-9 > .grow') + await page.click('.px-4 > .flex > .grid > .h-9 > .grow') - await this.page.waitForSelector('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)') - await this.page.click('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)') - return this.page; + await page.waitForSelector('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)') + await page.click('.grid > .block > .group:nth-child(5) > .grid > .grow:nth-child(1)') + return page; } public async askStream(req: Request): Promise { - if (this.writing) { + const [page, data = {gpt4times: 0} as PageData, done, destroy] = this.pagePool.get(); + if (!page) { const pt = new PassThrough(); - pt.write('Other conversation'); + pt.write('please wait init.....about 1 min'); pt.end(); return {text: pt} } - if (this.msgSize === 5) { - await this.remove(); - } - this.msgSize++; - if (!this.browser || !this.page) { - this.page = await this.init(); - } try { console.log('try find text input'); - await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000}) + await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', {timeout: 10000}) } catch (e) { console.error(e); } console.log('try to find input'); - await this.page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', { + await page.waitForSelector('.relative > .flex > .w-full > .text-th-primary-dark > div', { timeout: 10000, visible: true }) console.log('found'); - await this.page.click('.relative > .flex > .w-full > .text-th-primary-dark > div') - await this.page.focus('.relative > .flex > .w-full > .text-th-primary-dark > div') - await this.page.keyboard.type(req.prompt, {delay: 10}); - await this.page.keyboard.press('Enter'); - await this.page.waitForSelector('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div'); + await page.click('.relative > .flex > .w-full > .text-th-primary-dark > div') + await page.focus('.relative > .flex > .w-full > .text-th-primary-dark > div') + await page.keyboard.type(req.prompt, {delay: 10}); + await page.keyboard.press('Enter'); + await page.waitForSelector('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div'); // find markdown list container - const mdList = await this.page.$('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div'); + const mdList = await page.$('#__next > .flex > .relative > .relative > .w-full:nth-child(1) > div'); const md = mdList; // get latest markdown id - let id: number = this.msgSize * 4; + let id: number = 4; const selector = `div > .w-full:nth-child(${id}) > .flex > .flex > .post-markdown`; - await this.page.waitForSelector(selector); - const result = await this.page.$(selector) + await page.waitForSelector(selector); + const result = await page.$(selector) // get latest markdown text let oldText = ''; const pt = new PassThrough(); @@ -155,11 +144,11 @@ export class Forefrontnew extends Chat { pt.write(text.slice(oldText.length - text.length)); oldText = text; }, 100) - if (!this.page) { + if (!page) { return; } try { - await this.page.waitForSelector(`.w-full:nth-child(${id}) > .flex > .flex > .flex > .opacity-100`); + await page.waitForSelector(`.w-full > .flex > .flex > .flex > .opacity-100`); const text: any = await result?.evaluate(el => { return el.textContent; }); @@ -168,6 +157,14 @@ export class Forefrontnew extends Chat { } } finally { pt.end(); + await page.waitForSelector('.flex:nth-child(1) > div:nth-child(2) > .relative > .flex > .cursor-pointer') + await page.click('.flex:nth-child(1) > div:nth-child(2) > .relative > .flex > .cursor-pointer') + data.gpt4times += 1; + if (data.gpt4times >= 5) { + destroy(); + } else { + done(data); + } clearInterval(itl); } })().then(); diff --git a/model/index.ts b/model/index.ts index 0a8c617..3cad49d 100644 --- a/model/index.ts +++ b/model/index.ts @@ -1,7 +1,6 @@ import {Chat, ChatOptions} from "./base"; import {You} from "./you"; import {AiDream} from "./aidream"; -import {Phind} from "./phind"; import {Forefrontnew} from "./forefront"; import {Mcbbs} from "./mcbbs"; @@ -10,7 +9,6 @@ export enum Model { You = 'you', Forefront = 'forefront', AiDream = 'aidream', - Phind = 'phind', Mcbbs = 'mcbbs', } @@ -29,7 +27,6 @@ export class ChatModelFactory { this.modelMap.set(Model.You, new You(this.options)) this.modelMap.set(Model.Forefront, new Forefrontnew(this.options)) this.modelMap.set(Model.AiDream, new AiDream(this.options)) - this.modelMap.set(Model.Phind, new Phind(this.options)) this.modelMap.set(Model.Mcbbs, new Mcbbs(this.options)) } diff --git a/model/phind/index.ts b/model/phind/index.ts deleted file mode 100644 index a55b362..0000000 --- a/model/phind/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -import {Chat, ChatOptions, Request, Response, ResponseStream} from "../base"; -import {Page} from "puppeteer"; -import {FreeBrowser, freeBrowserPool} from "../../pool/puppeteer"; -import {Stream} from "stream"; - -export class Phind extends Chat { - private browser: FreeBrowser | undefined; - private page: Page | undefined = undefined; - - constructor(options?: ChatOptions) { - super(options); - } - - public async ask(req: Request): Promise { - return Promise.resolve({text: ''}); - } - - public async askStream(req: Request): Promise { - if (!this.browser) { - this.browser = freeBrowserPool.getRandom(); - } - if (!this.page) { - this.page = await this.browser.getPage('https://phind.com'); - await this.page.setViewport({width: 1920, height: 1080}) - // await this.page.waitForNavigation(); - } - - // await this.page.waitForSelector('.col-md-10 > .container-xl > .mb-3 > .input-group > .form-control') - // await this.page.click('.col-md-10 > .container-xl > .mb-3 > .input-group > .form-control') - // todo complete - return Promise.resolve({text: new Stream()}); - } - -} diff --git a/pool/puppeteer.ts b/pool/puppeteer.ts index ccb5503..1f3945b 100644 --- a/pool/puppeteer.ts +++ b/pool/puppeteer.ts @@ -1,103 +1,74 @@ import puppeteer, {Browser, Page, PuppeteerLaunchOptions} from "puppeteer"; -import fs from 'fs'; import path from "path"; import run from "node:test"; -import {v4} from "uuid"; const runPath = path.join(__dirname, 'run'); -export class FreeBrowser { - private browser: Browser | undefined = undefined; - private readonly options: PuppeteerLaunchOptions | undefined; - private urls: Set = new Set(); - private pages: Record = {}; - public readonly id: string; - - constructor(id: string, options?: PuppeteerLaunchOptions) { - this.options = { - // userDataDir: path.join(runPath, id), - ...options - }; - this.id = id; - } - - public async init() { - this.browser = await puppeteer.launch(this.options) - } - - - public async getPage(url: string): Promise { - if (!this.browser) { - throw new Error('Browser must init first') - } - if (this.pages[url]) { - return this.pages[url]; - } - const page = await this.browser.newPage(); - await page.goto(url) - - this.pages[url] = page; - return page; - } - - public async close() { - this.browser?.close(); - } +export interface PageInfo { + id: number; + ready: boolean; + page?: Page; + data?: T; } +type PrepareFunc = (browser: Browser) => Promise -class FreeBrowserPool { - private size: number = 0; - private readonly pool: FreeBrowser[]; - private debug: boolean = false; +export class BrowserPool { + private readonly pool: PageInfo[] = []; + private readonly size: number; + private readonly prepare: PrepareFunc - constructor() { - this.pool = []; + constructor(size: number, prepare: PrepareFunc,) { + this.size = size + this.prepare = prepare; + this.init(); } - public async init(size: number, debug: boolean) { - this.debug = debug; - console.log(`browser pool init size:${size}`) - if (!fs.existsSync(runPath)) { - fs.mkdirSync(runPath); - } - this.size = size; - for (let i = 0; i < size; i++) { - this.pool.push(await this.newBrowser()); - } - } - - public getRandom(): FreeBrowser { - return this.pool[Math.floor(Math.random() * this.pool.length)] - } - - private async newBrowser(): Promise { - const options: PuppeteerLaunchOptions = { - headless: this.debug ? false : 'new', - args: ['--no-sandbox'] - }; - if (process.env.http_proxy) { - options.args?.push(`--proxy-server=${process.env.http_proxy}`); - } - const browser = new FreeBrowser(v4(), options); - await browser.init(); - return browser; - } - - public async remove(id: string) { - let removed = false; - this.pool.filter(item => { - if (item.id === id) { - item.close(); - removed = true; - return false; + init() { + for (let i = 0; i < this.size; i++) { + const info: PageInfo = { + id: i, + ready: false, + } + this.initOne().then(page => { + info.page = page; + info.ready = true; + }).catch(e=>{ + console.error(e); + }) + this.pool.push(info) + } + } + + async initOne(): Promise { + const options: PuppeteerLaunchOptions = { + headless: process.env.DEBUG === "1" ? false : 'new', + args: ['--no-sandbox'], + }; + const browser = await puppeteer.launch(options) + return this.prepare(browser) + } + + //@ts-ignore + get(): [page: Page | undefined, data: T | undefined, done: (data: T) => void, destroy: () => void] { + for (const item of this.pool) { + if (item.ready) { + item.ready = false; + return [ + item.page, + item.data, + (data: T) => { + item.ready = true + item.data = data; + }, + () => { + this.initOne().then((page) => { + item.page = page + item.ready = true; + }) + } + ] } - return true; - }) - if (removed) { - this.pool.push(await this.newBrowser()); } } } - -export const freeBrowserPool = new FreeBrowserPool();