bigcode
f0444e5879
走生产双路径原形态(runtime=auto 全池 LRU,令牌内 go-fetch/溢出温池浏览器), 与 serp-profile-capacity.ts 的单号 sticky+fetch 互补。 控制器每 60s 决策(30min 滚动窗口):成功率<95%×0.5/<98%×0.7、验证码波×0.7、 内部门壳尝试占比>15%×0.85 抢先退避、池空×0.85;干净时每 5 拍 ×1.08 加速,区间 [3,400] req/min。 smoke(dp0,7min)验证发现并修复:发出/完成同刻记账(在途请求跨桶虚报失败,高并发下 会把控制器系统性压到下限)+ 退避最少失败数门槛(低速率单次偶发失败不触发)。
288 lines
15 KiB
TypeScript
288 lines
15 KiB
TypeScript
/** # 脚本说明
|
||
* SERP 双路径(fetch+浏览器)24h 自适应吞吐优化测试(用户 2026-10-30 目标):
|
||
* 在保证客户端成功率 >98% 的约束下,自适应逼近系统可持续的最大每分钟请求数。
|
||
* 与 serp-profile-capacity.ts(单号 sticky+fetch,绕过池/令牌)不同,本脚本走生产
|
||
* 双路径原形态(runtime=auto,无 sticky,全池 LRU 选号):
|
||
* GET /api/serp/json/search?q=...&num=10
|
||
* 令牌内走 go-fetch(全局限流 fetchPerSec),限流外并发走温池浏览器 —— 组合即生产形态。
|
||
* AIMD 控制器(每 60s 决策,30min 滚动窗口;加退避即时、加速每 5 分钟一步):
|
||
* 成功率<95% → 速率×0.5;<98% → ×0.7;验证码≥3/窗口 → ×0.7;
|
||
* 内部门壳尝试占比>15% → ×0.85(抢先退避,藏不住的门壳墙来临前减速);
|
||
* 池空>5/窗口 → ×0.85;成功率≥99.5% 且门壳占比<2% 且无池空/验证码 → ×1.08。
|
||
* 速率区间 [3, 400] req/min;并发 worker 默认 12(真实并发让溢出自然流向浏览器路径)。
|
||
* 判定(客户端,JSON 出口):ok = 200 且 organic>0;gate = 200 且 organic=0 且
|
||
* attempts 含 gate;empty = 200 且 organic=0 无门壳尝试;错误按响应 code 分类
|
||
* (CAPTCHA_DETECTED / SERP_POOL_EMPTY / 其余 httpErr);网络层失败记 netErr。
|
||
* 输出:data/<label>.jsonl 每分钟聚合、<label>-req.jsonl 逐请求、<label>-live.json
|
||
* 每 5min 快照、<label>-final.json 终局;控制台每 5min 摘要。
|
||
* 用法:SERP_TOKEN=xxx bun serp-dualpath-optimize.ts --hours 24 --label dp1 --rate0 15
|
||
*/
|
||
import fs from "node:fs";
|
||
|
||
/** # 成员变量 服务基址(同机直连 admin) **/
|
||
const BASE = "http://127.0.0.1:53100";
|
||
const args = process.argv.slice(2);
|
||
function argOf(k: string, d: string): string { const i = args.indexOf(k); return i >= 0 ? args[i + 1] : d; }
|
||
/** # 成员变量 测试时长(小时) **/
|
||
const HOURS = Number(argOf("--hours", "24"));
|
||
/** # 成员变量 输出标签(文件名前缀) **/
|
||
const LABEL = argOf("--label", "dp1");
|
||
/** # 成员变量 初始速率(req/min) **/
|
||
const RATE0 = Number(argOf("--rate0", "15"));
|
||
/** # 成员变量 速率下限/上限(req/min) **/
|
||
const RATE_MIN = Number(argOf("--min", "3"));
|
||
const RATE_MAX = Number(argOf("--max", "400"));
|
||
/** # 成员变量 并发 worker 数 **/
|
||
const WORKERS = Number(argOf("--workers", "12"));
|
||
/** # 成员变量 控制器窗口(分钟)与加速步间隔(分钟) **/
|
||
const WINDOW_MIN = 30;
|
||
const RAMP_EVERY_MIN = 5;
|
||
const TOKEN = process.env.SERP_TOKEN ?? "";
|
||
if (!TOKEN) { console.error("缺少 SERP_TOKEN"); process.exit(1); }
|
||
|
||
/** # 成员变量 输出目录(data/,相对脚本) **/
|
||
const DATA = new URL("./data/", import.meta.url).pathname;
|
||
fs.mkdirSync(DATA, { recursive: true });
|
||
const JSONL = `${DATA}${LABEL}.jsonl`;
|
||
const REQ_JSONL = `${DATA}${LABEL}-req.jsonl`;
|
||
const LIVE = `${DATA}${LABEL}-live.json`;
|
||
|
||
/** # 成员变量 长尾查询词表(A×B×C 组合,顺序消费避免重复) **/
|
||
const A = ["best", "top rated", "cheap", "review of", "how to choose a", "buy", "compare", "2026 guide for", "price of", "warranty for"];
|
||
const B = ["coffee maker", "running shoes", "laptop stand", "rice cooker", "air fryer", "standing desk", "office chair", "mechanical keyboard", "monitor arm", "webcam", "espresso machine", "vacuum cleaner", "electric kettle", "gaming mouse", "noise cancelling headphones", "desk lamp", "water bottle", "yoga mat", "blender", "space heater"];
|
||
const C = ["for home use", "for small office", "under 100 dollars", "in 2026", "with long warranty", "for beginners", "pro model", "on sale", "quiet operation", "energy efficient"];
|
||
const combos: string[] = [];
|
||
for (const a of A) for (const b of B) for (const c of C) combos.push(`${a} ${b} ${c}`);
|
||
let qi = Math.floor(Math.random() * combos.length);
|
||
function nextQuery(): string { const q = combos[qi % combos.length]; qi++; return q; }
|
||
|
||
// ============================================================================
|
||
// 自适应令牌桶(速率由控制器动态调整)
|
||
// ============================================================================
|
||
|
||
/** # 成员变量 当前速率(req/min) **/
|
||
let rate = RATE0;
|
||
/** # 成员变量 桶状态:令牌数/上次补充时刻(容量 = max(1, ceil(每秒速率))) **/
|
||
let bucket = { tokens: 0, lastMs: Date.now() };
|
||
|
||
/** # 函数
|
||
* 取一枚发送令牌(不足时等待;速率调整即时生效,无锁 —— 单线程事件循环内安全)。
|
||
*/
|
||
async function acquire(): Promise<void> {
|
||
for (;;) {
|
||
const rps = rate / 60;
|
||
const cap = Math.max(1, Math.ceil(rps));
|
||
const now = Date.now();
|
||
bucket.tokens = Math.min(cap, bucket.tokens + ((now - bucket.lastMs) / 1000) * rps);
|
||
bucket.lastMs = now;
|
||
if (bucket.tokens >= 1) { bucket.tokens -= 1; return; }
|
||
await Bun.sleep(200);
|
||
}
|
||
}
|
||
|
||
// ============================================================================
|
||
// 计数与控制器
|
||
// ============================================================================
|
||
|
||
/** # 成员变量 单分钟计数桶 **/
|
||
type Counter = {
|
||
sent: number; ok: number; gate: number; empty: number;
|
||
captcha: number; poolEmpty: number; httpErr: number; netErr: number;
|
||
fetch: number; browser: number; attemptsTotal: number; gateAttempts: number;
|
||
};
|
||
function newCounter(): Counter {
|
||
return { sent: 0, ok: 0, gate: 0, empty: 0, captcha: 0, poolEmpty: 0, httpErr: 0, netErr: 0, fetch: 0, browser: 0, attemptsTotal: 0, gateAttempts: 0 };
|
||
}
|
||
/** # 成员变量 当前分钟计数 / 滚动窗口(最近 WINDOW_MIN 个分钟桶) **/
|
||
let cur = newCounter();
|
||
const window: { ts: string; c: Counter }[] = [];
|
||
/** # 成员变量 全程累计(终局汇总用) **/
|
||
const totals = newCounter();
|
||
/** # 成员变量 控制器决策日志 **/
|
||
const decisions: Record<string, unknown>[] = [];
|
||
/** # 成员变量 成功指纹计数(分布公平性观察用) **/
|
||
const fpMap = new Map<string, number>();
|
||
|
||
/** # 函数
|
||
* 窗口聚合求和。
|
||
*/
|
||
function sumWindow(): Counter {
|
||
const s = newCounter();
|
||
for (const { c } of window) {
|
||
for (const k of Object.keys(s) as (keyof Counter)[]) s[k] += c[k];
|
||
}
|
||
return s;
|
||
}
|
||
|
||
/** # 函数
|
||
* 控制器决策(每 60s 调用):按窗口成功率/内部压力调整速率,返回决策记录或 null。
|
||
*/
|
||
function controlTick(): Record<string, unknown> | null {
|
||
const w = sumWindow();
|
||
const tot = w.sent;
|
||
if (tot < 10) return null; // 信号不足,保持
|
||
const success = w.ok / tot;
|
||
const gateRatio = w.attemptsTotal > 0 ? w.gateAttempts / w.attemptsTotal : 0;
|
||
const from = rate;
|
||
let to = from;
|
||
let reason = "hold";
|
||
// 最少失败数门槛:低速率下单次偶发失败不该触发退避(3 发 1 失败 = 67%,2 发 1 失败 = 50%)
|
||
const fails = tot - w.ok;
|
||
if (success < 0.95 && fails >= 3) { to = from * 0.5; reason = `success=${(success * 100).toFixed(1)}%<95%(失败 ${fails}) ×0.5`; }
|
||
else if (success < 0.98 && fails >= 2) { to = from * 0.7; reason = `success=${(success * 100).toFixed(1)}%<98%(失败 ${fails}) ×0.7`; }
|
||
else if (w.captcha >= 3) { to = from * 0.7; reason = `captcha 波(${w.captcha}) ×0.7`; }
|
||
else if (gateRatio > 0.15) { to = from * 0.85; reason = `门壳尝试占比 ${(gateRatio * 100).toFixed(1)}%>15% ×0.85`; }
|
||
else if (w.poolEmpty > 5) { to = from * 0.85; reason = `池空 ${w.poolEmpty} ×0.85`; }
|
||
else if (success >= 0.995 && gateRatio < 0.02 && w.poolEmpty === 0 && w.captcha === 0) {
|
||
// 加速步:每 RAMP_EVERY_MIN 个「干净节拍」至多一次(窗口滞后于用量,防预算烧穿前无信号)
|
||
rampSeq++;
|
||
if (rampSeq % RAMP_EVERY_MIN === 0) { to = from * 1.08 + 0.2; reason = `clean(success=${(success * 100).toFixed(1)}%,gate=${(gateRatio * 100).toFixed(1)}%) ×1.08`; }
|
||
}
|
||
to = Math.round(Math.min(RATE_MAX, Math.max(RATE_MIN, to)) * 10) / 10;
|
||
rate = to;
|
||
const rec = { ts: new Date().toISOString(), from: +from.toFixed(1), to: +to.toFixed(1), reason, rampSeq, rampTick: rampSeq % RAMP_EVERY_MIN, window: { sent: tot, ok: w.ok, gate: w.gate, captcha: w.captcha, poolEmpty: w.poolEmpty, success: +(success * 100).toFixed(2), gateRatio: +(gateRatio * 100).toFixed(2) } };
|
||
decisions.push(rec);
|
||
if (decisions.length > 500) decisions.splice(0, decisions.length - 500);
|
||
if (to !== from) console.log(`[${rec.ts}] 速率 ${rec.from} → ${rec.to} req/min(${reason})`);
|
||
return rec;
|
||
}
|
||
/** # 成员变量 控制器节拍序号(加速步按其取模) **/
|
||
let rampSeq = 0;
|
||
|
||
// ============================================================================
|
||
// 请求发射
|
||
// ============================================================================
|
||
|
||
/** # 函数
|
||
* 记逐请求行(紧凑:仅分类所需字段)。
|
||
*/
|
||
function recReq(o: Record<string, unknown>): void {
|
||
try { fs.appendFileSync(REQ_JSONL, JSON.stringify(o) + "\n"); } catch {}
|
||
}
|
||
|
||
/** # 函数
|
||
* 发射单次搜索请求并记账(成功/门壳/空结果/验证码/池空/上游失败/网络失败)。
|
||
* 路径与指纹取自响应体(search_parameters.runtime / search_metadata.profile.id),
|
||
* 内部门壳压力取 timing_stages.attempts 的 outcome 统计。
|
||
*/
|
||
async function fire(): Promise<void> {
|
||
const q = nextQuery();
|
||
const t = Date.now();
|
||
try {
|
||
const res = await fetch(`${BASE}/api/serp/json/search?q=${encodeURIComponent(q)}&num=10`, {
|
||
headers: { authorization: `Bearer ${TOKEN}` },
|
||
signal: AbortSignal.timeout(240_000),
|
||
});
|
||
const ms = Date.now() - t;
|
||
if (res.status === 401) { console.error("Token 无效(401),终止"); process.exit(1); }
|
||
if (!res.ok) {
|
||
const j = await res.json().catch(() => null) as any;
|
||
const code = String(j?.code ?? `HTTP${res.status}`);
|
||
// 发出与结果同刻记账:分钟桶边界处的在途请求计入完成那一桶,避免「发而不中」虚报
|
||
cur.sent++; totals.sent++;
|
||
if (code === "CAPTCHA_DETECTED") { cur.captcha++; totals.captcha++; }
|
||
else if (code === "SERP_POOL_EMPTY") { cur.poolEmpty++; totals.poolEmpty++; }
|
||
else { cur.httpErr++; totals.httpErr++; }
|
||
recReq({ ts: new Date().toISOString(), kind: code === "CAPTCHA_DETECTED" ? "captcha" : code === "SERP_POOL_EMPTY" ? "poolEmpty" : "httpErr", code, ms });
|
||
return;
|
||
}
|
||
const j = await res.json().catch(() => null) as any;
|
||
const organic = Array.isArray(j?.organic) ? j.organic.length : 0;
|
||
const path = j?.search_parameters?.runtime === "browser" ? "browser" : "fetch";
|
||
const fp = String(j?.search_metadata?.profile?.id ?? "");
|
||
const attempts: any[] = Array.isArray(j?.timing_stages?.attempts) ? j.timing_stages.attempts : [];
|
||
const gAtt = attempts.filter(a => a?.outcome === "gate" || a?.outcome === "redirect").length;
|
||
cur.sent++; totals.sent++;
|
||
cur.attemptsTotal += Math.max(1, attempts.length); totals.attemptsTotal += Math.max(1, attempts.length);
|
||
cur.gateAttempts += gAtt; totals.gateAttempts += gAtt;
|
||
cur[path]++; totals[path]++;
|
||
if (fp) fpMap.set(fp, (fpMap.get(fp) ?? 0) + 1);
|
||
if (organic > 0) {
|
||
cur.ok++; totals.ok++;
|
||
recReq({ ts: new Date().toISOString(), kind: "ok", path, fp, ms, attempts: attempts.length, organic });
|
||
} else if (gAtt > 0 || attempts.some(a => a?.outcome === "gate")) {
|
||
cur.gate++; totals.gate++;
|
||
recReq({ ts: new Date().toISOString(), kind: "gate", path, fp, ms, attempts: attempts.length, gateAttempts: gAtt });
|
||
} else {
|
||
cur.empty++; totals.empty++;
|
||
recReq({ ts: new Date().toISOString(), kind: "empty", path, fp, ms, attempts: attempts.length, organic: 0 });
|
||
}
|
||
} catch (e) {
|
||
cur.sent++; totals.sent++;
|
||
cur.netErr++; totals.netErr++;
|
||
recReq({ ts: new Date().toISOString(), kind: "netErr", error: String(e).slice(0, 140), ms: Date.now() - t });
|
||
}
|
||
}
|
||
|
||
/** # 函数
|
||
* worker 主循环:取令牌 → 发射,直到结束时刻。
|
||
*/
|
||
async function worker(endAt: number): Promise<void> {
|
||
while (Date.now() < endAt) {
|
||
await acquire();
|
||
if (Date.now() >= endAt) break;
|
||
await fire();
|
||
}
|
||
}
|
||
|
||
// ============================================================================
|
||
// 快照与汇总
|
||
// ============================================================================
|
||
|
||
/** # 函数
|
||
* 组装快照(窗口+全程+路径占比+指纹 Top20+决策尾部)。
|
||
*/
|
||
function snapshot(): Record<string, unknown> {
|
||
const w = sumWindow();
|
||
const topFps = [...fpMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20).map(([fp, n]) => ({ fp, n }));
|
||
return {
|
||
at: new Date().toISOString(),
|
||
hoursElapsed: +((Date.now() - startedAt) / 3600_000).toFixed(2),
|
||
rateReqMin: +rate.toFixed(1),
|
||
window: { ...w, successPct: w.sent ? +((w.ok / w.sent) * 100).toFixed(2) : null, gateAttemptPct: w.attemptsTotal ? +((w.gateAttempts / w.attemptsTotal) * 100).toFixed(2) : null },
|
||
totals: { ...totals, successPct: totals.sent ? +((totals.ok / totals.sent) * 100).toFixed(2) : null, gateAttemptPct: totals.attemptsTotal ? +((totals.gateAttempts / totals.attemptsTotal) * 100).toFixed(2) : null },
|
||
pathSplit: { fetch: totals.fetch, browser: totals.browser },
|
||
topFps,
|
||
decisions: decisions.slice(-20),
|
||
};
|
||
}
|
||
|
||
// ============================================================================
|
||
// 主流程:并行 worker + 每分钟聚合 + 控制器 + 每 5min 快照;到时终局收尾
|
||
// ============================================================================
|
||
|
||
const startedAt = Date.now();
|
||
const endAt = startedAt + HOURS * 3600_000;
|
||
fs.writeFileSync(JSONL, "");
|
||
fs.writeFileSync(REQ_JSONL, "");
|
||
console.log(`开始:label=${LABEL} hours=${HOURS} rate0=${RATE0} 区间=[${RATE_MIN},${RATE_MAX}] workers=${WORKERS} 出口=json(auto 双路径)`);
|
||
|
||
for (let i = 0; i < WORKERS; i++) void worker(endAt);
|
||
|
||
/** # 成员变量 每分钟定时器:滚动窗口推进 + 控制器决策 + 聚合行落盘 **/
|
||
const minuteTimer = setInterval(() => {
|
||
const row = { ts: new Date().toISOString(), rate: +rate.toFixed(1), ...cur };
|
||
try { fs.appendFileSync(JSONL, JSON.stringify(row) + "\n"); } catch {}
|
||
window.push({ ts: row.ts, c: cur });
|
||
if (window.length > WINDOW_MIN) window.shift();
|
||
cur = newCounter();
|
||
controlTick();
|
||
}, 60_000);
|
||
|
||
/** # 成员变量 每 5 分钟快照与控制台摘要 **/
|
||
const snapTimer = setInterval(() => {
|
||
const snap = snapshot();
|
||
fs.writeFileSync(LIVE, JSON.stringify(snap, null, 2));
|
||
const t = snap.totals as any; const w = snap.window as any;
|
||
console.log(`[${snap.at}] ${(snap.hoursElapsed as number).toFixed(1)}h rate=${snap.rateReqMin}/min | 窗口 成功率=${w.successPct}% 门壳尝试=${w.gateAttemptPct}% sent=${w.sent} (ok=${w.ok} gate=${w.gate} captcha=${w.captcha} poolEmpty=${w.poolEmpty} httpErr=${w.httpErr} netErr=${w.netErr}) | 全程 成功率=${t.successPct}% 路径 fetch=${snap.pathSplit.fetch}/browser=${snap.pathSplit.browser}`);
|
||
}, 5 * 60_000);
|
||
|
||
const endTimer = setTimeout(() => {
|
||
clearInterval(minuteTimer); clearInterval(snapTimer);
|
||
const snap = snapshot();
|
||
fs.writeFileSync(`${DATA}${LABEL}-final.json`, JSON.stringify(snap, null, 2));
|
||
console.log("测试结束,final 已写:" + `${DATA}${LABEL}-final.json`);
|
||
process.exit(0);
|
||
}, HOURS * 3600_000);
|
||
endTimer.unref?.();
|