bigcode
f17eef3f74
dp1 实测教训:速率 15→100/min 全程成功率 99.8-100%(100/min 时 99.9%),08:46 出现 12 验证码/30min(12 个不同指纹各 1 次,单 IP 逐个越限的常态噪声,/sorry 自动恢复 逐个接管),成功率仍有 99.3%,但「窗口含验证码≥3 就每分钟 ×0.7」把速率连砍 10 分钟 钉死在 3/min 下限 —— 窗口滞后放大过度反应。 v2:硬约束(成功率<95/98%)看 30min 窗即时砍不变;软信号(验证码/门壳占比/池空) 改 10min 短窗 + 事件段(episode)迟滞 —— 进段砍一次(验证码按占比 >2%×0.5、>0.5%×0.7), 退段才重置;干净爬坡改看短窗且验证码≤0.3% 不阻追涨。dp1 数据归档保留。
318 lines
16 KiB
TypeScript
318 lines
16 KiB
TypeScript
/** # 脚本说明
|
||
* SERP 双路径(fetch+浏览器)24h 自适应吞吐优化测试(用户 2026-10-30 目标):
|
||
* 在保证客户端成功率 >98% 的约束下,自适应逼近系统可持续的最大每分钟请求数。
|
||
* 与 serp-profile-capacity.ts(单号 sticky+fetch,绕过池/令牌)不同,本脚本走生产
|
||
* 双路径原形态(runtime=auto,无 sticky,全池 LRU 选号):
|
||
* GET /api/serp/json/search?q=...&num=10
|
||
* 令牌内走 go-fetch(全局限流 fetchPerSec),限流外并发走温池浏览器 —— 组合即生产形态。
|
||
* AIMD 控制器(每 60s 决策;硬约束看 30min 窗,软信号看 10min 短窗):
|
||
* 硬约束即时退避:成功率<95%(失败≥3)×0.5;<98%(失败≥2)×0.7;
|
||
* 软信号按事件段(episode)迟滞,每段波只砍一次(波退去才重置):
|
||
* 验证码占比>2% ×0.5、>0.5% ×0.7(10min 短窗;dp1 实测 0.5% 噪声时成功率
|
||
* 仍有 99.3%,按占比计而非按次数计,且不逐分钟连砍钉死下限);
|
||
* 门壳尝试占比>15% ×0.85;池空>5 ×0.85;
|
||
* 干净爬坡:短窗成功率≥99.5% 且门壳<2% 且无池空且验证码≤0.3%(高压力下
|
||
* 单 IP 逐个越限的常态噪声不阻止追涨)→ 每 5 拍 ×1.08。
|
||
* 速率区间 [3, 400] req/min;并发 worker 默认 12(真实并发让溢出自然流向浏览器路径)。
|
||
* 判定(客户端,JSON 出口):ok = 200 且 organic>0;gate = 200 且 organic=0 且
|
||
* attempts 含 gate;empty = 200 且 organic=0 无门壳尝试;错误按响应 code 分类
|
||
* (CAPTCHA_DETECTED / SERP_POOL_EMPTY / 其余 httpErr);网络层失败记 netErr。
|
||
* 输出:data/<label>.jsonl 每分钟聚合、<label>-req.jsonl 逐请求、<label>-live.json
|
||
* 每 5min 快照、<label>-final.json 终局;控制台每 5min 摘要。
|
||
* 用法:SERP_TOKEN=xxx bun serp-dualpath-optimize.ts --hours 24 --label dp1 --rate0 15
|
||
*/
|
||
import fs from "node:fs";
|
||
|
||
/** # 成员变量 服务基址(同机直连 admin) **/
|
||
const BASE = "http://127.0.0.1:53100";
|
||
const args = process.argv.slice(2);
|
||
function argOf(k: string, d: string): string { const i = args.indexOf(k); return i >= 0 ? args[i + 1] : d; }
|
||
/** # 成员变量 测试时长(小时) **/
|
||
const HOURS = Number(argOf("--hours", "24"));
|
||
/** # 成员变量 输出标签(文件名前缀) **/
|
||
const LABEL = argOf("--label", "dp1");
|
||
/** # 成员变量 初始速率(req/min) **/
|
||
const RATE0 = Number(argOf("--rate0", "15"));
|
||
/** # 成员变量 速率下限/上限(req/min) **/
|
||
const RATE_MIN = Number(argOf("--min", "3"));
|
||
const RATE_MAX = Number(argOf("--max", "400"));
|
||
/** # 成员变量 并发 worker 数 **/
|
||
const WORKERS = Number(argOf("--workers", "12"));
|
||
/** # 成员变量 控制器窗口(分钟)与加速步间隔(分钟) **/
|
||
const WINDOW_MIN = 30;
|
||
const RAMP_EVERY_MIN = 5;
|
||
const TOKEN = process.env.SERP_TOKEN ?? "";
|
||
if (!TOKEN) { console.error("缺少 SERP_TOKEN"); process.exit(1); }
|
||
|
||
/** # 成员变量 输出目录(data/,相对脚本) **/
|
||
const DATA = new URL("./data/", import.meta.url).pathname;
|
||
fs.mkdirSync(DATA, { recursive: true });
|
||
const JSONL = `${DATA}${LABEL}.jsonl`;
|
||
const REQ_JSONL = `${DATA}${LABEL}-req.jsonl`;
|
||
const LIVE = `${DATA}${LABEL}-live.json`;
|
||
|
||
/** # 成员变量 长尾查询词表(A×B×C 组合,顺序消费避免重复) **/
|
||
const A = ["best", "top rated", "cheap", "review of", "how to choose a", "buy", "compare", "2026 guide for", "price of", "warranty for"];
|
||
const B = ["coffee maker", "running shoes", "laptop stand", "rice cooker", "air fryer", "standing desk", "office chair", "mechanical keyboard", "monitor arm", "webcam", "espresso machine", "vacuum cleaner", "electric kettle", "gaming mouse", "noise cancelling headphones", "desk lamp", "water bottle", "yoga mat", "blender", "space heater"];
|
||
const C = ["for home use", "for small office", "under 100 dollars", "in 2026", "with long warranty", "for beginners", "pro model", "on sale", "quiet operation", "energy efficient"];
|
||
const combos: string[] = [];
|
||
for (const a of A) for (const b of B) for (const c of C) combos.push(`${a} ${b} ${c}`);
|
||
let qi = Math.floor(Math.random() * combos.length);
|
||
function nextQuery(): string { const q = combos[qi % combos.length]; qi++; return q; }
|
||
|
||
// ============================================================================
|
||
// 自适应令牌桶(速率由控制器动态调整)
|
||
// ============================================================================
|
||
|
||
/** # 成员变量 当前速率(req/min) **/
|
||
let rate = RATE0;
|
||
/** # 成员变量 桶状态:令牌数/上次补充时刻(容量 = max(1, ceil(每秒速率))) **/
|
||
let bucket = { tokens: 0, lastMs: Date.now() };
|
||
|
||
/** # 函数
|
||
* 取一枚发送令牌(不足时等待;速率调整即时生效,无锁 —— 单线程事件循环内安全)。
|
||
*/
|
||
async function acquire(): Promise<void> {
|
||
for (;;) {
|
||
const rps = rate / 60;
|
||
const cap = Math.max(1, Math.ceil(rps));
|
||
const now = Date.now();
|
||
bucket.tokens = Math.min(cap, bucket.tokens + ((now - bucket.lastMs) / 1000) * rps);
|
||
bucket.lastMs = now;
|
||
if (bucket.tokens >= 1) { bucket.tokens -= 1; return; }
|
||
await Bun.sleep(200);
|
||
}
|
||
}
|
||
|
||
// ============================================================================
|
||
// 计数与控制器
|
||
// ============================================================================
|
||
|
||
/** # 成员变量 单分钟计数桶 **/
|
||
type Counter = {
|
||
sent: number; ok: number; gate: number; empty: number;
|
||
captcha: number; poolEmpty: number; httpErr: number; netErr: number;
|
||
fetch: number; browser: number; attemptsTotal: number; gateAttempts: number;
|
||
};
|
||
function newCounter(): Counter {
|
||
return { sent: 0, ok: 0, gate: 0, empty: 0, captcha: 0, poolEmpty: 0, httpErr: 0, netErr: 0, fetch: 0, browser: 0, attemptsTotal: 0, gateAttempts: 0 };
|
||
}
|
||
/** # 成员变量 当前分钟计数 / 滚动窗口(最近 WINDOW_MIN 个分钟桶) **/
|
||
let cur = newCounter();
|
||
const window: { ts: string; c: Counter }[] = [];
|
||
/** # 成员变量 全程累计(终局汇总用) **/
|
||
const totals = newCounter();
|
||
/** # 成员变量 控制器决策日志 **/
|
||
const decisions: Record<string, unknown>[] = [];
|
||
/** # 成员变量 成功指纹计数(分布公平性观察用) **/
|
||
const fpMap = new Map<string, number>();
|
||
|
||
/** # 函数
|
||
* 窗口聚合求和(参数 n:取最近 n 个分钟桶;不传 = 全窗口)。
|
||
*/
|
||
function sumWindow(n = WINDOW_MIN): Counter {
|
||
const s = newCounter();
|
||
for (const { c } of window.slice(-n)) {
|
||
for (const k of Object.keys(s) as (keyof Counter)[]) s[k] += c[k];
|
||
}
|
||
return s;
|
||
}
|
||
|
||
/** # 成员变量 软信号事件段状态(迟滞:进段砍一次,退段重置;防同一段波逐分钟连砍) **/
|
||
let captchaWave = false;
|
||
let gateWave = false;
|
||
let poolEmptyWave = false;
|
||
|
||
/** # 函数
|
||
* 控制器决策(每 60s 调用):按窗口成功率/内部压力调整速率,返回决策记录或 null。
|
||
*/
|
||
function controlTick(): Record<string, unknown> | null {
|
||
const wH = sumWindow(30); // 硬约束窗口:成功率即约束本身
|
||
const tot = wH.sent;
|
||
if (tot < 10) return null; // 信号不足,保持
|
||
const wS = sumWindow(10); // 软信号短窗:波动退去快,防旧波滞留连砍(dp1 教训)
|
||
const success = wH.ok / tot;
|
||
const gateRatio = wH.attemptsTotal > 0 ? wH.gateAttempts / wH.attemptsTotal : 0;
|
||
const gateRatioS = wS.attemptsTotal > 0 ? wS.gateAttempts / wS.attemptsTotal : 0;
|
||
const capPctS = wS.sent > 0 ? wS.captcha / wS.sent : 0;
|
||
const from = rate;
|
||
let to = from;
|
||
let reason = "hold";
|
||
// 硬约束即时退避,最少失败数门槛:低速率下单次偶发失败不该触发(3 发 1 失败 = 67%)
|
||
const fails = tot - wH.ok;
|
||
if (success < 0.95 && fails >= 3) { to = from * 0.5; reason = `success=${(success * 100).toFixed(1)}%<95%(失败 ${fails}) ×0.5`; }
|
||
else if (success < 0.98 && fails >= 2) { to = from * 0.7; reason = `success=${(success * 100).toFixed(1)}%<98%(失败 ${fails}) ×0.7`; }
|
||
else {
|
||
// 软信号:事件段(episode)迟滞 —— 进段砍一次,退段重置;段内不重复砍
|
||
if (capPctS > 0.005 && !captchaWave) {
|
||
captchaWave = true;
|
||
to = capPctS > 0.02 ? from * 0.5 : from * 0.7;
|
||
reason = `验证码占比 ${(capPctS * 100).toFixed(2)}% ${capPctS > 0.02 ? "×0.5" : "×0.7"}`;
|
||
} else if (capPctS < 0.001) captchaWave = false;
|
||
if (to === from && gateRatioS > 0.15 && !gateWave) {
|
||
gateWave = true;
|
||
to = from * 0.85;
|
||
reason = `门壳尝试占比 ${(gateRatioS * 100).toFixed(1)}%>15% ×0.85`;
|
||
} else if (gateRatioS < 0.05) gateWave = false;
|
||
if (to === from && wS.poolEmpty > 5 && !poolEmptyWave) {
|
||
poolEmptyWave = true;
|
||
to = from * 0.85;
|
||
reason = `池空 ${wS.poolEmpty} ×0.85`;
|
||
} else if (wS.poolEmpty === 0) poolEmptyWave = false;
|
||
}
|
||
// 干净爬坡:每 RAMP_EVERY_MIN 个「干净节拍」至多一次(验证码≤0.3% 属高压力常态噪声,不阻追涨)
|
||
if (to === from && wS.sent >= 10) {
|
||
const successS = wS.ok / wS.sent;
|
||
if (successS >= 0.995 && gateRatioS < 0.02 && wS.poolEmpty === 0 && capPctS < 0.003) {
|
||
rampSeq++;
|
||
if (rampSeq % RAMP_EVERY_MIN === 0) { to = from * 1.08 + 0.2; reason = `clean(success=${(successS * 100).toFixed(1)}%,gate=${(gateRatioS * 100).toFixed(1)}%) ×1.08`; }
|
||
}
|
||
}
|
||
to = Math.round(Math.min(RATE_MAX, Math.max(RATE_MIN, to)) * 10) / 10;
|
||
rate = to;
|
||
const rec = { ts: new Date().toISOString(), from: +from.toFixed(1), to: +to.toFixed(1), reason, rampSeq, rampTick: rampSeq % RAMP_EVERY_MIN, window: { sent: tot, ok: wH.ok, gate: wH.gate, captcha: wH.captcha, poolEmpty: wH.poolEmpty, success: +(success * 100).toFixed(2), gateRatio: +(gateRatio * 100).toFixed(2), capPctS: +(capPctS * 100).toFixed(2) }, waves: { captcha: captchaWave, gate: gateWave, poolEmpty: poolEmptyWave } };
|
||
decisions.push(rec);
|
||
if (decisions.length > 500) decisions.splice(0, decisions.length - 500);
|
||
if (to !== from) console.log(`[${rec.ts}] 速率 ${rec.from} → ${rec.to} req/min(${reason})`);
|
||
return rec;
|
||
}
|
||
/** # 成员变量 控制器节拍序号(加速步按其取模) **/
|
||
let rampSeq = 0;
|
||
|
||
// ============================================================================
|
||
// 请求发射
|
||
// ============================================================================
|
||
|
||
/** # 函数
|
||
* 记逐请求行(紧凑:仅分类所需字段)。
|
||
*/
|
||
function recReq(o: Record<string, unknown>): void {
|
||
try { fs.appendFileSync(REQ_JSONL, JSON.stringify(o) + "\n"); } catch {}
|
||
}
|
||
|
||
/** # 函数
|
||
* 发射单次搜索请求并记账(成功/门壳/空结果/验证码/池空/上游失败/网络失败)。
|
||
* 路径与指纹取自响应体(search_parameters.runtime / search_metadata.profile.id),
|
||
* 内部门壳压力取 timing_stages.attempts 的 outcome 统计。
|
||
*/
|
||
async function fire(): Promise<void> {
|
||
const q = nextQuery();
|
||
const t = Date.now();
|
||
try {
|
||
const res = await fetch(`${BASE}/api/serp/json/search?q=${encodeURIComponent(q)}&num=10`, {
|
||
headers: { authorization: `Bearer ${TOKEN}` },
|
||
signal: AbortSignal.timeout(240_000),
|
||
});
|
||
const ms = Date.now() - t;
|
||
if (res.status === 401) { console.error("Token 无效(401),终止"); process.exit(1); }
|
||
if (!res.ok) {
|
||
const j = await res.json().catch(() => null) as any;
|
||
const code = String(j?.code ?? `HTTP${res.status}`);
|
||
// 发出与结果同刻记账:分钟桶边界处的在途请求计入完成那一桶,避免「发而不中」虚报
|
||
cur.sent++; totals.sent++;
|
||
if (code === "CAPTCHA_DETECTED") { cur.captcha++; totals.captcha++; }
|
||
else if (code === "SERP_POOL_EMPTY") { cur.poolEmpty++; totals.poolEmpty++; }
|
||
else { cur.httpErr++; totals.httpErr++; }
|
||
recReq({ ts: new Date().toISOString(), kind: code === "CAPTCHA_DETECTED" ? "captcha" : code === "SERP_POOL_EMPTY" ? "poolEmpty" : "httpErr", code, ms });
|
||
return;
|
||
}
|
||
const j = await res.json().catch(() => null) as any;
|
||
const organic = Array.isArray(j?.organic) ? j.organic.length : 0;
|
||
const path = j?.search_parameters?.runtime === "browser" ? "browser" : "fetch";
|
||
const fp = String(j?.search_metadata?.profile?.id ?? "");
|
||
const attempts: any[] = Array.isArray(j?.timing_stages?.attempts) ? j.timing_stages.attempts : [];
|
||
const gAtt = attempts.filter(a => a?.outcome === "gate" || a?.outcome === "redirect").length;
|
||
cur.sent++; totals.sent++;
|
||
cur.attemptsTotal += Math.max(1, attempts.length); totals.attemptsTotal += Math.max(1, attempts.length);
|
||
cur.gateAttempts += gAtt; totals.gateAttempts += gAtt;
|
||
cur[path]++; totals[path]++;
|
||
if (fp) fpMap.set(fp, (fpMap.get(fp) ?? 0) + 1);
|
||
if (organic > 0) {
|
||
cur.ok++; totals.ok++;
|
||
recReq({ ts: new Date().toISOString(), kind: "ok", path, fp, ms, attempts: attempts.length, organic });
|
||
} else if (gAtt > 0 || attempts.some(a => a?.outcome === "gate")) {
|
||
cur.gate++; totals.gate++;
|
||
recReq({ ts: new Date().toISOString(), kind: "gate", path, fp, ms, attempts: attempts.length, gateAttempts: gAtt });
|
||
} else {
|
||
cur.empty++; totals.empty++;
|
||
recReq({ ts: new Date().toISOString(), kind: "empty", path, fp, ms, attempts: attempts.length, organic: 0 });
|
||
}
|
||
} catch (e) {
|
||
cur.sent++; totals.sent++;
|
||
cur.netErr++; totals.netErr++;
|
||
recReq({ ts: new Date().toISOString(), kind: "netErr", error: String(e).slice(0, 140), ms: Date.now() - t });
|
||
}
|
||
}
|
||
|
||
/** # 函数
|
||
* worker 主循环:取令牌 → 发射,直到结束时刻。
|
||
*/
|
||
async function worker(endAt: number): Promise<void> {
|
||
while (Date.now() < endAt) {
|
||
await acquire();
|
||
if (Date.now() >= endAt) break;
|
||
await fire();
|
||
}
|
||
}
|
||
|
||
// ============================================================================
|
||
// 快照与汇总
|
||
// ============================================================================
|
||
|
||
/** # 函数
|
||
* 组装快照(窗口+全程+路径占比+指纹 Top20+决策尾部)。
|
||
*/
|
||
function snapshot(): Record<string, unknown> {
|
||
const w = sumWindow();
|
||
const topFps = [...fpMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20).map(([fp, n]) => ({ fp, n }));
|
||
return {
|
||
at: new Date().toISOString(),
|
||
hoursElapsed: +((Date.now() - startedAt) / 3600_000).toFixed(2),
|
||
rateReqMin: +rate.toFixed(1),
|
||
window: { ...w, successPct: w.sent ? +((w.ok / w.sent) * 100).toFixed(2) : null, gateAttemptPct: w.attemptsTotal ? +((w.gateAttempts / w.attemptsTotal) * 100).toFixed(2) : null },
|
||
totals: { ...totals, successPct: totals.sent ? +((totals.ok / totals.sent) * 100).toFixed(2) : null, gateAttemptPct: totals.attemptsTotal ? +((totals.gateAttempts / totals.attemptsTotal) * 100).toFixed(2) : null },
|
||
pathSplit: { fetch: totals.fetch, browser: totals.browser },
|
||
topFps,
|
||
decisions: decisions.slice(-20),
|
||
};
|
||
}
|
||
|
||
// ============================================================================
|
||
// 主流程:并行 worker + 每分钟聚合 + 控制器 + 每 5min 快照;到时终局收尾
|
||
// ============================================================================
|
||
|
||
const startedAt = Date.now();
|
||
const endAt = startedAt + HOURS * 3600_000;
|
||
fs.writeFileSync(JSONL, "");
|
||
fs.writeFileSync(REQ_JSONL, "");
|
||
console.log(`开始:label=${LABEL} hours=${HOURS} rate0=${RATE0} 区间=[${RATE_MIN},${RATE_MAX}] workers=${WORKERS} 出口=json(auto 双路径)`);
|
||
|
||
for (let i = 0; i < WORKERS; i++) void worker(endAt);
|
||
|
||
/** # 成员变量 每分钟定时器:滚动窗口推进 + 控制器决策 + 聚合行落盘 **/
|
||
const minuteTimer = setInterval(() => {
|
||
const row = { ts: new Date().toISOString(), rate: +rate.toFixed(1), ...cur };
|
||
try { fs.appendFileSync(JSONL, JSON.stringify(row) + "\n"); } catch {}
|
||
window.push({ ts: row.ts, c: cur });
|
||
if (window.length > WINDOW_MIN) window.shift();
|
||
cur = newCounter();
|
||
controlTick();
|
||
}, 60_000);
|
||
|
||
/** # 成员变量 每 5 分钟快照与控制台摘要 **/
|
||
const snapTimer = setInterval(() => {
|
||
const snap = snapshot();
|
||
fs.writeFileSync(LIVE, JSON.stringify(snap, null, 2));
|
||
const t = snap.totals as any; const w = snap.window as any;
|
||
console.log(`[${snap.at}] ${(snap.hoursElapsed as number).toFixed(1)}h rate=${snap.rateReqMin}/min | 窗口 成功率=${w.successPct}% 门壳尝试=${w.gateAttemptPct}% sent=${w.sent} (ok=${w.ok} gate=${w.gate} captcha=${w.captcha} poolEmpty=${w.poolEmpty} httpErr=${w.httpErr} netErr=${w.netErr}) | 全程 成功率=${t.successPct}% 路径 fetch=${snap.pathSplit.fetch}/browser=${snap.pathSplit.browser}`);
|
||
}, 5 * 60_000);
|
||
|
||
const endTimer = setTimeout(() => {
|
||
clearInterval(minuteTimer); clearInterval(snapTimer);
|
||
const snap = snapshot();
|
||
fs.writeFileSync(`${DATA}${LABEL}-final.json`, JSON.stringify(snap, null, 2));
|
||
console.log("测试结束,final 已写:" + `${DATA}${LABEL}-final.json`);
|
||
process.exit(0);
|
||
}, HOURS * 3600_000);
|
||
endTimer.unref?.();
|