390 lines
26 KiB
TypeScript
390 lines
26 KiB
TypeScript
/** # 脚本说明
|
||
* SERP 双路径(fetch+浏览器)24h 自适应吞吐优化测试(用户 2026-10-30 目标;
|
||
* 2026-11-02 用户要求可用率约束上调至 >99%):
|
||
* 在保证客户端成功率 >99% 的约束下,自适应逼近系统可持续的最大每分钟请求数。
|
||
* 与 serp-profile-capacity.ts(单号 sticky+fetch,绕过池/令牌)不同,本脚本走生产
|
||
* 双路径原形态(runtime=auto,无 sticky,全池公平负载选号):
|
||
* GET /api/serp/json/search?q=...&num=10
|
||
* 令牌内走 go-fetch(全局限流 fetchPerSec),限流外并发走温池浏览器 —— 组合即生产形态。
|
||
* AIMD 控制器 v5.2(dp6 全天 + dp7 前段实测教训驱动;v4 基础上的关键演进):
|
||
* 硬约束(成功率即约束本身):入段 = 浅破线(<99% 且失败 ≥8 且窗口 ≥200 样本)或
|
||
* 深破线(<96% 且失败 ≥4)×0.7/<96% ×0.5;段内续砍需短窗 <96% 且(非验证码失败
|
||
* ≥3 或验证码 ≥5/10min)且全程增量新失败 ≥2;硬砍地板 HARD_FLOOR=8;
|
||
* 涓流豁免(v5.1,dp7 实测:99% 线下比率陷阱锁死 degraded 砍到 3/min):
|
||
* 30min 窗零非验证码失败且短窗验证码 ≤2 = 涓流 —— 不入段、可退段(短窗 ≥98.5%)、
|
||
* 验证码涓流的唯一解药是爬坡摊薄(稀释论),砍速让分母变小占比更高;
|
||
* 软信号按事件段(episode)迟滞 + 绝对数护栏:验证码 10min 窗 ≥8 个(v5.1:5→8,只砍
|
||
* 波次规模)且占比>0.5% ×0.7(重波次 ≥12 且>2% ×0.5);门壳 ≥3 且占比>15% ×0.85;
|
||
* 池空>5 ×0.85;软砍下限 SOFT_FLOOR=40(v5.1:25→40,保稀释安全线);
|
||
* 干净爬坡:水位 = 30min 窗 ≥99%(v5.2:99.5%→99%,涓流时段高门槛不可达)且短窗 ≥99%
|
||
* 时记录;水位下方 ×1.3/拍快速回收,水位之上 ×1.08/5 拍;爬坡备选「验证码 ≤2 且
|
||
* 短窗 ≥98%」与涓流口径一致(dp6 低速恢复锁死 + dp7 涓流滞留修复)。
|
||
* 干净爬坡:不在退化段 且 短窗干净 且 无池空 且(验证码 0 个或样本 ≥300)→ 向「干净
|
||
* 水位」(30min 窗 ≥99.5% 时的最高速率)以 ×1.3/拍快速回收,水位之上每 5 拍 ×1.08。
|
||
* v5 修 dp6 实测「低速恢复锁死」:短窗成功率 ≥99.5% 在低速涓流下永不可达
|
||
* (05:00-06:10 滞留 70min 在 24-42/min,该小时实际 98.86%)→ 增加备选条件
|
||
* 「验证码 ≤1 个且短窗 ≥98.5%」(稀释论:同量涓流在高速率下占比可忽略,可爬)。
|
||
* 速率区间 [3, 400] req/min;并发 worker 默认 12(真实并发让溢出自然流向浏览器路径)。
|
||
* 判定(客户端,JSON 出口):ok = 200 且 organic>0;gate = 200 且 organic=0 且
|
||
* attempts 含 gate;empty = 200 且 organic=0 无门壳尝试;错误按响应 code 分类
|
||
* (CAPTCHA_DETECTED / SERP_POOL_EMPTY / 其余 httpErr);网络层失败记 netErr。
|
||
* 输出:data/<label>.jsonl 每分钟聚合、<label>-req.jsonl 逐请求、<label>-live.json
|
||
* 每 5min 快照、<label>-final.json 终局;控制台每 5min 摘要。
|
||
* 用法:SERP_TOKEN=xxx bun serp-dualpath-optimize.ts --hours 24 --label dp1 --rate0 15
|
||
*/
|
||
import fs from "node:fs";
|
||
|
||
/** # 成员变量 服务基址(同机直连 admin) **/
|
||
const BASE = "http://127.0.0.1:53100";
|
||
const args = process.argv.slice(2);
|
||
function argOf(k: string, d: string): string { const i = args.indexOf(k); return i >= 0 ? args[i + 1] : d; }
|
||
/** # 成员变量 测试时长(小时) **/
|
||
const HOURS = Number(argOf("--hours", "24"));
|
||
/** # 成员变量 输出标签(文件名前缀) **/
|
||
const LABEL = argOf("--label", "dp1");
|
||
/** # 成员变量 初始速率(req/min) **/
|
||
const RATE0 = Number(argOf("--rate0", "15"));
|
||
/** # 成员变量 速率下限/上限(req/min) **/
|
||
const RATE_MIN = Number(argOf("--min", "3"));
|
||
const RATE_MAX = Number(argOf("--max", "400"));
|
||
/** # 成员变量 并发 worker 数 **/
|
||
const WORKERS = Number(argOf("--workers", "12"));
|
||
/** # 成员变量 控制器窗口(分钟)与加速步间隔(分钟) **/
|
||
const WINDOW_MIN = 30;
|
||
const RAMP_EVERY_MIN = 5;
|
||
const TOKEN = process.env.SERP_TOKEN ?? "";
|
||
if (!TOKEN) { console.error("缺少 SERP_TOKEN"); process.exit(1); }
|
||
|
||
/** # 成员变量 输出目录(data/,相对脚本) **/
|
||
const DATA = new URL("./data/", import.meta.url).pathname;
|
||
fs.mkdirSync(DATA, { recursive: true });
|
||
const JSONL = `${DATA}${LABEL}.jsonl`;
|
||
const REQ_JSONL = `${DATA}${LABEL}-req.jsonl`;
|
||
const LIVE = `${DATA}${LABEL}-live.json`;
|
||
|
||
/** # 成员变量 长尾查询词表(4 维组合,2026-10-30 用户反馈改版):
|
||
* 旧 A×B×C(2000 组合)相似度太高,/goto 解析缓存命中虚高,同 Profile 并发 /goto
|
||
* 的真实负载被洗掉。改为 修饰×品类×限定×城市 4 维(40×60×40×60 = 576 万组合),
|
||
* 随机不重复消费 —— 每次搜索结果集与 /goto 链接基本不同,缓存命中回落到真实水平。
|
||
*/
|
||
const A = ["best", "top rated", "cheap", "review of", "how to choose a", "buy", "compare", "2026 guide for", "price of", "warranty for", "where to buy", "discount on", "is it worth buying", "reliability of", "problems with", "alternatives to", "user reviews for", "long term test of", "repair costs for", "energy use of", "setup guide for", "maintenance tips for", "refurbished", "open box deals on", "wholesale price of", "used value of", "safety rating of", "recalled models of", "accessories for", "replacement parts for", "manual pdf for", "dimensions of", "weight limit of", "installation cost of", "monthly deals on", "black friday price for", "cyber monday deal for", "student discount for", "military discount for", "trade in value of"];
|
||
const B = ["coffee maker", "running shoes", "laptop stand", "rice cooker", "air fryer", "standing desk", "office chair", "mechanical keyboard", "monitor arm", "webcam", "espresso machine", "vacuum cleaner", "electric kettle", "gaming mouse", "noise cancelling headphones", "desk lamp", "water bottle", "yoga mat", "blender", "space heater", "air purifier", "electric toothbrush", "smart thermostat", "robot vacuum", "ergonomic mouse", "usb-c hub", "portable ssd", "wireless charger", "dash cam", "action camera", "trail running shoes", "hiking backpack", "camping tent", "sleeping bag", "kayak paddle", "bicycle helmet", "electric scooter", "board games", "3d printer", "drawing tablet", "studio microphone", "audio interface", "midi keyboard", "electric guitar", "drum pad", "telescope", "binoculars", "bird feeder", "grill smoker", "pizza oven", "garden hose", "leaf blower", "chainsaw", "pressure washer", "snow shovel", "garage door opener", "stud finder", "laser level", "tool set", "sewing machine"];
|
||
const C = ["for home use", "for small office", "under 100 dollars", "in 2026", "with long warranty", "for beginners", "pro model", "on sale", "quiet operation", "energy efficient", "with fast shipping", "highly rated", "compact size", "heavy duty", "for students", "for professionals", "with extra accessories", "easy to clean", "portable version", "premium edition", "budget option", "luxury model", "with 2 year warranty", "top ranked", "editor's choice", "best seller", "new release", "on clearance", "with free returns", "for tall people", "for small spaces", "made in usa", "eco friendly", "with app control", "voice assistant compatible", "wifi enabled", "with hepa filter", "waterproof model", "lightweight version", "industrial grade"];
|
||
const D = ["in Austin tx", "in Denver co", "in Seattle wa", "in Portland or", "in Chicago il", "in Boston ma", "in Miami fl", "in Phoenix az", "in Dallas tx", "in Atlanta ga", "in New York ny", "in Los Angeles ca", "in Houston tx", "in Philadelphia pa", "in San Diego ca", "in Las Vegas nv", "in Nashville tn", "in Detroit mi", "in Minneapolis mn", "in Charlotte nc", "in Raleigh nc", "in Omaha ne", "in Boise id", "in Salt Lake City ut", "in Columbus oh", "in Indianapolis in", "in Milwaukee wi", "in Kansas City mo", "in St Louis mo", "in Cincinnati oh", "in Pittsburgh pa", "in New Orleans la", "in Memphis tn", "in Louisville ky", "in Albuquerque nm", "in Tucson az", "in Fresno ca", "in Sacramento ca", "in Portland me", "in Burlington vt", "in Providence ri", "in Hartford ct", "in Newark nj", "in Baltimore md", "in Richmond va", "in Charleston sc", "in Savannah ga", "in Orlando fl", "in Tampa fl", "in San Antonio tx", "in El Paso tx", "in Oklahoma City ok", "in Tulsa ok", "in Wichita ks", "in Des Moines ia", "in Little Rock ar", "in Jackson ms", "in Billings mt", "in Fargo nd", "in Charleston wv", "in Eugene or"];
|
||
const COMBO_TOTAL = A.length * B.length * C.length * D.length;
|
||
/** # 成员变量 已消费组合索引(随机不重复;24h 全程用量远低于组合空间,碰撞回退微乎其微) **/
|
||
const usedCombos = new Set<number>();
|
||
function nextQuery(): string {
|
||
let idx = Math.floor(Math.random() * COMBO_TOTAL);
|
||
while (usedCombos.has(idx)) idx = (idx + 1) % COMBO_TOTAL;
|
||
usedCombos.add(idx);
|
||
const d = idx % D.length;
|
||
const c = Math.floor(idx / D.length) % C.length;
|
||
const b = Math.floor(idx / (D.length * C.length)) % B.length;
|
||
const a = Math.floor(idx / (D.length * C.length * B.length)) % A.length;
|
||
return `${A[a]} ${B[b]} ${C[c]} ${D[d]}`;
|
||
}
|
||
|
||
// ============================================================================
|
||
// 自适应令牌桶(速率由控制器动态调整)
|
||
// ============================================================================
|
||
|
||
/** # 成员变量 当前速率(req/min) **/
|
||
let rate = RATE0;
|
||
/** # 成员变量 桶状态:令牌数/上次补充时刻(容量 = max(1, ceil(每秒速率))) **/
|
||
let bucket = { tokens: 0, lastMs: Date.now() };
|
||
|
||
/** # 函数
|
||
* 取一枚发送令牌(不足时等待;速率调整即时生效,无锁 —— 单线程事件循环内安全)。
|
||
*/
|
||
async function acquire(): Promise<void> {
|
||
for (;;) {
|
||
const rps = rate / 60;
|
||
const cap = Math.max(1, Math.ceil(rps));
|
||
const now = Date.now();
|
||
bucket.tokens = Math.min(cap, bucket.tokens + ((now - bucket.lastMs) / 1000) * rps);
|
||
bucket.lastMs = now;
|
||
if (bucket.tokens >= 1) { bucket.tokens -= 1; return; }
|
||
await Bun.sleep(200);
|
||
}
|
||
}
|
||
|
||
// ============================================================================
|
||
// 计数与控制器
|
||
// ============================================================================
|
||
|
||
/** # 成员变量 单分钟计数桶 **/
|
||
type Counter = {
|
||
sent: number; ok: number; gate: number; empty: number;
|
||
captcha: number; poolEmpty: number; httpErr: number; netErr: number;
|
||
fetch: number; browser: number; attemptsTotal: number; gateAttempts: number;
|
||
};
|
||
function newCounter(): Counter {
|
||
return { sent: 0, ok: 0, gate: 0, empty: 0, captcha: 0, poolEmpty: 0, httpErr: 0, netErr: 0, fetch: 0, browser: 0, attemptsTotal: 0, gateAttempts: 0 };
|
||
}
|
||
/** # 成员变量 当前分钟计数 / 滚动窗口(最近 WINDOW_MIN 个分钟桶) **/
|
||
let cur = newCounter();
|
||
const window: { ts: string; c: Counter }[] = [];
|
||
/** # 成员变量 全程累计(终局汇总用) **/
|
||
const totals = newCounter();
|
||
/** # 成员变量 控制器决策日志 **/
|
||
const decisions: Record<string, unknown>[] = [];
|
||
/** # 成员变量 成功指纹计数(分布公平性观察用) **/
|
||
const fpMap = new Map<string, number>();
|
||
|
||
/** # 函数
|
||
* 窗口聚合求和(参数 n:取最近 n 个分钟桶;不传 = 全窗口)。
|
||
*/
|
||
function sumWindow(n = WINDOW_MIN): Counter {
|
||
const s = newCounter();
|
||
for (const { c } of window.slice(-n)) {
|
||
for (const k of Object.keys(s) as (keyof Counter)[]) s[k] += c[k];
|
||
}
|
||
return s;
|
||
}
|
||
|
||
/** # 成员变量 软信号事件段状态(迟滞:进段砍一次,退段重置;防同一段波逐分钟连砍) **/
|
||
let captchaWave = false;
|
||
let gateWave = false;
|
||
let poolEmptyWave = false;
|
||
/** # 成员变量 硬约束退化段(进段砍一次;段内连砍需增量新失败;30min 窗恢复≥99% 退段) **/
|
||
let degraded = false;
|
||
/** # 成员变量 上次硬砍时的全程累计失败数(段内连砍要求全程新失败 ≥2,dp5 教训:同一波旧失败被短窗反复引用,10:24-10:28 五分钟连砍五刀 175→30/min) **/
|
||
let failsAtLastCut = 0;
|
||
/** # 成员变量 干净水位:30min 窗 ≥99.5% 且短窗 ≥99% 时达到过的最高速率(深跌后向水位快速回收;dp5 教训:×1.08/5 拍从 3/min 爬回需 3 小时,温池 150 ready 全程空转) **/
|
||
let maxCleanRate = 0;
|
||
/** # 成员变量 软砍速率下限(req/min):背景验证码供给恒定,速率越低比率越高,比率规则在低位永真 → 钉死地板(dp5 实测 12:00-17:00 自 52/min 自残至 3/min)。软规则不砍破此位。v5.1 上调 40:99% 目标下 25/min 时涓流(~14 个/时)占比已达 0.93% 贴线,40/min 摊薄到 0.58% 留出安全余量(稀释论) **/
|
||
const SOFT_FLOOR = 40;
|
||
/** # 成员变量 硬砍速率下限(req/min;v5.2 新增):dp7 实测 99% 线下小样本噪音(4 个失败)即可驱动 ×0.5 级联到 3/min —— 硬规则也设地板,真退化到 8/min 已足够压缩暴露量,3/min 只会恶化涓流占比(稀释论) **/
|
||
const HARD_FLOOR = 8;
|
||
|
||
/** # 函数
|
||
* 控制器决策(每 60s 调用):按窗口成功率/内部压力调整速率,返回决策记录或 null。
|
||
* v5(dp6 教训):目标上调 99%;爬坡备选条件「验证码 ≤1 且短窗 ≥98.5%」修低速恢复锁死。
|
||
* v4(dp5 教训):①硬砍连砍需「增量新失败」(全程失败数较上次砍时新增 ≥2),同一波旧失败不再反复引用;
|
||
* ②软信号加绝对数护栏(10min 窗内验证码 ≥3 / 门壳 ≥3 才允许比率规则),低速率噪音不再触发;
|
||
* ③软砍下限 SOFT_FLOOR,消除比率陷阱钉死地板;④爬坡:向干净水位快速回收 ×1.3/拍,水位之上 ×1.08/5 拍。
|
||
*/
|
||
function controlTick(): Record<string, unknown> | null {
|
||
const wH = sumWindow(30); // 硬约束窗口:成功率即约束本身
|
||
const tot = wH.sent;
|
||
if (tot < 10) return null; // 信号不足,保持
|
||
const wS = sumWindow(10); // 软信号短窗:波动退去快,防旧波滞留连砍(dp1 教训)
|
||
const success = wH.ok / tot;
|
||
const gateRatio = wH.attemptsTotal > 0 ? wH.gateAttempts / wH.attemptsTotal : 0;
|
||
const gateRatioS = wS.attemptsTotal > 0 ? wS.gateAttempts / wS.attemptsTotal : 0;
|
||
const capPctS = wS.sent > 0 ? wS.captcha / wS.sent : 0;
|
||
const from = rate;
|
||
let to = from;
|
||
let reason = "hold";
|
||
// 硬约束(v5.1,dp7 前 1.5h 实测教训:99% 线下比率陷阱重现在入段/退段/续砍三级 ——
|
||
// 低速涓流永远压住 99% 窗口线,degraded 锁死 → 砍到 3/min 浏览器饿死;稀释论:
|
||
// 验证码涓流的唯一解药是爬坡摊薄,砍速让分母变小占比更高,是毒药):
|
||
// 涓流豁免 = 30min 窗零非验证码失败 且 短窗验证码 ≤2 个(涓流规模,非波次);
|
||
// ① 退段:窗口 ≥99%,或涓流豁免且短窗 ≥98.5%;② 入段:涓流不入段;
|
||
// ③ 段内续砍需真证据:非验证码失败 ≥1(我方故障)或验证码 ≥5/10min(波次规模)
|
||
const fails = tot - wH.ok;
|
||
const allFails = totals.sent - totals.ok;
|
||
const successS = wS.sent > 0 ? wS.ok / wS.sent : 1;
|
||
const nonCapFails = wH.gate + wH.empty + wH.poolEmpty + wH.httpErr + wH.netErr;
|
||
const trickle = nonCapFails === 0 && wS.captcha <= 2;
|
||
if (success >= 0.99 || (trickle && successS >= 0.985)) degraded = false;
|
||
// 入段(v5.2 加量下限,dp7 实测教训:冷启动波次在 ~90 样本窗口上 4 个失败即被 ×0.5,
|
||
// 小样本噪音驱动级联):浅破线(99%~96%)需窗口 ≥200 样本且 ≥8 失败;深破线(<96%)
|
||
// 仍快速响应(≥4 失败即可,真崩溃不等样本攒够)
|
||
const entry = !degraded && !trickle && ((success < 0.99 && fails >= 8 && tot >= 200) || (success < 0.96 && fails >= 4));
|
||
// 段内续砍(v5.2 提证据门槛):非验证码失败 ≥3(1~2 个门壳是波次伴生噪音,dp7 实测
|
||
// gate=1 即解锁续砍)或验证码 ≥5/10min(波次规模);短窗样本 ≥30
|
||
const ongoing = degraded && wS.sent >= 30 && successS < 0.96 && (nonCapFails >= 3 || wS.captcha >= 5) && allFails - failsAtLastCut >= 2;
|
||
if (entry || ongoing) {
|
||
degraded = true;
|
||
failsAtLastCut = allFails;
|
||
to = success < 0.96 ? from * 0.5 : from * 0.7;
|
||
to = Math.max(HARD_FLOOR, to);
|
||
reason = `success=${(success * 100).toFixed(1)}%<${success < 0.96 ? 96 : 99}%(失败 ${fails}${entry ? "" : ";短窗<96%+新失败≥2"}) ×${success < 0.96 ? 0.5 : 0.7}`;
|
||
} else {
|
||
// 软信号:事件段迟滞 + 绝对数护栏(v5.1:验证码护栏 3→8 个/10min —— 只砍波次规模,
|
||
// 涓流交给稀释;低速率下 1~2 个噪音即高比率,不得触发;dp5 教训)
|
||
if (wS.captcha >= 8 && capPctS > 0.005 && !captchaWave) {
|
||
captchaWave = true;
|
||
const heavy = wS.captcha >= 12 && capPctS > 0.02;
|
||
to = Math.max(SOFT_FLOOR, from * (heavy ? 0.5 : 0.7));
|
||
reason = `验证码 ${wS.captcha} 个占比 ${(capPctS * 100).toFixed(2)}% ${heavy ? "×0.5" : "×0.7"}`;
|
||
} else if (wS.captcha === 0) captchaWave = false;
|
||
if (to === from && wS.gate >= 3 && gateRatioS > 0.15 && !gateWave) {
|
||
gateWave = true;
|
||
to = Math.max(SOFT_FLOOR, from * 0.85);
|
||
reason = `门壳 ${wS.gate} 占比 ${(gateRatioS * 100).toFixed(1)}%>15% ×0.85`;
|
||
} else if (gateRatioS < 0.05) gateWave = false;
|
||
if (to === from && wS.poolEmpty > 5 && !poolEmptyWave) {
|
||
poolEmptyWave = true;
|
||
to = Math.max(SOFT_FLOOR, from * 0.85);
|
||
reason = `池空 ${wS.poolEmpty} ×0.85`;
|
||
} else if (wS.poolEmpty === 0) poolEmptyWave = false;
|
||
}
|
||
// 干净水位记账(v5.2:门槛 99.5%→99%,99% 线下 ≥99.5% 的 30min 窗在涓流时段不可达,
|
||
// 水位永远立不起来 → 深跌后只剩 ×1.08/5 拍龟速爬坡,dp7 实测):窗口 ≥99% 且短窗 ≥99%
|
||
// 时的速率即为「实测可行」
|
||
if (success >= 0.99 && successS >= 0.99 && wS.sent >= 10) maxCleanRate = Math.max(maxCleanRate, from);
|
||
// 干净爬坡:退化段中不爬(30min 窗未恢复 99%);深跌后向水位快速回收 ×1.3/拍,水位之上 ×1.08/5 拍;
|
||
// 爬坡验证码条件:主条件短窗成功率 ≥99.5%,备选「验证码 ≤2 且短窗 ≥98%」(v5.1:与涓流
|
||
// 豁免口径一致,修 dp6 实测低速恢复锁死 + dp7 实测涓流滞留 —— 爬坡摊薄后比率自然回落);
|
||
// 短窗 0 个验证码恒可爬,或样本 ≥300 时比率已无意义(绝对护栏统一交给软砍规则)
|
||
if (to === from && !degraded && wS.sent >= 10) {
|
||
const cleanS = successS >= 0.995 || (wS.captcha <= 2 && successS >= 0.98);
|
||
if (cleanS && gateRatioS < 0.02 && wS.poolEmpty === 0 && (wS.captcha === 0 || wS.sent >= 300)) {
|
||
rampSeq++;
|
||
if (maxCleanRate > 0 && from < maxCleanRate * 0.75) {
|
||
to = from * 1.3 + 0.2;
|
||
reason = `clean-fast(水位 ${maxCleanRate.toFixed(0)}) ×1.3`;
|
||
} else if (rampSeq % RAMP_EVERY_MIN === 0) { to = from * 1.08 + 0.2; reason = `clean(success=${(successS * 100).toFixed(1)}%,gate=${(gateRatioS * 100).toFixed(1)}%) ×1.08`; }
|
||
}
|
||
}
|
||
to = Math.round(Math.min(RATE_MAX, Math.max(RATE_MIN, to)) * 10) / 10;
|
||
rate = to;
|
||
const rec = { ts: new Date().toISOString(), from: +from.toFixed(1), to: +to.toFixed(1), reason, rampSeq, rampTick: rampSeq % RAMP_EVERY_MIN, window: { sent: tot, ok: wH.ok, gate: wH.gate, captcha: wH.captcha, poolEmpty: wH.poolEmpty, success: +(success * 100).toFixed(2), gateRatio: +(gateRatio * 100).toFixed(2), capPctS: +(capPctS * 100).toFixed(2) }, waves: { captcha: captchaWave, gate: gateWave, poolEmpty: poolEmptyWave } };
|
||
decisions.push(rec);
|
||
if (decisions.length > 500) decisions.splice(0, decisions.length - 500);
|
||
if (to !== from) console.log(`[${rec.ts}] 速率 ${rec.from} → ${rec.to} req/min(${reason})`);
|
||
return rec;
|
||
}
|
||
/** # 成员变量 控制器节拍序号(加速步按其取模) **/
|
||
let rampSeq = 0;
|
||
|
||
// ============================================================================
|
||
// 请求发射
|
||
// ============================================================================
|
||
|
||
/** # 函数
|
||
* 记逐请求行(紧凑:仅分类所需字段)。
|
||
*/
|
||
function recReq(o: Record<string, unknown>): void {
|
||
try { fs.appendFileSync(REQ_JSONL, JSON.stringify(o) + "\n"); } catch {}
|
||
}
|
||
|
||
/** # 函数
|
||
* 发射单次搜索请求并记账(成功/门壳/空结果/验证码/池空/上游失败/网络失败)。
|
||
* 路径与指纹取自响应体(search_parameters.runtime / search_metadata.profile.id),
|
||
* 内部门壳压力取 timing_stages.attempts 的 outcome 统计。
|
||
*/
|
||
async function fire(): Promise<void> {
|
||
const q = nextQuery();
|
||
const t = Date.now();
|
||
try {
|
||
const res = await fetch(`${BASE}/api/serp/json/search?q=${encodeURIComponent(q)}&num=10`, {
|
||
headers: { authorization: `Bearer ${TOKEN}` },
|
||
signal: AbortSignal.timeout(240_000),
|
||
});
|
||
const ms = Date.now() - t;
|
||
if (res.status === 401) { console.error("Token 无效(401),终止"); process.exit(1); }
|
||
if (!res.ok) {
|
||
const j = await res.json().catch(() => null) as any;
|
||
const code = String(j?.code ?? `HTTP${res.status}`);
|
||
// 发出与结果同刻记账:分钟桶边界处的在途请求计入完成那一桶,避免「发而不中」虚报
|
||
cur.sent++; totals.sent++;
|
||
if (code === "CAPTCHA_DETECTED") { cur.captcha++; totals.captcha++; }
|
||
else if (code === "SERP_POOL_EMPTY") { cur.poolEmpty++; totals.poolEmpty++; }
|
||
else { cur.httpErr++; totals.httpErr++; }
|
||
recReq({ ts: new Date().toISOString(), kind: code === "CAPTCHA_DETECTED" ? "captcha" : code === "SERP_POOL_EMPTY" ? "poolEmpty" : "httpErr", code, ms });
|
||
return;
|
||
}
|
||
const j = await res.json().catch(() => null) as any;
|
||
const organic = Array.isArray(j?.organic) ? j.organic.length : 0;
|
||
const path = j?.search_parameters?.runtime === "browser" ? "browser" : "fetch";
|
||
const fp = String(j?.search_metadata?.profile?.id ?? "");
|
||
const attempts: any[] = Array.isArray(j?.timing_stages?.attempts) ? j.timing_stages.attempts : [];
|
||
const gAtt = attempts.filter(a => a?.outcome === "gate" || a?.outcome === "redirect").length;
|
||
cur.sent++; totals.sent++;
|
||
cur.attemptsTotal += Math.max(1, attempts.length); totals.attemptsTotal += Math.max(1, attempts.length);
|
||
cur.gateAttempts += gAtt; totals.gateAttempts += gAtt;
|
||
cur[path]++; totals[path]++;
|
||
if (fp) fpMap.set(fp, (fpMap.get(fp) ?? 0) + 1);
|
||
if (organic > 0) {
|
||
cur.ok++; totals.ok++;
|
||
recReq({ ts: new Date().toISOString(), kind: "ok", path, fp, ms, attempts: attempts.length, organic });
|
||
} else if (gAtt > 0 || attempts.some(a => a?.outcome === "gate")) {
|
||
cur.gate++; totals.gate++;
|
||
recReq({ ts: new Date().toISOString(), kind: "gate", path, fp, ms, attempts: attempts.length, gateAttempts: gAtt });
|
||
} else {
|
||
cur.empty++; totals.empty++;
|
||
recReq({ ts: new Date().toISOString(), kind: "empty", path, fp, ms, attempts: attempts.length, organic: 0 });
|
||
}
|
||
} catch (e) {
|
||
cur.sent++; totals.sent++;
|
||
cur.netErr++; totals.netErr++;
|
||
recReq({ ts: new Date().toISOString(), kind: "netErr", error: String(e).slice(0, 140), ms: Date.now() - t });
|
||
}
|
||
}
|
||
|
||
/** # 函数
|
||
* worker 主循环:取令牌 → 发射,直到结束时刻。
|
||
*/
|
||
async function worker(endAt: number): Promise<void> {
|
||
while (Date.now() < endAt) {
|
||
await acquire();
|
||
if (Date.now() >= endAt) break;
|
||
await fire();
|
||
}
|
||
}
|
||
|
||
// ============================================================================
|
||
// 快照与汇总
|
||
// ============================================================================
|
||
|
||
/** # 函数
|
||
* 组装快照(窗口+全程+路径占比+指纹 Top20+决策尾部)。
|
||
*/
|
||
function snapshot(): Record<string, unknown> {
|
||
const w = sumWindow();
|
||
const topFps = [...fpMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20).map(([fp, n]) => ({ fp, n }));
|
||
return {
|
||
at: new Date().toISOString(),
|
||
hoursElapsed: +((Date.now() - startedAt) / 3600_000).toFixed(2),
|
||
rateReqMin: +rate.toFixed(1),
|
||
window: { ...w, successPct: w.sent ? +((w.ok / w.sent) * 100).toFixed(2) : null, gateAttemptPct: w.attemptsTotal ? +((w.gateAttempts / w.attemptsTotal) * 100).toFixed(2) : null },
|
||
totals: { ...totals, successPct: totals.sent ? +((totals.ok / totals.sent) * 100).toFixed(2) : null, gateAttemptPct: totals.attemptsTotal ? +((totals.gateAttempts / totals.attemptsTotal) * 100).toFixed(2) : null },
|
||
pathSplit: { fetch: totals.fetch, browser: totals.browser },
|
||
topFps,
|
||
decisions: decisions.slice(-20),
|
||
};
|
||
}
|
||
|
||
// ============================================================================
|
||
// 主流程:并行 worker + 每分钟聚合 + 控制器 + 每 5min 快照;到时终局收尾
|
||
// ============================================================================
|
||
|
||
const startedAt = Date.now();
|
||
const endAt = startedAt + HOURS * 3600_000;
|
||
fs.writeFileSync(JSONL, "");
|
||
fs.writeFileSync(REQ_JSONL, "");
|
||
console.log(`开始:label=${LABEL} hours=${HOURS} rate0=${RATE0} 区间=[${RATE_MIN},${RATE_MAX}] workers=${WORKERS} 出口=json(auto 双路径)`);
|
||
|
||
for (let i = 0; i < WORKERS; i++) void worker(endAt);
|
||
|
||
/** # 成员变量 每分钟定时器:滚动窗口推进 + 控制器决策 + 聚合行落盘 **/
|
||
const minuteTimer = setInterval(() => {
|
||
const row = { ts: new Date().toISOString(), rate: +rate.toFixed(1), ...cur };
|
||
try { fs.appendFileSync(JSONL, JSON.stringify(row) + "\n"); } catch {}
|
||
window.push({ ts: row.ts, c: cur });
|
||
if (window.length > WINDOW_MIN) window.shift();
|
||
cur = newCounter();
|
||
controlTick();
|
||
}, 60_000);
|
||
|
||
/** # 成员变量 每 5 分钟快照与控制台摘要 **/
|
||
const snapTimer = setInterval(() => {
|
||
const snap = snapshot();
|
||
fs.writeFileSync(LIVE, JSON.stringify(snap, null, 2));
|
||
const t = snap.totals as any; const w = snap.window as any;
|
||
console.log(`[${snap.at}] ${(snap.hoursElapsed as number).toFixed(1)}h rate=${snap.rateReqMin}/min | 窗口 成功率=${w.successPct}% 门壳尝试=${w.gateAttemptPct}% sent=${w.sent} (ok=${w.ok} gate=${w.gate} captcha=${w.captcha} poolEmpty=${w.poolEmpty} httpErr=${w.httpErr} netErr=${w.netErr}) | 全程 成功率=${t.successPct}% 路径 fetch=${snap.pathSplit.fetch}/browser=${snap.pathSplit.browser}`);
|
||
}, 5 * 60_000);
|
||
|
||
const endTimer = setTimeout(() => {
|
||
clearInterval(minuteTimer); clearInterval(snapTimer);
|
||
const snap = snapshot();
|
||
fs.writeFileSync(`${DATA}${LABEL}-final.json`, JSON.stringify(snap, null, 2));
|
||
console.log("测试结束,final 已写:" + `${DATA}${LABEL}-final.json`);
|
||
process.exit(0);
|
||
}, HOURS * 3600_000);
|
||
endTimer.unref?.();
|