199 lines
9.6 KiB
TypeScript
199 lines
9.6 KiB
TypeScript
/** # 脚本说明
|
||
* SERP 单号日容量「真实极限」测试器(用户 2026-10-30 要求:真实跑一天,不要算出来的数):
|
||
* SERP_TOKEN=xxx bun serp-profile-max.ts --hours 24 --label capmax --spec "fp:组名,fp:组名"
|
||
* 每号自适应控制器:能跑多快跑多快,被 Google 拦就退避,恢复就再加速 ——
|
||
* 干净 200:间隔 ×0.72 收缩(下限 5s ≈ 720/h 上限);
|
||
* 门壳 / 上游硬失败:间隔 ×2.2 放大(上限 10 分钟),连续 ≥5 判入狱转探测(1 次/5 分钟);
|
||
* 验证码:直接入狱探测,首个干净 200 出狱,间隔重置 30s 重新爬坡。
|
||
* 跑满 24h 后每号的干净 200 总数 = 该号真实一天能执行的搜索量(实测,非推算)。
|
||
* 记录:data/<label>.jsonl 逐条 + <label>-live.json 每 30 分钟快照 + <label>-final.json 终表。
|
||
*/
|
||
import fs from "node:fs";
|
||
|
||
/** # 成员变量 服务基址(同机直连) **/
|
||
const BASE = "http://127.0.0.1:53100";
|
||
const args = process.argv.slice(2);
|
||
function argOf(k: string, d: string): string { const i = args.indexOf(k); return i >= 0 ? args[i + 1] : d; }
|
||
/** # 成员变量 测试时长(小时) **/
|
||
const HOURS = Number(argOf("--hours", "24"));
|
||
const LABEL = argOf("--label", "capmax");
|
||
/** # 成员变量 号表:指纹:组名(逗号分隔) **/
|
||
const SPEC = argOf("--spec", "");
|
||
/** # 成员变量 间隔边界:下限 5s(=720/h 理论上限),初始 30s,上限 10 分钟 **/
|
||
const FLOOR_MS = 5000, START_MS = 30_000, CEIL_MS = 600_000;
|
||
/** # 成员变量 探测模式间隔:5 分钟 **/
|
||
const PROBE_MS = 300_000;
|
||
const TOKEN = process.env.SERP_TOKEN ?? "";
|
||
if (!TOKEN || !SPEC) { console.error("缺少 SERP_TOKEN 或 --spec"); process.exit(1); }
|
||
|
||
/** # 成员变量 输出目录(data/,相对脚本) **/
|
||
const DATA = new URL("./data/", import.meta.url).pathname;
|
||
fs.mkdirSync(DATA, { recursive: true });
|
||
const JSONL = `${DATA}${LABEL}.jsonl`;
|
||
const LIVE = `${DATA}${LABEL}-live.json`;
|
||
|
||
/** # 成员变量 长尾查询词表(A×B×C 组合,顺序消费避免重复) **/
|
||
const A = ["best", "top rated", "cheap", "review of", "how to choose a", "buy", "compare", "2026 guide for", "price of", "warranty for"];
|
||
const B = ["coffee maker", "running shoes", "laptop stand", "rice cooker", "air fryer", "standing desk", "office chair", "mechanical keyboard", "monitor arm", "webcam", "espresso machine", "vacuum cleaner", "electric kettle", "gaming mouse", "noise cancelling headphones", "desk lamp", "water bottle", "yoga mat", "blender", "space heater"];
|
||
const C = ["for home use", "for small office", "under 100 dollars", "in 2026", "with long warranty", "for beginners", "pro model", "on sale", "quiet operation", "energy efficient"];
|
||
const combos: string[] = [];
|
||
for (const a of A) for (const b of B) for (const c of C) combos.push(`${a} ${b} ${c}`);
|
||
let qi = Math.floor(Math.random() * combos.length);
|
||
function nextQuery(): string { const q = combos[qi % combos.length]; qi++; return q; }
|
||
|
||
/** # 成员变量 单号运行时状态 **/
|
||
type Slot = {
|
||
fp: string; group: string;
|
||
/** # 成员变量 当前发射间隔(ms;探测模式下含义=探测间隔) **/
|
||
intervalMs: number;
|
||
sent: number; ok: number; gate: number; captcha: number; httpErr: number; netErr: number;
|
||
/** # 成员变量 连续被拦计数(门壳/上游失败累计) **/
|
||
consecBlock: number;
|
||
probeMode: boolean; jails: number; jailStart: number;
|
||
/** # 成员变量 每次入狱→出狱的恢复记录(分钟) **/
|
||
recoveries: { kind: string; flaggedAt: string; recoveredAt: string; minutes: number }[];
|
||
/** # 成员变量 干净 200 峰值间隔达到的最快速率(req/h) **/
|
||
fastest: number;
|
||
gone: boolean;
|
||
};
|
||
|
||
const slots: Slot[] = SPEC.split(",").map((s) => {
|
||
const [fp, group] = s.trim().split(":");
|
||
return {
|
||
fp, group: group ?? fp, intervalMs: START_MS,
|
||
sent: 0, ok: 0, gate: 0, captcha: 0, httpErr: 0, netErr: 0,
|
||
consecBlock: 0, probeMode: false, jails: 0, jailStart: 0,
|
||
recoveries: [], fastest: 0, gone: false,
|
||
};
|
||
});
|
||
|
||
/** # 函数
|
||
* 写单条 JSONL 记录。
|
||
*/
|
||
function rec(o: Record<string, unknown>): void {
|
||
try { fs.appendFileSync(JSONL, JSON.stringify(o) + "\n"); } catch {}
|
||
}
|
||
|
||
/** # 函数
|
||
* 入狱:转探测模式,间隔固定 5 分钟,直到首个干净 200。
|
||
*/
|
||
function enterJail(s: Slot, kind: string, at: number): void {
|
||
if (!s.probeMode) {
|
||
s.probeMode = true; s.jails++; s.jailStart = at; s.intervalMs = PROBE_MS;
|
||
console.log(`[${new Date().toISOString()}] ${s.fp}(${s.group}) 入狱(${kind}),第 ${s.jails} 次,探测 1/5min`);
|
||
}
|
||
}
|
||
|
||
/** # 函数
|
||
* 发射单次 sticky 请求并执行自适应调整。
|
||
*/
|
||
async function fire(s: Slot): Promise<void> {
|
||
const q = nextQuery();
|
||
const url = `${BASE}/api/serp/html/search?q=${encodeURIComponent(q)}&num=10&runtime=fetch&profile=${s.fp}`;
|
||
s.sent++;
|
||
const t = Date.now();
|
||
try {
|
||
const res = await fetch(url, { headers: { authorization: `Bearer ${TOKEN}` }, signal: AbortSignal.timeout(120000) });
|
||
const gate = res.headers.get("x-serp-gate") === "1";
|
||
if (!res.ok) {
|
||
const j = await res.json().catch(() => null) as any;
|
||
const code = String(j?.code ?? "");
|
||
const captcha = code === "CAPTCHA_DETECTED";
|
||
const upstreamFail = code === "UPSTREAM_FAILED" || code === "SERP_UPSTREAM_FAILED";
|
||
if (captcha) {
|
||
s.captcha++;
|
||
enterJail(s, "captcha", Date.now());
|
||
rec({ ts: new Date().toISOString(), fp: s.fp, group: s.group, kind: "captcha", code, sent: s.sent, ms: Date.now() - t });
|
||
} else if (code === "SERP_POOL_EMPTY" && String(j?.message ?? "").includes("不存在")) {
|
||
s.gone = true;
|
||
rec({ ts: new Date().toISOString(), fp: s.fp, group: s.group, kind: "gone", code, sent: s.sent, ms: Date.now() - t });
|
||
} else {
|
||
s.httpErr++;
|
||
if (upstreamFail) { s.consecBlock++; s.intervalMs = Math.min(CEIL_MS, Math.round(s.intervalMs * 2.2)); }
|
||
rec({ ts: new Date().toISOString(), fp: s.fp, group: s.group, kind: "httpErr", code, intervalMs: s.intervalMs, sent: s.sent, ms: Date.now() - t });
|
||
if (s.consecBlock >= 5) enterJail(s, "upstream", Date.now());
|
||
}
|
||
return;
|
||
}
|
||
if (gate) {
|
||
s.gate++;
|
||
s.consecBlock++;
|
||
s.intervalMs = Math.min(CEIL_MS, Math.round(s.intervalMs * 2.2));
|
||
if (s.consecBlock >= 5) enterJail(s, "gate", Date.now());
|
||
rec({ ts: new Date().toISOString(), fp: s.fp, group: s.group, kind: "gate", intervalMs: s.intervalMs, sent: s.sent, ms: Date.now() - t });
|
||
return;
|
||
}
|
||
// 干净 200:出狱 / 收缩间隔加速
|
||
const wasJailed = s.probeMode;
|
||
s.consecBlock = 0;
|
||
if (wasJailed) {
|
||
s.probeMode = false;
|
||
s.recoveries.push({
|
||
kind: "jail", flaggedAt: new Date(s.jailStart).toISOString(),
|
||
recoveredAt: new Date().toISOString(),
|
||
minutes: +((Date.now() - s.jailStart) / 60000).toFixed(1),
|
||
});
|
||
console.log(`[${new Date().toISOString()}] ${s.fp}(${s.group}) 出狱:${s.recoveries.at(-1)!.minutes} 分钟,间隔重置 30s 重新爬坡`);
|
||
s.intervalMs = START_MS;
|
||
} else {
|
||
s.intervalMs = Math.max(FLOOR_MS, Math.round(s.intervalMs * 0.72));
|
||
}
|
||
s.ok++;
|
||
s.fastest = Math.max(s.fastest, Math.round(3600_000 / s.intervalMs));
|
||
rec({ ts: new Date().toISOString(), fp: s.fp, group: s.group, kind: "ok200", intervalMs: s.intervalMs, sent: s.sent, ms: Date.now() - t });
|
||
} catch (e) {
|
||
s.netErr++;
|
||
rec({ ts: new Date().toISOString(), fp: s.fp, group: s.group, kind: "netErr", error: String(e).slice(0, 160), sent: s.sent, ms: Date.now() - t });
|
||
}
|
||
}
|
||
|
||
/** # 函数
|
||
* 单号主循环:按当前间隔(人性化抖动 ±15%)发射,直到时长耗尽。
|
||
*/
|
||
async function runSlot(s: Slot): Promise<void> {
|
||
const endAt = Date.now() + HOURS * 3600_000;
|
||
while (Date.now() < endAt && !s.gone) {
|
||
await fire(s);
|
||
if (s.gone) break;
|
||
const waitMs = s.intervalMs * (0.85 + Math.random() * 0.3);
|
||
const remain = endAt - Date.now();
|
||
if (remain <= 0) break;
|
||
await Bun.sleep(Math.min(waitMs, remain));
|
||
}
|
||
console.log(`[${new Date().toISOString()}] ${s.fp}(${s.group}) 结束:干净=${s.ok} 门壳=${s.gate} 验证码=${s.captcha} 入狱=${s.jails} 次 最快=${s.fastest}/h gone=${s.gone}`);
|
||
}
|
||
|
||
/** # 函数
|
||
* 每 30 分钟写活跃快照(主读数:每号干净 200 累计 = 真实日容量进度)。
|
||
*/
|
||
function snapshot(): Record<string, unknown> {
|
||
const now = new Date().toISOString();
|
||
const list = slots.map((s) => ({
|
||
fp: s.fp, group: s.group, clean: s.ok, gate: s.gate, captcha: s.captcha,
|
||
jails: s.jails, probing: s.probeMode, intervalMs: s.intervalMs,
|
||
currentRate: Math.round(3600_000 / s.intervalMs), fastest: s.fastest, gone: s.gone,
|
||
recoveries: s.recoveries,
|
||
}));
|
||
const snap = { at: now, hoursElapsed: +((Date.now() - startedAt) / 3600_000).toFixed(2), totalClean: slots.reduce((a, s) => a + s.ok, 0), slots: list };
|
||
fs.writeFileSync(LIVE, JSON.stringify(snap, null, 2));
|
||
return snap;
|
||
}
|
||
|
||
/** # 主流程:并行起所有号;每 30 分钟摘要;到时写终表收尾 **/
|
||
const startedAt = Date.now();
|
||
fs.writeFileSync(JSONL, "");
|
||
console.log(`开始:label=${LABEL} hours=${HOURS} 号数=${slots.length} 规格=${SPEC}`);
|
||
for (const s of slots) void runSlot(s);
|
||
const snapTimer = setInterval(() => {
|
||
const snap = snapshot();
|
||
console.log(`[${snap.at}] ` + (snap.slots as any[]).map((v) => `${v.fp}(${v.group}): clean=${v.clean} jail=${v.jails}${v.probing ? "(探测中)" : ""} cur=${v.currentRate}/h peak=${v.fastest}/h`).join(" | "));
|
||
}, 30 * 60_000);
|
||
const endTimer = setTimeout(() => {
|
||
clearInterval(snapTimer);
|
||
const snap = snapshot();
|
||
fs.writeFileSync(`${DATA}${LABEL}-final.json`, JSON.stringify(snap, null, 2));
|
||
console.log("测试结束,final 已写:" + `${DATA}${LABEL}-final.json`);
|
||
process.exit(0);
|
||
}, HOURS * 3600_000);
|
||
endTimer.unref?.();
|