94 lines
4.4 KiB
TypeScript
94 lines
4.4 KiB
TypeScript
/**
|
||
* # 脚本 文章分类归属修复（一次性，幂等，需外网可达 www.dusuniot.com）
|
||
* 背景：WP 导入只带文章不带 category 归属，news/case-study/gateway/technical-brief
|
||
* 四个文章分类在本库文章数均为 0，归档页是空的。
|
||
* 做法：抓取原站这四个分类的归档页（含 /page/N/ 翻页），提取 /<cat>/<slug>/ 文章链接，
|
||
* 与本地已发布文章按 slug 匹配，缺失的 content_terms 归属补插。
|
||
* 不在本地库的原站文章只计数不插入（避免归档里再出死链）。
|
||
* 重复执行安全：插入前先查存在。
|
||
*/
|
||
import { SQL } from "bun";
|
||
const sql = new SQL(process.env.DATABASE_URL || "postgres://dusuniot:dusuniot@127.0.0.1:25432/dusuniot");
|
||
|
||
/** # 成员变量 抓取 UA（原站对无 UA 的 curl 返回空体） **/
|
||
const UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36";
|
||
|
||
/** # 类型 归档抓取目标 **/
|
||
const TARGETS = [
|
||
{ catId: 98, catSlug: "case-study", start: "https://www.dusuniot.com/case-study/success-stories/" }, // 原站 /case-study/ 301 到 success-stories
|
||
{ catId: 14, catSlug: "news", start: "https://www.dusuniot.com/news/" },
|
||
{ catId: 119, catSlug: "gateway", start: "https://www.dusuniot.com/gateway/", alsoBlogH3: true }, // 该分类文章 permalink 是 /blog/<slug>/（主分类 blog），卡片在 h3>a
|
||
{ catId: 120, catSlug: "technical-brief", start: "https://www.dusuniot.com/resources/technical-brief/" },
|
||
];
|
||
|
||
/** # 函数 抓一个页面 HTML（失败返回空串） */
|
||
async function get(url: string): Promise<string> {
|
||
try {
|
||
const res = await fetch(url, { headers: { "user-agent": UA } });
|
||
if (res.status !== 200) return "";
|
||
return await res.text();
|
||
} catch {
|
||
return "";
|
||
}
|
||
}
|
||
|
||
/** # 函数 从归档 HTML 提取该分类下的文章 slug（去重、排除分页页码/feed 与归档自身）
|
||
* 分类路径允许一段可选前缀（如 /resources/technical-brief/<slug>/、/blog/gateway/<slug>/）
|
||
*/
|
||
function extractSlugs(html: string, catSlug: string): string[] {
|
||
const re = new RegExp(`https://www\\.dusuniot\\.com/(?:[a-z0-9-]+/)?${catSlug}/([a-z0-9-]+)/`, "gi");
|
||
const out = new Set<string>();
|
||
let m: RegExpExecArray | null;
|
||
while ((m = re.exec(html))) {
|
||
const slug = m[1]!;
|
||
if (slug === "page" || slug === "feed" || slug === "amp" || /^\d+$/.test(slug)) continue;
|
||
out.add(slug);
|
||
}
|
||
return [...out];
|
||
}
|
||
|
||
/** # 函数 从归档 HTML 提取 h3 文章卡片的 /blog/<slug>/ 链接（gateway 归档专用：文章 permalink 无分类段） */
|
||
function extractBlogH3Slugs(html: string): string[] {
|
||
const re = /<h3[^>]*>\s*<a[^>]*href="https:\/\/www\.dusuniot\.com\/blog\/([a-z0-9-]+)\/"/gi;
|
||
const out = new Set<string>();
|
||
let m: RegExpExecArray | null;
|
||
while ((m = re.exec(html))) out.add(m[1]!);
|
||
return [...out];
|
||
}
|
||
|
||
for (const t of TARGETS) {
|
||
// 首页 + 从首页发现的 /page/N/ 翻页（上限 40 页防失控）
|
||
let html = await get(t.start);
|
||
const pages = new Set<string>();
|
||
for (let i = 2; i <= 40; i++) {
|
||
const u = t.start.replace(/\/$/, "") + `/page/${i}/`;
|
||
// 只有首页 HTML 里真实出现翻页链接才抓
|
||
if (html.includes(`/${t.catSlug}/page/${i}/`) || html.includes(`/page/${i}/`)) pages.add(u);
|
||
}
|
||
const slugs = new Set<string>(extractSlugs(html, t.catSlug));
|
||
if ("alsoBlogH3" in t && t.alsoBlogH3) for (const s of extractBlogH3Slugs(html)) slugs.add(s);
|
||
for (const u of pages) {
|
||
html = await get(u);
|
||
for (const s of extractSlugs(html, t.catSlug)) slugs.add(s);
|
||
if ("alsoBlogH3" in t && t.alsoBlogH3) for (const s of extractBlogH3Slugs(html)) slugs.add(s);
|
||
await Bun.sleep(300);
|
||
}
|
||
// 与本地已发布文章匹配并补归属
|
||
let inserted = 0;
|
||
let missing = 0;
|
||
for (const slug of slugs) {
|
||
const [local] = await sql`SELECT id FROM contents WHERE slug = ${slug} AND type = 'post' AND status = 'published' LIMIT 1`;
|
||
if (!local) {
|
||
missing++;
|
||
continue;
|
||
}
|
||
const [dup] = await sql`SELECT 1 FROM content_terms WHERE content_id = ${local.id} AND category_id = ${t.catId}`;
|
||
if (dup) continue;
|
||
await sql`INSERT INTO content_terms (content_id, category_id) VALUES (${local.id}, ${t.catId})`;
|
||
inserted++;
|
||
}
|
||
console.log(`[${t.catSlug}] 原站文章 ${slugs.size}（翻页 ${pages.size + 1} 页）；本地命中并补归属 ${inserted}，本地缺失 ${missing}`);
|
||
}
|
||
console.log("完成");
|
||
process.exit(0);
|