"""抓取 danding.vip 站点页面正文,保存为带标题的文本文件,供知识库导入。"""
import re
import time
import urllib.parse
from pathlib import Path

import requests
from bs4 import BeautifulSoup

OUT = Path(__file__).parent / "pages"
OUT.mkdir(exist_ok=True)

BASE = "https://www.danding.vip"

PATHS = [
    "/",
    "/常见问题FAQ",
    "/支持与反馈",
    "/更新日志",
    "/简单使用/下载与安装",
    "/简单使用/快速入门",
    "/简单使用/快速入门_V2",
    "/简单使用/功能详解",
    "/简单使用/升级说明",
    "/功能帮助/一键配置",
    "/功能帮助/云端配置共享",
    "/功能帮助/任务历史记录",
    "/功能帮助/半自动资源库",
    "/功能帮助/卡屏重启功能",
    "/功能帮助/定时调度",
    "/功能帮助/御魂手动组队步骤",
    "/功能帮助/控制台命令功能",
    "/功能帮助/通知方式",
    "/功能帮助/预设更换功能",
    "/脚本进阶/任务次数限制",
    "/脚本进阶/用户积分详解",
    "/脚本进阶/超级多开功能",
    "/其它/如何禁止代理",
    "/赞助",
]


def slug(path: str) -> str:
    return re.sub(r"[/ ]", "_", path.strip("/")) or "index"


def extract(url: str) -> tuple[str, str] | None:
    resp = requests.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0"})
    resp.raise_for_status()
    resp.encoding = "utf-8"
    soup = BeautifulSoup(resp.text, "html.parser")
    main = soup.select_one("main") or soup.body
    if not main:
        return None
    for tag in main.select("script, style, nav, footer"):
        tag.decompose()
    title_el = main.select_one("h1")
    title = title_el.get_text(strip=True) if title_el else url
    lines = []
    for el in main.descendants:
        if el.name in ("h1", "h2", "h3", "h4"):
            lines.append("\n" + "#" * int(el.name[1]) + " " + el.get_text(strip=True))
        elif el.name == "p":
            t = el.get_text(" ", strip=True)
            if t:
                lines.append(t)
        elif el.name in ("li",):
            t = el.get_text(" ", strip=True)
            if t:
                lines.append("- " + t)
        elif el.name in ("td",) :
            pass
    text = "\n".join(lines)
    text = re.sub(r"\n{3,}", "\n\n", text)
    if len(text.strip()) < 50:
        return None
    return title, text.strip()


def main() -> None:
    ok, fail = [], []
    for path in PATHS:
        url = BASE + urllib.parse.quote(path, safe="/") + (".html" if path != "/" else "")
        try:
            result = extract(url)
            time.sleep(0.5)
            if not result:
                fail.append((path, "empty"))
                continue
            title, text = result
            header = f"# {title}\n> 来源: {url}\n\n"
            (OUT / (slug(path) + ".txt")).write_text(header + text, encoding="utf-8")
            ok.append((path, len(text)))
            print(f"OK  {path} ({len(text)} chars)")
        except Exception as e:  # noqa: BLE001
            fail.append((path, str(e)))
            print(f"FAIL {path}: {e}")
    print(f"\n{len(ok)} ok, {len(fail)} failed")
    for p, r in fail:
        print("  FAIL:", p, r)


if __name__ == "__main__":
    main()