Mr.Xia
c2da507f43
- pyproject 注册 nonebot_plugin_wtfllm 插件 - 新增 wtfllm_gate 插件:仅在群 621016172 启用 wtfllm 响应 - 新增 kb_build:抓取 danding.vip 文档并导入 wtfllm 知识库的脚本 - requirements: 升级 nonebot2 2.5.0 / pydantic 2.13.5,补 aiohttp 与 wtfllm 依赖
105 lines
3.2 KiB
Python
105 lines
3.2 KiB
Python
"""抓取 danding.vip 站点页面正文,保存为带标题的文本文件,供知识库导入。"""
|
|
import re
|
|
import time
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
OUT = Path(__file__).parent / "pages"
|
|
OUT.mkdir(exist_ok=True)
|
|
|
|
BASE = "https://www.danding.vip"
|
|
|
|
PATHS = [
|
|
"/",
|
|
"/常见问题FAQ",
|
|
"/支持与反馈",
|
|
"/更新日志",
|
|
"/简单使用/下载与安装",
|
|
"/简单使用/快速入门",
|
|
"/简单使用/快速入门_V2",
|
|
"/简单使用/功能详解",
|
|
"/简单使用/升级说明",
|
|
"/功能帮助/一键配置",
|
|
"/功能帮助/云端配置共享",
|
|
"/功能帮助/任务历史记录",
|
|
"/功能帮助/半自动资源库",
|
|
"/功能帮助/卡屏重启功能",
|
|
"/功能帮助/定时调度",
|
|
"/功能帮助/御魂手动组队步骤",
|
|
"/功能帮助/控制台命令功能",
|
|
"/功能帮助/通知方式",
|
|
"/功能帮助/预设更换功能",
|
|
"/脚本进阶/任务次数限制",
|
|
"/脚本进阶/用户积分详解",
|
|
"/脚本进阶/超级多开功能",
|
|
"/其它/如何禁止代理",
|
|
"/赞助",
|
|
]
|
|
|
|
|
|
def slug(path: str) -> str:
|
|
return re.sub(r"[/ ]", "_", path.strip("/")) or "index"
|
|
|
|
|
|
def extract(url: str) -> tuple[str, str] | None:
|
|
resp = requests.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0"})
|
|
resp.raise_for_status()
|
|
resp.encoding = "utf-8"
|
|
soup = BeautifulSoup(resp.text, "html.parser")
|
|
main = soup.select_one("main") or soup.body
|
|
if not main:
|
|
return None
|
|
for tag in main.select("script, style, nav, footer"):
|
|
tag.decompose()
|
|
title_el = main.select_one("h1")
|
|
title = title_el.get_text(strip=True) if title_el else url
|
|
lines = []
|
|
for el in main.descendants:
|
|
if el.name in ("h1", "h2", "h3", "h4"):
|
|
lines.append("\n" + "#" * int(el.name[1]) + " " + el.get_text(strip=True))
|
|
elif el.name == "p":
|
|
t = el.get_text(" ", strip=True)
|
|
if t:
|
|
lines.append(t)
|
|
elif el.name in ("li",):
|
|
t = el.get_text(" ", strip=True)
|
|
if t:
|
|
lines.append("- " + t)
|
|
elif el.name in ("td",) :
|
|
pass
|
|
text = "\n".join(lines)
|
|
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
if len(text.strip()) < 50:
|
|
return None
|
|
return title, text.strip()
|
|
|
|
|
|
def main() -> None:
|
|
ok, fail = [], []
|
|
for path in PATHS:
|
|
url = BASE + urllib.parse.quote(path, safe="/") + (".html" if path != "/" else "")
|
|
try:
|
|
result = extract(url)
|
|
time.sleep(0.5)
|
|
if not result:
|
|
fail.append((path, "empty"))
|
|
continue
|
|
title, text = result
|
|
header = f"# {title}\n> 来源: {url}\n\n"
|
|
(OUT / (slug(path) + ".txt")).write_text(header + text, encoding="utf-8")
|
|
ok.append((path, len(text)))
|
|
print(f"OK {path} ({len(text)} chars)")
|
|
except Exception as e: # noqa: BLE001
|
|
fail.append((path, str(e)))
|
|
print(f"FAIL {path}: {e}")
|
|
print(f"\n{len(ok)} ok, {len(fail)} failed")
|
|
for p, r in fail:
|
|
print(" FAIL:", p, r)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|