feat(wtfllm): 接入 nonebot-plugin-wtfllm AI 帮助助手
- pyproject 注册 nonebot_plugin_wtfllm 插件 - 新增 wtfllm_gate 插件:仅在群 621016172 启用 wtfllm 响应 - 新增 kb_build:抓取 danding.vip 文档并导入 wtfllm 知识库的脚本 - requirements: 升级 nonebot2 2.5.0 / pydantic 2.13.5,补 aiohttp 与 wtfllm 依赖
This commit is contained in:
104
kb_build/crawl.py
Normal file
104
kb_build/crawl.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""抓取 danding.vip 站点页面正文,保存为带标题的文本文件,供知识库导入。"""
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
OUT = Path(__file__).parent / "pages"
|
||||
OUT.mkdir(exist_ok=True)
|
||||
|
||||
BASE = "https://www.danding.vip"
|
||||
|
||||
PATHS = [
|
||||
"/",
|
||||
"/常见问题FAQ",
|
||||
"/支持与反馈",
|
||||
"/更新日志",
|
||||
"/简单使用/下载与安装",
|
||||
"/简单使用/快速入门",
|
||||
"/简单使用/快速入门_V2",
|
||||
"/简单使用/功能详解",
|
||||
"/简单使用/升级说明",
|
||||
"/功能帮助/一键配置",
|
||||
"/功能帮助/云端配置共享",
|
||||
"/功能帮助/任务历史记录",
|
||||
"/功能帮助/半自动资源库",
|
||||
"/功能帮助/卡屏重启功能",
|
||||
"/功能帮助/定时调度",
|
||||
"/功能帮助/御魂手动组队步骤",
|
||||
"/功能帮助/控制台命令功能",
|
||||
"/功能帮助/通知方式",
|
||||
"/功能帮助/预设更换功能",
|
||||
"/脚本进阶/任务次数限制",
|
||||
"/脚本进阶/用户积分详解",
|
||||
"/脚本进阶/超级多开功能",
|
||||
"/其它/如何禁止代理",
|
||||
"/赞助",
|
||||
]
|
||||
|
||||
|
||||
def slug(path: str) -> str:
|
||||
return re.sub(r"[/ ]", "_", path.strip("/")) or "index"
|
||||
|
||||
|
||||
def extract(url: str) -> tuple[str, str] | None:
|
||||
resp = requests.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0"})
|
||||
resp.raise_for_status()
|
||||
resp.encoding = "utf-8"
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
main = soup.select_one("main") or soup.body
|
||||
if not main:
|
||||
return None
|
||||
for tag in main.select("script, style, nav, footer"):
|
||||
tag.decompose()
|
||||
title_el = main.select_one("h1")
|
||||
title = title_el.get_text(strip=True) if title_el else url
|
||||
lines = []
|
||||
for el in main.descendants:
|
||||
if el.name in ("h1", "h2", "h3", "h4"):
|
||||
lines.append("\n" + "#" * int(el.name[1]) + " " + el.get_text(strip=True))
|
||||
elif el.name == "p":
|
||||
t = el.get_text(" ", strip=True)
|
||||
if t:
|
||||
lines.append(t)
|
||||
elif el.name in ("li",):
|
||||
t = el.get_text(" ", strip=True)
|
||||
if t:
|
||||
lines.append("- " + t)
|
||||
elif el.name in ("td",) :
|
||||
pass
|
||||
text = "\n".join(lines)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
if len(text.strip()) < 50:
|
||||
return None
|
||||
return title, text.strip()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ok, fail = [], []
|
||||
for path in PATHS:
|
||||
url = BASE + urllib.parse.quote(path, safe="/") + (".html" if path != "/" else "")
|
||||
try:
|
||||
result = extract(url)
|
||||
time.sleep(0.5)
|
||||
if not result:
|
||||
fail.append((path, "empty"))
|
||||
continue
|
||||
title, text = result
|
||||
header = f"# {title}\n> 来源: {url}\n\n"
|
||||
(OUT / (slug(path) + ".txt")).write_text(header + text, encoding="utf-8")
|
||||
ok.append((path, len(text)))
|
||||
print(f"OK {path} ({len(text)} chars)")
|
||||
except Exception as e: # noqa: BLE001
|
||||
fail.append((path, str(e)))
|
||||
print(f"FAIL {path}: {e}")
|
||||
print(f"\n{len(ok)} ok, {len(fail)} failed")
|
||||
for p, r in fail:
|
||||
print(" FAIL:", p, r)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user