From ff24840594402229d03eb490a44468b77e0cf8c1 Mon Sep 17 00:00:00 2001 From: Elaine <348749595@qq.com> Date: Fri, 21 Aug 2026 08:16:17 +0800 Subject: [PATCH] feat(deepwiki): automate Gita wiki generation Assisted-by: Codex:GPT-5 --- .gitignore | 3 + .../roncarve/DEEPWIKI-GITEA-INTEGRATION.md | 233 ++++++++++++++++++ contrib/roncarve/README.md | 2 + contrib/roncarve/compose.production.yaml | 83 +++++++ .../deepwiki-integration/.dockerignore | 3 + .../roncarve/deepwiki-integration/Dockerfile | 16 ++ .../deepwiki-config/embedder.json | 22 ++ .../deepwiki-config/generator.json | 15 ++ .../deepwiki-integration/deepwiki.env.example | 11 + .../gita_deepwiki/__init__.py | 3 + .../gita_deepwiki/artifacts.py | 20 ++ .../deepwiki-integration/gita_deepwiki/cli.py | 109 ++++++++ .../gita_deepwiki/config.py | 115 +++++++++ .../gita_deepwiki/database.py | 227 +++++++++++++++++ .../gita_deepwiki/deepwiki.py | 118 +++++++++ .../gita_deepwiki/gitea.py | 121 +++++++++ .../gita_deepwiki/http_client.py | 61 +++++ .../gita_deepwiki/rendering.py | 144 +++++++++++ .../gita_deepwiki/server.py | 113 +++++++++ .../gita_deepwiki/service.py | 150 +++++++++++ .../gita_deepwiki/snapshot.py | 119 +++++++++ .../gita_deepwiki/webhook.py | 87 +++++++ .../integration.env.example | 6 + .../deepwiki-integration/tests/__init__.py | 0 .../tests/test_artifacts.py | 27 ++ .../tests/test_database.py | 53 ++++ .../tests/test_deepwiki.py | 55 +++++ .../deepwiki-integration/tests/test_gitea.py | 47 ++++ .../tests/test_rendering.py | 67 +++++ .../tests/test_service.py | 102 ++++++++ .../tests/test_snapshot.py | 72 ++++++ .../tests/test_webhook.py | 71 ++++++ 32 files changed, 2275 insertions(+) create mode 100644 contrib/roncarve/DEEPWIKI-GITEA-INTEGRATION.md create mode 100644 contrib/roncarve/deepwiki-integration/.dockerignore create mode 100644 contrib/roncarve/deepwiki-integration/Dockerfile create mode 100644 contrib/roncarve/deepwiki-integration/deepwiki-config/embedder.json create mode 100644 contrib/roncarve/deepwiki-integration/deepwiki-config/generator.json create mode 100644 contrib/roncarve/deepwiki-integration/deepwiki.env.example create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/__init__.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/artifacts.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/cli.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/config.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/database.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/deepwiki.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/gitea.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/http_client.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/rendering.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/server.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/service.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/snapshot.py create mode 100644 contrib/roncarve/deepwiki-integration/gita_deepwiki/webhook.py create mode 100644 contrib/roncarve/deepwiki-integration/integration.env.example create mode 100644 contrib/roncarve/deepwiki-integration/tests/__init__.py create mode 100644 contrib/roncarve/deepwiki-integration/tests/test_artifacts.py create mode 100644 contrib/roncarve/deepwiki-integration/tests/test_database.py create mode 100644 contrib/roncarve/deepwiki-integration/tests/test_deepwiki.py create mode 100644 contrib/roncarve/deepwiki-integration/tests/test_gitea.py create mode 100644 contrib/roncarve/deepwiki-integration/tests/test_rendering.py create mode 100644 contrib/roncarve/deepwiki-integration/tests/test_service.py create mode 100644 contrib/roncarve/deepwiki-integration/tests/test_snapshot.py create mode 100644 contrib/roncarve/deepwiki-integration/tests/test_webhook.py diff --git a/.gitignore b/.gitignore index 2d2036f..d9c5837 100644 --- a/.gitignore +++ b/.gitignore @@ -123,3 +123,6 @@ Makefile.local # Local deployment secrets /contrib/roncarve/.env +/contrib/roncarve/deepwiki-integration/deepwiki.env +/contrib/roncarve/deepwiki-integration/integration.env +/contrib/roncarve/deepwiki-integration/**/__pycache__/ diff --git a/contrib/roncarve/DEEPWIKI-GITEA-INTEGRATION.md b/contrib/roncarve/DEEPWIKI-GITEA-INTEGRATION.md new file mode 100644 index 0000000..a12c01a --- /dev/null +++ b/contrib/roncarve/DEEPWIKI-GITEA-INTEGRATION.md @@ -0,0 +1,233 @@ +# Gita 自动文档生成方案 + +## 1. 目标与边界 + +本方案在不修改 Gita 核心仓库模型、不覆盖人工 Wiki 页面、不公开 AI 服务端口的前提下,为 Gita 仓库提供按默认分支提交自动生成文档的能力。 + +目标: + +- 默认分支发生 `push` 后自动排队生成文档。 +- 每次生成严格绑定 Webhook 中的 commit SHA。 +- 生成结果写入 Gita 内置 Wiki,并保留源码版本信息。 +- Gita 访问令牌不离开服务器;代码片段仅通过 TLS 发送到明确配置的 + DeepSeek 与 Embedding API。 +- 服务重启后未完成任务可以恢复,连续推送会合并为最新任务。 +- 人工 Wiki 页面与外部 Wiki 配置不会被静默覆盖。 + +非目标: + +- 第一阶段不把 DeepWiki UI 嵌入 Gita 页面。 +- 第一阶段不开放仓库问答和 Codemap 公网入口。 +- 不将 DeepWiki 源码编译进 Gita 的 Go 二进制。 + +## 2. 总体架构 + +```text +Gita push + | + | system webhook, HMAC-SHA256 + v +gita-deepwiki-adapter + |-- SQLite durable queue + |-- read-only /gitea-data/git/repositories + |-- export exact commit to /work/repos// + | + +--> DeepWiki API (internal only) + | |-- DeepSeek API: deepseek-v4-flash + | +-- OpenAI-compatible Embedding API + | + +--> Gita API: create/update managed Wiki pages +``` + +生产容器均加入既有 `gateway_gateway` Docker 网络,但 DeepWiki 和适配器不映射宿主机端口。Gita 通过容器 DNS 访问 `http://gita-deepwiki-adapter:8080/hooks/gitea`。DeepWiki 仅通过 HTTPS 调用外部模型 API。 + +## 3. 组件职责 + +### 3.1 Gita + +- 发送实例级 `push` Webhook。 +- 提供只读裸仓库存储。 +- 通过 API 提供 Wiki 创建、更新和删除能力。 +- 继续负责用户认证与 Wiki 访问权限。 + +### 3.2 gita-deepwiki-adapter + +- 在解析 JSON 前校验 `X-Gitea-Signature`。 +- 仅接受默认分支非删除 push。 +- 对仓库名、commit SHA 和仓库存储路径做白名单与边界校验。 +- 使用 SQLite 持久化任务;启动时恢复中断任务。 +- 同一仓库连续推送时废弃尚未执行的旧任务。 +- 从只读裸仓库执行 `git archive `,不修改 Gita Git 数据。 +- 删除仓库 Wiki 缓存后提交 DeepWiki 生成任务。 +- 轮询任务状态,读取生成结果并改写为不可变 commit 源码链接。 +- 只维护带内部标记且记录在本地清单中的 `AI-Generated-*` 页面。 + +### 3.3 DeepWiki + +- 仅分析适配器生成的本地代码快照。 +- 使用 commit SHA 作为本地路径末级目录,使 RAG 索引天然按提交隔离。 +- Wiki 缓存由适配器在每次任务前显式失效。 +- 最大仓库任务并发数和页面生成并发数均设为 1。 + +### 3.4 外部模型服务 + +- DeepSeek 官方 OpenAI 兼容接口的 `deepseek-v4-flash` 用于中文文档生成。 +- 独立的 OpenAI 兼容 Embedding API 用于代码向量化。 +- 生成密钥与 Embedding 密钥分离,任一服务都不会收到 Gita API 令牌。 +- DeepSeek 官方 API 不提供 Embeddings,因此不能使用同一个 DeepSeek 接口完成代码索引。 + +## 4. Webhook 接口 + +### `POST /hooks/gitea` + +必需请求头: + +- `X-Gitea-Event: push` +- `X-Gitea-Signature: ` +- `X-Gitea-Delivery: ` + +处理规则: + +1. 请求体超过配置上限时返回 `413`。 +2. 签名缺失或不匹配时返回 `401`。 +3. 非 `push` 事件返回 `202` 并忽略。 +4. 非默认分支、删除分支或无效 SHA 返回 `202` 并忽略。 +5. 有效事件持久化后返回 `202`,文档生成异步执行。 + +### `GET /health` + +返回进程、工作线程和 SQLite 状态。该接口仅用于容器健康检查。 + +### `GET /status` + +返回各任务状态数量和最近任务,不返回令牌、Webhook 请求体或代码内容。 + +## 5. 任务与缓存模型 + +任务唯一版本由以下字段确定: + +```text +repository-id + commit-sha + language + generator-model +``` + +SQLite 状态机: + +```text +queued -> running -> completed + -> retry_wait -> queued + -> failed +queued -> superseded +``` + +策略: + +- 每个仓库只保留最新的未执行 commit。 +- 正在执行的 commit 不被强行终止;新 commit 在其后执行。 +- 失败任务按指数退避重试,超过上限后保留错误摘要。 +- 服务启动时将遗留的 `running` 任务恢复为 `queued`。 +- 代码快照在任务结束后删除;成功同步 Wiki 后删除该 commit 的向量索引,当前 Wiki 缓存保留供故障排查。 + +## 6. Wiki 页面所有权 + +适配器创建: + +- `AI-Generated-Documentation`:生成信息、源码 commit 和目录。 +- `AI-Generated-`:DeepWiki 页面正文。 + +每页包含不可见所有权标记: + +```html + +``` + +覆盖规则: + +- 只有 SQLite 清单记录的页面,或正文中存在同仓库所有权标记的页面,才允许更新。 +- 同名但没有所有权标记的人工页面会导致任务失败,不会覆盖。 +- 新版本不再包含的受管页面会被删除;人工页面不会参与清理。 + +## 7. 权限与密钥 + +- Webhook Secret 为独立 256-bit 随机值,只保存在 `/opt/gita/deepwiki/integration.env`。 +- Gita API 使用独立 `deepwiki-bot` 服务账号;长期运行令牌仅授予 `write:repository`。 +- 安装实例级 Webhook 时临时生成 `write:admin` 令牌,通过 + `GITA_ADMIN_API_TOKEN` 只注入一次性命令,安装后立即吊销,不写入环境文件。 +- Gita 令牌仅注入适配器,不注入 DeepWiki。 +- DeepSeek 和 Embedding API 密钥仅注入 DeepWiki,不注入 Gita 或适配器。 +- Gita 仓库目录以只读方式挂载到适配器。 +- DeepWiki 工作目录以只读方式挂载到 DeepWiki。 +- DeepWiki 不开放宿主机端口,外部模型请求强制使用 HTTPS。 +- 日志不得打印令牌、完整 Webhook 请求体或带凭据 URL。 +- `integration.env` 权限为 `0600`。 + +实例级自动处理全部仓库需要服务账号具有相应仓库 Wiki 写权限。当前部署使用专用管理员服务账号实现实例级覆盖;如果以后需要更严格的租户边界,应改为组织级 Webhook和组织团队账号。 + +## 8. 资源限制 + +生产服务器为 8 vCPU、15 GiB RAM、无 Swap。配置限制: + +- Adapter:1 CPU、384 MiB。 +- DeepWiki:3 CPU、3 GiB,任务并发 1,页面并发 1。 +- 生成与向量计算由外部 API 承担,不部署本地模型容器。 + +`deepseek-v4-flash` 使用官方 `https://api.deepseek.com` 接口。后续可在不改变适配器的情况下替换生成模型或 Embedding API。 + +## 9. 部署步骤 + +1. 创建 `/opt/gita/deepwiki/{state,work,data}`。 +2. 创建权限为 `0600` 的 `integration.env` 和 `deepwiki.env`;后者保存 + DeepSeek 与 Embedding API 密钥。 +3. 创建 `deepwiki-bot`,生成长期 `write:repository` 运行令牌。 +4. 验证 DeepSeek Chat Completions 与 Embedding API 可用。 +5. 启动 DeepWiki 和 Adapter,检查两个容器健康状态。 +6. 临时生成 `write:admin` 令牌,使用管理脚本幂等创建实例级 push + Webhook,随后立即吊销临时令牌。 +7. 向测试仓库默认分支推送提交。 +8. 检查任务完成、Wiki 页面、commit 链接和页面访问权限。 + +生产命令: + +```bash +cd /opt/gita/app +docker compose -f contrib/roncarve/compose.production.yaml up -d deepwiki gita-deepwiki-adapter +printf '%s\n' "$GITA_ADMIN_API_TOKEN" | \ + docker compose -f contrib/roncarve/compose.production.yaml exec -T \ + gita-deepwiki-adapter sh -c \ + 'IFS= read -r GITA_ADMIN_API_TOKEN; export GITA_ADMIN_API_TOKEN; exec python -m gita_deepwiki.cli install-webhook' +``` + +`GITA_ADMIN_API_TOKEN` 仅存在于当前管理会话;安装成功后必须从 Gita +账号令牌列表删除。适配器常驻容器只读取 `integration.env` 中的运行令牌。 + +## 10. 验证 + +本地自动验证: + +- Webhook HMAC 正确、错误和缺失场景。 +- 默认分支过滤、删除 push 过滤和 SHA 校验。 +- 路径穿越防护与只读裸仓库导出。 +- 队列去重、恢复和重试。 +- DeepWiki API 提交、轮询与缓存读取。 +- Wiki 新建、更新、冲突保护和过期受管页面清理。 +- Markdown 源码链接改写。 + +生产冒烟验证: + +- `/health` 在容器内返回 `200`。 +- DeepWiki `/health` 返回 `200`。 +- DeepSeek `deepseek-v4-flash` 能完成最小 Chat Completions 请求。 +- Embedding API 能为最小文本返回非空且维度一致的向量。 +- 系统 Webhook 存在且仅订阅 `push`。 +- 测试推送后出现绑定正确 commit SHA 的 `AI-Generated-Documentation` 页面。 + +## 11. 回滚 + +停止 AI 服务不会影响 Gita: + +```bash +docker compose -f contrib/roncarve/compose.production.yaml stop gita-deepwiki-adapter deepwiki +``` + +随后删除名为 `Gita DeepWiki automatic documentation` 的系统 Webhook。保留 `/opt/gita/deepwiki` 可继续恢复任务;删除受管 Wiki 页面属于数据操作,不纳入自动回滚。 + +Gita、MariaDB、Runner 和既有网关配置不依赖 AI 服务,回滚不需要重建 Gita 容器。 diff --git a/contrib/roncarve/README.md b/contrib/roncarve/README.md index 2e81524..76a181e 100644 --- a/contrib/roncarve/README.md +++ b/contrib/roncarve/README.md @@ -30,6 +30,8 @@ docker compose -f contrib/roncarve/compose.local.yaml down Gita Actions、镜像保留策略和 RKE2 测试环境发布配置见 [DEVOPS-RKE2-TEST.md](DEVOPS-RKE2-TEST.md)。 +DeepWiki 自动文档生成的架构、安全边界、部署和回滚流程见 [DEEPWIKI-GITEA-INTEGRATION.md](DEEPWIKI-GITEA-INTEGRATION.md)。生产环境的 AI 服务密钥分别保存在 `/opt/gita/deepwiki/integration.env` 与 `/opt/gita/deepwiki/deepwiki.env`,不写入主 `.env`。 + Runner 任务镜像由 `runner/job-image.Dockerfile` 构建,基础镜像固定 digest,只增加 Helm 和 kubectl 两个部署工具。构建上下文中的二进制文件应分别来自固定版本的 Helm 镜像和当前 RKE2 安装目录。 2026-08-17 已在测试机完成 7 镜像构建和 Helm revision 8 发布,全部工作负载 Ready,HTTPS 冒烟测试、认证配置接口和动态模块加载均验证通过。旧动态模块现在返回 404 而不是首页 HTML,入口页禁止缓存,当前前端可在后续发布出现过期模块时单次自动刷新。镜像仓库按服务仅保留当前与上一构建两个标签;Runner 在每个镜像推送后释放本地标签,并在部署前清空 BuildKit 缓存。 diff --git a/contrib/roncarve/compose.production.yaml b/contrib/roncarve/compose.production.yaml index 56899b5..dab460e 100644 --- a/contrib/roncarve/compose.production.yaml +++ b/contrib/roncarve/compose.production.yaml @@ -64,6 +64,7 @@ services: GITEA__server__SSH_PORT: "2222" GITEA__server__ROOT_URL: https://git.roncarve.com/ GITEA__security__INSTALL_LOCK: "true" + GITEA__security__ALLOWED_HOST_LIST: external,gita-deepwiki-adapter GITEA__service__DISABLE_REGISTRATION: "true" GITEA__service__ENABLE_NOTIFY_MAIL: "true" GITEA__service__ENABLE_PASSKEY_AUTHENTICATION: "false" @@ -113,6 +114,88 @@ services: networks: - gateway + deepwiki: + image: ghcr.io/asyncfuncai/deepwiki-open@sha256:f49829fd92d0614b7c5d9607c40abda5c8175defc51031456a432de1a82b10d7 + container_name: gita-deepwiki + hostname: deepwiki + restart: unless-stopped + cpus: 3 + mem_limit: 3g + mem_reservation: 512m + env_file: + - /opt/gita/deepwiki/deepwiki.env + environment: + PORT: "8001" + NODE_ENV: production + SERVER_BASE_URL: http://localhost:8001 + OPENAI_BASE_URL: https://api.deepseek.com + DEEPWIKI_EMBEDDER_TYPE: openai + DEEPWIKI_MAX_CONCURRENT_WIKI_TASKS: "1" + DEEPWIKI_WIKI_PAGE_CONCURRENCY: "1" + DEEPWIKI_WIKI_PAGE_RETRIES: "2" + volumes: + - /opt/gita/deepwiki/data:/root/.adalflow + - /opt/gita/deepwiki/work:/work:ro + - ./deepwiki-integration/deepwiki-config/generator.json:/app/api/config/generator.json:ro + - ./deepwiki-integration/deepwiki-config/embedder.json:/app/api/config/embedder.json:ro + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8001/health"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 60s + networks: + - gateway + + gita-deepwiki-adapter: + build: + context: deepwiki-integration + dockerfile: Dockerfile + image: roncarve/gita-deepwiki-adapter:1.0.0 + container_name: gita-deepwiki-adapter + hostname: gita-deepwiki-adapter + restart: unless-stopped + user: "1000:1000" + cpus: 1 + mem_limit: 384m + env_file: + - /opt/gita/deepwiki/integration.env + environment: + GITA_URL: http://gitea:3000 + DEEPWIKI_URL: http://deepwiki:8001 + GITA_REPOSITORY_ROOT: /gitea-data/git/repositories + GITA_DEEPWIKI_WORK_ROOT: /work + GITA_DEEPWIKI_DATABASE: /state/jobs.sqlite3 + DEEPWIKI_DATA_ROOT: /deepwiki-data + GITA_DEEPWIKI_WEBHOOK_URL: http://gita-deepwiki-adapter:8080/hooks/gitea + DEEPWIKI_PROVIDER: openai + DEEPWIKI_MODEL: deepseek-v4-flash + DEEPWIKI_LANGUAGE: zh + GITA_DEEPWIKI_DEBOUNCE_SECONDS: "30" + GITA_DEEPWIKI_RETRY_LIMIT: "3" + volumes: + - /opt/gita/data:/gitea-data:ro + - /opt/gita/deepwiki/work:/work + - /opt/gita/deepwiki/state:/state + - /opt/gita/deepwiki/data:/deepwiki-data + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request; urllib.request.urlopen('http://localhost:8080/health', timeout=5).read() + interval: 20s + timeout: 10s + retries: 5 + start_period: 10s + depends_on: + gita: + condition: service_started + deepwiki: + condition: service_healthy + networks: + - gateway + networks: gateway: external: true diff --git a/contrib/roncarve/deepwiki-integration/.dockerignore b/contrib/roncarve/deepwiki-integration/.dockerignore new file mode 100644 index 0000000..f84a9af --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/.dockerignore @@ -0,0 +1,3 @@ +__pycache__ +*.pyc +tests diff --git a/contrib/roncarve/deepwiki-integration/Dockerfile b/contrib/roncarve/deepwiki-integration/Dockerfile new file mode 100644 index 0000000..eeb2d15 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-slim-bookworm@sha256:a116514e19457bcb7af7efe9c3dd0b9b71e85b317694e7882a1c52aa15a78134 + +ARG DEBIAN_MIRROR=https://mirrors.cloud.tencent.com +RUN sed -i "s#http://deb.debian.org#${DEBIAN_MIRROR}#g" /etc/apt/sources.list.d/debian.sources \ + && apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY gita_deepwiki ./gita_deepwiki + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +EXPOSE 8080 +ENTRYPOINT ["python", "-m", "gita_deepwiki.cli"] diff --git a/contrib/roncarve/deepwiki-integration/deepwiki-config/embedder.json b/contrib/roncarve/deepwiki-integration/deepwiki-config/embedder.json new file mode 100644 index 0000000..eb48c51 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/deepwiki-config/embedder.json @@ -0,0 +1,22 @@ +{ + "embedder": { + "client_class": "OpenAIClient", + "initialize_kwargs": { + "api_key": "${EMBEDDING_API_KEY}", + "base_url": "${EMBEDDING_BASE_URL}" + }, + "batch_size": 25, + "model_kwargs": { + "model": "${EMBEDDING_MODEL}", + "encoding_format": "float" + } + }, + "retriever": { + "top_k": 20 + }, + "text_splitter": { + "split_by": "word", + "chunk_size": 350, + "chunk_overlap": 100 + } +} diff --git a/contrib/roncarve/deepwiki-integration/deepwiki-config/generator.json b/contrib/roncarve/deepwiki-integration/deepwiki-config/generator.json new file mode 100644 index 0000000..fe5d3a6 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/deepwiki-config/generator.json @@ -0,0 +1,15 @@ +{ + "default_provider": "openai", + "providers": { + "openai": { + "client_class": "OpenAIClient", + "default_model": "deepseek-v4-flash", + "supportsCustomModel": false, + "models": { + "deepseek-v4-flash": { + "temperature": 0.2 + } + } + } + } +} diff --git a/contrib/roncarve/deepwiki-integration/deepwiki.env.example b/contrib/roncarve/deepwiki-integration/deepwiki.env.example new file mode 100644 index 0000000..2d8dfe6 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/deepwiki.env.example @@ -0,0 +1,11 @@ +DEEPWIKI_AUTH_MODE=true +DEEPWIKI_AUTH_CODE=replace-with-shared-deepwiki-auth-code + +# DeepWiki uses its OpenAI-compatible client for DeepSeek generation. +OPENAI_API_KEY=replace-with-deepseek-api-key + +# DeepSeek does not expose an embeddings API. Configure a separate +# OpenAI-compatible embeddings endpoint and model. +EMBEDDING_API_KEY=replace-with-embedding-api-key +EMBEDDING_BASE_URL=https://replace-with-embedding-api-base-url/v1 +EMBEDDING_MODEL=replace-with-embedding-model-id diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/__init__.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/__init__.py new file mode 100644 index 0000000..11b2c3d --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/__init__.py @@ -0,0 +1,3 @@ +"""Gita to DeepWiki automatic documentation integration.""" + +__version__ = "1.0.0" diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/artifacts.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/artifacts.py new file mode 100644 index 0000000..0819f5d --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/artifacts.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import re +from pathlib import Path + + +_SHA = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") + + +class ArtifactCleaner: + def __init__(self, deepwiki_data_root: Path): + self.database_root = (deepwiki_data_root / "databases").resolve() + + def remove_index(self, commit_sha: str) -> None: + if not _SHA.fullmatch(commit_sha): + raise ValueError("refusing to clean an invalid commit SHA") + path = (self.database_root / f"{commit_sha}.pkl").resolve() + if self.database_root not in path.parents: + raise ValueError("DeepWiki index path escapes database root") + path.unlink(missing_ok=True) diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/cli.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/cli.py new file mode 100644 index 0000000..0459836 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/cli.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import argparse +import logging +import signal +import threading + +from .artifacts import ArtifactCleaner +from .config import Config +from .database import Database +from .deepwiki import DeepWikiClient +from .gitea import GiteaClient +from .server import AdapterServer +from .service import DocumentationProcessor, Worker +from .snapshot import SnapshotManager + + +HOOK_NAME = "Gita DeepWiki automatic documentation" + + +def serve(config: Config) -> None: + database = Database(config.database_path) + snapshots = SnapshotManager( + config.repository_root, config.work_root, config.max_snapshot_bytes + ) + deepwiki = DeepWikiClient(config) + gitea = GiteaClient(config) + artifacts = ArtifactCleaner(config.deepwiki_data_root) + processor = DocumentationProcessor( + config, database, snapshots, deepwiki, gitea, artifacts + ) + worker = Worker(config, database, processor) + worker.start() + server = AdapterServer(config, database, worker) + + def stop(_signum, _frame) -> None: + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + logging.info("adapter listening on %s:%s", config.listen_host, config.listen_port) + try: + server.serve_forever(poll_interval=0.5) + finally: + worker.stop() + worker.join(timeout=10) + server.server_close() + + +def install_webhook(config: Config) -> None: + if not config.gitea_admin_token: + raise ValueError( + "GITA_ADMIN_API_TOKEN is required only while installing the system webhook" + ) + gitea = GiteaClient(config, token=config.gitea_admin_token) + create_payload = { + "type": "gitea", + "name": HOOK_NAME, + "config": { + "url": config.webhook_target_url, + "content_type": "json", + "secret": config.webhook_secret, + "is_system_webhook": "true", + }, + "events": ["push"], + "branch_filter": "*", + "active": True, + } + existing = next( + (hook for hook in gitea.list_system_hooks() if hook.get("name") == HOOK_NAME), + None, + ) + if existing is None: + hook = gitea.create_system_hook(create_payload) + print(f"created system webhook id={hook.get('id')}") + return + hook_id = int(existing["id"]) + gitea.update_system_hook( + hook_id, + { + "config": create_payload["config"], + "events": create_payload["events"], + "branch_filter": create_payload["branch_filter"], + "active": True, + "name": HOOK_NAME, + }, + ) + print(f"updated system webhook id={hook_id}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Gita DeepWiki integration") + parser.add_argument( + "command", choices=("serve", "install-webhook"), nargs="?", default="serve" + ) + args = parser.parse_args() + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + config = Config.from_env() + if args.command == "install-webhook": + install_webhook(config) + else: + serve(config) + + +if __name__ == "__main__": + main() diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/config.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/config.py new file mode 100644 index 0000000..b1ff7bd --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/config.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _required(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise ValueError(f"missing required environment variable: {name}") + return value + + +def _boolean(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def _integer(name: str, default: int, minimum: int = 0) -> int: + value = int(os.environ.get(name, str(default))) + if value < minimum: + raise ValueError(f"{name} must be at least {minimum}") + return value + + +def _number(name: str, default: float, minimum: float = 0) -> float: + value = float(os.environ.get(name, str(default))) + if value < minimum: + raise ValueError(f"{name} must be at least {minimum}") + return value + + +@dataclass(frozen=True) +class Config: + webhook_secret: str + gitea_url: str + gitea_token: str + deepwiki_url: str + repository_root: Path + work_root: Path + database_path: Path + deepwiki_data_root: Path = Path("/deepwiki-data") + gitea_admin_token: str = "" + listen_host: str = "0.0.0.0" + listen_port: int = 8080 + webhook_target_url: str = "http://gita-deepwiki-adapter:8080/hooks/gitea" + deepwiki_auth_code: str = "" + provider: str = "openai" + model: str = "deepseek-v4-flash" + language: str = "zh" + comprehensive: bool = True + auto_enable_wiki: bool = True + debounce_seconds: int = 30 + poll_interval_seconds: float = 2 + generation_timeout_seconds: int = 7200 + retry_limit: int = 3 + retry_base_seconds: int = 60 + max_request_bytes: int = 2 * 1024 * 1024 + max_snapshot_bytes: int = 2 * 1024 * 1024 * 1024 + + @classmethod + def from_env(cls) -> "Config": + return cls( + webhook_secret=_required("GITA_DEEPWIKI_WEBHOOK_SECRET"), + gitea_url=_required("GITA_URL").rstrip("/"), + gitea_token=_required("GITA_API_TOKEN"), + deepwiki_url=os.environ.get( + "DEEPWIKI_URL", "http://deepwiki:8001" + ).rstrip("/"), + repository_root=Path( + os.environ.get( + "GITA_REPOSITORY_ROOT", "/gitea-data/git/repositories" + ) + ), + work_root=Path(os.environ.get("GITA_DEEPWIKI_WORK_ROOT", "/work")), + database_path=Path( + os.environ.get("GITA_DEEPWIKI_DATABASE", "/state/jobs.sqlite3") + ), + deepwiki_data_root=Path( + os.environ.get("DEEPWIKI_DATA_ROOT", "/deepwiki-data") + ), + gitea_admin_token=os.environ.get("GITA_ADMIN_API_TOKEN", "").strip(), + listen_host=os.environ.get("GITA_DEEPWIKI_LISTEN_HOST", "0.0.0.0"), + listen_port=_integer("GITA_DEEPWIKI_LISTEN_PORT", 8080, 1), + webhook_target_url=os.environ.get( + "GITA_DEEPWIKI_WEBHOOK_URL", + "http://gita-deepwiki-adapter:8080/hooks/gitea", + ), + deepwiki_auth_code=os.environ.get("DEEPWIKI_AUTH_CODE", ""), + provider=os.environ.get("DEEPWIKI_PROVIDER", "openai"), + model=os.environ.get("DEEPWIKI_MODEL", "deepseek-v4-flash"), + language=os.environ.get("DEEPWIKI_LANGUAGE", "zh"), + comprehensive=_boolean("DEEPWIKI_COMPREHENSIVE", True), + auto_enable_wiki=_boolean("GITA_DEEPWIKI_AUTO_ENABLE_WIKI", True), + debounce_seconds=_integer("GITA_DEEPWIKI_DEBOUNCE_SECONDS", 30), + poll_interval_seconds=_number("DEEPWIKI_POLL_INTERVAL_SECONDS", 2, 0.1), + generation_timeout_seconds=_integer( + "DEEPWIKI_GENERATION_TIMEOUT_SECONDS", 7200, 30 + ), + retry_limit=_integer("GITA_DEEPWIKI_RETRY_LIMIT", 3, 1), + retry_base_seconds=_integer( + "GITA_DEEPWIKI_RETRY_BASE_SECONDS", 60, 1 + ), + max_request_bytes=_integer( + "GITA_DEEPWIKI_MAX_REQUEST_BYTES", 2 * 1024 * 1024, 1024 + ), + max_snapshot_bytes=_integer( + "GITA_DEEPWIKI_MAX_SNAPSHOT_BYTES", + 2 * 1024 * 1024 * 1024, + 1024, + ), + ) diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/database.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/database.py new file mode 100644 index 0000000..26d7df1 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/database.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import sqlite3 +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from collections.abc import Iterator + + +@dataclass(frozen=True) +class Job: + id: int + delivery_id: str + repo_id: int + owner: str + repo: str + html_url: str + default_branch: str + commit_sha: str + status: str + attempts: int + available_at: float + error: str | None + + +class Database: + def __init__(self, path: Path): + self.path = path + self.path.parent.mkdir(parents=True, exist_ok=True) + self._initialize() + + @contextmanager + def _connect(self) -> Iterator[sqlite3.Connection]: + connection = sqlite3.connect(self.path, timeout=30) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA journal_mode = WAL") + try: + with connection: + yield connection + finally: + connection.close() + + def _initialize(self) -> None: + with self._connect() as connection: + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + delivery_id TEXT NOT NULL, + repo_id INTEGER NOT NULL, + owner TEXT NOT NULL, + repo TEXT NOT NULL, + html_url TEXT NOT NULL, + default_branch TEXT NOT NULL, + commit_sha TEXT NOT NULL, + status TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + available_at REAL NOT NULL, + error TEXT, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + UNIQUE(repo_id, commit_sha) + ); + CREATE INDEX IF NOT EXISTS jobs_ready_idx + ON jobs(status, available_at, id); + CREATE TABLE IF NOT EXISTS managed_pages ( + repo_id INTEGER NOT NULL, + title TEXT NOT NULL, + PRIMARY KEY(repo_id, title) + ); + """ + ) + connection.execute( + "UPDATE jobs SET status = 'queued', available_at = ?, " + "error = 'recovered after adapter restart', updated_at = ? " + "WHERE status = 'running'", + (time.time(), time.time()), + ) + + def enqueue( + self, + *, + delivery_id: str, + repo_id: int, + owner: str, + repo: str, + html_url: str, + default_branch: str, + commit_sha: str, + debounce_seconds: int, + ) -> tuple[int, bool]: + now = time.time() + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + existing = connection.execute( + "SELECT id FROM jobs WHERE repo_id = ? AND commit_sha = ?", + (repo_id, commit_sha), + ).fetchone() + if existing: + return int(existing["id"]), False + connection.execute( + "UPDATE jobs SET status = 'superseded', updated_at = ? " + "WHERE repo_id = ? AND status IN ('queued', 'retry_wait')", + (now, repo_id), + ) + cursor = connection.execute( + """ + INSERT INTO jobs ( + delivery_id, repo_id, owner, repo, html_url, + default_branch, commit_sha, status, available_at, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', ?, ?, ?) + """, + ( + delivery_id, + repo_id, + owner, + repo, + html_url, + default_branch, + commit_sha, + now + debounce_seconds, + now, + now, + ), + ) + return int(cursor.lastrowid), True + + def claim(self) -> Job | None: + now = time.time() + with self._connect() as connection: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT * FROM jobs WHERE status IN ('queued', 'retry_wait') " + "AND available_at <= ? ORDER BY available_at, id LIMIT 1", + (now,), + ).fetchone() + if row is None: + return None + connection.execute( + "UPDATE jobs SET status = 'running', attempts = attempts + 1, " + "error = NULL, updated_at = ? WHERE id = ?", + (now, row["id"]), + ) + claimed = connection.execute( + "SELECT * FROM jobs WHERE id = ?", (row["id"],) + ).fetchone() + return self._job(claimed) + + def complete(self, job_id: int) -> None: + self._set_terminal(job_id, "completed", None) + + def fail(self, job_id: int, error: str) -> None: + self._set_terminal(job_id, "failed", error) + + def retry(self, job_id: int, error: str, delay_seconds: int) -> None: + now = time.time() + with self._connect() as connection: + connection.execute( + "UPDATE jobs SET status = 'retry_wait', available_at = ?, " + "error = ?, updated_at = ? WHERE id = ?", + (now + delay_seconds, error[:2000], now, job_id), + ) + + def _set_terminal(self, job_id: int, status: str, error: str | None) -> None: + now = time.time() + with self._connect() as connection: + connection.execute( + "UPDATE jobs SET status = ?, error = ?, updated_at = ? WHERE id = ?", + (status, error[:2000] if error else None, now, job_id), + ) + + def counts(self) -> dict[str, int]: + with self._connect() as connection: + rows = connection.execute( + "SELECT status, COUNT(*) AS count FROM jobs GROUP BY status" + ).fetchall() + return {str(row["status"]): int(row["count"]) for row in rows} + + def recent(self, limit: int = 20) -> list[dict[str, object]]: + with self._connect() as connection: + rows = connection.execute( + "SELECT id, repo_id, owner, repo, commit_sha, status, attempts, " + "error, updated_at FROM jobs ORDER BY id DESC LIMIT ?", + (limit,), + ).fetchall() + return [dict(row) for row in rows] + + def managed_pages(self, repo_id: int) -> set[str]: + with self._connect() as connection: + rows = connection.execute( + "SELECT title FROM managed_pages WHERE repo_id = ?", (repo_id,) + ).fetchall() + return {str(row["title"]) for row in rows} + + def add_managed_page(self, repo_id: int, title: str) -> None: + with self._connect() as connection: + connection.execute( + "INSERT OR IGNORE INTO managed_pages(repo_id, title) VALUES (?, ?)", + (repo_id, title), + ) + + def remove_managed_page(self, repo_id: int, title: str) -> None: + with self._connect() as connection: + connection.execute( + "DELETE FROM managed_pages WHERE repo_id = ? AND title = ?", + (repo_id, title), + ) + + @staticmethod + def _job(row: sqlite3.Row) -> Job: + return Job( + id=int(row["id"]), + delivery_id=str(row["delivery_id"]), + repo_id=int(row["repo_id"]), + owner=str(row["owner"]), + repo=str(row["repo"]), + html_url=str(row["html_url"]), + default_branch=str(row["default_branch"]), + commit_sha=str(row["commit_sha"]), + status=str(row["status"]), + attempts=int(row["attempts"]), + available_at=float(row["available_at"]), + error=str(row["error"]) if row["error"] is not None else None, + ) diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/deepwiki.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/deepwiki.py new file mode 100644 index 0000000..82f311f --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/deepwiki.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import time +from typing import Any, Callable +from urllib.parse import quote, urlencode + +from .config import Config +from .http_client import HTTPClientError, request_json + + +class DeepWikiClient: + def __init__( + self, + config: Config, + *, + request: Callable[..., Any] = request_json, + sleep: Callable[[float], None] = time.sleep, + ): + self.config = config + self.request = request + self.sleep = sleep + + def health(self) -> bool: + try: + result = self.request("GET", f"{self.config.deepwiki_url}/health", timeout=5) + return isinstance(result, dict) and result.get("status") == "healthy" + except HTTPClientError: + return False + + def generate( + self, + *, + owner: str, + repo: str, + snapshot_path: str, + ) -> dict[str, Any]: + self._invalidate(owner, repo) + submitted = self.request( + "POST", + f"{self.config.deepwiki_url}/wiki/tasks", + payload={ + "repo_url": snapshot_path, + "type": "local", + "owner": owner, + "repo": repo, + "comprehensive": self.config.comprehensive, + "provider": self.config.provider, + "model": self.config.model, + "language": self.config.language, + "excluded_dirs": [], + "excluded_files": [], + "included_dirs": [], + "included_files": [], + }, + timeout=30, + ) + if not isinstance(submitted, dict) or not submitted.get("task_id"): + raise RuntimeError("DeepWiki returned an invalid task response") + task_id = str(submitted["task_id"]) + if not submitted.get("from_cache"): + self._wait(task_id) + return self._cache(owner, repo) + + def _invalidate(self, owner: str, repo: str) -> None: + query = { + "owner": owner, + "repo": repo, + "repo_type": "local", + "language": self.config.language, + } + if self.config.deepwiki_auth_code: + query["authorization_code"] = self.config.deepwiki_auth_code + try: + self.request( + "DELETE", + f"{self.config.deepwiki_url}/api/wiki_cache?{urlencode(query)}", + timeout=30, + ) + except HTTPClientError as exc: + if exc.status != 404: + raise + + def _wait(self, task_id: str) -> None: + deadline = time.monotonic() + self.config.generation_timeout_seconds + encoded = quote(task_id, safe="") + while time.monotonic() < deadline: + status = self.request( + "GET", + f"{self.config.deepwiki_url}/wiki/tasks/{encoded}", + timeout=30, + ) + if not isinstance(status, dict): + raise RuntimeError("DeepWiki returned an invalid task status") + state = status.get("status") + if state == "completed": + return + if state == "failed": + raise RuntimeError(f"DeepWiki generation failed: {status.get('error')}") + self.sleep(self.config.poll_interval_seconds) + raise TimeoutError("DeepWiki generation timed out") + + def _cache(self, owner: str, repo: str) -> dict[str, Any]: + query = urlencode( + { + "owner": owner, + "repo": repo, + "repo_type": "local", + "language": self.config.language, + } + ) + result = self.request( + "GET", + f"{self.config.deepwiki_url}/api/wiki_cache?{query}", + timeout=30, + ) + if not isinstance(result, dict): + raise RuntimeError("DeepWiki completed without a Wiki cache") + return result diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/gitea.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/gitea.py new file mode 100644 index 0000000..1db9c07 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/gitea.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import base64 +from typing import Any, Callable +from urllib.parse import quote, urlencode + +from .config import Config +from .http_client import request_json + + +class GiteaClient: + def __init__( + self, + config: Config, + *, + token: str | None = None, + request: Callable[..., Any] = request_json, + ): + self.config = config + self.request = request + self.headers = {"Authorization": f"token {token or config.gitea_token}"} + + def repository(self, owner: str, repo: str) -> dict[str, Any]: + result = self._request("GET", self._repo_path(owner, repo)) + if not isinstance(result, dict): + raise RuntimeError("Gita returned invalid repository metadata") + return result + + def enable_wiki(self, owner: str, repo: str) -> None: + self._request("PATCH", self._repo_path(owner, repo), {"has_wiki": True}) + + def list_wiki_pages(self, owner: str, repo: str) -> list[dict[str, Any]]: + pages: list[dict[str, Any]] = [] + page = 1 + while True: + query = urlencode({"page": page, "limit": 50}) + result = self._request( + "GET", f"{self._repo_path(owner, repo)}/wiki/pages?{query}" + ) + if not isinstance(result, list): + raise RuntimeError("Gita returned an invalid Wiki page list") + pages.extend(item for item in result if isinstance(item, dict)) + if len(result) < 50: + return pages + page += 1 + + def wiki_page(self, owner: str, repo: str, title: str) -> dict[str, Any]: + result = self._request( + "GET", f"{self._repo_path(owner, repo)}/wiki/page/{quote(title, safe='')}" + ) + if not isinstance(result, dict): + raise RuntimeError("Gita returned invalid Wiki page data") + return result + + def create_wiki_page( + self, owner: str, repo: str, title: str, content: str, message: str + ) -> None: + self._request( + "POST", + f"{self._repo_path(owner, repo)}/wiki/new", + self._wiki_payload(title, content, message), + ) + + def update_wiki_page( + self, owner: str, repo: str, title: str, content: str, message: str + ) -> None: + self._request( + "PATCH", + f"{self._repo_path(owner, repo)}/wiki/page/{quote(title, safe='')}", + self._wiki_payload(title, content, message), + ) + + def delete_wiki_page(self, owner: str, repo: str, title: str) -> None: + self._request( + "DELETE", + f"{self._repo_path(owner, repo)}/wiki/page/{quote(title, safe='')}", + ) + + def list_system_hooks(self) -> list[dict[str, Any]]: + result = self._request("GET", "/api/v1/admin/hooks?type=system&limit=50") + if not isinstance(result, list): + raise RuntimeError("Gita returned an invalid system Webhook list") + return [item for item in result if isinstance(item, dict)] + + def create_system_hook(self, payload: dict[str, Any]) -> dict[str, Any]: + result = self._request("POST", "/api/v1/admin/hooks", payload) + if not isinstance(result, dict): + raise RuntimeError("Gita returned invalid system Webhook data") + return result + + def update_system_hook(self, hook_id: int, payload: dict[str, Any]) -> None: + self._request("PATCH", f"/api/v1/admin/hooks/{hook_id}", payload) + + def _repo_path(self, owner: str, repo: str) -> str: + return f"/api/v1/repos/{quote(owner, safe='')}/{quote(repo, safe='')}" + + def _request( + self, method: str, path: str, payload: dict[str, Any] | None = None + ) -> Any: + return self.request( + method, + f"{self.config.gitea_url}{path}", + payload=payload, + headers=self.headers, + timeout=30, + ) + + @staticmethod + def _wiki_payload(title: str, content: str, message: str) -> dict[str, str]: + encoded = base64.b64encode(content.encode("utf-8")).decode("ascii") + return {"title": title, "content_base64": encoded, "message": message} + + +def decode_wiki_content(page: dict[str, Any]) -> str: + raw = page.get("content_base64") + if not isinstance(raw, str): + return "" + try: + return base64.b64decode(raw, validate=True).decode("utf-8") + except (ValueError, UnicodeDecodeError): + return "" diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/http_client.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/http_client.py new file mode 100644 index 0000000..75e3612 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/http_client.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import json +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from urllib.request import Request, urlopen + + +class HTTPClientError(RuntimeError): + def __init__(self, message: str, status: int | None = None): + super().__init__(message) + self.status = status + + +def request_json( + method: str, + url: str, + *, + payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + timeout: float = 30, +) -> Any: + data = None + request_headers = {"Accept": "application/json", **(headers or {})} + if payload is not None: + data = json.dumps(payload).encode("utf-8") + request_headers["Content-Type"] = "application/json" + request = Request(url, data=data, method=method, headers=request_headers) + try: + with urlopen(request, timeout=timeout) as response: + raw = response.read() + except HTTPError as exc: + detail = exc.read(2048).decode("utf-8", errors="replace") + raise HTTPClientError( + f"{method} {_safe_url(url)} returned {exc.code}: {detail}", exc.code + ) from exc + except URLError as exc: + raise HTTPClientError( + f"{method} {_safe_url(url)} failed: {exc.reason}" + ) from exc + if not raw: + return None + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise HTTPClientError( + f"{method} {_safe_url(url)} returned invalid JSON" + ) from exc + + +def _safe_url(url: str) -> str: + parts = urlsplit(url) + sensitive = {"token", "access_token", "authorization_code"} + query = urlencode( + [ + (key, "***" if key.lower() in sensitive else value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + ] + ) + return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment)) diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/rendering.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/rendering.py new file mode 100644 index 0000000..37f7d86 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/rendering.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import hashlib +import re +from datetime import UTC, datetime +from typing import Any +from urllib.parse import quote + +from .database import Job + + +def ownership_marker(repo_id: int) -> str: + return f"" + + +def render_pages( + cache: dict[str, Any], job: Job, *, provider: str, model: str +) -> dict[str, str]: + generated = cache.get("generated_pages") + structure = cache.get("wiki_structure") + if not isinstance(generated, dict) or not isinstance(structure, dict): + raise ValueError("DeepWiki cache is missing generated pages or structure") + + ordered = structure.get("pages") + if not isinstance(ordered, list): + ordered = list(generated.values()) + + pages: dict[str, str] = {} + toc: list[tuple[str, str]] = [] + used_titles: set[str] = set() + for item in ordered: + if not isinstance(item, dict): + continue + page_id = str(item.get("id", "page")) + page = generated.get(page_id, item) + if not isinstance(page, dict): + continue + title = _managed_title(page_id, used_titles) + used_titles.add(title) + display_title = str(page.get("title") or page_id) + content = str(page.get("content") or "") + file_paths = page.get("filePaths") + if not isinstance(file_paths, list): + file_paths = [] + content = rewrite_source_links( + content, + [str(path) for path in file_paths], + job.html_url, + job.commit_sha, + ) + pages[title] = _page_header(job, display_title, provider, model) + content + toc.append((display_title, title)) + + if not pages: + raise ValueError("DeepWiki cache contains no generated pages") + overview = "\n".join(f"- [{label}]({title})" for label, title in toc) + overview_title = "AI-Generated-Documentation" + pages[overview_title] = ( + _page_header(job, "AI 自动生成文档", provider, model) + + "该目录由 Gita 在默认分支推送后自动更新。人工维护的 Wiki 页面不会被覆盖。\n\n" + + "## 目录\n\n" + + overview + + "\n" + ) + return pages + + +def rewrite_source_links( + content: str, + file_paths: list[str], + html_url: str, + commit_sha: str, +) -> str: + processed = content + by_basename: dict[str, str] = {} + for path in sorted(set(file_paths), key=len, reverse=True): + normalized = path.replace("\\", "/").lstrip("./") + if not normalized: + continue + by_basename.setdefault(normalized.rsplit("/", 1)[-1], normalized) + url = _source_url(html_url, commit_sha, normalized) + processed = processed.replace(f"]({path})", f"]({url})") + processed = processed.replace(f"](./{normalized})", f"]({url})") + citation = re.compile( + r"\[" + + re.escape(normalized) + + r"(?::(\d+)(?:-(\d+))?)?\]\(\)" + ) + + def replace_citation(match: re.Match[str]) -> str: + start, end = match.group(1), match.group(2) + label = normalized + anchor = "" + if start: + label += f":{start}" + (f"-{end}" if end else "") + anchor = f"#L{start}" + (f"-L{end}" if end else "") + return f"[{label}]({url}{anchor})" + + processed = citation.sub(replace_citation, processed) + + prefixed = re.compile( + r"\[(Sources?|Source):\s*([^\[\]\s():]+?)(?::(\d+)(?:-(\d+))?)?\]\(\)", + re.IGNORECASE, + ) + + def replace_prefixed(match: re.Match[str]) -> str: + token = match.group(2) + path = token if "/" in token else by_basename.get(token) + if not path: + return match.group(0) + start, end = match.group(3), match.group(4) + anchor = f"#L{start}" if start else "" + if start and end: + anchor += f"-L{end}" + label = path + (f":{start}" if start else "") + (f"-{end}" if end else "") + return f"{match.group(1)}: [{label}]({_source_url(html_url, commit_sha, path)}{anchor})" + + return prefixed.sub(replace_prefixed, processed) + + +def _page_header(job: Job, title: str, provider: str, model: str) -> str: + generated_at = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S UTC") + commit_url = f"{job.html_url.rstrip('/')}/commit/{job.commit_sha}" + return ( + f"{ownership_marker(job.repo_id)}\n\n" + f"# {title}\n\n" + f"> 自动生成时间:{generated_at} " + f"\n> 源码版本:[`{job.commit_sha[:12]}`]({commit_url}) " + f"\n> 模型:`{provider}/{model}`\n\n" + ) + + +def _managed_title(page_id: str, used: set[str]) -> str: + slug = re.sub(r"[^A-Za-z0-9_-]+", "-", page_id).strip("-") or "page" + title = f"AI-Generated-{slug[:80]}" + if title not in used: + return title + suffix = hashlib.sha256(page_id.encode("utf-8")).hexdigest()[:8] + return f"{title[:90]}-{suffix}" + + +def _source_url(html_url: str, commit_sha: str, path: str) -> str: + encoded_path = quote(path, safe="/") + return f"{html_url.rstrip('/')}/src/commit/{commit_sha}/{encoded_path}" diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/server.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/server.py new file mode 100644 index 0000000..ad2c001 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/server.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import json +import logging +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +from .config import Config +from .database import Database +from .webhook import parse_push, verify_signature + + +logger = logging.getLogger(__name__) + + +class AdapterServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, config: Config, database: Database, worker): + self.config = config + self.database = database + self.worker = worker + super().__init__((config.listen_host, config.listen_port), AdapterHandler) + + +class AdapterHandler(BaseHTTPRequestHandler): + server: AdapterServer + + def do_GET(self) -> None: + if self.path == "/health": + healthy = self.server.worker.is_alive() + self._json( + HTTPStatus.OK if healthy else HTTPStatus.SERVICE_UNAVAILABLE, + {"status": "healthy" if healthy else "unhealthy"}, + ) + return + if self.path == "/status": + self._json( + HTTPStatus.OK, + { + "counts": self.server.database.counts(), + "recent": self.server.database.recent(), + }, + ) + return + self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) + + def do_POST(self) -> None: + if self.path != "/hooks/gitea": + self._json(HTTPStatus.NOT_FOUND, {"error": "not found"}) + return + raw_length = self.headers.get("Content-Length", "") + try: + length = int(raw_length) + except ValueError: + self._json(HTTPStatus.BAD_REQUEST, {"error": "invalid content length"}) + return + if length < 0 or length > self.server.config.max_request_bytes: + self._json(HTTPStatus.REQUEST_ENTITY_TOO_LARGE, {"error": "payload too large"}) + return + body = self.rfile.read(length) + signature = self.headers.get("X-Gitea-Signature", "") + if not verify_signature( + body, signature, self.server.config.webhook_secret + ): + self._json(HTTPStatus.UNAUTHORIZED, {"error": "invalid signature"}) + return + try: + event, reason = parse_push( + body, + event_name=self.headers.get("X-Gitea-Event", ""), + delivery_id=self.headers.get("X-Gitea-Delivery", ""), + ) + except ValueError as exc: + self._json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) + return + if event is None: + self._json(HTTPStatus.ACCEPTED, {"status": "ignored", "reason": reason}) + return + job_id, created = self.server.database.enqueue( + delivery_id=event.delivery_id, + repo_id=event.repo_id, + owner=event.owner, + repo=event.repo, + html_url=event.html_url, + default_branch=event.default_branch, + commit_sha=event.commit_sha, + debounce_seconds=self.server.config.debounce_seconds, + ) + logger.info( + "webhook accepted job=%s repository=%s/%s commit=%s created=%s", + job_id, + event.owner, + event.repo, + event.commit_sha[:12], + created, + ) + self._json( + HTTPStatus.ACCEPTED, + {"status": "queued" if created else "duplicate", "job_id": job_id}, + ) + + def log_message(self, message: str, *args: Any) -> None: + logger.info("http %s - %s", self.client_address[0], message % args) + + def _json(self, status: HTTPStatus, payload: dict[str, Any]) -> None: + encoded = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_response(status.value) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/service.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/service.py new file mode 100644 index 0000000..d3b8c83 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/service.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import logging +import threading +from pathlib import Path + +from .artifacts import ArtifactCleaner +from .config import Config +from .database import Database, Job +from .deepwiki import DeepWikiClient +from .gitea import GiteaClient, decode_wiki_content +from .rendering import ownership_marker, render_pages +from .snapshot import SnapshotManager + + +logger = logging.getLogger(__name__) + + +class DocumentationProcessor: + def __init__( + self, + config: Config, + database: Database, + snapshots: SnapshotManager, + deepwiki: DeepWikiClient, + gitea: GiteaClient, + artifacts: ArtifactCleaner, + ): + self.config = config + self.database = database + self.snapshots = snapshots + self.deepwiki = deepwiki + self.gitea = gitea + self.artifacts = artifacts + + def process(self, job: Job) -> None: + snapshot: Path | None = None + try: + snapshot = self.snapshots.create(job) + cache = self.deepwiki.generate( + owner=job.owner, + repo=job.repo, + snapshot_path=str(snapshot), + ) + pages = render_pages( + cache, + job, + provider=self.config.provider, + model=self.config.model, + ) + self._ensure_internal_wiki(job) + self._sync(job, pages) + self.artifacts.remove_index(job.commit_sha) + finally: + if snapshot is not None: + self.snapshots.remove(snapshot) + + def _ensure_internal_wiki(self, job: Job) -> None: + repository = self.gitea.repository(job.owner, job.repo) + if repository.get("external_wiki"): + raise RuntimeError("repository uses an external Wiki; refusing to replace it") + if repository.get("has_wiki"): + return + if not self.config.auto_enable_wiki: + raise RuntimeError("repository Wiki is disabled") + self.gitea.enable_wiki(job.owner, job.repo) + + def _sync(self, job: Job, pages: dict[str, str]) -> None: + existing = { + str(page.get("title")): page + for page in self.gitea.list_wiki_pages(job.owner, job.repo) + if page.get("title") + } + managed = self.database.managed_pages(job.repo_id) + marker = ownership_marker(job.repo_id) + message = f"docs(ai): update from {job.commit_sha[:12]}" + + for title, content in pages.items(): + if title in existing: + if title not in managed: + current = self.gitea.wiki_page(job.owner, job.repo, title) + if marker not in decode_wiki_content(current): + raise RuntimeError( + f"Wiki page conflict: {title} is not managed by DeepWiki" + ) + self.gitea.update_wiki_page( + job.owner, job.repo, title, content, message + ) + else: + self.gitea.create_wiki_page( + job.owner, job.repo, title, content, message + ) + self.database.add_managed_page(job.repo_id, title) + + stale = self.database.managed_pages(job.repo_id) - set(pages) + for title in stale: + if title in existing: + current = self.gitea.wiki_page(job.owner, job.repo, title) + if marker in decode_wiki_content(current): + self.gitea.delete_wiki_page(job.owner, job.repo, title) + else: + logger.warning("managed page marker missing; preserving %s", title) + self.database.remove_managed_page(job.repo_id, title) + + +class Worker(threading.Thread): + def __init__( + self, + config: Config, + database: Database, + processor: DocumentationProcessor, + ): + super().__init__(name="deepwiki-worker", daemon=True) + self.config = config + self.database = database + self.processor = processor + self.stop_event = threading.Event() + + def run(self) -> None: + while not self.stop_event.is_set(): + job = self.database.claim() + if job is None: + self.stop_event.wait(1) + continue + logger.info( + "processing job=%s repository=%s/%s commit=%s", + job.id, + job.owner, + job.repo, + job.commit_sha[:12], + ) + try: + self.processor.process(job) + except Exception as exc: + logger.exception("job=%s failed", job.id) + error = f"{type(exc).__name__}: {exc}" + if job.attempts >= self.config.retry_limit: + self.database.fail(job.id, error) + else: + delay = min( + self.config.retry_base_seconds * (2 ** (job.attempts - 1)), + 3600, + ) + self.database.retry(job.id, error, delay) + else: + self.database.complete(job.id) + logger.info("job=%s completed", job.id) + + def stop(self) -> None: + self.stop_event.set() diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/snapshot.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/snapshot.py new file mode 100644 index 0000000..1a3e250 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/snapshot.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import shutil +import subprocess +import tarfile +from pathlib import Path, PurePosixPath + +from .database import Job + + +class SnapshotManager: + def __init__(self, repository_root: Path, work_root: Path, max_bytes: int): + self.repository_root = repository_root.resolve() + self.snapshot_root = (work_root / "repos").resolve() + self.max_bytes = max_bytes + self.snapshot_root.mkdir(parents=True, exist_ok=True) + + def create(self, job: Job) -> Path: + source = self._repository_path(job.owner, job.repo) + destination = (self.snapshot_root / str(job.repo_id) / job.commit_sha).resolve() + self._ensure_within(destination, self.snapshot_root) + if destination.exists(): + shutil.rmtree(destination) + destination.mkdir(parents=True) + + self._git( + source, + ["cat-file", "-e", f"{job.commit_sha}^{{commit}}"], + ) + process = subprocess.Popen( + [ + "git", + f"--git-dir={source}", + "archive", + "--format=tar", + job.commit_sha, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert process.stdout is not None + assert process.stderr is not None + try: + self._extract(process.stdout, destination) + except Exception: + process.kill() + process.wait() + process.stdout.close() + process.stderr.close() + shutil.rmtree(destination, ignore_errors=True) + raise + process.stdout.close() + stderr = process.stderr.read().decode("utf-8", errors="replace") + process.stderr.close() + if process.wait() != 0: + shutil.rmtree(destination, ignore_errors=True) + raise RuntimeError(f"git archive failed: {stderr[:1000]}") + return destination + + def remove(self, path: Path) -> None: + resolved = path.resolve() + self._ensure_within(resolved, self.snapshot_root) + if resolved.exists(): + shutil.rmtree(resolved) + parent = resolved.parent + if parent != self.snapshot_root and parent.exists() and not any(parent.iterdir()): + parent.rmdir() + + def _repository_path(self, owner: str, repo: str) -> Path: + candidates = [ + self.repository_root / owner / f"{repo}.git", + self.repository_root / owner.lower() / f"{repo.lower()}.git", + ] + for candidate in candidates: + resolved = candidate.resolve() + self._ensure_within(resolved, self.repository_root) + if resolved.is_dir(): + return resolved + raise FileNotFoundError(f"bare repository not found for {owner}/{repo}") + + def _extract(self, stream, destination: Path) -> None: + total = 0 + with tarfile.open(fileobj=stream, mode="r|") as archive: + for member in archive: + relative = PurePosixPath(member.name) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError("git archive contains an unsafe path") + target = (destination / Path(*relative.parts)).resolve() + self._ensure_within(target, destination) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + if not member.isfile(): + continue + total += member.size + if total > self.max_bytes: + raise ValueError("repository snapshot exceeds configured size limit") + target.parent.mkdir(parents=True, exist_ok=True) + source = archive.extractfile(member) + if source is None: + raise ValueError(f"could not read archive member: {member.name}") + with source, target.open("wb") as output: + shutil.copyfileobj(source, output) + + @staticmethod + def _git(repository: Path, arguments: list[str]) -> None: + result = subprocess.run( + ["git", f"--git-dir={repository}", *arguments], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"git command failed: {result.stderr[:1000]}") + + @staticmethod + def _ensure_within(path: Path, root: Path) -> None: + if path != root and root not in path.parents: + raise ValueError(f"path escapes configured root: {path}") diff --git a/contrib/roncarve/deepwiki-integration/gita_deepwiki/webhook.py b/contrib/roncarve/deepwiki-integration/gita_deepwiki/webhook.py new file mode 100644 index 0000000..d0b2617 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/gita_deepwiki/webhook.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +import re +from dataclasses import dataclass +from typing import Any + + +_NAME = re.compile(r"^[A-Za-z0-9_.-]+$") +_SHA = re.compile(r"^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$") + + +@dataclass(frozen=True) +class PushEvent: + delivery_id: str + repo_id: int + owner: str + repo: str + html_url: str + default_branch: str + commit_sha: str + + +def verify_signature(body: bytes, signature: str, secret: str) -> bool: + if not signature or not secret: + return False + expected = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, signature.strip().lower()) + + +def parse_push( + body: bytes, + *, + event_name: str, + delivery_id: str, +) -> tuple[PushEvent | None, str]: + if event_name != "push": + return None, "event ignored" + try: + payload: dict[str, Any] = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("invalid JSON payload") from exc + + repository = payload.get("repository") + if not isinstance(repository, dict): + raise ValueError("payload is missing repository") + + try: + repo_id = int(repository["id"]) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError("repository.id must be an integer") from exc + if repo_id <= 0: + raise ValueError("repository.id must be positive") + + owner_obj = repository.get("owner") + owner = owner_obj.get("username", "") if isinstance(owner_obj, dict) else "" + repo = str(repository.get("name", "")) + default_branch = str(repository.get("default_branch", "")) + html_url = str(repository.get("html_url", "")).rstrip("/") + commit_sha = str(payload.get("after", "")).lower() + ref = str(payload.get("ref", "")) + + if not _NAME.fullmatch(owner) or not _NAME.fullmatch(repo): + raise ValueError("repository owner or name contains unsupported characters") + if not default_branch or ref != f"refs/heads/{default_branch}": + return None, "non-default branch ignored" + if not _SHA.fullmatch(commit_sha): + raise ValueError("after must be a 40 or 64 character commit SHA") + if set(commit_sha) == {"0"}: + return None, "branch deletion ignored" + if not html_url.startswith(("http://", "https://")): + raise ValueError("repository.html_url must be an HTTP URL") + + return ( + PushEvent( + delivery_id=delivery_id, + repo_id=repo_id, + owner=owner, + repo=repo, + html_url=html_url, + default_branch=default_branch, + commit_sha=commit_sha, + ), + "queued", + ) diff --git a/contrib/roncarve/deepwiki-integration/integration.env.example b/contrib/roncarve/deepwiki-integration/integration.env.example new file mode 100644 index 0000000..ed83e07 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/integration.env.example @@ -0,0 +1,6 @@ +GITA_DEEPWIKI_WEBHOOK_SECRET=replace-with-64-character-random-hex +GITA_API_TOKEN=replace-with-write-repository-runtime-token +DEEPWIKI_AUTH_CODE=replace-with-shared-deepwiki-auth-code + +# Supply this only to the one-shot install-webhook command, then revoke it. +# GITA_ADMIN_API_TOKEN=replace-with-temporary-write-admin-token diff --git a/contrib/roncarve/deepwiki-integration/tests/__init__.py b/contrib/roncarve/deepwiki-integration/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/contrib/roncarve/deepwiki-integration/tests/test_artifacts.py b/contrib/roncarve/deepwiki-integration/tests/test_artifacts.py new file mode 100644 index 0000000..ba4139a --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/tests/test_artifacts.py @@ -0,0 +1,27 @@ +import tempfile +import unittest +from pathlib import Path + +from gita_deepwiki.artifacts import ArtifactCleaner + + +class ArtifactCleanerTest(unittest.TestCase): + def test_removes_only_commit_index(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + databases = root / "databases" + databases.mkdir() + sha = "a" * 40 + index = databases / f"{sha}.pkl" + index.write_bytes(b"index") + ArtifactCleaner(root).remove_index(sha) + self.assertFalse(index.exists()) + + def test_rejects_unsafe_name(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "invalid commit SHA"): + ArtifactCleaner(Path(directory)).remove_index("../escape") + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/roncarve/deepwiki-integration/tests/test_database.py b/contrib/roncarve/deepwiki-integration/tests/test_database.py new file mode 100644 index 0000000..fd9c719 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/tests/test_database.py @@ -0,0 +1,53 @@ +import tempfile +import unittest +from pathlib import Path + +from gita_deepwiki.database import Database + + +class DatabaseTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.database = Database(Path(self.temp.name) / "jobs.sqlite3") + + def tearDown(self): + self.temp.cleanup() + + def enqueue(self, sha: str): + return self.database.enqueue( + delivery_id=sha[:8], + repo_id=1, + owner="team", + repo="demo", + html_url="https://git.example.com/team/demo", + default_branch="main", + commit_sha=sha, + debounce_seconds=0, + ) + + def test_duplicate_commit_is_idempotent(self): + first_id, first_created = self.enqueue("a" * 40) + second_id, second_created = self.enqueue("a" * 40) + self.assertTrue(first_created) + self.assertFalse(second_created) + self.assertEqual(first_id, second_id) + + def test_new_commit_supersedes_queued_commit(self): + self.enqueue("a" * 40) + self.enqueue("b" * 40) + job = self.database.claim() + self.assertIsNotNone(job) + self.assertEqual("b" * 40, job.commit_sha) + self.assertEqual(1, self.database.counts()["superseded"]) + + def test_managed_page_manifest(self): + self.database.add_managed_page(1, "AI-Generated-Documentation") + self.assertEqual( + {"AI-Generated-Documentation"}, self.database.managed_pages(1) + ) + self.database.remove_managed_page(1, "AI-Generated-Documentation") + self.assertEqual(set(), self.database.managed_pages(1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/roncarve/deepwiki-integration/tests/test_deepwiki.py b/contrib/roncarve/deepwiki-integration/tests/test_deepwiki.py new file mode 100644 index 0000000..387cedc --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/tests/test_deepwiki.py @@ -0,0 +1,55 @@ +import tempfile +import unittest +from pathlib import Path + +from gita_deepwiki.config import Config +from gita_deepwiki.deepwiki import DeepWikiClient +from gita_deepwiki.http_client import HTTPClientError + + +class DeepWikiClientTest(unittest.TestCase): + def config(self): + root = Path(tempfile.gettempdir()) + return Config( + webhook_secret="secret", + gitea_url="http://gitea", + gitea_token="token", + deepwiki_url="http://deepwiki", + repository_root=root, + work_root=root, + database_path=root / "jobs.sqlite3", + poll_interval_seconds=0.1, + generation_timeout_seconds=30, + ) + + def test_generation_invalidates_waits_and_reads_cache(self): + calls = [] + statuses = iter(["indexing", "completed"]) + + def request(method, url, **kwargs): + calls.append((method, url, kwargs.get("payload"))) + if method == "DELETE": + raise HTTPClientError("not found", 404) + if method == "POST": + return {"task_id": "local_team_demo", "created": True} + if "/wiki/tasks/" in url: + return {"status": next(statuses)} + return { + "wiki_structure": {"pages": []}, + "generated_pages": {}, + } + + client = DeepWikiClient(self.config(), request=request, sleep=lambda _: None) + result = client.generate( + owner="team", repo="demo", snapshot_path="/work/repos/1/sha" + ) + self.assertIn("wiki_structure", result) + self.assertEqual("DELETE", calls[0][0]) + self.assertEqual("POST", calls[1][0]) + self.assertEqual("local", calls[1][2]["type"]) + self.assertEqual("openai", calls[1][2]["provider"]) + self.assertEqual("deepseek-v4-flash", calls[1][2]["model"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/roncarve/deepwiki-integration/tests/test_gitea.py b/contrib/roncarve/deepwiki-integration/tests/test_gitea.py new file mode 100644 index 0000000..5eb78e2 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/tests/test_gitea.py @@ -0,0 +1,47 @@ +import tempfile +import unittest +from pathlib import Path + +from gita_deepwiki.cli import install_webhook +from gita_deepwiki.config import Config +from gita_deepwiki.gitea import GiteaClient + + +class GiteaClientTest(unittest.TestCase): + def config(self, **overrides): + root = Path(tempfile.gettempdir()) + values = { + "webhook_secret": "secret", + "gitea_url": "http://gitea", + "gitea_token": "runtime-token", + "deepwiki_url": "http://deepwiki", + "repository_root": root, + "work_root": root, + "database_path": root / "jobs.sqlite3", + } + values.update(overrides) + return Config(**values) + + def test_explicit_token_overrides_runtime_token(self): + calls = [] + + def request(method, url, **kwargs): + calls.append((method, url, kwargs)) + return [] + + client = GiteaClient( + self.config(), token="one-shot-admin-token", request=request + ) + client.list_system_hooks() + self.assertEqual( + "token one-shot-admin-token", + calls[0][2]["headers"]["Authorization"], + ) + + def test_webhook_install_requires_one_shot_admin_token(self): + with self.assertRaisesRegex(ValueError, "GITA_ADMIN_API_TOKEN"): + install_webhook(self.config()) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/roncarve/deepwiki-integration/tests/test_rendering.py b/contrib/roncarve/deepwiki-integration/tests/test_rendering.py new file mode 100644 index 0000000..7adb382 --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/tests/test_rendering.py @@ -0,0 +1,67 @@ +import unittest + +from gita_deepwiki.database import Job +from gita_deepwiki.rendering import render_pages, rewrite_source_links + + +class RenderingTest(unittest.TestCase): + def job(self): + return Job( + id=1, + delivery_id="delivery", + repo_id=42, + owner="team", + repo="demo", + html_url="https://git.example.com/team/demo", + default_branch="main", + commit_sha="a" * 40, + status="running", + attempts=1, + available_at=0, + error=None, + ) + + def test_rewrites_relative_links_and_citations(self): + content = "[source](src/main.go) and [src/main.go:10-12]()" + rewritten = rewrite_source_links( + content, + ["src/main.go"], + self.job().html_url, + self.job().commit_sha, + ) + source = ( + "https://git.example.com/team/demo/src/commit/" + + "a" * 40 + + "/src/main.go" + ) + self.assertIn(f"]({source})", rewritten) + self.assertIn(f"]({source}#L10-L12)", rewritten) + + def test_renders_owned_pages_and_overview(self): + cache = { + "wiki_structure": { + "pages": [{"id": "overview", "title": "Overview"}] + }, + "generated_pages": { + "overview": { + "id": "overview", + "title": "Overview", + "content": "Body", + "filePaths": [], + } + }, + } + pages = render_pages( + cache, + self.job(), + provider="openai", + model="deepseek-v4-flash", + ) + self.assertIn("AI-Generated-Documentation", pages) + self.assertIn("AI-Generated-overview", pages) + self.assertIn("repo-id=42", pages["AI-Generated-overview"]) + self.assertIn("a" * 12, pages["AI-Generated-overview"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/roncarve/deepwiki-integration/tests/test_service.py b/contrib/roncarve/deepwiki-integration/tests/test_service.py new file mode 100644 index 0000000..cbe96fa --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/tests/test_service.py @@ -0,0 +1,102 @@ +import base64 +import tempfile +import unittest +from pathlib import Path + +from gita_deepwiki.config import Config +from gita_deepwiki.database import Database, Job +from gita_deepwiki.service import DocumentationProcessor + + +class FakeGitea: + def __init__(self): + self.pages = {} + self.deleted = [] + + def list_wiki_pages(self, _owner, _repo): + return [{"title": title} for title in self.pages] + + def wiki_page(self, _owner, _repo, title): + content = self.pages[title] + return { + "content_base64": base64.b64encode(content.encode()).decode() + } + + def create_wiki_page(self, _owner, _repo, title, content, _message): + self.pages[title] = content + + def update_wiki_page(self, _owner, _repo, title, content, _message): + self.pages[title] = content + + def delete_wiki_page(self, _owner, _repo, title): + self.deleted.append(title) + self.pages.pop(title, None) + + +class ServiceTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + root = Path(self.temp.name) + self.database = Database(root / "jobs.sqlite3") + self.gitea = FakeGitea() + config = Config( + webhook_secret="secret", + gitea_url="http://gitea", + gitea_token="token", + deepwiki_url="http://deepwiki", + repository_root=root, + work_root=root, + database_path=root / "jobs.sqlite3", + ) + self.processor = DocumentationProcessor( + config, + self.database, + None, + None, + self.gitea, + None, + ) + + def tearDown(self): + self.temp.cleanup() + + def job(self): + return Job( + id=1, + delivery_id="delivery", + repo_id=42, + owner="team", + repo="demo", + html_url="https://git.example.com/team/demo", + default_branch="main", + commit_sha="a" * 40, + status="running", + attempts=1, + available_at=0, + error=None, + ) + + def test_sync_does_not_overwrite_manual_page(self): + self.gitea.pages["AI-Generated-page"] = "manual content" + with self.assertRaisesRegex(RuntimeError, "not managed"): + self.processor._sync( + self.job(), {"AI-Generated-page": "generated content"} + ) + self.assertEqual("manual content", self.gitea.pages["AI-Generated-page"]) + + def test_sync_updates_owned_and_deletes_stale_page(self): + marker = "" + self.gitea.pages["AI-Generated-old"] = marker + "old" + self.database.add_managed_page(42, "AI-Generated-old") + self.processor._sync( + self.job(), {"AI-Generated-new": marker + "new"} + ) + self.assertIn("AI-Generated-new", self.gitea.pages) + self.assertEqual(["AI-Generated-old"], self.gitea.deleted) + self.assertEqual( + {"AI-Generated-new"}, self.database.managed_pages(42) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/roncarve/deepwiki-integration/tests/test_snapshot.py b/contrib/roncarve/deepwiki-integration/tests/test_snapshot.py new file mode 100644 index 0000000..24efd0c --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/tests/test_snapshot.py @@ -0,0 +1,72 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path + +from gita_deepwiki.database import Job +from gita_deepwiki.snapshot import SnapshotManager + + +class SnapshotTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + source = self.root / "source" + source.mkdir() + self.git(source, "init") + self.git(source, "config", "user.name", "Test") + self.git(source, "config", "user.email", "test@example.com") + (source / "README.md").write_text("hello\n", encoding="utf-8") + self.git(source, "add", "README.md") + self.git(source, "commit", "-m", "initial") + self.sha = self.git(source, "rev-parse", "HEAD").strip() + bare = self.root / "repositories" / "team" / "demo.git" + bare.parent.mkdir(parents=True) + subprocess.run( + ["git", "clone", "--bare", str(source), str(bare)], + check=True, + capture_output=True, + ) + + def tearDown(self): + self.temp.cleanup() + + def test_exports_exact_commit(self): + manager = SnapshotManager( + self.root / "repositories", self.root / "work", 10 * 1024 * 1024 + ) + destination = manager.create(self.job()) + self.assertEqual("hello\n", (destination / "README.md").read_text()) + self.assertFalse((destination / ".git").exists()) + manager.remove(destination) + self.assertFalse(destination.exists()) + + def job(self): + return Job( + id=1, + delivery_id="delivery", + repo_id=42, + owner="team", + repo="demo", + html_url="https://git.example.com/team/demo", + default_branch="main", + commit_sha=self.sha, + status="running", + attempts=1, + available_at=0, + error=None, + ) + + @staticmethod + def git(directory: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(directory), *arguments], + check=True, + capture_output=True, + text=True, + ) + return result.stdout + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/roncarve/deepwiki-integration/tests/test_webhook.py b/contrib/roncarve/deepwiki-integration/tests/test_webhook.py new file mode 100644 index 0000000..c5b06ad --- /dev/null +++ b/contrib/roncarve/deepwiki-integration/tests/test_webhook.py @@ -0,0 +1,71 @@ +import hashlib +import hmac +import json +import unittest + +from gita_deepwiki.webhook import parse_push, verify_signature + + +class WebhookTest(unittest.TestCase): + def payload(self, *, ref: str = "refs/heads/main", after: str = "a" * 40): + return json.dumps( + { + "ref": ref, + "after": after, + "repository": { + "id": 42, + "name": "demo", + "owner": {"username": "team"}, + "default_branch": "main", + "html_url": "https://git.example.com/team/demo", + }, + } + ).encode() + + def test_signature(self): + body = self.payload() + secret = "secret" + signature = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest() + self.assertTrue(verify_signature(body, signature, secret)) + self.assertFalse(verify_signature(body, "0" * 64, secret)) + + def test_default_branch_push(self): + event, reason = parse_push( + self.payload(), event_name="push", delivery_id="delivery" + ) + self.assertEqual("queued", reason) + self.assertIsNotNone(event) + self.assertEqual(42, event.repo_id) + self.assertEqual("a" * 40, event.commit_sha) + + def test_non_default_branch_is_ignored(self): + event, reason = parse_push( + self.payload(ref="refs/heads/feature"), + event_name="push", + delivery_id="delivery", + ) + self.assertIsNone(event) + self.assertEqual("non-default branch ignored", reason) + + def test_branch_deletion_is_ignored(self): + event, reason = parse_push( + self.payload(after="0" * 40), + event_name="push", + delivery_id="delivery", + ) + self.assertIsNone(event) + self.assertEqual("branch deletion ignored", reason) + + def test_invalid_repository_name_is_rejected(self): + payload = json.loads(self.payload()) + payload["repository"]["name"] = "../escape" + with self.assertRaisesRegex(ValueError, "unsupported characters"): + parse_push( + json.dumps(payload).encode(), + event_name="push", + delivery_id="delivery", + ) + + +if __name__ == "__main__": + unittest.main()