import from clockmost-infra projects/drive-rag/app (WP-53 phase 2)

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01GZZ5ztbBNFM4DjL1zF4dua

.gitea/workflows/ci.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+      - run: uv sync --frozen --group server
+      # no unit tests yet: the app must at least import (model + DB load in lifespan)
+      - run: uv run python -c "import server.app"
+
+  deploy:
+    needs: test
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    container:
+      image: catthehacker/ubuntu:act-latest
+      volumes: [/var/run/docker.sock:/var/run/docker.sock]
+    steps:
+      - uses: actions/checkout@v4
+      - name: build, push, promote (shared deploy.sh, WP-53)
+        env:
+          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+          INFRA_DEPLOY_KEY: ${{ secrets.INFRA_DEPLOY_KEY }}
+          INFRA_REPO_URL: ${{ secrets.INFRA_REPO_URL }}
+          INFRA_KNOWN_HOSTS: ${{ secrets.INFRA_KNOWN_HOSTS }}
+          NTFY_TOKEN: ${{ secrets.NTFY_TOKEN }}
+          COMMIT_MSG: ${{ github.event.head_commit.message }}
+        run: curl -fsSL https://git.clockmost.com/Benylin/ci-workflows/raw/branch/main/deploy.sh | bash -s -- drive-rag

.gitignore

@@ -0,0 +1,5 @@
+.venv/
+__pycache__/
+data/
+*.db
+.env

Dockerfile

@@ -0,0 +1,22 @@
+# drive-rag backend — FastAPI + model2vec + sqlite-vec.
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    UV_NO_CACHE=1 \
+    HF_HOME=/models \
+    DRIVE_DB=/data/drive.db
+
+WORKDIR /app
+
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-dev --group server
+
+# bake the embedding model into the image so the container is offline-capable
+RUN uv run python -c "from model2vec import StaticModel; StaticModel.from_pretrained('minishlab/potion-retrieval-32M')"
+
+COPY server/ ./server/
+
+EXPOSE 8000
+CMD ["uv", "run", "--no-dev", "--group", "server", "uvicorn", "server.app:app", \
+     "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -0,0 +1,7 @@
+# drive-rag
+
+RAG over Mario's local Google Drive sync (docs, PDFs, the Obsidian vault). felt-core's search_documents skill is the only caller; answers via felt-llm (local).
+
+Deployed on clockmost: push to `main` -> CI builds `git.clockmost.com/benylin/drive-rag:<sha>` and bumps its tag in the infra repo; the deploy loop health-checks it and auto-reverts on failure (Felt WP-53). The index (`/data`) is mounted on the server, not in this repo.
+
+Until 2026-10-02 this code lived vendored inside the infra repo (`projects/drive-rag/app`).

pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "drive-rag"
+version = "0.1.0"
+description = "RAG over Mario's local Google Drive sync (docs, PDFs, the Obsidian vault), answered by Gemini via ninerouter"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "httpx>=0.28.1",
+    "model2vec>=0.9.0",
+    "openpyxl>=3.1.5",
+    "pypdf>=5.1.0",
+    "python-docx>=1.1.2",
+    "sqlite-vec>=0.1.9",
+    "tqdm>=4.70.0",
+]
+
+[dependency-groups]
+server = [
+    "fastapi>=0.118",
+    "uvicorn[standard]>=0.34",
+]
+
+[tool.uv]
+package = false

server/app.py

@@ -0,0 +1,107 @@
+"""drive-rag backend — hybrid retrieval + a Gemini answer, called internally
+by felt-core's search_documents skill (not a human-facing chat UI, so this is
+a plain JSON request/response, not SSE like poe2rag's).
+
+Env:
+  DRIVE_DB          path to drive.db            (default ./data/drive.db)
+  EMBED_MODEL       model2vec model name        (default minishlab/potion-retrieval-32M)
+  LLM_BASE_URL      OpenAI-compatible endpoint  (default http://ninerouter:20128/v1)
+  LLM_API_KEY       key for that endpoint
+  LLM_MODEL         model id                    (default ag/gemini-3.8-flash)
+"""
+
+from __future__ import annotations
+
+import os
+from contextlib import asynccontextmanager
+
+import httpx
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+from .retrieve import Hit, connect, search
+
+DB = os.getenv("DRIVE_DB", "data/drive.db")
+EMBED_MODEL = os.getenv("EMBED_MODEL", "minishlab/potion-retrieval-32M")
+LLM_BASE_URL = os.getenv("LLM_BASE_URL", "http://ninerouter:20128/v1")
+LLM_API_KEY = os.getenv("LLM_API_KEY", "")
+LLM_MODEL = os.getenv("LLM_MODEL", "ag/gemini-3.8-flash")
+TOP_K = int(os.getenv("TOP_K", "6"))
+
+SYSTEM = (
+    "You are searching Mario's personal documents (his local Google Drive sync "
+    "and Obsidian vault) to answer a question. Answer using ONLY the numbered "
+    "passages below.\n"
+    "Style:\n"
+    "- Lead with the direct answer in one sentence. No preamble.\n"
+    "- Keep it short — a few sentences or a tight bullet list.\n"
+    "- Cite each claim with its passage number, e.g. [2].\n"
+    "- Always mention which file(s) the answer came from by name, so Mario can "
+    "go open it directly.\n"
+    "- If the passages don't actually answer the question, say so plainly — do "
+    "NOT guess or invent something that sounds plausible."
+)
+
+state: dict = {}
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    from model2vec import StaticModel
+
+    state["db"] = connect(DB)
+    state["embed"] = StaticModel.from_pretrained(EMBED_MODEL)
+    state["http"] = httpx.AsyncClient(timeout=120)
+    yield
+    await state["http"].aclose()
+    state["db"].close()
+
+
+app = FastAPI(title="drive-rag", lifespan=lifespan)
+
+
+class Ask(BaseModel):
+    question: str
+
+
+def embed_query(q: str) -> list[float]:
+    return state["embed"].encode([q])[0].astype("float32").tolist()
+
+
+def build_prompt(question: str, hits: list[Hit]) -> str:
+    blocks = []
+    for i, h in enumerate(hits, 1):
+        loc = f"{h.title}" + (f" — {h.section}" if h.section else "")
+        blocks.append(f"[{i}] (file: {h.url}, {loc})\n{h.text}")
+    return "\n\n".join(blocks) + f"\n\n---\nQuestion: {question}"
+
+
+@app.get("/api/health")
+async def health():
+    n = state["db"].execute("SELECT count(*) FROM chunks").fetchone()[0]
+    return {"ok": True, "chunks": n, "model": LLM_MODEL}
+
+
+@app.post("/api/ask")
+async def ask(body: Ask):
+    hits = search(state["db"], embed_query(body.question), body.question, k=TOP_K)
+    sources = [
+        {"n": i, "title": h.title, "url": h.url, "section": h.section} for i, h in enumerate(hits, 1)
+    ]
+
+    if not hits:
+        return {"answer": "No matching documents found.", "sources": []}
+
+    payload = {
+        "model": LLM_MODEL,
+        "stream": False,
+        "messages": [
+            {"role": "system", "content": SYSTEM},
+            {"role": "user", "content": build_prompt(body.question, hits)},
+        ],
+    }
+    headers = {"Authorization": f"Bearer {LLM_API_KEY}"} if LLM_API_KEY else {}
+    response = await state["http"].post(f"{LLM_BASE_URL}/chat/completions", json=payload, headers=headers)
+    response.raise_for_status()
+    answer = response.json()["choices"][0]["message"]["content"]
+    return {"answer": answer, "sources": sources}

server/retrieve.py

@@ -0,0 +1,89 @@
+"""Hybrid retrieval over drive.db: semantic (sqlite-vec) + keyword (FTS5),
+fused with Reciprocal Rank Fusion — identical logic to poe2-rag's retriever,
+generic enough to reuse verbatim (source/title/url/section/text/score)."""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+import struct
+from dataclasses import dataclass
+
+import sqlite_vec
+
+RRF_K = 60
+POOL = 40
+
+
+@dataclass
+class Hit:
+    id: int
+    source: str
+    title: str
+    url: str
+    section: str
+    text: str
+    score: float
+
+
+def connect(db_path: str) -> sqlite3.Connection:
+    db = sqlite3.connect(db_path, check_same_thread=False)
+    db.enable_load_extension(True)
+    sqlite_vec.load(db)
+    db.enable_load_extension(False)
+    db.row_factory = sqlite3.Row
+    return db
+
+
+def _fts_query(q: str) -> str:
+    terms = re.findall(r"[A-Za-z0-9]+", q)
+    return " OR ".join(f'"{t}"' for t in terms) if terms else '""'
+
+
+def search(db: sqlite3.Connection, query_vec: list[float], query_text: str, k: int = 8) -> list[Hit]:
+    vblob = struct.pack(f"{len(query_vec)}f", *query_vec)
+    vec_ids = [
+        r[0]
+        for r in db.execute(
+            "SELECT rowid FROM vec_chunks WHERE embedding MATCH ? AND k = ? ORDER BY distance",
+            (vblob, POOL),
+        )
+    ]
+    try:
+        fts_ids = [
+            r[0]
+            for r in db.execute(
+                "SELECT rowid FROM fts_chunks WHERE fts_chunks MATCH ? ORDER BY rank LIMIT ?",
+                (_fts_query(query_text), POOL),
+            )
+        ]
+    except sqlite3.OperationalError:
+        fts_ids = []
+
+    ranks: dict[int, float] = {}
+    for lst in (vec_ids, fts_ids):
+        for rank, cid in enumerate(lst):
+            ranks[cid] = ranks.get(cid, 0.0) + 1.0 / (RRF_K + rank)
+
+    top = sorted(ranks, key=ranks.get, reverse=True)[:k]
+    if not top:
+        return []
+    rows = {
+        r["id"]: r
+        for r in db.execute(
+            f"SELECT * FROM chunks WHERE id IN ({','.join('?' * len(top))})", top
+        )
+    }
+    return [
+        Hit(
+            id=cid,
+            source=rows[cid]["source"],
+            title=rows[cid]["title"],
+            url=rows[cid]["url"],
+            section=rows[cid]["section"],
+            text=rows[cid]["text"],
+            score=ranks[cid],
+        )
+        for cid in top
+        if cid in rows
+    ]