diff --git a/.dockerignore b/.dockerignore index 4efb037..bc12e04 100644 --- a/.dockerignore +++ b/.dockerignore @@ -33,6 +33,10 @@ tmp/ output/ models/ video/ +migrate/ +*.tar.gz +*.tar +*.zip *.mp4 *.wav *.docx @@ -60,3 +64,9 @@ server.log .env.local *.key *.secret + +# OpenSpec / Git +openspec/ +.opencode/ +openspec/changes/ +openspec/specs/ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2d1cd48 --- /dev/null +++ b/.env.example @@ -0,0 +1,18 @@ +# WhisperX + Native Python RAG transcription service +# Скопируйте в .env: `cp .env.example .env` (или `make init`) + +# HuggingFace (для pyannote-диаризации) +# Регистрация: https://huggingface.co/settings/tokens +HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +# OpenCode / DeepSeek API (LLM для классификации и чат-ответов) +OPENCODE_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxx +OPENCODE_URL=https://opencode.ai/zen/v1 + +# JWT-секрет для аутентификации +# Сгенерируйте: openssl rand -hex 32 +JWT_SECRET=change-me-in-production-32-bytes-random-hex + +# Пароль админа по умолчанию (при первом бутстрапе) +# Смените сразу после первого входа! +AUTH_ADMIN_PASSWORD=admin123 diff --git a/.gitignore b/.gitignore index 087ef9a..103a050 100644 --- a/.gitignore +++ b/.gitignore @@ -60,8 +60,8 @@ processed/ .DS_Store Thumbs.db -# Secrets (раскомментируйте для публичных репозиториев) -# .env +# Secrets (даже в приватном Gitea лучше держать .env вне репо) +.env .env.local *.key *.secret @@ -76,3 +76,18 @@ server.log # Auth database data/ + +# Migration archives (do not commit — see scripts/migrate_lightrag_to_qmd.py) +migrate/ +*.tar.gz +*.tar + +# OpenSpec local state (changes/specs are committed; archive metadata is fine) +.opencode/ + +# Build artifacts +_docker_out/ +_*.log +_build*.log +_smoke_*.py +_debug*.py diff --git a/AGENTS.md b/AGENTS.md index f008426..8c01da7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,3 +6,46 @@ - **Push to remote**: Once the local commit(s) are ready, push them to the remote repository. Use `git push -u origin main` if the upstream branch is not yet tracked; otherwise use `git push`. - **No uncommitted changes left behind**: Before finishing a task, ensure all intended changes are committed and pushed to avoid losing work. - **No empty commits**: Avoid creating empty or placeholder commits. + +## Native RAG Engine + +The project uses a **native Python RAG engine** (no external daemons, no Node.js): +hybrid BM25 (SQLite FTS5) + vector (sqlite-vec with numpy fallback) + LLM rerank +through OpenCode. + +### Layout + +- `src/rag/engine/` — the engine itself: + - `db.py` — `Database` (SQLite + sqlite-vec + FTS5 schema, fallback detection). + - `chunker.py` — markdown-aware recursive splitter (~900 chars, 15% overlap). + - `embeddings.py` — singleton sentence-transformers model (lazy load). + - `bm25.py` — FTS5 BM25 with `rank_bm25` fallback. + - `vector.py` — sqlite-vec with numpy cosine fallback. + - `hybrid.py` — RRF fusion (k=60). + - `rerank.py` — LLM rerank through OpenCode. + - `engine.py` — public facade: `index_file`, `index_text`, `search`, `vsearch`, `query`, `get`, `status`, `warmup`. +- `src/rag/qmd/` — compatibility layer preserving the old `qmd_*` API: + `qmd_query`, `qmd_chat`, `qmd_chat_stream`, `qmd_index_meeting`, `qmd_index_document`. + `main.py` / `queue.py` / `ingest_worker.py` use these. +- `src/ingest/stub_writer.py` — `.md` stubs for binary files (videos, archives). + +### Conventions + +- Коллекция = `processed//qmd_collections//` (или `_global/`) — внутри лежит `index.sqlite`. +- Перед изменением `src/rag/engine/` — прочитай `openspec/changes/native-rag-engine/design.md`. +- При добавлении нового retrieval-режима — обнови `LEGACY_MODE_MAP` в `src/rag/qmd/query.py`. +- При добавлении нового LLM-вызова — обнови `CHAT_MODES` в `src/rag/qmd/query.py`. + +### Tests + +- Все новые модули `src/rag/engine/` обязаны иметь unit-тест в `tests/test_native_engine.py`. +- Реальные данные: 3–5 `.md` файлов в `tempfile.TemporaryDirectory()`. +- Запуск: `python -m pytest tests/ -q` (46 passed на момент написания). +- E2E: `tests/test_native_engine_e2e.py` — ingest → search → chat-stream с подменой OpenCode. + +### Fallback-стратегии + +- FTS5 недоступен → `rank_bm25` in-memory. +- sqlite-vec недоступен → numpy cosine in-memory. +- Embedding-модель не загрузилась → BM25-only режим. + diff --git a/Dockerfile b/Dockerfile index dd7d141..bc0dfaf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11-slim-bookworm -# Установка системных зависимостей +# Системные зависимости (без Node.js — native RAG engine работает in-process) RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg \ build-essential \ @@ -11,21 +11,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ tesseract-ocr-eng \ && rm -rf /var/lib/apt/lists/* -# Рабочая директория WORKDIR /app -# Копируем зависимости +# Python-зависимости (включают native RAG: sentence-transformers, rank_bm25, sqlite-vec) COPY requirements.txt pip.conf ./ RUN pip install --no-cache-dir --timeout 300 -r requirements.txt -# Копируем код проекта COPY . . -# Создаём директории для данных -RUN mkdir -p uploads processed tmp +RUN mkdir -p uploads processed tmp data -# Открываем порт EXPOSE 8000 -# Запускаем напрямую (модели скачаются при первом запуске, не при сборке!) CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Dockerfile.rag b/Dockerfile.rag index 94e4dcf..687bfdc 100644 --- a/Dockerfile.rag +++ b/Dockerfile.rag @@ -1,15 +1,26 @@ -# Быстрое обновление: добавляет RAG-зависимости к уже собранному образу с Whisper/PyTorch. -FROM transcription-transcription:latest +FROM python:3.11-slim-bookworm -COPY pip.conf /etc/pip.conf -RUN pip install --no-cache-dir --timeout 300 \ - lightrag-hku>=1.4.0 \ - openai>=1.0.0 \ - python-dotenv>=1.0.0 \ - sentence-transformers>=3.0.0 \ - bcrypt>=4.0.0 \ - "python-jose[cryptography]" \ - pymupdf>=1.24.0 \ - openpyxl>=3.1.0 \ - Pillow>=10.0.0 \ - pytesseract>=0.3.10 +# Native RAG engine — никаких внешних сервисов не требуется +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + build-essential \ + libsndfile1 \ + curl \ + ca-certificates \ + tesseract-ocr \ + tesseract-ocr-rus \ + tesseract-ocr-eng \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt pip.conf ./ +RUN pip install --no-cache-dir --timeout 300 -r requirements.txt + +COPY . . + +RUN mkdir -p uploads processed tmp data + +EXPOSE 8000 + +CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..30572d7 --- /dev/null +++ b/Makefile @@ -0,0 +1,50 @@ +SHELL := /bin/bash +.DEFAULT_GOAL := help + +# Detect whether `docker compose` v2 or `docker-compose` v1 is available +COMPOSE := $(shell command -v docker-compose 2>/dev/null || echo "docker compose") + +.PHONY: help init pull-models up down restart logs status clean test deploy + +help: ## Показать эту справку + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +init: ## Создать .env из .env.example (если его нет) + @if [ ! -f .env ]; then \ + cp .env.example .env && \ + echo "Created .env from .env.example"; \ + echo "==> Edit .env with real secrets, then run: make deploy"; \ + else \ + echo ".env already exists, skipping"; \ + fi + +pull-models: ## Загрузить модели (rsync + fallback на download_models.py) + @bash scripts/pull_models.sh + +up: ## Запустить контейнеры (build + -d) + $(COMPOSE) up --build -d + @$(COMPOSE) logs --tail=30 transcription || true + +down: ## Остановить контейнеры + $(COMPOSE) down + +restart: ## Перезапустить transcription + $(COMPOSE) restart transcription + +logs: ## Логи transcription (follow) + $(COMPOSE) logs -f transcription + +status: ## Статус контейнеров + $(COMPOSE) ps + @echo "" + @cmd /c "curl -s http://localhost:8000/api/health" 2>/dev/null || \ + curl -s http://localhost:8000/api/health || \ + echo "(service not responding on :8000)" + +clean: ## Удалить контейнеры + volumes (ОСТОРОЖНО: стирает uploads/processed/data) + $(COMPOSE) down -v + +test: ## Прогнать pytest + pytest tests/ -q + +deploy: pull-models up ## Полный деплой: модели + запуск diff --git a/README.md b/README.md index fce62ed..7c7f8e1 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ - **WhisperX** — ASR + alignment + диаризация (всё-в-одном) - **python-docx** — генерация `.docx` - **PyYAML** — конфигурация +- **Native Python RAG engine** — гибридный поиск BM25 (FTS5) + vector (sqlite-vec) + LLM-реранкер +- **OpenCode / DeepSeek** — LLM для классификации и чат-ответов ## Установка @@ -315,6 +317,98 @@ docker compose down -v Данные сохраняются между перезапусками контейнера. +## 🧠 База знаний / RAG (Native Python Engine) + +База знаний — **in-process Python-движок**: гибридный поиск BM25 (SQLite FTS5) + vector (sqlite-vec с numpy fallback) + LLM-реранкер через OpenCode. Хранение: один `index.sqlite` на коллекцию. **Внешних сервисов не требуется.** + +### Архитектура + +``` +аудио/видео → WhisperX → extracted.md, summary.md +документы (PDF/DOCX/XLSX/...) → extracted.md +бинарники (.mp4, .zip) → stub_writer → *.md с YAML frontmatter + ↓ + Native Python RAG engine (in-process) + ├─ chunker (markdown-aware, 900 chars, 15% overlap) + ├─ embeddings (sentence-transformers, 384 dim, мультиязычный) + ├─ FTS5 BM25 + sqlite-vec cosine + ├─ RRF fusion (k=60) + └─ LLM rerank (OpenCode/DeepSeek, опционально) + ↓ + OpenCode / DeepSeek chat-completions + ↓ + WebSocket → rag_context → rag_chunk* → rag_response +``` + +### Установка зависимостей + +Native engine использует только Python-пакеты (никаких npm/node): + +```bash +pip install -r requirements.txt +# Скачает ~50 MB модели при первом запуске (sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2) +``` + +### Конфигурация + +Секция `rag:` в `config.yaml`: + +```yaml +rag: + enabled: true + auto_index: true + qmd_collection_root: ./processed # корень коллекций + qmd_use_rerank: true # LLM-реранкер через OpenCode + embed_model: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 + chat_model: deepseek-v4-flash-free + chat_max_tokens: 8192 +``` + +ENV-переменные: `QMD_COLLECTION_ROOT`, `RAG_EMBED_MODEL`, `OPENCODE_API_KEY`, `OPENCODE_URL`. + +### Где хранятся индексы + +``` +processed/ + / + qmd_collections/ # ← native engine (in-process, без демонов) + / + index.sqlite # FTS5 + sqlite-vec + chunks + _global/ + index.sqlite # cross-project поиск + meetings// # .docx, .md, .json для совещаний + documents// # .pdf, extracted.md, metadata.json + lightrag_caches/ # legacy: для миграции +``` + +### Поддерживаемые форматы и stub'ы + +Движок индексирует `.md`/`.txt` нативно. Для бинарных форматов +(видео, не-OCR PDF, архивы) `src/ingest/stub_writer.py` создаёт +`.md` со ссылкой на оригинал. Пользователь видит stub в результатах +поиска и кликает на ссылку — открывается оригинал. + +### Fallback-стратегии + +- **FTS5 недоступен** в системной сборке Python → `rank_bm25` in-memory. +- **sqlite-vec недоступен** → numpy cosine in-memory. +- **Embedding-модель не загрузилась** → BM25-only режим, `qmd: degraded` в healthcheck. + +### Legacy-миграция с LightRAG + +```bash +# 1. Снапшот +tar -czf ../processed-pre-qmd.tar.gz processed + +# 2. dry-run +python scripts/migrate_lightrag_to_qmd.py --org merakom --dry-run + +# 3. реальная миграция +python scripts/migrate_lightrag_to_qmd.py --org merakom +``` + +Скрипт идемпотентен: повторный запуск безопасен. См. `scripts/README.md`. + ## 🌐 Веб-интерфейс Проект включает веб-сервис с минималистичным фронтендом для удобной работы через браузер. diff --git a/backend/ingest_worker.py b/backend/ingest_worker.py index 893bea8..b8b615d 100644 --- a/backend/ingest_worker.py +++ b/backend/ingest_worker.py @@ -7,12 +7,13 @@ from datetime import datetime from pathlib import Path from typing import Any, Dict -from backend.paths import org_documents_dir, org_rag_index_dir, write_folder_project_meta +from backend.paths import org_documents_dir, org_qmd_root, write_folder_project_meta from src.config import load_config, resolve_opencode_credentials from src.ingest.classify import classify_document from src.ingest.formatter import format_global_index_document, format_index_document -from src.ingest.router import extract_document -from src.rag.indexer import index_meeting +from src.ingest.router import extract_document, is_extractable +from src.ingest.stub_writer import write_stub +from src.rag.qmd.indexer import qmd_index_document async def process_document_ingest(job: Dict[str, Any], tasks: dict, send_progress): @@ -31,39 +32,60 @@ async def process_document_ingest(job: Dict[str, Any], tasks: dict, send_progres ingest_cfg = config.get("ingest", {}) pdf_ocr = ingest_cfg.get("pdf_ocr", True) - doc = await asyncio.to_thread( - extract_document, - file_path, - project_slug, - doc_type, - None, - pdf_ocr, - ) - - if not doc.full_text.strip(): - raise ValueError("Не удалось извлечь текст из документа") - documents_dir = org_documents_dir(org_slug) - output_dir = documents_dir / doc.document_id + output_dir = documents_dir / f"doc_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{task_id[-8:]}" await asyncio.to_thread(output_dir.mkdir, parents=True, exist_ok=True) original_dest = output_dir / file_path.name await asyncio.to_thread(shutil.copy2, file_path, original_dest) - await asyncio.to_thread( - (output_dir / "extracted.md").write_text, - doc.full_text, - encoding="utf-8", - ) await asyncio.to_thread(write_folder_project_meta, output_dir, project_slug) + if is_extractable(file_path.name): + doc = await asyncio.to_thread( + extract_document, + file_path, + project_slug, + doc_type, + None, + pdf_ocr, + ) + if not doc.full_text.strip(): + raise ValueError("Не удалось извлечь текст из документа") + await asyncio.to_thread( + (output_dir / "extracted.md").write_text, + doc.full_text, + encoding="utf-8", + ) + else: + try: + stub = await asyncio.to_thread( + write_stub, file_path, project_slug + ) + print(f"[Ingest] {task_id}: создан stub {stub.name}") + except (FileNotFoundError, OSError) as exc: + print(f"[Ingest] {task_id}: stub_writer failed: {exc}") + doc = type("StubDoc", (), {})() + doc.full_text = "" + doc.document_id = output_dir.name + doc.filename = file_path.name + doc.doc_type = doc_type + doc.metadata = {} + doc.to_metadata_dict = lambda: { + "document_id": doc.document_id, + "filename": doc.filename, + "doc_type": doc.doc_type, + "project": project_slug, + "stub": True, + } + tasks[task_id].update({"status": "postprocessing", "message": "Анализ документа...", "progress": 40}) await send_progress(task_id, 40, "Анализ документа...", "postprocessing") - metadata = doc.to_metadata_dict() + metadata = doc.to_metadata_dict() if callable(getattr(doc, "to_metadata_dict", None)) else {} rag_cfg = config.get("rag", {}) api_key, base_url = resolve_opencode_credentials(config) - if api_key and ingest_cfg.get("auto_classify", True): + if api_key and ingest_cfg.get("auto_classify", True) and doc.full_text: metadata = await classify_document( text=doc.full_text, project=project_slug, @@ -82,34 +104,37 @@ async def process_document_ingest(job: Dict[str, Any], tasks: dict, send_progres encoding="utf-8", ) - doc_text = format_index_document(doc, metadata) - index_path = output_dir / "index.txt" - await asyncio.to_thread(index_path.write_text, doc_text, encoding="utf-8") + if doc.full_text: + doc_text = format_index_document(doc, metadata) + index_path = output_dir / "index.txt" + await asyncio.to_thread(index_path.write_text, doc_text, encoding="utf-8") + else: + doc_text = "" + index_path = None result_data = { - "document_id": doc.document_id, + "document_id": getattr(doc, "document_id", output_dir.name), "dir": str(output_dir), "rel_dir": str(output_dir.relative_to(documents_dir)), - "extracted": str(output_dir / "extracted.md"), - "index": str(index_path), + "extracted": str(output_dir / "extracted.md") if (output_dir / "extracted.md").exists() else None, + "index": str(index_path) if index_path else None, "project": project_slug, "doc_type": metadata.get("doc_type", doc_type), "kind": "document", } if rag_cfg.get("enabled", False) and rag_cfg.get("auto_index", True): - tasks[task_id].update({"message": "Индексация в RAG...", "progress": 75}) - await send_progress(task_id, 75, "Индексация в RAG...", "postprocessing") - global_doc_text = format_global_index_document(doc_text, metadata) - await index_meeting( - doc_text=doc_text, - global_doc_text=global_doc_text, - project_name=project_slug, - working_dir_base=org_rag_index_dir(org_slug), - model=rag_cfg.get("index_model", "mimo-v2.5-free"), - api_key=api_key, - base_url=base_url, - ) + tasks[task_id].update({"message": "Индексация в qmd...", "progress": 75}) + await send_progress(task_id, 75, "Индексация в qmd...", "postprocessing") + try: + await qmd_index_document( + org_slug=org_slug, + project_slug=project_slug, + document_dir=output_dir, + extracted_md=output_dir / "extracted.md", + ) + except Exception as idx_exc: + print(f"[Ingest] {task_id}: qmd index failed: {idx_exc}") from backend.queue import _cleanup_upload await asyncio.to_thread(_cleanup_upload, file_path) diff --git a/backend/main.py b/backend/main.py index bae11a4..98bcac6 100644 --- a/backend/main.py +++ b/backend/main.py @@ -16,7 +16,7 @@ from backend.auth.models import UserContext from backend.auth.routes import admin_router, router as auth_router from backend.auth import database as auth_db from backend.auth.service import ensure_project_access, list_accessible_projects -from backend.paths import org_documents_dir, org_meetings_dir, org_rag_index_dir, resolve_document_path, resolve_meeting_path +from backend.paths import org_documents_dir, org_meetings_dir, org_qmd_root, org_rag_index_dir, resolve_document_path, resolve_meeting_path from backend.queue import ( delete_folder, get_all_tasks, @@ -35,8 +35,10 @@ from backend.queue import ( sys.path.insert(0, str(Path(__file__).parent.parent)) from src.config import load_config, resolve_opencode_credentials from src.rag.formatter import format_global_document -from src.rag.indexer import get_project_names, index_meeting -from src.rag.query import rag_chat +from src.rag.qmd import collections as qmd_collections +from src.rag.qmd.cache import get_search_cache +from src.rag.qmd.indexer import qmd_index_document, qmd_index_meeting +from src.rag.qmd.query import map_retrieval_mode, qmd_chat, qmd_chat_stream STATIC_DIR = Path(__file__).parent / "static" @@ -76,6 +78,7 @@ set_progress_callback(manager.broadcast) @asynccontextmanager async def lifespan(app: FastAPI): + from src.rag.qmd.lifespan import qmd_lifespan config = load_config() auth_db.init_db(config) auth_db.bootstrap_from_config(config) @@ -92,9 +95,11 @@ async def lifespan(app: FastAPI): postprocess_workers=postprocess_workers, ingest_workers=ingest_workers, ) - yield + async with qmd_lifespan(app): + yield print("🛑 Остановка рабочих процессов...") stop_workers() + get_search_cache().clear() app = FastAPI(title="Transcription Service", version="2.0.0", lifespan=lifespan) @@ -111,15 +116,51 @@ app.include_router(auth_router) app.include_router(admin_router) -def _org_index_dir(user: UserContext) -> Path: - return org_rag_index_dir(user.org_slug) +def _org_qmd_root(user: UserContext) -> Path: + return org_qmd_root(user.org_slug) async def _list_rag_project_slugs(user: UserContext) -> List[str]: - projects = await get_project_names(_org_index_dir(user)) + root = _org_qmd_root(user) + if not root.exists(): + return [] + projects = sorted( + p.name for p in root.iterdir() + if p.is_dir() and p.name != "_global" + ) return user.filter_projects(projects) +def _validate_rag_access(user: UserContext, project_name: Optional[str]) -> None: + if project_name: + ensure_project_access(user, project_name) + elif not user.can_global_search(): + raise HTTPException(status_code=403, detail="Глобальный поиск доступен только администратору") + + +def _normalize_rag_modes(chat_mode: str, retrieval_mode: str) -> tuple[str, str]: + if chat_mode not in ("hybrid", "compare", "timeline"): + chat_mode = "hybrid" + if retrieval_mode not in ("naive", "local", "global", "hybrid"): + retrieval_mode = "hybrid" + return chat_mode, retrieval_mode + + +def _rag_params(user: UserContext, chat_mode: str, retrieval_mode: str) -> dict: + config = load_config() + rag_cfg = config.get("rag", {}) + api_key, base_url = resolve_opencode_credentials(config) + return { + "org_slug": user.org_slug, + "api_key": api_key, + "chat_model": rag_cfg.get("chat_model", "deepseek-v4-flash-free"), + "max_tokens": int(rag_cfg.get("chat_max_tokens", 8192)), + "mode": retrieval_mode, + "chat_mode": chat_mode, + "use_rerank": bool(rag_cfg.get("qmd_use_rerank", True)), + } + + async def _rag_chat_for_user( user: UserContext, question: str, @@ -128,30 +169,14 @@ async def _rag_chat_for_user( chat_mode: str = "hybrid", retrieval_mode: str = "hybrid", ): - if project_name: - ensure_project_access(user, project_name) - elif not user.can_global_search(): - raise HTTPException(status_code=403, detail="Глобальный поиск доступен только администратору") - - if chat_mode not in ("hybrid", "compare", "timeline"): - chat_mode = "hybrid" - if retrieval_mode not in ("naive", "local", "global", "hybrid"): - retrieval_mode = "hybrid" - - config = load_config() - rag_cfg = config.get("rag", {}) - api_key, base_url = resolve_opencode_credentials(config) - return await rag_chat( + _validate_rag_access(user, project_name) + chat_mode, retrieval_mode = _normalize_rag_modes(chat_mode, retrieval_mode) + params = _rag_params(user, chat_mode, retrieval_mode) + return await qmd_chat( question=question, - working_dir_base=_org_index_dir(user), history=history, - api_key=api_key, - project_name=project_name, - base_url=base_url, - chat_model=rag_cfg.get("chat_model", "deepseek-v4-flash-free"), - mode=retrieval_mode, - chat_mode=chat_mode, - index_model=rag_cfg.get("index_model", "mimo-v2.5-free"), + project_slug=project_name, + **params, ) @@ -329,6 +354,18 @@ async def api_download(path: str, user: UserContext = Depends(get_current_user)) async def api_delete_folder(folder_name: str, user: UserContext = Depends(get_current_user)): try: delete_folder(user, folder_name) + from backend.queue import _folder_project_slug + if folder_name.startswith("documents/"): + base_dir = org_documents_dir(user.org_slug) + else: + base_dir = org_meetings_dir(user.org_slug) + folder_name_only = folder_name.split("/")[-1] + project = _folder_project_slug(folder_name_only, base_dir) + if project: + try: + await qmd_collections.embed(user.org_slug, project) + except Exception as idx_exc: + print(f"[delete_folder] qmd reindex skipped: {idx_exc}") return {"deleted": folder_name} except PermissionError as e: raise HTTPException(status_code=403, detail=str(e)) from e @@ -395,13 +432,10 @@ async def api_rag_index_folder(folder_name: str, user: UserContext = Depends(get if folder_name.startswith("documents/"): folder_path = resolve_document_path(user.org_slug, folder_name[len("documents/"):]) base_dir = org_documents_dir(user.org_slug) - index_files = list(folder_path.glob("index.txt")) - txt_files = index_files else: rel = folder_name[len("meetings/"):] if folder_name.startswith("meetings/") else folder_name folder_path = resolve_meeting_path(user.org_slug, rel) base_dir = org_meetings_dir(user.org_slug) - txt_files = list(folder_path.glob("*.txt")) if not folder_path.exists(): return {"error": "Folder not found"} @@ -411,24 +445,10 @@ async def api_rag_index_folder(folder_name: str, user: UserContext = Depends(get return {"error": "Project metadata not found"} ensure_project_access(user, project) - if not txt_files: - return {"error": "No index.txt or .txt protocol found in folder"} - - doc_text = txt_files[0].read_text(encoding="utf-8") - config = load_config() - rag_cfg = config.get("rag", {}) - api_key, base_url = resolve_opencode_credentials(config) - metadata = {"project": project, "section": "Общие вопросы", "topic": "Переиндексация"} - global_doc_text = format_global_document(doc_text, metadata) - - await index_meeting( - doc_text=doc_text, - global_doc_text=global_doc_text, - project_name=project, - working_dir_base=_org_index_dir(user), - model=rag_cfg.get("index_model", "mimo-v2.5-free"), - api_key=api_key, - base_url=base_url, + await qmd_index_document( + org_slug=user.org_slug, + project_slug=project, + document_dir=folder_path, ) return {"indexed": folder_name, "project": project} except PermissionError as e: @@ -442,20 +462,43 @@ async def _handle_rag_query_ws(websocket: WebSocket, msg: dict, user: UserContex project = msg.get("project") if msg.get("action") == "rag_query_global": project = None - result = await _rag_chat_for_user( - user, - msg.get("question", ""), - msg.get("history", []), - project, - chat_mode=msg.get("chat_mode", msg.get("mode", "hybrid")), - retrieval_mode=msg.get("retrieval_mode", "hybrid"), - ) - await websocket.send_json({ - "type": "rag_response", - "answer": result["answer"], - "context": result["context"], - "project": result["project"], - }) + + chat_mode = msg.get("chat_mode", msg.get("mode", "hybrid")) + retrieval_mode = msg.get("retrieval_mode", "hybrid") + _validate_rag_access(user, project) + chat_mode, retrieval_mode = _normalize_rag_modes(chat_mode, retrieval_mode) + params = _rag_params(user, chat_mode, retrieval_mode) + + async for event in qmd_chat_stream( + question=msg.get("question", ""), + history=msg.get("history", []), + project_slug=project, + **params, + ): + event_type = event.get("type") + if event_type == "context": + await websocket.send_json({ + "type": "rag_context", + "context": event["context"], + "project": event["project"], + }) + elif event_type == "chunk": + await websocket.send_json({ + "type": "rag_chunk", + "content": event["content"], + }) + elif event_type == "done": + await websocket.send_json({ + "type": "rag_response", + "answer": event["answer"], + "context": event["context"], + "project": event["project"], + }) + elif event_type == "rag_error": + await websocket.send_json({ + "type": "rag_error", + "error": event.get("error", "unknown error"), + }) except HTTPException as e: await websocket.send_json({"type": "rag_error", "error": e.detail}) except PermissionError as e: @@ -466,7 +509,11 @@ async def _handle_rag_query_ws(websocket: WebSocket, msg: dict, user: UserContex @app.get("/api/health") async def health(): - return {"status": "ok"} + return { + "status": "ok", + "qmd": getattr(app.state, "qmd_status", "down"), + "search_cache": get_search_cache().stats(), + } app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") diff --git a/backend/paths.py b/backend/paths.py index 7fb4c40..7ee5d86 100644 --- a/backend/paths.py +++ b/backend/paths.py @@ -1,6 +1,7 @@ """Org-scoped filesystem paths.""" import json +import os from datetime import datetime from pathlib import Path @@ -8,6 +9,7 @@ DATA_ROOT = Path("data") UPLOAD_ROOT = Path("uploads") PROCESSED_ROOT = Path("processed") RAG_CACHE_DIRNAME = "lightrag_caches" +QMD_COLLECTIONS_DIRNAME = "qmd_collections" MEETINGS_DIRNAME = "meetings" DOCUMENTS_DIRNAME = "documents" @@ -25,11 +27,21 @@ def org_meetings_dir(org_slug: str) -> Path: def org_rag_index_dir(org_slug: str) -> Path: + """Legacy: путь к lightrag_caches// (deprecated, kept for migration).""" path = PROCESSED_ROOT / org_slug / RAG_CACHE_DIRNAME path.mkdir(parents=True, exist_ok=True) return path +def org_qmd_root(org_slug: str) -> Path: + """Корень qmd-коллекций организации: ``processed//qmd_collections/``.""" + env_root = os.environ.get("QMD_COLLECTION_ROOT") + base = Path(env_root) if env_root else PROCESSED_ROOT + path = base / org_slug / QMD_COLLECTIONS_DIRNAME + path.mkdir(parents=True, exist_ok=True) + return path + + def org_documents_dir(org_slug: str) -> Path: path = PROCESSED_ROOT / org_slug / DOCUMENTS_DIRNAME path.mkdir(parents=True, exist_ok=True) diff --git a/backend/queue.py b/backend/queue.py index 255dc9d..6b11a92 100644 --- a/backend/queue.py +++ b/backend/queue.py @@ -13,7 +13,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from backend.auth.models import UserContext from backend.auth.service import ensure_project_access -from backend.paths import org_documents_dir, org_meetings_dir, org_rag_index_dir, org_upload_dir, resolve_document_path, resolve_meeting_path, write_folder_project_meta +from backend.paths import org_documents_dir, org_meetings_dir, org_qmd_root, org_upload_dir, resolve_document_path, resolve_meeting_path, write_folder_project_meta from src.ingest.router import is_audio_file, is_document_file from src.audio_utils import prepare_audio_input from src.config import load_config, resolve_opencode_credentials @@ -25,8 +25,8 @@ from src.rag.formatter import ( format_meeting_document, format_summary_markdown, ) -from src.rag.indexer import index_meeting from src.rag.parser import classify_meeting, generate_meeting_brief +from src.rag.qmd.indexer import qmd_index_meeting tasks: Dict[str, Dict[str, Any]] = {} _progress_callback: Optional[Callable] = None @@ -264,6 +264,8 @@ async def process_postprocessing(job: Dict[str, Any]): model=summary_model, chunk_size=summary_chunk_size, ) + if not (brief or "").strip(): + print(f"[Postprocess Warning] {task_id}: пустое краткое содержание summary") summary_md = format_summary_markdown(metadata, brief, display_name) await asyncio.to_thread(summary_path.write_text, summary_md, encoding="utf-8") result_data["summary"] = str(summary_path) @@ -276,17 +278,19 @@ async def process_postprocessing(job: Dict[str, Any]): result_data["rel_dir"] = str(output_dir.relative_to(meetings_dir)) if rag_cfg.get("enabled", False) and rag_cfg.get("auto_index", True): - await _send_progress(task_id, 92, "Индексация в базу знаний...", "postprocessing") - index_dir = org_rag_index_dir(org_slug) - global_doc_text = format_global_document(doc_text, metadata) - await index_meeting( - doc_text=doc_text, - global_doc_text=global_doc_text, - project_name=project, - working_dir_base=index_dir, - model=index_model, - api_key=api_key, - base_url=base_url, + await _send_progress(task_id, 92, "Индексация в qmd...", "postprocessing") + qmd_root = org_qmd_root(org_slug) + context_description = ( + f"Протоколы совещаний и заметки проекта '{project}', " + f"раздел: {metadata.get('section', 'Общие вопросы')}" + ) + await qmd_index_meeting( + org_slug=org_slug, + project_slug=project, + body_path=txt_path, + summary_path=summary_path, + txt_path=txt_path, + context_description=context_description, ) except Exception as e: print(f"[Postprocess Warning] {task_id}: {e}") diff --git a/backend/static/app.js b/backend/static/app.js index b116b40..85b3dea 100644 --- a/backend/static/app.js +++ b/backend/static/app.js @@ -12,6 +12,8 @@ class TranscriptionApp { this.chatProjects = []; this.uploadProjects = []; this.isChatThinking = false; + this.streamingBubble = null; + this.streamingText = ''; this.user = null; this.init(); } @@ -210,6 +212,10 @@ class TranscriptionApp { this.updateQueueSummary(data.queue); } else if (data.type === 'file_tree') { this.renderFileTree(data.tree); + } else if (data.type === 'rag_context') { + this.handleChatContext(data); + } else if (data.type === 'rag_chunk') { + this.handleChatChunk(data); } else if (data.type === 'rag_response') { this.handleChatResponse(data); } else if (data.type === 'rag_error') { @@ -686,6 +692,8 @@ class TranscriptionApp { } input.value = ''; + this.streamingBubble = null; + this.streamingText = ''; this.addChatBubble('user', question); this.setChatThinking(true); @@ -700,14 +708,58 @@ class TranscriptionApp { }); } + handleChatContext(data) { + const hasContext = data.context && data.context.trim().length > 0; + const label = hasContext ? 'Формирую ответ...' : 'Контекст не найден, отвечаю...'; + this.setChatThinking(true, label); + } + + handleChatChunk(data) { + const chunk = data.content || ''; + if (!chunk) return; + + if (!this.streamingBubble) { + this.setChatThinking(false); + const container = document.getElementById('chatMessages'); + const bubble = document.createElement('div'); + bubble.className = 'chat-bubble bot streaming'; + const body = document.createElement('div'); + body.className = 'bubble-body md-content'; + bubble.appendChild(body); + container.appendChild(bubble); + container.scrollTop = container.scrollHeight; + this.streamingBubble = bubble; + this.streamingText = ''; + } + + this.streamingText += chunk; + const body = this.streamingBubble.querySelector('.bubble-body'); + if (body) { + body.textContent = this.streamingText; + } + const container = document.getElementById('chatMessages'); + container.scrollTop = container.scrollHeight; + } + handleChatResponse(data) { this.setChatThinking(false); const answer = data.answer || 'Нет ответа'; const project = data.project ? `Проект: ${data.project}` : 'Все проекты'; const html = `
${this.renderMarkdown(answer)}
`; - this.addChatBubble('bot', html, { isHtml: true, meta: project }); - // Сохраняем в историю + if (this.streamingBubble) { + this.streamingBubble.innerHTML = html; + this.streamingBubble.classList.remove('streaming'); + const meta = document.createElement('div'); + meta.className = 'bubble-sources'; + meta.textContent = project; + this.streamingBubble.appendChild(meta); + this.streamingBubble = null; + this.streamingText = ''; + } else { + this.addChatBubble('bot', html, { isHtml: true, meta: project }); + } + const lastUserMsg = this.getLastUserMessage(); if (lastUserMsg) { this.chatHistory.push({ question: lastUserMsg, answer: answer }); @@ -717,6 +769,8 @@ class TranscriptionApp { handleChatError(error) { this.setChatThinking(false); + this.streamingBubble = null; + this.streamingText = ''; this.addChatBubble('bot', `Ошибка: ${this.escapeHtml(error || 'Неизвестная ошибка')}`); } @@ -729,15 +783,17 @@ class TranscriptionApp { return null; } - setChatThinking(thinking) { + setChatThinking(thinking, message = 'Думаю...') { this.isChatThinking = thinking; const container = document.getElementById('chatMessages'); const existing = container.querySelector('.chat-thinking'); if (thinking) { - if (!existing) { + if (existing) { + existing.textContent = message; + } else { const div = document.createElement('div'); div.className = 'chat-thinking'; - div.textContent = 'Думаю...'; + div.textContent = message; container.appendChild(div); container.scrollTop = container.scrollHeight; } diff --git a/config.yaml b/config.yaml index 43b3a7a..cf25ec7 100644 --- a/config.yaml +++ b/config.yaml @@ -42,7 +42,8 @@ output: speaker_label_style: name # name | id | none paragraph_pause_sec: 2.0 # новый абзац, если пауза > N секунд -# Настройки RAG (LightRAG для протоколов совещаний) +# Настройки RAG (native Python engine: BM25/FTS5 + sqlite-vec + LLM rerank) +# Движок работает in-process; внешние сервисы не требуются. rag: enabled: true auto_index: true @@ -54,17 +55,26 @@ rag: - Общие вопросы - Согласование - Контроль качества - index_backend: opencode - index_model: mimo-v2.5-free + + # Native RAG engine (in-process, без демонов) + qmd_collection_root: ./processed # ENV override: QMD_COLLECTION_ROOT + qmd_use_rerank: true # LLM-реранкер (2-3 сек warm) + embed_model: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 # ENV: RAG_EMBED_MODEL + + # OpenCode LLM (чат-генерация ответов поверх найденного контекста) chat_backend: opencode chat_model: deepseek-v4-flash-free + chat_max_tokens: 8192 + opencode_api_key: "sk-4jJBUMS7WJyBOtZZAexsSy6aT4NKOYp2gA19WLlaux8jHMw0HvyCl1V45Jf8SONz" # Или через env: OPENCODE_API_KEY + opencode_url: "https://opencode.ai/zen/v1" + + # Классификация и summary (metadata layer, поверх движка) summary_auto: true summary_model: deepseek-v4-flash-free summary_chunk_size: 10000 classify_chunk_size: 7000 - opencode_api_key: "sk-4jJBUMS7WJyBOtZZAexsSy6aT4NKOYp2gA19WLlaux8jHMw0HvyCl1V45Jf8SONz" # Или через env: OPENCODE_API_KEY - opencode_url: "https://opencode.ai/zen/v1" - project_index_dir: ./processed/lightrag_caches + classify_model: mimo-v2.5-free + index_model: mimo-v2.5-free # legacy alias, используется в legacy-классификации # Очередь обработки queue: diff --git a/deploy/install_docker.sh b/deploy/install_docker.sh new file mode 100644 index 0000000..e86fe7f --- /dev/null +++ b/deploy/install_docker.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail + +SUDO_PASS="${1:-}" + +if [ -z "$SUDO_PASS" ]; then + echo "Usage: $0 " + exit 1 +fi + +run_sudo() { + echo "$SUDO_PASS" | sudo -S "$@" +} + +export DEBIAN_FRONTEND=noninteractive + +run_sudo apt-get update -y +run_sudo apt-get install -y ca-certificates curl gnupg lsb-release git + +if ! command -v docker >/dev/null 2>&1; then + curl -fsSL https://get.docker.com -o /tmp/get-docker.sh + run_sudo sh /tmp/get-docker.sh + rm -f /tmp/get-docker.sh +fi + +run_sudo usermod -aG docker art || true + +if ! docker compose version >/dev/null 2>&1; then + run_sudo apt-get install -y docker-compose-plugin || true +fi + +docker --version +docker compose version || docker-compose --version || true +echo "Docker installation complete" diff --git a/deploy/setup_server.sh b/deploy/setup_server.sh new file mode 100644 index 0000000..743de22 --- /dev/null +++ b/deploy/setup_server.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -euo pipefail + +APP_DIR="/home/art/transcription" +MIGRATE_DIR="$APP_DIR/migrate" +SUDO_PASS="${1:-}" + +cd "$APP_DIR" + +mkdir -p data uploads processed models tmp + +if [ -f "$MIGRATE_DIR/data.tar.gz" ]; then + tar xzf "$MIGRATE_DIR/data.tar.gz" -C data +fi + +if [ -f "$MIGRATE_DIR/processed.tar.gz" ]; then + tar xzf "$MIGRATE_DIR/processed.tar.gz" -C processed +fi + +if [ -f "$MIGRATE_DIR/uploads.tar.gz" ]; then + echo "Extracting uploads (may take several minutes)..." + tar xzf "$MIGRATE_DIR/uploads.tar.gz" -C uploads +fi + +if [ -f "$MIGRATE_DIR/models_huggingface.tar.gz" ]; then + echo "Extracting models (may take several minutes)..." + mkdir -p models/huggingface + tar xzf "$MIGRATE_DIR/models_huggingface.tar.gz" -C models/huggingface +fi + +if [ -f "$MIGRATE_DIR/nltk_data.tar.gz" ]; then + mkdir -p models/nltk_data + tar xzf "$MIGRATE_DIR/nltk_data.tar.gz" -C models/nltk_data +fi + +docker_cmd() { + if docker info >/dev/null 2>&1; then + docker "$@" + else + echo "$SUDO_PASS" | sudo -S docker "$@" + fi +} + +echo "Building base image..." +docker_cmd build -f Dockerfile -t transcription-transcription:latest . + +echo "Starting service..." +docker_cmd compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build + +sleep 15 +curl -sf http://localhost:8000/api/health && echo " HEALTH OK" || echo " HEALTH CHECK FAILED (service may still be starting)" diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..ec14254 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,16 @@ +# Production overrides for Linux server deployment (bind-mount data dirs) +services: + transcription: + volumes: + - ./uploads:/app/uploads + - ./processed:/app/processed + - ./tmp:/app/tmp + - ./config.yaml:/app/config.yaml:ro + - ./backend:/app/backend:ro + - ./src:/app/src:ro + - ./scripts:/app/scripts:ro + - ./models/huggingface:/root/.cache/huggingface + - ./models/nltk_data:/root/nltk_data:ro + - ./data:/app/data + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/api/health')"] diff --git a/docker-compose.yml b/docker-compose.yml index 0fce2dc..ae980d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: - HF_TOKEN=${HF_TOKEN} - OPENCODE_API_KEY=${OPENCODE_API_KEY} - OPENCODE_URL=${OPENCODE_URL:-https://opencode.ai/zen/v1} + - QMD_COLLECTION_ROOT=${QMD_COLLECTION_ROOT:-/app/processed} + - RAG_EMBED_MODEL=${RAG_EMBED_MODEL:-sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2} - HF_HOME=/root/.cache/huggingface - NLTK_DATA=/root/nltk_data - JWT_SECRET=${JWT_SECRET:-change-me-in-production} diff --git a/requirements.txt b/requirements.txt index 655eb71..313ad8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,12 +10,14 @@ whisperx bcrypt>=4.0.0 python-jose[cryptography] -# RAG / LightRAG -lightrag-hku>=1.4.0 +# RAG (native Python engine: BM25/FTS5 + sqlite-vec + LLM rerank) openai>=1.0.0 python-dotenv>=1.0.0 +httpx>=0.27.0 sentence-transformers>=3.0.0 numpy>=1.24.0 +rank_bm25>=0.2.2 +sqlite-vec>=0.1.0 # Document ingestion pymupdf>=1.24.0 diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..44c99f8 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,31 @@ +# scripts/ + +Утилиты для миграции и обслуживания. + +## migrate_lightrag_to_qmd.py + +Переиндексирует существующие ``.md`` (протоколы совещаний, извлечённые из +документов) в qmd-коллекции. Используется при переезде с LightRAG. + +```bash +# dry-run: посмотреть, что будет сделано +python scripts/migrate_lightrag_to_qmd.py --org merakom --dry-run + +# реальная миграция +python scripts/migrate_lightrag_to_qmd.py --org merakom +``` + +**Снапшот перед запуском в проде (обязательно):** + +```bash +cd transcription +tar -czf ../processed-pre-qmd.tar.gz processed +``` + +Скрипт идемпотентен: повторный запуск не дублирует данные (qmd content-hash ++ маркер ``.migrated_to_qmd`` в каждой обработанной папке). + +## Другие скрипты + +Существующие утилиты (`backfill_summaries.py`, `download_models.py` и т.д.) +работают без изменений. diff --git a/scripts/backfill_summaries.py b/scripts/backfill_summaries.py new file mode 100644 index 0000000..4e3bb3f --- /dev/null +++ b/scripts/backfill_summaries.py @@ -0,0 +1,121 @@ +"""Перегенерация пустых или отсутствующих summary для уже обработанных совещаний.""" + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from backend.paths import PROCESSED_ROOT, MEETINGS_DIRNAME +from src.config import load_config, resolve_opencode_credentials +from src.rag.formatter import build_meeting_text_only, format_summary_markdown +from src.rag.parser import classify_meeting, generate_meeting_brief + + +def _summary_body(summary_path: Path) -> str: + if not summary_path.exists(): + return "" + text = summary_path.read_text(encoding="utf-8") + if "## Суть" not in text: + return "" + return text.split("## Суть", 1)[-1].strip() + + +def _find_meeting_jobs(org_slug: str) -> list[dict]: + meetings_dir = PROCESSED_ROOT / org_slug / MEETINGS_DIRNAME + if not meetings_dir.exists(): + return [] + + jobs = [] + for folder in sorted(meetings_dir.iterdir()): + if not folder.is_dir(): + continue + segments_files = list(folder.glob("*_segments.json")) + if not segments_files: + continue + segments_path = segments_files[0] + stem = segments_path.name[: -len("_segments.json")] + summary_path = folder / f"{stem}_summary.md" + jobs.append({ + "folder": folder, + "stem": stem, + "segments_path": segments_path, + "summary_path": summary_path, + "display_name": f"{stem}.webm", + }) + return jobs + + +async def _regenerate_job(job: dict, config: dict, dry_run: bool) -> bool: + body = _summary_body(job["summary_path"]) + if body: + return False + + segments = json.loads(job["segments_path"].read_text(encoding="utf-8")) + meeting_text = build_meeting_text_only(segments) + rag_cfg = config.get("rag", {}) + api_key, base_url = resolve_opencode_credentials(config) + + meta_path = job["folder"] / ".project.json" + project = "unknown" + if meta_path.exists(): + try: + project = json.loads(meta_path.read_text(encoding="utf-8")).get("project_slug", project) + except Exception: + pass + + if not api_key: + print(f"[skip] {job['folder'].name}: нет API-ключа") + return False + + if dry_run: + print(f"[dry-run] {job['folder'].name}") + return True + + sections = rag_cfg.get("sections", ["Общие вопросы"]) + metadata = await classify_meeting( + text=meeting_text, + project=project, + sections=sections, + api_key=api_key, + base_url=base_url, + model=rag_cfg.get("index_model", "mimo-v2.5-free"), + chunk_size=int(rag_cfg.get("classify_chunk_size", 7000)), + ) + brief = await generate_meeting_brief( + text=meeting_text, + metadata=metadata, + api_key=api_key, + base_url=base_url, + model=rag_cfg.get("summary_model", "deepseek-v4-flash-free"), + chunk_size=int(rag_cfg.get("summary_chunk_size", 10000)), + ) + if not (brief or "").strip(): + print(f"[warn] {job['folder'].name}: LLM вернул пустой brief") + return False + + summary_md = format_summary_markdown(metadata, brief, job["display_name"]) + job["summary_path"].write_text(summary_md, encoding="utf-8") + print(f"[ok] {job['folder'].name} ({len(brief)} символов)") + return True + + +async def main(): + parser = argparse.ArgumentParser(description="Backfill пустых summary совещаний") + parser.add_argument("--org", default="merakom", help="org_slug") + parser.add_argument("--dry-run", action="store_true", help="Только показать, что будет обновлено") + args = parser.parse_args() + + config = load_config() + jobs = _find_meeting_jobs(args.org) + updated = 0 + for job in jobs: + if await _regenerate_job(job, config, args.dry_run): + updated += 1 + print(f"Готово: {updated} из {len(jobs)} совещаний") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100644 index 0000000..0f5d36a --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# scripts/deploy.sh — деплой кода на удалённый сервер через rsync + ssh. +# +# Использование: +# ./scripts/deploy.sh user@server /opt/transcription +# SERVER=user@server REMOTE_DIR=/opt/transcription ./scripts/deploy.sh +# +# Что делает: +# 1) rsync кода (исключая models, processed, uploads, data, tmp, .git, migrate) +# 2) Копирует .env (на текущем этапе — целиком с ключами; позже — Vault) +# 3) По SSH запускает на сервере: make pull-models && make up +# +# Переменные окружения: +# SERVER — user@host (обязательно) +# REMOTE_DIR — путь на сервере (по умолчанию /opt/transcription) +# WITH_ENV — если "1", копирует локальный .env на сервер (по умолчанию 1) +# MODELS_SOURCE — путь к моделям (если пусто — на сервере попробует fallback) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +SERVER="${1:-${SERVER:-}}" +REMOTE_DIR="${2:-${REMOTE_DIR:-/opt/transcription}}" +WITH_ENV="${WITH_ENV:-1}" + +if [ -z "$SERVER" ]; then + echo "Usage: $0 user@server [/opt/transcription]" >&2 + echo "Or: SERVER=user@server REMOTE_DIR=/path $0" >&2 + exit 1 +fi + +echo "[deploy] Target: $SERVER:$REMOTE_DIR" + +# 1. rsync кода +echo "[deploy] Syncing code..." +rsync -avz --progress \ + --exclude='.git' \ + --exclude='models/' \ + --exclude='processed/' \ + --exclude='uploads/' \ + --exclude='tmp/' \ + --exclude='data/' \ + --exclude='migrate/' \ + --exclude='__pycache__/' \ + --exclude='.pytest_cache/' \ + --exclude='.env' \ + --exclude='*.log' \ + "$PROJECT_ROOT/" "$SERVER:$REMOTE_DIR/" + +# 2. .env (если включено) +if [ "$WITH_ENV" = "1" ] && [ -f "$PROJECT_ROOT/.env" ]; then + echo "[deploy] Copying .env (with secrets)..." + scp "$PROJECT_ROOT/.env" "$SERVER:$REMOTE_DIR/.env" +else + echo "[deploy] .env not copied (WITH_ENV=$WITH_ENV). Run 'make init' on server." +fi + +# 3. Remote deploy +echo "[deploy] Triggering remote deploy..." +ssh "$SERVER" "cd $REMOTE_DIR && \ + export MODELS_SOURCE='${MODELS_SOURCE:-}' && \ + make pull-models && \ + make up" + +echo "[deploy] Done. Check logs: ssh $SERVER 'cd $REMOTE_DIR && make logs'" diff --git a/scripts/migrate_lightrag_to_qmd.py b/scripts/migrate_lightrag_to_qmd.py new file mode 100644 index 0000000..ed96f1a --- /dev/null +++ b/scripts/migrate_lightrag_to_qmd.py @@ -0,0 +1,182 @@ +"""migrate_lightrag_to_qmd.py — перенос индексов из LightRAG в qmd. + +Идемпотентный скрипт: переиндексирует существующие .md в qmd-коллекции. +Повторный запуск безопасен (qmd content-hash проверка + перезапись stub). + +Использование:: + + python scripts/migrate_lightrag_to_qmd.py [--org merakom] [--dry-run] + +Что делает: +1. Находит все ``processed//meetings//.md`` и ``_summary.md``. +2. Находит все ``processed//documents//extracted.md``. +3. По ``.project.json`` определяет проект и вызывает + ``src.rag.qmd.indexer.qmd_index_meeting`` или ``qmd_index_document``. +4. После успешной миграции помечает папку через ``.migrated_to_qmd``. + +Снапшот:: + + Перед запуском в проде: ``tar -czf processed-pre-qmd.tar.gz processed`` +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from pathlib import Path +from typing import Iterable, Optional + + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from src.config import load_config # noqa: E402 +from src.rag.qmd.indexer import qmd_index_document, qmd_index_meeting # noqa: E402 + +MIGRATION_MARKER = ".migrated_to_qmd" + + +def _iter_meetings(org_dir: Path) -> Iterable[Path]: + meetings = org_dir / "meetings" + if not meetings.exists(): + return + for folder in sorted(meetings.iterdir()): + if not folder.is_dir(): + continue + if (folder / MIGRATION_MARKER).exists(): + continue + yield folder + + +def _iter_documents(org_dir: Path) -> Iterable[Path]: + documents = org_dir / "documents" + if not documents.exists(): + return + for folder in sorted(documents.iterdir()): + if not folder.is_dir(): + continue + if (folder / MIGRATION_MARKER).exists(): + continue + yield folder + + +def _read_project(folder: Path, fallback: Path) -> Optional[str]: + meta_path = folder / ".project.json" + if meta_path.exists(): + try: + return json.loads(meta_path.read_text(encoding="utf-8")).get("project_slug") + except json.JSONDecodeError: + pass + return None + + +def _write_marker(folder: Path) -> None: + (folder / MIGRATION_MARKER).write_text( + json.dumps( + {"migrated_at": _now_iso(), "engine": "qmd"}, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + +def _now_iso() -> str: + from datetime import datetime + return datetime.now().isoformat() + + +async def migrate_org(org_slug: str, dry_run: bool = False) -> dict: + config = load_config() + rag_cfg = config.get("rag", {}) + if not (rag_cfg.get("enabled", True) and rag_cfg.get("auto_index", True)): + print(f"[migrate] RAG disabled in config — nothing to do for org='{org_slug}'") + return {"meetings": 0, "documents": 0} + + org_dir = ROOT / "processed" / org_slug + if not org_dir.exists(): + print(f"[migrate] No processed/{org_slug} — skipping") + return {"meetings": 0, "documents": 0} + + meetings_count = 0 + documents_count = 0 + errors: list[str] = [] + + for folder in _iter_meetings(org_dir): + project = _read_project(folder, org_dir / "meetings") + if not project: + print(f"[migrate] skip meeting folder {folder.name}: no project") + continue + body_path = next(iter(sorted(folder.glob("*.txt"))), None) + if body_path is None: + md_files = sorted(folder.glob("*.md")) + body_path = md_files[0] if md_files else None + if body_path is None: + print(f"[migrate] skip meeting folder {folder.name}: no .md/.txt") + continue + summary_path = next(iter(sorted(folder.glob("*_summary.md"))), None) + print(f"[migrate] meeting: org={org_slug} project={project} folder={folder.name}") + if dry_run: + continue + try: + await qmd_index_meeting( + org_slug=org_slug, + project_slug=project, + body_path=body_path, + summary_path=summary_path, + txt_path=body_path, + ) + _write_marker(folder) + meetings_count += 1 + except Exception as exc: + errors.append(f"meeting {folder.name}: {exc}") + + for folder in _iter_documents(org_dir): + project = _read_project(folder, org_dir / "documents") + if not project: + print(f"[migrate] skip document folder {folder.name}: no project") + continue + extracted = folder / "extracted.md" + print(f"[migrate] document: org={org_slug} project={project} folder={folder.name}") + if dry_run: + continue + try: + await qmd_index_document( + org_slug=org_slug, + project_slug=project, + document_dir=folder, + extracted_md=extracted, + ) + _write_marker(folder) + documents_count += 1 + except Exception as exc: + errors.append(f"document {folder.name}: {exc}") + + if errors: + print(f"[migrate] {len(errors)} errors:") + for err in errors: + print(f" - {err}") + + return {"meetings": meetings_count, "documents": documents_count, "errors": len(errors)} + + +def main() -> int: + parser = argparse.ArgumentParser(description="Migrate LightRAG caches to qmd collections.") + parser.add_argument("--org", help="Org slug (default: bootstrap org from config)") + parser.add_argument("--dry-run", action="store_true", help="Show what would be done") + args = parser.parse_args() + + org_slug = args.org + if not org_slug: + config = load_config() + org_slug = config.get("auth", {}).get("bootstrap", {}).get("org_slug", "merakom") + + print(f"[migrate] target org: {org_slug} dry-run: {args.dry_run}") + result = asyncio.run(migrate_org(org_slug, dry_run=args.dry_run)) + print(f"[migrate] done: {result}") + return 0 if not result.get("errors") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/pull_models.sh b/scripts/pull_models.sh new file mode 100644 index 0000000..91af614 --- /dev/null +++ b/scripts/pull_models.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# scripts/pull_models.sh — загрузка моделей (rsync из источника + fallback на download). +# +# Использование: +# MODELS_SOURCE=user@host:/opt/transcription/models/ bash scripts/pull_models.sh +# bash scripts/pull_models.sh # если MODELS_SOURCE не задан — попробует скачать из интернета +# +# Переменные окружения: +# MODELS_SOURCE — rsync-источник (user@host:path) — приоритетный способ +# MODELS_DIR — локальная папка (по умолчанию ./models) +# FORCE_DOWNLOAD — если "1", игнорирует кэш и качает заново + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MODELS_DIR="${MODELS_DIR:-$PROJECT_ROOT/models}" +MARKER="$MODELS_DIR/huggingface/hub/models--Systran--faster-whisper-large-v3" + +# Если модели уже есть — выходим +if [ -d "$MARKER" ] && [ "${FORCE_DOWNLOAD:-0}" != "1" ]; then + echo "[pull_models] Models already present at $MODELS_DIR — skipping" + exit 0 +fi + +# 1. Приоритет: rsync +if [ -n "${MODELS_SOURCE:-}" ]; then + echo "[pull_models] Pulling models from $MODELS_SOURCE ..." + mkdir -p "$MODELS_DIR/huggingface" + rsync -avz --progress "$MODELS_SOURCE/huggingface/" "$MODELS_DIR/huggingface/" + rsync -avz --progress "$MODELS_SOURCE/nltk_data/" "$MODELS_DIR/nltk_data/" 2>/dev/null || true + echo "[pull_models] Done." + exit 0 +fi + +# 2. Fallback: скачать из интернета (если rsync не задан) +echo "[pull_models] MODELS_SOURCE not set, falling back to download_models.py ..." +cd "$PROJECT_ROOT" + +# Проверяем, что python доступен +if ! command -v python >/dev/null 2>&1; then + echo "[pull_models] ERROR: python not found in PATH" >&2 + exit 1 +fi + +# Устанавливаем минимальный набор для download +pip install --quiet --no-cache-dir openai-whisper 2>/dev/null || true +pip install --quiet --no-cache-dir "whisperx @ git+https://github.com/m-bain/whisperx.git" 2>/dev/null || true + +python scripts/download_models.py || { + echo "[pull_models] ERROR: download_models.py failed." >&2 + echo "Hint: set MODELS_SOURCE=user@host:/opt/transcription/models/ and retry." >&2 + exit 1 +} +echo "[pull_models] Downloaded to $MODELS_DIR" diff --git a/scripts/smoke_rag.py b/scripts/smoke_rag.py new file mode 100644 index 0000000..0820ffe --- /dev/null +++ b/scripts/smoke_rag.py @@ -0,0 +1,67 @@ +"""Ручной smoke-test движка через CLI: ingest → query → exit.""" +import asyncio +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from src.rag.engine import get_or_create_engine # noqa: E402 + + +SAMPLES = { + "plan.md": ( + "# План 3-го этажа\n\n" + "План 3-го этажа жилого дома. Оси: А, Б, В, Г. Размеры между осями А и Б: 5400 мм.\n" + "Квартиры: 301, 302, 303.\n" + ), + "auth.md": ( + "# Авторизация\n\n" + "Авторизация работает через JWT-токены с TTL 24 часа.\n" + "Refresh-токен живёт 30 дней.\n" + ), + "schedule.md": ( + "# График работ\n\n" + "Строительство начинается 1 июня 2026. Срок сдачи — 30 ноября 2027.\n" + ), +} + + +async def main() -> int: + with tempfile.TemporaryDirectory() as tmp: + coll = Path(tmp) / "demo" / "qmd_collections" / "demo" + coll.mkdir(parents=True) + eng = get_or_create_engine(coll) + eng.warmup() + + # Ingest + for name, text in SAMPLES.items(): + result = eng.index_text(text, source_path=name) + print(f" + {name}: chunks={result.chunks_indexed} vectors={result.vectors_indexed} skipped={result.skipped}") + + # Status + print("\nStatus:", eng.status()) + + # Search BM25 + print('\n--- BM25 "авторизация JWT" ---') + for h in eng.search("авторизация JWT"): + print(f" {h.score:.3f} {h.file_path}: {h.snippet(60)}") + + # Search vector + print('\n--- Vector "как устроен вход в систему" ---') + for h in eng.vsearch("как устроен вход в систему"): + print(f" {h.score:.3f} {h.file_path}: {h.snippet(60)}") + + # Hybrid query + print('\n--- Hybrid "когда сдача объекта" ---') + for h in eng.query("когда сдача объекта", limit=3, use_rerank=False): + print(f" {h.score:.3f} {h.source_annotation()}: {h.snippet(60)}") + + eng.close() + print("\nAll good [OK]") + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/src/config.py b/src/config.py index 8432f36..e4d04b6 100644 --- a/src/config.py +++ b/src/config.py @@ -2,7 +2,7 @@ import os from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, Optional, Tuple import yaml @@ -43,3 +43,25 @@ def resolve_opencode_credentials(config: Dict[str, Any] | None = None) -> tuple[ "OPENCODE_URL", "https://opencode.ai/zen/v1" ) return api_key, base_url + + +def resolve_qmd_config(config: Dict[str, Any] | None = None) -> Dict[str, Any]: + """Возвращает словарь настроек native RAG engine (config → env → defaults). + + Ключи: ``collection_root``, ``use_rerank``, ``embed_model``, + ``chat_model``, ``chat_max_tokens``, ``enabled``, ``auto_index``. + """ + if config is None: + config = load_config() + rag = config.get("rag", {}) + return { + "collection_root": os.environ.get("QMD_COLLECTION_ROOT") or rag.get("qmd_collection_root", "./processed"), + "use_rerank": bool(rag.get("qmd_use_rerank", True)), + "embed_model": os.environ.get("RAG_EMBED_MODEL") or rag.get( + "embed_model", "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" + ), + "chat_model": rag.get("chat_model", "deepseek-v4-flash-free"), + "chat_max_tokens": int(rag.get("chat_max_tokens", 8192)), + "enabled": bool(rag.get("enabled", True)), + "auto_index": bool(rag.get("auto_index", True)), + } diff --git a/src/ingest/router.py b/src/ingest/router.py index de49b59..e898803 100644 --- a/src/ingest/router.py +++ b/src/ingest/router.py @@ -10,6 +10,19 @@ from src.ingest.extractors.xlsx_extractor import extract_csv, extract_xlsx from src.ingest.models import AUDIO_EXTENSIONS, SUPPORTED_DOCUMENT_EXTENSIONS, NormalizedDocument +_EXTRACTABLE_SUFFIXES = { + ".txt", + ".md", + ".markdown", + ".docx", + ".doc", + ".pdf", + ".xlsx", + ".xls", + ".csv", +} + + def is_audio_file(filename: str) -> bool: return Path(filename).suffix.lower() in AUDIO_EXTENSIONS @@ -18,6 +31,11 @@ def is_document_file(filename: str) -> bool: return Path(filename).suffix.lower() in SUPPORTED_DOCUMENT_EXTENSIONS +def is_extractable(filename: str) -> bool: + """True если файл поддерживается extract_document (конвертируется в .md).""" + return Path(filename).suffix.lower() in _EXTRACTABLE_SUFFIXES + + def resolve_upload_kind(filename: str) -> str: """Return 'document', 'audio', or raise ValueError.""" if is_document_file(filename): diff --git a/src/ingest/stub_writer.py b/src/ingest/stub_writer.py new file mode 100644 index 0000000..9522920 --- /dev/null +++ b/src/ingest/stub_writer.py @@ -0,0 +1,84 @@ +"""Stub-генератор для бинарных файлов, которые не конвертируются в текст. + +Создаёт ``.md`` с YAML frontmatter (source/size/mime/kind/project) рядом +с оригиналом — qmd индексирует stub, и пользователь получает ссылку на +оригинал в результатах поиска. +""" + +from __future__ import annotations + +import mimetypes +from datetime import datetime +from pathlib import Path +from typing import Optional + + +STUB_BODY = "Бинарный файл; для просмотра откройте оригинал.\n" + + +def _detect_mime(file_path: Path) -> str: + mime, _ = mimetypes.guess_type(file_path.name) + return mime or "application/octet-stream" + + +def _infer_kind(file_path: Path) -> str: + """Грубая эвристика по расширению: video/audio/image/archive/other.""" + suffix = file_path.suffix.lower().lstrip(".") + video = {"mp4", "webm", "avi", "mkv", "mov"} + audio = {"wav", "mp3", "m4a", "ogg", "flac"} + image = {"png", "jpg", "jpeg", "gif", "bmp", "tiff", "webp"} + archive = {"zip", "tar", "gz", "bz2", "7z", "rar"} + if suffix in video: + return "video" + if suffix in audio: + return "audio" + if suffix in image: + return "image" + if suffix in archive: + return "archive" + return "other" + + +def write_stub( + file_path: Path, + project: str, + kind: Optional[str] = None, +) -> Path: + """Создаёт ``.md`` stub и возвращает путь к нему. + + Поведение: + * ``file_path`` должен существовать (для получения size/mime); + * frontmatter содержит: ``source`` (имя файла), ``size`` (байт), + ``mime``, ``kind``, ``project``, ``original_name``, ``created_at``; + * body — одна строка-заглушка; + * перезаписывается, если уже существует. + """ + file_path = Path(file_path) + if not file_path.exists(): + raise FileNotFoundError(f"Stub source not found: {file_path}") + + try: + size = file_path.stat().st_size + except OSError as exc: + raise OSError(f"Cannot stat {file_path}: {exc}") from exc + + mime = _detect_mime(file_path) + resolved_kind = kind or _infer_kind(file_path) + stub_path = file_path.with_suffix(file_path.suffix + ".md") + original_name = file_path.name + + frontmatter_lines = [ + "---", + f"source: {original_name}", + f"size: {size}", + f"mime: {mime}", + f"kind: {resolved_kind}", + f"project: {project}", + f"original_name: {original_name}", + f"created_at: {datetime.now().isoformat()}", + "---", + "", + ] + content = "\n".join(frontmatter_lines) + STUB_BODY + stub_path.write_text(content, encoding="utf-8") + return stub_path diff --git a/src/rag/engine/__init__.py b/src/rag/engine/__init__.py new file mode 100644 index 0000000..e00e0aa --- /dev/null +++ b/src/rag/engine/__init__.py @@ -0,0 +1,36 @@ +"""Native Python RAG engine. + +Гибридный движок BM25 (FTS5) + vector (sqlite-vec с numpy fallback) + LLM-реранкер +через OpenCode. Хранение: один ``index.sqlite`` на коллекцию. Полностью in-process, +без внешних демонов. + +Публичный API: + Engine.index_file(path, project_slug, org_slug) -> dict + Engine.index_text(text, source_path, ...) -> dict + Engine.search(query, collection, limit=8) -> list[Hit] + Engine.vsearch(query, collection, limit=8) -> list[Hit] + Engine.query(query, collection, limit=8, use_rerank=True) -> list[Hit] + Engine.get(doc_id) -> str + Engine.status() -> dict + Engine.warmup() -> None +""" + +from .bm25 import Hit +from .engine import ( + Engine, + IndexResult, + all_engines, + get_or_create_engine, + invalidate_engine, +) +from . import embeddings + +__all__ = [ + "Engine", + "IndexResult", + "Hit", + "get_or_create_engine", + "invalidate_engine", + "all_engines", + "embeddings", +] diff --git a/src/rag/engine/bm25.py b/src/rag/engine/bm25.py new file mode 100644 index 0000000..3ad841f --- /dev/null +++ b/src/rag/engine/bm25.py @@ -0,0 +1,90 @@ +"""BM25-поиск через SQLite FTS5 с numpy fallback.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional + +import numpy as np + +from .db import Database + + +@dataclass +class Hit: + rowid: int + file_path: str + doc_id: str + chunk_index: int + start_line: int + end_line: int + content: str + score: float + + def snippet(self, max_chars: int = 200) -> str: + text = self.content.strip().replace("\n", " ") + return text[:max_chars] + ("…" if len(text) > max_chars else "") + + def source_annotation(self) -> str: + return f"[source: {self.file_path}:{self.start_line}-{self.end_line}]" + + +def bm25_search(db: Database, query: str, limit: int = 8) -> List[Hit]: + """BM25-поиск: FTS5 если есть, иначе rank_bm25 в памяти.""" + if not query or not query.strip(): + return [] + if db.has_fts5: + return _bm25_fts5(db, query, limit) + return _bm25_rank(db, query, limit) + + +def _bm25_fts5(db: Database, query: str, limit: int) -> List[Hit]: + rows = db.fts_search(query, limit) + hits: List[Hit] = [] + for row in rows: + hits.append(Hit( + rowid=row["rowid"], + file_path=row["file_path"], + doc_id=row["doc_id"], + chunk_index=row["chunk_index"], + start_line=row["start_line"], + end_line=row["end_line"], + content=row["content"], + score=float(-row["score"]), # bm25() возвращает отрицательное; инвертируем для "выше = лучше" + )) + return hits + + +def _bm25_rank(db: Database, query: str, limit: int) -> List[Hit]: + """In-memory rank_bm25 fallback.""" + try: + from rank_bm25 import BM25Okapi + except ImportError: + return [] + rows = db.all_chunks() + if not rows: + return [] + from .db import re_split_term + tokenized_corpus = [re_split_term(row["content"]) for row in rows] + bm25 = BM25Okapi(tokenized_corpus) + query_terms = re_split_term(query) + if not query_terms: + return [] + scores = bm25.get_scores(query_terms) + order = np.argsort(-scores)[:limit] + hits: List[Hit] = [] + for idx in order: + if scores[idx] <= 0: + continue + row = rows[int(idx)] + hits.append(Hit( + rowid=row["id"], + file_path=row["file_path"], + doc_id=row["doc_id"], + chunk_index=row["chunk_index"], + start_line=row["start_line"], + end_line=row["end_line"], + content=row["content"], + score=float(scores[idx]), + )) + return hits diff --git a/src/rag/engine/chunker.py b/src/rag/engine/chunker.py new file mode 100644 index 0000000..e41d7c8 --- /dev/null +++ b/src/rag/engine/chunker.py @@ -0,0 +1,115 @@ +"""Markdown-aware рекурсивный чанкер. + +Режет текст на куски ~``max_chars`` с ``overlap`` (по умолчанию 15%) по границам +``\\n\\n`` → ``\\n`` → ``". "`` → ``" "``. Сохраняет код-блоки (```...```) целиком. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import List + + +@dataclass +class Chunk: + text: str + start_line: int + end_line: int + + +_SEPARATORS = ["\n\n", "\n", ". ", " "] + + +def chunk_text(text: str, max_chars: int = 900, overlap: int = 135) -> List[Chunk]: + """Режет ``text`` на чанки. Возвращает список ``Chunk`` с координатами строк.""" + if not text or not text.strip(): + return [] + if max_chars <= 0: + max_chars = 900 + if overlap < 0: + overlap = 0 + if overlap >= max_chars: + overlap = max_chars // 6 + + # Сначала режем по абзацам; если абзац > max_chars — рекурсивно рубим дальше. + raw_chunks = _split_recursive(text.strip(), max_chars) + + # Склеиваем мелкие куски, пока они влезают в max_chars. + merged = _merge_small(raw_chunks, max_chars) + + # Добавляем overlap: каждый следующий чанк начинается с хвоста предыдущего. + overlapped = _apply_overlap(merged, overlap) + + # Считаем номера строк. + return _annotate_lines(overlapped, text) + + +def _split_recursive(text: str, max_chars: int, sep_index: int = 0) -> List[str]: + if len(text) <= max_chars or sep_index >= len(_SEPARATORS): + if len(text) > max_chars: + return [text[i:i + max_chars] for i in range(0, len(text), max_chars)] + return [text] if text else [] + + sep = _SEPARATORS[sep_index] + if sep not in text: + return _split_recursive(text, max_chars, sep_index + 1) + + parts = text.split(sep) + out: List[str] = [] + for i, part in enumerate(parts): + if not part: + continue + if i > 0: + part = sep + part + if len(part) <= max_chars: + out.append(part) + else: + out.extend(_split_recursive(part, max_chars, sep_index + 1)) + return out + + +def _merge_small(chunks: List[str], max_chars: int) -> List[str]: + merged: List[str] = [] + buf = "" + for chunk in chunks: + if not chunk: + continue + candidate = (buf + "\n" + chunk).strip() if buf else chunk + if len(candidate) <= max_chars: + buf = candidate + else: + if buf: + merged.append(buf) + if len(chunk) > max_chars: + buf = chunk[:max_chars] + else: + buf = chunk + if buf: + merged.append(buf) + return merged + + +def _apply_overlap(chunks: List[str], overlap: int) -> List[str]: + if overlap <= 0 or len(chunks) <= 1: + return chunks + result = [chunks[0]] + for prev, curr in zip(chunks, chunks[1:]): + tail = prev[-overlap:] if len(prev) > overlap else prev + result.append((tail + "\n" + curr).strip()) + return result + + +def _annotate_lines(chunks: List[str], original: str) -> List[Chunk]: + """Грубая аннотация: считаем сколько строк оригинала прошёл каждый чанк.""" + out: List[Chunk] = [] + line_no = 1 + for chunk in chunks: + lines_in_chunk = chunk.count("\n") + 1 + out.append(Chunk( + text=chunk, + start_line=line_no, + end_line=line_no + lines_in_chunk - 1, + )) + line_no += lines_in_chunk + return out diff --git a/src/rag/engine/db.py b/src/rag/engine/db.py new file mode 100644 index 0000000..e38f818 --- /dev/null +++ b/src/rag/engine/db.py @@ -0,0 +1,259 @@ +"""SQLite-обёртка для коллекции: open, schema init, execute, fallback detection. + +Один файл ``index.sqlite`` хранит: +- таблицу ``chunks`` (id, doc_id, file_path, chunk_index, content, start_line, end_line, file_hash, mtime); +- virtual table ``chunks_fts`` (FTS5) по content+file_path; +- virtual table ``chunks_vec`` (sqlite-vec) по embedding. + +Fallback-стратегии: +- нет FTS5 в сборке Python → in-memory rank_bm25 (медленно, но работает); +- нет sqlite-vec → in-memory numpy cosine. +""" + +from __future__ import annotations + +import hashlib +import logging +import sqlite3 +import struct +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterable, Iterator, List, Optional, Sequence, Tuple + +import numpy as np + +from . import embeddings + +logger = logging.getLogger(__name__) + + +def _detect_fts5(conn: sqlite3.Connection) -> bool: + try: + conn.execute("CREATE VIRTUAL TABLE _fts5_probe USING fts5(c)") + conn.execute("DROP TABLE _fts5_probe") + return True + except sqlite3.OperationalError: + return False + + +def _detect_sqlite_vec(conn: sqlite3.Connection) -> bool: + try: + import sqlite_vec + conn.enable_load_extension(True) + sqlite_vec.load(conn) + conn.execute("CREATE VIRTUAL TABLE _vec_probe USING vec0(embedding float[4])") + conn.execute("DROP TABLE _vec_probe") + return True + except Exception as exc: + logger.debug("sqlite-vec unavailable: %s", exc) + return False + + +class Database: + """Обёртка над sqlite3.Connection с фиксированной схемой.""" + + def __init__(self, path: Path) -> None: + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self._conn = sqlite3.connect(str(self.path), check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA synchronous=NORMAL") + self.has_fts5 = _detect_fts5(self._conn) + self.has_vec = _detect_sqlite_vec(self._conn) + self._init_schema() + + def close(self) -> None: + try: + self._conn.close() + except Exception: + pass + + def __enter__(self) -> "Database": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def _init_schema(self) -> None: + self._conn.executescript( + """ + CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + doc_id TEXT NOT NULL, + file_path TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + content TEXT NOT NULL, + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + file_hash TEXT NOT NULL, + mtime REAL NOT NULL, + UNIQUE(file_path, chunk_index) + ); + CREATE INDEX IF NOT EXISTS idx_chunks_doc ON chunks(doc_id); + CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path); + """ + ) + if self.has_fts5: + self._conn.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(" + "content, file_path, content='chunks', content_rowid='id', " + "tokenize='unicode61 remove_diacritics 2'" + ")" + ) + self._conn.execute( + "CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN " + "INSERT INTO chunks_fts(rowid, content, file_path) VALUES (new.id, new.content, new.file_path); " + "END;" + ) + self._conn.execute( + "CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN " + "INSERT INTO chunks_fts(chunks_fts, rowid, content, file_path) VALUES('delete', old.id, old.content, old.file_path); " + "END;" + ) + self._conn.execute( + "CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN " + "INSERT INTO chunks_fts(chunks_fts, rowid, content, file_path) VALUES('delete', old.id, old.content, old.file_path); " + "INSERT INTO chunks_fts(rowid, content, file_path) VALUES (new.id, new.content, new.file_path); " + "END;" + ) + if self.has_vec: + self._conn.execute( + f"CREATE VIRTUAL TABLE IF NOT EXISTS chunks_vec USING vec0(" + f"embedding float[{embeddings.EMBED_DIM}], id INTEGER" + ")" + ) + self._conn.commit() + + @contextmanager + def transaction(self) -> Iterator[sqlite3.Connection]: + try: + yield self._conn + self._conn.commit() + except Exception: + self._conn.rollback() + raise + + def execute(self, sql: str, params: Sequence[Any] = ()) -> sqlite3.Cursor: + return self._conn.execute(sql, params) + + def executemany(self, sql: str, params_list: Iterable[Sequence[Any]]) -> sqlite3.Cursor: + return self._conn.executemany(sql, params_list) + + def fetchall(self, sql: str, params: Sequence[Any] = ()) -> List[sqlite3.Row]: + return list(self._conn.execute(sql, params).fetchall()) + + def insert_chunks( + self, + doc_id: str, + file_path: str, + chunks: List[Tuple[int, str, int, int]], + file_hash: str, + mtime: float, + ) -> List[int]: + """Вставка/обновление чанков. Возвращает список rowid. + + ``chunks`` — список ``(chunk_index, content, start_line, end_line)``. + """ + ids: List[int] = [] + with self.transaction(): + self.execute("DELETE FROM chunks WHERE file_path = ?", (file_path,)) + for chunk_index, content, start_line, end_line in chunks: + cur = self.execute( + "INSERT INTO chunks(doc_id, file_path, chunk_index, content, start_line, end_line, file_hash, mtime) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (doc_id, file_path, chunk_index, content, start_line, end_line, file_hash, mtime), + ) + ids.append(int(cur.lastrowid)) + return ids + + def insert_vectors(self, rowids: List[int], vectors: np.ndarray) -> None: + """Вставка эмбеддингов в chunks_vec (вызывается после insert_chunks).""" + if not self.has_vec or not rowids: + return + from sqlite_vec import serialize_float32 + with self.transaction(): + for rowid, vec in zip(rowids, vectors): + self.execute( + "INSERT INTO chunks_vec(id, embedding) VALUES (?, ?)", + (rowid, sqlite3.Binary(serialize_float32(vec.tolist()))), + ) + + def get_chunk_by_rowid(self, rowid: int) -> Optional[sqlite3.Row]: + rows = self.fetchall("SELECT * FROM chunks WHERE id = ?", (rowid,)) + return rows[0] if rows else None + + def get_chunks_by_doc(self, doc_id: str) -> List[sqlite3.Row]: + return self.fetchall( + "SELECT * FROM chunks WHERE doc_id = ? ORDER BY chunk_index", + (doc_id,), + ) + + def all_chunks(self) -> List[sqlite3.Row]: + return self.fetchall("SELECT * FROM chunks") + + def file_hash(self, file_path: str) -> Optional[str]: + rows = self.fetchall( + "SELECT file_hash FROM chunks WHERE file_path = ? LIMIT 1", + (file_path,), + ) + return rows[0]["file_hash"] if rows else None + + def delete_by_file(self, file_path: str) -> None: + with self.transaction(): + self.execute("DELETE FROM chunks WHERE file_path = ?", (file_path,)) + + def count(self) -> Tuple[int, int]: + chunks = self.execute("SELECT COUNT(*) AS c FROM chunks").fetchone()["c"] + files = self.execute("SELECT COUNT(DISTINCT file_path) AS c FROM chunks").fetchone()["c"] + return int(chunks), int(files) + + def fts_search(self, query: str, limit: int) -> List[sqlite3.Row]: + if not self.has_fts5 or not query.strip(): + return [] + return self.fetchall( + "SELECT c.id AS rowid, c.file_path, c.doc_id, c.chunk_index, c.start_line, c.end_line, c.content, " + " bm25(chunks_fts) AS score " + "FROM chunks_fts f " + "JOIN chunks c ON c.id = f.rowid " + "WHERE chunks_fts MATCH ? " + "ORDER BY score ASC LIMIT ?", + (_escape_fts(query), limit), + ) + + def vec_search(self, query_vec: np.ndarray, limit: int) -> List[Tuple[int, float]]: + if not self.has_vec or query_vec is None: + return [] + from sqlite_vec import serialize_float32 + # sqlite-vec 0.1.x knn queries require explicit "AND k = ?" (no LIMIT support). + cur = self._conn.execute( + "SELECT id, distance FROM chunks_vec WHERE embedding MATCH ? AND k = ?", + (sqlite3.Binary(serialize_float32(query_vec.tolist())), int(limit)), + ) + # sqlite-vec distance: меньше = ближе; конвертируем в similarity. + return [(int(row[0]), float(row[1])) for row in cur.fetchall()] + + def close_and_remove(self) -> None: + self.close() + try: + self.path.unlink() + except OSError: + pass + + +def _escape_fts(query: str) -> str: + """Простая защита от FTS5-синтаксиса: разбиваем на термы, добавляем звёздочки.""" + terms = [t for t in re_split_term(query) if t] + return " OR ".join(f'"{t}"*' for t in terms) + + +def re_split_term(query: str) -> List[str]: + import re + return re.findall(r"[\w\u0400-\u04ff]{2,}", query or "") + + +def hash_file(path: Path) -> str: + h = hashlib.sha1() + h.update(str(path).encode("utf-8")) + h.update(path.read_bytes() if path.exists() else b"") + return h.hexdigest() diff --git a/src/rag/engine/embeddings.py b/src/rag/engine/embeddings.py new file mode 100644 index 0000000..784f540 --- /dev/null +++ b/src/rag/engine/embeddings.py @@ -0,0 +1,59 @@ +"""Embedding-генератор через sentence-transformers (lazy singleton). + +Используется ``paraphrase-multilingual-MiniLM-L12-v2`` — 384 dim, мультиязычный +(рус/англ), ~50 MB, CPU-friendly. +""" + +from __future__ import annotations + +import os +from threading import Lock +from typing import List, Optional + +import numpy as np + +DEFAULT_MODEL = os.environ.get( + "RAG_EMBED_MODEL", + "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", +) +EMBED_DIM = 384 + + +_model = None +_lock = Lock() + + +def get_model(): + """Lazy-load + cache embedding-модели (singleton).""" + global _model + if _model is not None: + return _model + with _lock: + if _model is None: + from sentence_transformers import SentenceTransformer + _model = SentenceTransformer(DEFAULT_MODEL) + return _model + + +def is_loaded() -> bool: + return _model is not None + + +def warmup() -> None: + """Прогрев модели; используется в FastAPI lifespan.""" + get_model() + + +def embed_texts(texts: List[str]) -> np.ndarray: + """Возвращает ``np.ndarray`` формы ``(len(texts), EMBED_DIM)``, dtype=float32.""" + if not texts: + return np.zeros((0, EMBED_DIM), dtype=np.float32) + model = get_model() + vectors = model.encode(texts, convert_to_numpy=True, show_progress_bar=False) + return np.asarray(vectors, dtype=np.float32) + + +def embed_query(query: str) -> Optional[np.ndarray]: + if not query or not query.strip(): + return None + return embed_texts([query])[0] diff --git a/src/rag/engine/engine.py b/src/rag/engine/engine.py new file mode 100644 index 0000000..33d33ac --- /dev/null +++ b/src/rag/engine/engine.py @@ -0,0 +1,238 @@ +"""Главный фасад native RAG engine. + +Использование:: + + engine = Engine.from_paths(project_dir=Path("processed/merakom/qmd_collections/2026")) + engine.warmup() + engine.index_text("...", source_path="foo.md", doc_id="foo") + hits = engine.query("как устроена авторизация") +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional + +import numpy as np + +from . import embeddings +from .bm25 import Hit, bm25_search +from .chunker import Chunk, chunk_text +from .db import Database, hash_file +from .hybrid import rrf_fuse +from .rerank import llm_rerank +from .vector import vector_search + + +logger = logging.getLogger(__name__) + + +@dataclass +class IndexResult: + doc_id: str + file_path: str + chunks_indexed: int + vectors_indexed: int + skipped: bool + + +class Engine: + """In-process RAG: BM25 + vector + LLM rerank.""" + + def __init__(self, db: Database) -> None: + self.db = db + + @classmethod + def from_paths(cls, project_dir: Path) -> "Engine": + """Открывает/создаёт ``index.sqlite`` в ``project_dir``.""" + project_dir = Path(project_dir) + project_dir.mkdir(parents=True, exist_ok=True) + return cls(Database(project_dir / "index.sqlite")) + + def warmup(self) -> None: + """Прогревает embedding-модель (вызывать из FastAPI lifespan).""" + try: + embeddings.warmup() + except Exception as exc: + logger.warning("[engine] warmup failed: %s", exc) + + def is_ready(self) -> bool: + return embeddings.is_loaded() + + # ---- Indexing ---- + def index_text( + self, + text: str, + *, + source_path: str, + doc_id: Optional[str] = None, + file_hash: Optional[str] = None, + mtime: Optional[float] = None, + max_chars: int = 900, + overlap: int = 135, + ) -> IndexResult: + doc_id = doc_id or _doc_id_from_path(source_path) + mtime = mtime if mtime is not None else _now() + file_hash = file_hash or hashlib_text(text) + + existing_hash = self.db.file_hash(source_path) + if existing_hash == file_hash: + return IndexResult( + doc_id=doc_id, file_path=source_path, + chunks_indexed=0, vectors_indexed=0, skipped=True, + ) + + chunks = chunk_text(text, max_chars=max_chars, overlap=overlap) + if not chunks: + return IndexResult( + doc_id=doc_id, file_path=source_path, + chunks_indexed=0, vectors_indexed=0, skipped=True, + ) + + chunk_records = [ + (i, ch.text, ch.start_line, ch.end_line) for i, ch in enumerate(chunks) + ] + rowids = self.db.insert_chunks(doc_id, source_path, chunk_records, file_hash, mtime) + + vectors_indexed = 0 + try: + vectors = embeddings.embed_texts([ch.text for ch in chunks]) + self.db.insert_vectors(rowids, vectors) + vectors_indexed = len(rowids) + except Exception as exc: + logger.warning("[engine] embedding failed for %s: %s", source_path, exc) + + return IndexResult( + doc_id=doc_id, file_path=source_path, + chunks_indexed=len(rowids), vectors_indexed=vectors_indexed, skipped=False, + ) + + def index_file( + self, + file_path: Path, + *, + doc_id: Optional[str] = None, + max_chars: int = 900, + overlap: int = 135, + ) -> IndexResult: + file_path = Path(file_path) + if not file_path.exists(): + raise FileNotFoundError(file_path) + text = file_path.read_text(encoding="utf-8", errors="replace") + mtime = file_path.stat().st_mtime + file_hash = hash_file(file_path) + return self.index_text( + text, + source_path=str(file_path), + doc_id=doc_id, + file_hash=file_hash, + mtime=mtime, + max_chars=max_chars, + overlap=overlap, + ) + + def delete(self, file_path: str) -> int: + cur = self.db.execute("SELECT COUNT(*) AS c FROM chunks WHERE file_path = ?", (file_path,)) + before = int(cur.fetchone()["c"]) + self.db.delete_by_file(file_path) + return before + + # ---- Search ---- + def search(self, query: str, limit: int = 8) -> List[Hit]: + return bm25_search(self.db, query, limit=limit) + + def vsearch(self, query: str, limit: int = 8) -> List[Hit]: + return vector_search(self.db, query, limit=limit) + + def query( + self, + query_text: str, + limit: int = 8, + use_rerank: bool = True, + ) -> List[Hit]: + bm25_hits = bm25_search(self.db, query_text, limit=max(limit * 3, 20)) + vec_hits = vector_search(self.db, query_text, limit=max(limit * 3, 20)) + fused = rrf_fuse(bm25_hits, vec_hits, k=60) + if use_rerank and len(fused) > 1: + reranked = llm_rerank(query_text, fused[:20]) + if reranked is not None: + fused = reranked + return fused[:limit] + + def get(self, doc_id: str) -> str: + rows = self.db.get_chunks_by_doc(doc_id) + return "\n\n".join(row["content"] for row in rows) + + def status(self) -> Dict[str, Any]: + chunks, files = self.db.count() + return { + "chunks": chunks, + "files": files, + "embedding_dim": embeddings.EMBED_DIM, + "engine": "sqlite-vec" if self.db.has_vec else "numpy", + "fts": "fts5" if self.db.has_fts5 else "rank_bm25", + "embedding_loaded": embeddings.is_loaded(), + } + + def close(self) -> None: + self.db.close() + + +def _doc_id_from_path(path: str) -> str: + import hashlib + return hashlib.sha1(path.encode("utf-8")).hexdigest()[:16] + + +def _now() -> float: + import time + return time.time() + + +def hashlib_text(text: str) -> str: + import hashlib + return hashlib.sha1((text or "").encode("utf-8")).hexdigest() + + +# ---- Registry: один Engine на (org, project_slug) ---- +_REGISTRY: Dict[str, Engine] = {} +_REGISTRY_LOCKS: Dict[str, "_EngineLock"] = {} + + +class _EngineLock: + def __init__(self) -> None: + import threading + self._lock = threading.Lock() + + def __enter__(self): + self._lock.acquire() + return self + + def __exit__(self, *args): + self._lock.release() + + +def get_or_create_engine(project_dir: Path) -> Engine: + """Singleton Engine на ``project_dir`` (потокобезопасно).""" + key = str(Path(project_dir).resolve()) + if key in _REGISTRY: + return _REGISTRY[key] + lock = _REGISTRY_LOCKS.setdefault(key, _EngineLock()) + with lock: + if key not in _REGISTRY: + _REGISTRY[key] = Engine.from_paths(Path(project_dir)) + return _REGISTRY[key] + + +def invalidate_engine(project_dir: Path) -> None: + key = str(Path(project_dir).resolve()) + engine = _REGISTRY.pop(key, None) + _REGISTRY_LOCKS.pop(key, None) + if engine is not None: + engine.close() + + +def all_engines() -> List[Engine]: + return list(_REGISTRY.values()) diff --git a/src/rag/engine/hybrid.py b/src/rag/engine/hybrid.py new file mode 100644 index 0000000..ffdbb0f --- /dev/null +++ b/src/rag/engine/hybrid.py @@ -0,0 +1,36 @@ +"""Reciprocal Rank Fusion (RRF) — фьюжн нескольких списков выдач. + +``score(d) = sum(1 / (k + rank_i(d)))`` где rank_i начинается с 1. +``k=60`` — стандарт (Cormack et al., 2009). +""" + +from __future__ import annotations + +from typing import Dict, Iterable, List + +from .bm25 import Hit + + +def rrf_fuse(*lists: Iterable[Hit], k: int = 60) -> List[Hit]: + """Фьюзит несколько списков ``Hit`` через RRF.""" + scores: Dict[int, float] = {} + by_rowid: Dict[int, Hit] = {} + for lst in lists: + for rank, hit in enumerate(lst, start=1): + scores[hit.rowid] = scores.get(hit.rowid, 0.0) + 1.0 / (k + rank) + by_rowid[hit.rowid] = hit + sorted_ids = sorted(scores.keys(), key=lambda rid: -scores[rid]) + result: List[Hit] = [] + for rid in sorted_ids: + base = by_rowid[rid] + result.append(Hit( + rowid=base.rowid, + file_path=base.file_path, + doc_id=base.doc_id, + chunk_index=base.chunk_index, + start_line=base.start_line, + end_line=base.end_line, + content=base.content, + score=scores[rid], + )) + return result diff --git a/src/rag/engine/rerank.py b/src/rag/engine/rerank.py new file mode 100644 index 0000000..39259bd --- /dev/null +++ b/src/rag/engine/rerank.py @@ -0,0 +1,105 @@ +"""LLM-реранкер через OpenCode/DeepSeek. + +Отправляет top-N кандидатов с промптом «верни JSON-список rowid, отсортированных +по релевантности». При любой ошибке возвращает ``None`` → caller использует +нереранкнутый список. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from typing import List, Optional + +from .bm25 import Hit + +logger = logging.getLogger(__name__) + + +RERANK_PROMPT = """Ты — реранкер для поисковой выдачи. Тебе дан запрос и {n} фрагментов документов. +Верни JSON-список ``rowid`` (целые числа) В ПОРЯДКЕ убывания релевантности запросу. +Не добавляй пояснений, только JSON. + +Запрос: {query} + +Фрагменты: +{chunks} + +Верни ТОЛЬКО JSON-массив rowid, например: ``[42, 17, 5]`` +""" + + +def llm_rerank( + query: str, + hits: List[Hit], + api_key: str = "", + base_url: str = "https://opencode.ai/zen/v1", + model: str = "deepseek-v4-flash-free", + top_k: int = 20, +) -> Optional[List[Hit]]: + """Отправляет ``top_k`` чанков в LLM и возвращает пересортированный список. + + Возвращает ``None`` если запрос не удался. + """ + if not hits: + return [] + api_key = api_key or os.environ.get("OPENCODE_API_KEY", "") + if not api_key: + logger.warning("[rerank] OPENCODE_API_KEY not set, skipping") + return None + + candidates = hits[:top_k] + chunks_text = "\n\n".join( + f"rowid={h.rowid}: {h.snippet(400)}" for h in candidates + ) + prompt = RERANK_PROMPT.format(n=len(candidates), query=query, chunks=chunks_text) + + try: + from openai import AsyncOpenAI + client = AsyncOpenAI(base_url=base_url, api_key=api_key) + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=0.0, + max_tokens=512, + ) + content = (response.choices[0].message.content or "").strip() + order_ids = _parse_ids(content) + if not order_ids: + return None + except Exception as exc: + logger.warning("[rerank] LLM call failed: %s", exc) + return None + + by_id = {h.rowid: h for h in candidates} + result: List[Hit] = [] + for rid in order_ids: + hit = by_id.get(int(rid)) + if hit is None: + continue + result.append(hit) + for h in candidates: + if h.rowid not in {r.rowid for r in result}: + result.append(h) + return result + + +def _parse_ids(content: str) -> List[int]: + match = re.search(r"\[[^\]]*\]", content, re.DOTALL) + if not match: + return [] + try: + data = json.loads(match.group(0)) + except json.JSONDecodeError: + return [] + if not isinstance(data, list): + return [] + out: List[int] = [] + for item in data: + try: + out.append(int(item)) + except (TypeError, ValueError): + continue + return out diff --git a/src/rag/engine/vector.py b/src/rag/engine/vector.py new file mode 100644 index 0000000..c8d442c --- /dev/null +++ b/src/rag/engine/vector.py @@ -0,0 +1,80 @@ +"""Vector-семантический поиск через sqlite-vec с numpy fallback.""" + +from __future__ import annotations + +from typing import List, Optional + +import numpy as np + +from .bm25 import Hit +from .db import Database +from . import embeddings + + +def vector_search( + db: Database, + query: str, + limit: int = 8, +) -> List[Hit]: + """Vector-поиск: sqlite-vec если есть, иначе numpy cosine в памяти.""" + if not query or not query.strip(): + return [] + qv = embeddings.embed_query(query) + if qv is None: + return [] + if db.has_vec: + return _vec_sqlite_vec(db, qv, limit) + return _vec_numpy(db, qv, limit) + + +def _vec_sqlite_vec(db: Database, qv: np.ndarray, limit: int) -> List[Hit]: + pairs = db.vec_search(qv, limit) + hits: List[Hit] = [] + for rowid, distance in pairs: + # distance в sqlite-vec = L2 (или cosine, зависит от версии); нормализуем к [0,1] + similarity = 1.0 / (1.0 + float(distance)) + row = db.get_chunk_by_rowid(rowid) + if row is None: + continue + hits.append(Hit( + rowid=rowid, + file_path=row["file_path"], + doc_id=row["doc_id"], + chunk_index=row["chunk_index"], + start_line=row["start_line"], + end_line=row["end_line"], + content=row["content"], + score=similarity, + )) + return hits + + +def _vec_numpy(db: Database, qv: np.ndarray, limit: int) -> List[Hit]: + rows = db.all_chunks() + if not rows: + return [] + texts = [row["content"] for row in rows] + vectors = embeddings.embed_texts(texts) + if vectors.shape[0] == 0: + return [] + qv_norm = qv / (np.linalg.norm(qv) + 1e-12) + mat_norm = vectors / (np.linalg.norm(vectors, axis=1, keepdims=True) + 1e-12) + sims = mat_norm @ qv_norm + order = np.argsort(-sims)[:limit] + hits: List[Hit] = [] + for idx in order: + sim = float(sims[idx]) + if sim <= 0: + continue + row = rows[int(idx)] + hits.append(Hit( + rowid=row["id"], + file_path=row["file_path"], + doc_id=row["doc_id"], + chunk_index=row["chunk_index"], + start_line=row["start_line"], + end_line=row["end_line"], + content=row["content"], + score=sim, + )) + return hits diff --git a/src/rag/formatter.py b/src/rag/formatter.py index d94d611..219a107 100644 --- a/src/rag/formatter.py +++ b/src/rag/formatter.py @@ -1,4 +1,4 @@ -"""Форматирование документа совещания для индексации в LightRAG.""" +"""Форматирование документа совещания для индексации в qmd (knowledge base).""" import json from datetime import datetime @@ -12,7 +12,7 @@ def format_meeting_document( metadata: Dict[str, Any], source_filename: str, ) -> str: - """Собирает текстовый документ для вставки в LightRAG. + """Собирает текстовый документ для индексации в qmd. Сохраняет полную расшифровку + метаданные + извлечённые сущности. """ diff --git a/src/rag/indexer.py b/src/rag/indexer.py deleted file mode 100644 index f1a86e6..0000000 --- a/src/rag/indexer.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Индексация протоколов совещаний в LightRAG (проектные индексы + глобальный).""" - -import os -from pathlib import Path -from typing import Optional - -from dotenv import load_dotenv -from lightrag import LightRAG -from lightrag.utils import EmbeddingFunc -from openai import AsyncOpenAI -from sentence_transformers import SentenceTransformer - -load_dotenv() - -# ------------------------------------------------------------------ -# Shared embedding model (lazy-loaded) -# ------------------------------------------------------------------ -_embed_model_instance: Optional[SentenceTransformer] = None - - -def _get_embed_model() -> SentenceTransformer: - global _embed_model_instance - if _embed_model_instance is None: - _embed_model_instance = SentenceTransformer( - "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" - ) - return _embed_model_instance - - -async def _embed_func(texts: list[str]): - model = _get_embed_model() - embeddings = model.encode(texts, convert_to_numpy=True) - return embeddings - - -EMBED_CONFIG = EmbeddingFunc( - embedding_dim=384, - max_token_size=512, - func=_embed_func, -) - - -# ------------------------------------------------------------------ -# LLM for indexing (mimo-v2.5-free via OpenCode) -# ------------------------------------------------------------------ -def _get_opencode_llm_func( - model: str = "mimo-v2.5-free", - api_key: str = "", - base_url: str = "https://opencode.ai/zen/v1", -): - if not api_key: - raise ValueError( - "OPENCODE_API_KEY не задан. Укажите rag.opencode_api_key в config.yaml " - "или переменную окружения OPENCODE_API_KEY." - ) - client = AsyncOpenAI(base_url=base_url, api_key=api_key) - - async def llm_func(prompt, system_prompt=None, history_messages=[], **kwargs): - messages = [] - if system_prompt: - messages.append({"role": "system", "content": system_prompt}) - if history_messages: - messages.extend(history_messages) - messages.append({"role": "user", "content": prompt}) - - response = await client.chat.completions.create( - model=model, - messages=messages, - temperature=kwargs.get("temperature", 0.3), - max_tokens=kwargs.get("max_tokens", 1024), - ) - content = response.choices[0].message.content - return content if content is not None else "" - - return llm_func - - -# ------------------------------------------------------------------ -# Project index helpers -# ------------------------------------------------------------------ -def _normalize_project_name(name: str) -> str: - """Нормализует имя проекта для использования в пути.""" - import re - name = re.sub(r'[^\w\-_]', '_', name) - return name.strip('_') or "default" - - -def get_project_index_dir(working_dir_base: Path, project_name: str) -> Path: - """Возвращает путь к кэшу индекса проекта.""" - norm = _normalize_project_name(project_name) - return working_dir_base / norm - - -def get_global_index_dir(working_dir_base: Path) -> Path: - """Возвращает путь к глобальному индексу.""" - return working_dir_base / "_global" - - -async def get_project_rag( - project_name: str, - working_dir_base: Path, - model: str = "mimo-v2.5-free", - api_key: str = "", - base_url: str = "https://opencode.ai/zen/v1", -) -> LightRAG: - """Возвращает инициализированный LightRAG для проекта.""" - cache_dir = get_project_index_dir(working_dir_base, project_name) - cache_dir.mkdir(parents=True, exist_ok=True) - - llm_func = _get_opencode_llm_func(model, api_key=api_key, base_url=base_url) - - rag = LightRAG( - working_dir=str(cache_dir), - llm_model_func=llm_func, - embedding_func=EMBED_CONFIG, - ) - await rag.initialize_storages() - return rag - - -async def get_global_rag( - working_dir_base: Path, - model: str = "mimo-v2.5-free", - api_key: str = "", - base_url: str = "https://opencode.ai/zen/v1", -) -> LightRAG: - """Возвращает инициализированный LightRAG для глобального индекса.""" - cache_dir = get_global_index_dir(working_dir_base) - cache_dir.mkdir(parents=True, exist_ok=True) - - llm_func = _get_opencode_llm_func(model, api_key=api_key, base_url=base_url) - - rag = LightRAG( - working_dir=str(cache_dir), - llm_model_func=llm_func, - embedding_func=EMBED_CONFIG, - ) - await rag.initialize_storages() - return rag - - -# ------------------------------------------------------------------ -# Indexing API -# ------------------------------------------------------------------ -async def index_meeting( - doc_text: str, - global_doc_text: str, - project_name: str, - working_dir_base: Path, - model: str = "mimo-v2.5-free", - api_key: str = "", - base_url: str = "https://opencode.ai/zen/v1", -): - """Индексирует документ в проектный и глобальный индексы. - - Args: - doc_text: текст для проектного индекса. - global_doc_text: текст для глобального индекса (с явным project). - project_name: имя проекта. - working_dir_base: базовая директория для lightrag_caches. - model: модель LLM для индексации. - """ - # Проектный индекс - rag_project = await get_project_rag( - project_name, working_dir_base, model, api_key=api_key, base_url=base_url - ) - await rag_project.ainsert(doc_text) - - # Глобальный индекс - rag_global = await get_global_rag( - working_dir_base, model, api_key=api_key, base_url=base_url - ) - await rag_global.ainsert(global_doc_text) - - -async def get_project_names(working_dir_base: Path) -> list[str]: - """Возвращает список проектов, для которых есть индекс.""" - if not working_dir_base.exists(): - return [] - projects = [] - for item in working_dir_base.iterdir(): - if item.is_dir() and item.name != "_global": - projects.append(item.name) - return sorted(projects) diff --git a/src/rag/parser.py b/src/rag/parser.py index 7505ec7..0c90edf 100644 --- a/src/rag/parser.py +++ b/src/rag/parser.py @@ -10,6 +10,11 @@ from openai import AsyncOpenAI DEFAULT_CLASSIFY_CHUNK_SIZE = 7000 DEFAULT_SUMMARY_CHUNK_SIZE = 10000 CHUNK_OVERLAP = 300 +# Reasoning-модели тратят max_tokens на reasoning; content остаётся пустым при малых лимитах +CLASSIFY_MAX_TOKENS = 4096 +BRIEF_MAX_TOKENS = 8192 +CHUNK_BRIEF_MAX_TOKENS = 4096 +REDUCE_BRIEF_MAX_TOKENS = 8192 CLASSIFY_PROMPT_TEMPLATE = """Ты — ассистент по анализу протоколов совещаний. @@ -210,6 +215,15 @@ def _merge_metadata(parts: List[Dict[str, Any]], project: str) -> Dict[str, Any] return merged +def _message_text(message: Any) -> str: + """Извлекает текст ответа LLM; reasoning-модели могут вернуть content=None.""" + content = (message.content or "").strip() + if content: + return content + reasoning = getattr(message, "reasoning_content", None) or getattr(message, "reasoning", None) + return (reasoning or "").strip() + + async def _llm_complete( client: AsyncOpenAI, model: str, @@ -223,7 +237,7 @@ async def _llm_complete( temperature=temperature, max_tokens=max_tokens, ) - return (response.choices[0].message.content or "").strip() + return _message_text(response.choices[0].message) async def _classify_chunk( @@ -238,7 +252,7 @@ async def _classify_chunk( sections=", ".join(sections), text=text, ) - content = await _llm_complete(client, model, prompt, temperature=0.2, max_tokens=1024) + content = await _llm_complete(client, model, prompt, temperature=0.2, max_tokens=CLASSIFY_MAX_TOKENS) return _parse_json_response(content, project) @@ -286,21 +300,28 @@ async def generate_meeting_brief( if len(chunks) == 1: prompt = BRIEF_SUMMARY_PROMPT.format(project=project, topic=topic, text=chunks[0]) - content = await _llm_complete(client, model, prompt, temperature=0.3, max_tokens=1024) + content = await _llm_complete(client, model, prompt, temperature=0.3, max_tokens=BRIEF_MAX_TOKENS) return content or metadata.get("summary", "") partials: List[str] = [] total = len(chunks) for idx, chunk in enumerate(chunks, start=1): prompt = CHUNK_BRIEF_PROMPT.format(part=idx, total=total, text=chunk) - partial = await _llm_complete(client, model, prompt, temperature=0.3, max_tokens=512) + partial = await _llm_complete( + client, model, prompt, temperature=0.3, max_tokens=CHUNK_BRIEF_MAX_TOKENS + ) if partial: partials.append(f"[Часть {idx}/{total}]\n{partial}") + if not partials: + return metadata.get("summary", "") + reduce_prompt = REDUCE_BRIEF_PROMPT.format( project=project, topic=topic, partials="\n\n".join(partials), ) - content = await _llm_complete(client, model, reduce_prompt, temperature=0.3, max_tokens=1024) + content = await _llm_complete( + client, model, reduce_prompt, temperature=0.3, max_tokens=REDUCE_BRIEF_MAX_TOKENS + ) return content or metadata.get("summary", "") diff --git a/src/rag/qmd/__init__.py b/src/rag/qmd/__init__.py new file mode 100644 index 0000000..9d1c931 --- /dev/null +++ b/src/rag/qmd/__init__.py @@ -0,0 +1,6 @@ +"""qmd knowledge-base integration (Hermes Agent, Nous Research). + +Заменяет LightRAG. Локальный движок BM25 (FTS5) + vector (sqlite-vec) + LLM-реранкер +с хранением в одном .sqlite на коллекцию. Запускается как MCP-сервер через CLI +``qmd mcp --http --daemon``. +""" diff --git a/src/rag/qmd/cache.py b/src/rag/qmd/cache.py new file mode 100644 index 0000000..74cefa9 --- /dev/null +++ b/src/rag/qmd/cache.py @@ -0,0 +1,105 @@ +"""TTL-кэш результатов поиска qmd с инвалидацией по mtime индекса. + +Ключ: ``hash(question) + (org_slug, project_slug, mode, use_rerank)``. +Значение инвалидируется либо по TTL (60 сек), либо при изменении mtime +``index.sqlite`` коллекции. +""" + +from __future__ import annotations + +import hashlib +import os +import time +from pathlib import Path +from typing import Any, Optional, Tuple + +CacheKey = Tuple[str, str, Optional[str], str, bool] + + +def _question_hash(question: str) -> str: + return hashlib.sha1((question or "").strip().lower().encode("utf-8")).hexdigest()[:16] + + +def _index_mtime(collection_root: Optional[Path] = None) -> float: + """Возвращает mtime самого свежего ``index.sqlite`` в коллекциях org. + + 0.0 если ничего не найдено (например, qmd ещё не запускался). + """ + from src.rag.qmd import collections + root = collections.get_root(collection_root) + if not root.exists(): + return 0.0 + latest = 0.0 + for sqlite_path in root.rglob("index.sqlite"): + try: + mtime = sqlite_path.stat().st_mtime + if mtime > latest: + latest = mtime + except OSError: + continue + return latest + + +class SearchCache: + """Потокобезопасный (single-loop asyncio) кэш результатов поиска.""" + + def __init__(self, ttl_seconds: int = 60) -> None: + self.ttl = ttl_seconds + self._store: dict[CacheKey, tuple[float, float, str]] = {} + + def _build_key( + self, + question: str, + args: tuple[str, Optional[str], str, bool], + ) -> CacheKey: + org_slug, project_slug, mode, use_rerank = args + return ( + _question_hash(question), + org_slug, + project_slug, + mode, + bool(use_rerank), + ) + + def get(self, question: str, args: tuple[str, Optional[str], str, bool]) -> Optional[str]: + key = self._build_key(question, args) + entry = self._store.get(key) + if entry is None: + return None + stored_at, stored_mtime, value = entry + current_mtime = _index_mtime() + if current_mtime > stored_mtime: + self._store.pop(key, None) + return None + if (time.monotonic() - stored_at) > self.ttl: + self._store.pop(key, None) + return None + return value + + def set( + self, + question: str, + args: tuple[str, Optional[str], str, bool], + value: str, + ) -> None: + if not value: + return + key = self._build_key(question, args) + self._store[key] = (time.monotonic(), _index_mtime(), value) + + def clear(self) -> None: + self._store.clear() + + def stats(self) -> dict: + return {"entries": len(self._store), "ttl_seconds": self.ttl} + + +_search_cache = None + + +def get_search_cache(): + """Глобальный singleton (для FastAPI dependency / healthcheck).""" + global _search_cache + if _search_cache is None: + _search_cache = SearchCache(ttl_seconds=60) + return _search_cache diff --git a/src/rag/qmd/collections.py b/src/rag/qmd/collections.py new file mode 100644 index 0000000..15f44a1 --- /dev/null +++ b/src/rag/qmd/collections.py @@ -0,0 +1,121 @@ +"""Управление qmd-коллекциями: per-org / per-project + глобальная. + +Коллекция — это каталог с ``index.sqlite`` под +``///`` или ``//_global/``. + +Хранение реализовано в :mod:`src.rag.engine` (sqlite-vec + FTS5). +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import List, Optional + +logger = logging.getLogger(__name__) + + +def get_root(collection_root: Optional[Path] = None) -> Path: + """Возвращает базовый путь коллекций (env → config → default).""" + if collection_root is not None: + return collection_root + env_root = os.environ.get("QMD_COLLECTION_ROOT") + if env_root: + return Path(env_root) + return Path("processed") + + +def get_org_root(org_slug: str, collection_root: Optional[Path] = None) -> Path: + norm = _normalize(org_slug) or "default" + return get_root(collection_root) / norm + + +def get_project_collection_dir( + org_slug: str, + project_slug: str, + collection_root: Optional[Path] = None, +) -> Path: + return get_org_root(org_slug, collection_root) / "qmd_collections" / _normalize(project_slug) + + +def get_global_collection_dir( + org_slug: str, + collection_root: Optional[Path] = None, +) -> Path: + return get_org_root(org_slug, collection_root) / "qmd_collections" / "_global" + + +def collection_uri(org_slug: str, project_slug: str) -> str: + """qmd:// URI для коллекции.""" + if not project_slug or project_slug == "_global": + return f"qmd://{_normalize(org_slug)}/_global" + return f"qmd://{_normalize(org_slug)}/{_normalize(project_slug)}" + + +def _normalize(name: str) -> str: + import re + cleaned = re.sub(r"[^\w\-_]", "_", (name or "").strip()) + return cleaned.strip("_") or "default" + + +async def ensure_collection( + org_slug: str, + project_slug: str, + collection_root: Optional[Path] = None, +) -> Path: + """Создаёт каталог коллекции (engine создаст .sqlite при первом index).""" + if not project_slug or project_slug == "_global": + path = get_global_collection_dir(org_slug, collection_root) + else: + path = get_project_collection_dir(org_slug, project_slug, collection_root) + path.mkdir(parents=True, exist_ok=True) + return path + + +async def add_context( + org_slug: str, + project_slug: str, + description: str, +) -> None: + """Сохраняет описание контекста рядом с index.sqlite (best-effort).""" + if not project_slug or project_slug == "_global": + path = get_global_collection_dir(org_slug) + else: + path = get_project_collection_dir(org_slug, project_slug) + path.mkdir(parents=True, exist_ok=True) + context_path = path / "context.md" + try: + with context_path.open("a", encoding="utf-8") as f: + f.write(f"- {description}\n") + except OSError as exc: + logger.debug("add_context: %s", exc) + + +async def embed( + org_slug: str, + project_slug: str, + force: bool = False, +) -> int: + """Переиндексация коллекции: проходит по всем ``*.md`` в каталоге коллекции. + + Возвращает число проиндексированных файлов. + """ + from src.rag.engine import get_or_create_engine + if not project_slug or project_slug == "_global": + path = get_global_collection_dir(org_slug) + else: + path = get_project_collection_dir(org_slug, project_slug) + if not path.exists(): + return 0 + engine = get_or_create_engine(path) + engine.warmup() + indexed = 0 + for md_file in sorted(path.rglob("*.md")): + try: + result = engine.index_file(md_file) + if not result.skipped: + indexed += 1 + except Exception as exc: + logger.warning("embed: failed to index %s: %s", md_file, exc) + return indexed diff --git a/src/rag/qmd/indexer.py b/src/rag/qmd/indexer.py new file mode 100644 index 0000000..bdddb0c --- /dev/null +++ b/src/rag/qmd/indexer.py @@ -0,0 +1,126 @@ +"""Высокоуровневые операции индексации: meetings и documents. + +Использует native RAG engine (``src.rag.engine``). На каждый файл: +1) ``Engine.index_file`` добавляет в проектную коллекцию; +2) копия с префиксом проекта добавляется в глобальную коллекцию org. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + +from src.rag.engine import Engine, get_or_create_engine +from src.rag.qmd import collections + + +def _global_doc_text(doc_text: str, project_slug: str) -> str: + """Добавляет префикс с проектом для глобальной коллекции.""" + return f"=== Проект: {project_slug} ===\n\n{doc_text}" + + +def _engine_for(org_slug: str, project_slug: Optional[str]) -> Engine: + if not project_slug or project_slug == "_global": + path = collections.get_global_collection_dir(org_slug) + else: + path = collections.get_project_collection_dir(org_slug, project_slug) + path.mkdir(parents=True, exist_ok=True) + return get_or_create_engine(path) + + +async def qmd_index_meeting( + org_slug: str, + project_slug: str, + *, + body_path: Path, + summary_path: Optional[Path] = None, + txt_path: Optional[Path] = None, + context_description: Optional[str] = None, +) -> Dict[str, Any]: + """Индексирует протокол совещания: ``.md`` + ``_summary.md``.""" + project_engine = _engine_for(org_slug, project_slug) + global_engine = _engine_for(org_slug, "_global") + + written: List[Path] = [] + files = [body_path, summary_path, txt_path] + for src in files: + if src and src.exists(): + result = project_engine.index_file(src) + if not result.skipped: + written.append(src) + if context_description: + await collections.add_context(org_slug, project_slug, context_description) + # Глобальный индекс: добавляем тот же файл с префиксом проекта. + text = src.read_text(encoding="utf-8") + prefixed = _global_doc_text(text, project_slug) + global_path = global_engine.db.path.parent / src.name + global_path.write_text(prefixed, encoding="utf-8") + global_engine.index_file(global_path, doc_id=f"{project_slug}_{src.stem}") + global_path.unlink(missing_ok=True) + + return { + "project_dir": str(project_engine.db.path.parent), + "files": [str(p) for p in written], + "chunks_indexed": sum( + 1 for p in written for _ in project_engine.db.fetchall( + "SELECT id FROM chunks WHERE file_path = ?", (str(p),) + ) + ), + } + + +async def qmd_index_document( + org_slug: str, + project_slug: str, + *, + document_dir: Path, + extracted_md: Optional[Path] = None, + context_description: Optional[str] = None, +) -> Dict[str, Any]: + """Индексирует произвольный документ: PDF/DOCX/XLSX/CSV/TXT/MD или stub.""" + project_engine = _engine_for(org_slug, project_slug) + global_engine = _engine_for(org_slug, "_global") + + written: List[Path] = [] + sources: List[Path] = [] + if extracted_md and extracted_md.exists(): + sources = [extracted_md] + if not sources and document_dir.exists(): + sources = sorted(document_dir.glob("*.md")) + + if not sources: + return {"project_dir": str(document_dir), "files": [], "chunks_indexed": 0} + + for src in sources: + result = project_engine.index_file(src) + if not result.skipped: + written.append(src) + if context_description: + await collections.add_context(org_slug, project_slug, context_description) + text = src.read_text(encoding="utf-8") + prefixed = _global_doc_text(text, project_slug) + global_path = global_engine.db.path.parent / f"{project_slug}__{src.name}" + global_path.write_text(prefixed, encoding="utf-8") + global_engine.index_file(global_path, doc_id=f"{project_slug}__{src.stem}") + global_path.unlink(missing_ok=True) + + return { + "project_dir": str(document_dir), + "files": [str(p) for p in written], + "chunks_indexed": sum( + 1 for p in written for _ in project_engine.db.fetchall( + "SELECT id FROM chunks WHERE file_path = ?", (str(p),) + ) + ), + } + + +def build_global_doc_text(doc_text: str, project_slug: str) -> str: + """Совместимость со старым API ``format_global_document``.""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + header = ( + f"=== ДОКУМЕНТ (Проект: {project_slug}, ID: {project_slug}_{timestamp}) ===\n" + f"Проиндексировано: {datetime.now().isoformat()}\n\n" + ) + return header + _global_doc_text(doc_text, project_slug) diff --git a/src/rag/qmd/lifespan.py b/src/rag/qmd/lifespan.py new file mode 100644 index 0000000..75c3244 --- /dev/null +++ b/src/rag/qmd/lifespan.py @@ -0,0 +1,57 @@ +"""FastAPI lifespan: прогрев native RAG engine. + +Больше не запускает внешний демон — движок in-process. + +Использование:: + + from contextlib import asynccontextmanager + from fastapi import FastAPI + from src.rag.qmd.lifespan import qmd_lifespan + + @asynccontextmanager + async def lifespan(app: FastAPI): + async with qmd_lifespan(app): + yield + + app = FastAPI(lifespan=lifespan) +""" + +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def qmd_lifespan(app: Any): + """Async context manager: прогревает embedding-модель при старте. + + ``app.state.qmd_status`` устанавливается в ``"ready"`` если warmup успешен, + иначе ``"degraded"`` (BM25-only fallback). + """ + app.state.qmd_status = "degraded" + app.state.qmd_daemon_pid = None + + try: + from src.rag.engine import embeddings + embeddings.warmup() + app.state.qmd_status = "ready" + logger.info("[qmd] native engine ready (in-process)") + except Exception as exc: + logger.warning("[qmd] warmup failed: %s — degraded mode (BM25-only)", exc) + app.state.qmd_status = "degraded" + + try: + yield + finally: + try: + from src.rag.engine import all_engines + for engine in all_engines(): + engine.close() + except Exception: + pass + app.state.qmd_status = "down" + logger.info("[qmd] engines closed") diff --git a/src/rag/qmd/mcp_client.py b/src/rag/qmd/mcp_client.py new file mode 100644 index 0000000..89f63d6 --- /dev/null +++ b/src/rag/qmd/mcp_client.py @@ -0,0 +1,175 @@ +"""In-process клиент к native RAG engine (замена старого MCP-клиента). + +Сохраняет совместимый API со старым ``mcp_client.py``: + qmd_search(query, collection) -> list[dict] + qmd_vsearch(query, collection) -> list[dict] + qmd_query(query, collection) -> str + qmd_get(docid) -> str + qmd_status() -> dict + +Класс ``QmdMcpError`` оставлен для обратной совместимости (больше не бросается, +но импорты не ломаются). +""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +class QmdMcpError(RuntimeError): + """Совместимый exception (больше не используется, оставлен для API).""" + + +def _collection_to_dir(org_slug: str, collection: Optional[str]) -> Path: + """``qmd://org/project`` → ``//qmd_collections/``. + + Также принимает ``Path`` напрямую (для тестов). + """ + from src.rag.qmd.collections import get_project_collection_dir, get_global_collection_dir + if collection is None or collection == "": + return get_global_collection_dir(org_slug) + if isinstance(collection, Path): + return collection + name = str(collection).rstrip("/").split("/")[-1] + if name == "_global" or not name: + return get_global_collection_dir(org_slug) + return get_project_collection_dir(org_slug, name) + + +def _engine_for(org_slug: str, collection: Optional[str]): + from src.rag.engine import get_or_create_engine + directory = _collection_to_dir(org_slug, collection) + return get_or_create_engine(directory), directory + + +def _hit_to_dict(hit, score_key: str = "score") -> Dict[str, Any]: + return { + "rowid": hit.rowid, + "docid": hit.doc_id, + "path": hit.file_path, + "file_path": hit.file_path, + "chunk_index": hit.chunk_index, + "start_line": hit.start_line, + "end_line": hit.end_line, + "score": float(hit.score), + "snippet": hit.snippet(400), + "text": hit.content, + "source": hit.source_annotation(), + } + + +async def qmd_search( + query: str, + collection: Optional[str] = None, + limit: int = 8, + org_slug: Optional[str] = None, +) -> List[Dict[str, Any]]: + org = org_slug or _default_org() + engine, _ = _engine_for(org, collection) + engine.warmup() + return [_hit_to_dict(h) for h in engine.search(query, limit=limit)] + + +async def qmd_vsearch( + query: str, + collection: Optional[str] = None, + limit: int = 8, + org_slug: Optional[str] = None, +) -> List[Dict[str, Any]]: + org = org_slug or _default_org() + engine, _ = _engine_for(org, collection) + engine.warmup() + return [_hit_to_dict(h) for h in engine.vsearch(query, limit=limit)] + + +async def qmd_query( + query: str, + collection: Optional[str] = None, + limit: int = 8, + use_rerank: bool = True, + org_slug: Optional[str] = None, +) -> str: + org = org_slug or _default_org() + engine, _ = _engine_for(org, collection) + engine.warmup() + hits = engine.query(query, limit=limit, use_rerank=use_rerank) + if not hits: + return "" + parts = [] + for h in hits: + parts.append( + f"[source: {h.file_path}:{h.start_line}-{h.end_line}] " + f"(score={h.score:.3f})\n{h.content}" + ) + return "\n\n---\n\n".join(parts) + + +async def qmd_get(docid: str, org_slug: Optional[str] = None) -> str: + """Возвращает полный документ по doc_id (ищет в любой коллекции org).""" + org = org_slug or _default_org() + from src.rag.qmd.collections import get_org_root + org_root = get_org_root(org) + if not org_root.exists(): + return "" + import sqlite3 + for db_path in org_root.rglob("index.sqlite"): + try: + conn = sqlite3.connect(str(db_path)) + row = conn.execute( + "SELECT content FROM chunks WHERE doc_id = ? ORDER BY chunk_index", + (docid,), + ).fetchall() + conn.close() + except sqlite3.OperationalError: + continue + if row: + return "\n\n".join(r[0] for r in row) + return "" + + +async def qmd_status(org_slug: Optional[str] = None) -> Dict[str, Any]: + """Агрегированный статус по всем коллекциям org.""" + org = org_slug or _default_org() + from src.rag.qmd.collections import get_org_root + org_root = get_org_root(org) + result: Dict[str, Any] = { + "collections": [], + "total_chunks": 0, + "total_files": 0, + "engines": set(), + "fts": set(), + "embedding_loaded": False, + } + if not org_root.exists(): + return _finalize_status(result) + for db_path in org_root.rglob("index.sqlite"): + try: + from src.rag.engine import Engine + eng = Engine.from_paths(db_path.parent) + st = eng.status() + result["collections"].append({"path": str(db_path.parent), **st}) + result["total_chunks"] += st.get("chunks", 0) + result["total_files"] += st.get("files", 0) + result["engines"].add(st.get("engine", "?")) + result["fts"].add(st.get("fts", "?")) + result["embedding_loaded"] = result["embedding_loaded"] or st.get("embedding_loaded", False) + eng.close() + except Exception as exc: + logger.debug("status: failed to open %s: %s", db_path, exc) + return _finalize_status(result) + + +def _finalize_status(result: Dict[str, Any]) -> Dict[str, Any]: + result["engines"] = sorted(result["engines"]) + result["fts"] = sorted(result["fts"]) + return result + + +def _default_org() -> str: + return os.environ.get("DEFAULT_ORG_SLUG", "merakom") diff --git a/src/rag/qmd/query.py b/src/rag/qmd/query.py new file mode 100644 index 0000000..671f45d --- /dev/null +++ b/src/rag/qmd/query.py @@ -0,0 +1,284 @@ +"""Поиск и чат-генерация ответов через native RAG engine. + +Совместимо с WebSocket-API: ``{"type": "context"|"chunk"|"done"}``. +Поддерживает режимы bm25/vector/hybrid и маппинг legacy-имён. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, AsyncIterator, Dict, List, Optional + +from openai import AsyncOpenAI + +from src.rag.qmd import mcp_client +from src.rag.qmd.cache import SearchCache + +logger = logging.getLogger(__name__) + +DEFAULT_CHAT_MAX_TOKENS = 8192 +DEFAULT_CHAT_MODEL = "deepseek-v4-flash-free" + +CHAT_MODES: Dict[str, Dict[str, str]] = { + "hybrid": { + "system": ( + "Ты — ассистент по базе знаний строительной компании. " + "Отвечай на основе контекста. Указывай источники (файл, страница, лист). " + "Если данных нет — так и скажи." + ), + "instruction": "Ответь на вопрос:", + }, + "compare": { + "system": ( + "Ты — аналитик. Сопоставь информацию из разных документов в контексте. " + "Найди совпадения, расхождения, противоречия. Структурируй ответ по пунктам." + ), + "instruction": "Сопоставь и сравни информацию по запросу:", + }, + "timeline": { + "system": ( + "Ты — аналитик хронологии проекта. " + "Восстанови timeline событий, решений, изменений по датам из контекста. " + "Сортируй по времени, указывай источник каждого события." + ), + "instruction": "Построй хронологию / timeline по запросу:", + }, +} + + +LEGACY_MODE_MAP: Dict[str, str] = { + "naive": "bm25", + "local": "vector", + "global": "bm25", + "hybrid": "hybrid", + "bm25": "bm25", + "vector": "vector", +} + +_cache = SearchCache(ttl_seconds=60) + + +def _message_text(message: Any) -> str: + content = (getattr(message, "content", None) or "").strip() + if content: + return content + reasoning = getattr(message, "reasoning_content", None) or getattr(message, "reasoning", None) + return (reasoning or "").strip() + + +def _resolve_credentials() -> tuple[str, str]: + api_key = os.environ.get("OPENCODE_API_KEY", "") + base_url = os.environ.get("OPENCODE_URL", "https://opencode.ai/zen/v1") + return api_key, base_url + + +def _build_chat_messages( + question: str, + context: str, + history: List[Dict[str, str]], + chat_mode: str, +) -> List[Dict[str, str]]: + mode_cfg = CHAT_MODES.get(chat_mode, CHAT_MODES["hybrid"]) + messages: List[Dict[str, str]] = [{"role": "system", "content": mode_cfg["system"]}] + for entry in history or []: + if "question" in entry and "answer" in entry: + messages.append({"role": "user", "content": entry["question"]}) + messages.append({"role": "assistant", "content": entry["answer"]}) + user_prompt = ( + f"Контекст из базы знаний:\n---\n{context}\n---\n\n" + f"{mode_cfg['instruction']} {question}\n" + ) + messages.append({"role": "user", "content": user_prompt}) + return messages + + +def _project_for_query(project_slug: Optional[str], mode: str) -> Optional[str]: + if mode == "global" or not project_slug or project_slug == "_global": + return None + return project_slug + + +def map_retrieval_mode(mode: str) -> str: + return LEGACY_MODE_MAP.get(mode, "hybrid") + + +async def qmd_query( + question: str, + org_slug: str, + project_slug: Optional[str] = None, + mode: str = "hybrid", + use_rerank: bool = True, +) -> str: + """Совместимо со старым ``rag_chat`` — возвращает контекст как строку.""" + retrieval_mode = map_retrieval_mode(mode) + target = _project_for_query(project_slug, retrieval_mode) + cache_key_args = (org_slug, target, retrieval_mode, use_rerank) + cached = _cache.get(question, cache_key_args) + if cached is not None: + return cached + + # Резолвим каталог коллекции и используем engine напрямую (без + # round-trip через mcp_client, чтобы сохранить Path-идентичность). + from src.rag.qmd.collections import get_project_collection_dir, get_global_collection_dir + if target is None: + coll_dir = get_global_collection_dir(org_slug) + else: + coll_dir = get_project_collection_dir(org_slug, target) + coll_dir.mkdir(parents=True, exist_ok=True) + from src.rag.engine import get_or_create_engine + engine = get_or_create_engine(coll_dir) + engine.warmup() + + if retrieval_mode == "bm25": + hits = engine.search(question, limit=20) + context = _format_bm25(_hits_to_dicts(hits)) + elif retrieval_mode == "vector": + hits = engine.vsearch(question, limit=20) + context = _format_bm25(_hits_to_dicts(hits)) + else: + hits = engine.query(question, limit=20, use_rerank=use_rerank) + context = _format_bm25(_hits_to_dicts(hits)) + + _cache.set(question, cache_key_args, context) + return context + + +def _hits_to_dicts(hits) -> list: + return [ + { + "file_path": h.file_path, + "docid": h.doc_id, + "score": h.score, + "snippet": h.snippet(400), + "text": h.content, + "source": h.source_annotation(), + } + for h in hits + ] + + +def _format_bm25(hits: List[Dict[str, Any]]) -> str: + if not hits: + return "" + parts: List[str] = [] + for hit in hits: + path = hit.get("file_path", hit.get("docid", "?")) + score = hit.get("score") + snippet = hit.get("snippet", hit.get("text", "")) + score_str = f" (score={score:.3f})" if isinstance(score, (int, float)) else "" + parts.append(f"[source: {path}]{score_str}\n{snippet}") + return "\n\n---\n\n".join(parts) + + +async def _generate_answer( + question: str, + context: str, + history: List[Dict[str, str]], + chat_mode: str, + chat_model: str, + max_tokens: int, + stream: bool, +): + api_key, base_url = _resolve_credentials() + if not api_key: + raise ValueError( + "OPENCODE_API_KEY не задан. Укажите rag.opencode_api_key в config.yaml " + "или переменную окружения OPENCODE_API_KEY." + ) + client = AsyncOpenAI(base_url=base_url, api_key=api_key) + messages = _build_chat_messages(question, context, history, chat_mode) + return await client.chat.completions.create( + model=chat_model, + messages=messages, + temperature=0.4, + max_tokens=max_tokens, + stream=stream, + ) + + +async def qmd_chat( + question: str, + org_slug: str, + history: List[Dict[str, str]], + api_key: str = "", + project_slug: Optional[str] = None, + chat_model: str = DEFAULT_CHAT_MODEL, + mode: str = "hybrid", + chat_mode: str = "hybrid", + max_tokens: int = DEFAULT_CHAT_MAX_TOKENS, + use_rerank: bool = True, +) -> Dict[str, Any]: + """Совместимо со старым ``rag_chat``: возвращает dict с ``answer``/``context``.""" + if api_key: + os.environ["OPENCODE_API_KEY"] = api_key + context = await qmd_query( + question=question, + org_slug=org_slug, + project_slug=project_slug, + mode=mode, + use_rerank=use_rerank, + ) + response = await _generate_answer( + question, context, history, chat_mode, chat_model, max_tokens, stream=False + ) + answer = _message_text(response.choices[0].message) + return { + "answer": answer, + "context": context, + "project": project_slug, + "chat_mode": chat_mode, + } + + +async def qmd_chat_stream( + question: str, + org_slug: str, + history: List[Dict[str, str]], + api_key: str = "", + project_slug: Optional[str] = None, + chat_model: str = DEFAULT_CHAT_MODEL, + mode: str = "hybrid", + chat_mode: str = "hybrid", + max_tokens: int = DEFAULT_CHAT_MAX_TOKENS, + use_rerank: bool = True, +) -> AsyncIterator[Dict[str, Any]]: + """Стрим: context → chunk* → done (payload совместим с WebSocket).""" + if api_key: + os.environ["OPENCODE_API_KEY"] = api_key + context = await qmd_query( + question=question, + org_slug=org_slug, + project_slug=project_slug, + mode=mode, + use_rerank=use_rerank, + ) + yield { + "type": "context", + "context": context, + "project": project_slug, + "chat_mode": chat_mode, + } + answer_parts: List[str] = [] + try: + stream = await _generate_answer( + question, context, history, chat_mode, chat_model, max_tokens, stream=True + ) + async for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if delta and getattr(delta, "content", None): + answer_parts.append(delta.content) + yield {"type": "chunk", "content": delta.content} + except ValueError as exc: + yield {"type": "rag_error", "error": str(exc)} + return + + yield { + "type": "done", + "answer": "".join(answer_parts), + "context": context, + "project": project_slug, + "chat_mode": chat_mode, + } diff --git a/src/rag/query.py b/src/rag/query.py deleted file mode 100644 index 5ecea1e..0000000 --- a/src/rag/query.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Запросы к RAG и генерация ответов чат-бота.""" - -from pathlib import Path -from typing import Any, Dict, List, Optional - -from lightrag import QueryParam -from openai import AsyncOpenAI - -from src.rag.indexer import get_global_rag, get_project_rag - -CHAT_MODES = { - "hybrid": { - "system": ( - "Ты — ассистент по базе знаний строительной компании. " - "Отвечай на основе контекста. Указывай источники (файл, страница, лист). " - "Если данных нет — так и скажи." - ), - "instruction": "Ответь на вопрос:", - }, - "compare": { - "system": ( - "Ты — аналитик. Сопоставь информацию из разных документов в контексте. " - "Найди совпадения, расхождения, противоречия. Структурируй ответ по пунктам." - ), - "instruction": "Сопоставь и сравни информацию по запросу:", - }, - "timeline": { - "system": ( - "Ты — аналитик хронологии проекта. " - "Восстанови timeline событий, решений, изменений по датам из контекста. " - "Сортируй по времени, указывай источник каждого события." - ), - "instruction": "Построй хронологию / timeline по запросу:", - }, -} - - -async def retrieve_context( - question: str, - working_dir_base: Path, - project_name: Optional[str] = None, - mode: str = "hybrid", - api_key: str = "", - base_url: str = "https://opencode.ai/zen/v1", - index_model: str = "mimo-v2.5-free", -) -> str: - if project_name: - rag = await get_project_rag( - project_name, - working_dir_base, - model=index_model, - api_key=api_key, - base_url=base_url, - ) - else: - rag = await get_global_rag( - working_dir_base, - model=index_model, - api_key=api_key, - base_url=base_url, - ) - - param = QueryParam(mode=mode if mode in ("naive", "local", "global", "hybrid") else "hybrid", only_need_context=True) - context = await rag.aquery(question, param=param) - return context if context else "" - - -async def generate_chat_response( - question: str, - context: str, - history: List[Dict[str, str]], - api_key: str, - base_url: str = "https://opencode.ai/zen/v1", - model: str = "deepseek-v4-flash-free", - chat_mode: str = "hybrid", -) -> str: - if not api_key: - raise ValueError( - "OPENCODE_API_KEY не задан. Укажите rag.opencode_api_key в config.yaml " - "или переменную окружения OPENCODE_API_KEY." - ) - client = AsyncOpenAI(base_url=base_url, api_key=api_key) - - mode_cfg = CHAT_MODES.get(chat_mode, CHAT_MODES["hybrid"]) - messages = [{"role": "system", "content": mode_cfg["system"]}] - - for h in history: - messages.append({"role": "user", "content": h["question"]}) - messages.append({"role": "assistant", "content": h["answer"]}) - - user_prompt = f"""Контекст из базы знаний: ---- -{context} ---- - -{mode_cfg["instruction"]} {question} -""" - messages.append({"role": "user", "content": user_prompt}) - - response = await client.chat.completions.create( - model=model, - messages=messages, - temperature=0.4, - max_tokens=2048, - ) - content = response.choices[0].message.content - return content if content is not None else "" - - -async def rag_chat( - question: str, - working_dir_base: Path, - history: List[Dict[str, str]], - api_key: str, - project_name: Optional[str] = None, - base_url: str = "https://opencode.ai/zen/v1", - chat_model: str = "deepseek-v4-flash-free", - mode: str = "hybrid", - index_model: str = "mimo-v2.5-free", - chat_mode: str = "hybrid", -) -> Dict[str, Any]: - retrieval_mode = mode if mode in ("naive", "local", "global", "hybrid") else "hybrid" - context = await retrieve_context( - question=question, - working_dir_base=working_dir_base, - project_name=project_name, - mode=retrieval_mode, - api_key=api_key, - base_url=base_url, - index_model=index_model, - ) - - answer = await generate_chat_response( - question=question, - context=context, - history=history, - api_key=api_key, - base_url=base_url, - model=chat_model, - chat_mode=chat_mode, - ) - - return { - "answer": answer, - "context": context, - "project": project_name, - "chat_mode": chat_mode, - } diff --git a/tests/test_ingest.py b/tests/test_ingest.py index f1a48b9..c747107 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -12,7 +12,14 @@ sys.path.insert(0, str(ROOT)) from src.ingest.classify import _parse_json from src.ingest.formatter import format_index_document from src.ingest.models import NormalizedDocument, DocumentChunk -from src.ingest.router import extract_document, is_audio_file, is_document_file, resolve_upload_kind +from src.ingest.router import ( + extract_document, + is_audio_file, + is_document_file, + is_extractable, + resolve_upload_kind, +) +from src.ingest.stub_writer import write_stub def test_is_audio_file(): @@ -85,6 +92,25 @@ def test_resolve_upload_kind(): pass +def test_is_extractable(): + assert is_extractable("spec.pdf") is True + assert is_extractable("data.csv") is True + assert is_extractable("notes.md") is True + assert is_extractable("video.mp4") is False + assert is_extractable("archive.zip") is False + + +def test_write_stub_for_binary(): + with tempfile.TemporaryDirectory() as tmp: + mp4 = Path(tmp) / "movie.mp4" + mp4.write_bytes(b"\x00" * 8) + stub = write_stub(mp4, project="2026") + assert stub.exists() + text = stub.read_text(encoding="utf-8") + assert "kind: video" in text + assert "project: 2026" in text + + if __name__ == "__main__": class IngestTestCase(unittest.TestCase): def test_audio(self): @@ -108,4 +134,10 @@ if __name__ == "__main__": def test_route(self): test_resolve_upload_kind() + def test_extractable(self): + test_is_extractable() + + def test_stub(self): + test_write_stub_for_binary() + unittest.main(verbosity=2) diff --git a/tests/test_native_engine.py b/tests/test_native_engine.py new file mode 100644 index 0000000..4598a83 --- /dev/null +++ b/tests/test_native_engine.py @@ -0,0 +1,177 @@ +"""Real-data tests for native RAG engine (no mocks, in-process). + +Использует sentence-transformers реальную модель (~50 MB скачивается при первом +запуске). Время выполнения: ~10 сек (cold) + ~1 сек на warm-кейсы. +""" + +import tempfile +import unittest +from pathlib import Path + +from src.rag.engine import Engine, get_or_create_engine, invalidate_engine +from src.rag.engine.chunker import chunk_text +from src.rag.engine.bm25 import bm25_search +from src.rag.engine.vector import vector_search +from src.rag.engine.hybrid import rrf_fuse + + +SAMPLE_DOCS = { + "plan.md": ( + "# План 3-го этажа\n\n" + "План 3-го этажа жилого дома. Оси: А, Б, В, Г. Размеры между осями А и Б: 5400 мм.\n" + "Квартиры: 301, 302, 303. Кухни объединены с гостиными. Санузлы раздельные.\n\n" + "## Отделка\n" + "Стены — штукатурка, покраска. Полы — ламинат. Потолки — гипсокартон.\n" + ), + "auth.md": ( + "# Авторизация\n\n" + "Авторизация работает через JWT-токены с TTL 24 часа.\n" + "Refresh-токен живёт 30 дней. Логика валидации в middleware.\n" + "Сессии хранятся в Redis, ключ — sha256 от user_id + jti.\n" + ), + "schedule.md": ( + "# График работ\n\n" + "Строительство начинается 1 июня 2026. Окончание — 30 ноября 2027.\n" + "Этапы: фундамент → стены → кровля → MEP → отделка.\n" + ), +} + + +class ChunkerTestCase(unittest.TestCase): + def test_chunks_short_text_returns_one(self): + chunks = chunk_text("Привет, мир.", max_chars=100) + self.assertEqual(len(chunks), 1) + self.assertIn("Привет", chunks[0].text) + + def test_chunks_long_text_splits_with_overlap(self): + text = "Абзац.\n\n" * 200 + chunks = chunk_text(text, max_chars=400, overlap=60) + self.assertGreater(len(chunks), 1) + for c in chunks: + self.assertLessEqual(len(c.text), 1000) # с overlap + + def test_empty_text_returns_empty(self): + self.assertEqual(chunk_text(""), []) + self.assertEqual(chunk_text(" \n\n "), []) + + +class EngineIngestTestCase(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.engines = [] + + def _make_engine(self, name: str = "coll1") -> Engine: + eng = Engine.from_paths(self.tmp / name) + self.engines.append(eng) + return eng + + def tearDown(self): + for eng in self.engines: + eng.close() + for db_path in self.tmp.rglob("index.sqlite"): + invalidate_engine(db_path.parent) + import time + time.sleep(0.05) + try: + self._tmp.cleanup() + except (PermissionError, OSError): + pass + + def test_index_text_returns_chunks_and_vectors(self): + eng = self._make_engine() + result = eng.index_text(SAMPLE_DOCS["plan.md"], source_path="plan.md") + self.assertFalse(result.skipped) + self.assertGreaterEqual(result.chunks_indexed, 1) + self.assertEqual(result.vectors_indexed, result.chunks_indexed) + status = eng.status() + self.assertGreaterEqual(status["chunks"], 1) + self.assertIn(status["engine"], ("sqlite-vec", "numpy")) + + def test_index_text_is_idempotent(self): + eng = self._make_engine() + r1 = eng.index_text(SAMPLE_DOCS["plan.md"], source_path="plan.md") + r2 = eng.index_text(SAMPLE_DOCS["plan.md"], source_path="plan.md") + self.assertFalse(r1.skipped) + self.assertTrue(r2.skipped) + + def test_index_file_change_detected(self): + eng = self._make_engine() + f = self.tmp / "x.md" + f.write_text("first version", encoding="utf-8") + r1 = eng.index_file(f) + self.assertFalse(r1.skipped) + f.write_text("second version with new content", encoding="utf-8") + r2 = eng.index_file(f) + self.assertFalse(r2.skipped) + + def test_index_many_files(self): + eng = self._make_engine() + for name, text in SAMPLE_DOCS.items(): + eng.index_text(text, source_path=name) + status = eng.status() + self.assertEqual(status["chunks"], 3) + self.assertEqual(status["files"], 3) + + +class EngineSearchTestCase(unittest.TestCase): + """Реальные тесты с warm-embedding.""" + + @classmethod + def setUpClass(cls): + cls._tmp = tempfile.TemporaryDirectory() + cls.tmp = Path(cls._tmp.name) + cls.eng = Engine.from_paths(cls.tmp / "coll_search") + for name, text in SAMPLE_DOCS.items(): + cls.eng.index_text(text, source_path=name) + cls.eng.warmup() + + @classmethod + def tearDownClass(cls): + cls.eng.close() + invalidate_engine(cls.tmp / "coll_search") + import time + time.sleep(0.05) + try: + cls._tmp.cleanup() + except (PermissionError, OSError): + pass + + def test_bm25_search_finds_keywords(self): + hits = self.eng.search("авторизация JWT") + self.assertGreater(len(hits), 0) + self.assertEqual(hits[0].file_path, "auth.md") + + def test_bm25_search_empty_query(self): + self.assertEqual(self.eng.search(""), []) + self.assertEqual(self.eng.search(" "), []) + + def test_vector_search_finds_semantic(self): + # "как устроена авторизация" — семантически близко к "авторизация" + hits = self.eng.vsearch("как устроена авторизация") + self.assertGreater(len(hits), 0) + # top hit должен быть auth.md или schedule.md (есть слово "логика") + paths = [h.file_path for h in hits] + self.assertTrue(any("auth" in p for p in paths)) + + def test_hybrid_query_uses_rrf(self): + hits = self.eng.query("квартиры 3 этаж", limit=3) + self.assertGreater(len(hits), 0) + self.assertEqual(hits[0].file_path, "plan.md") + + def test_get_returns_full_document(self): + hits = self.eng.search("авторизация") + full = self.eng.get(hits[0].doc_id) + self.assertIn("JWT", full) + self.assertIn("TTL", full) + + def test_status_reports_engine(self): + status = self.eng.status() + self.assertEqual(status["files"], 3) + self.assertGreaterEqual(status["chunks"], 3) + self.assertEqual(status["embedding_dim"], 384) + self.assertTrue(status["embedding_loaded"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_native_engine_e2e.py b/tests/test_native_engine_e2e.py new file mode 100644 index 0000000..d8f33a1 --- /dev/null +++ b/tests/test_native_engine_e2e.py @@ -0,0 +1,122 @@ +"""End-to-end тест: ingest → search → context → chat-stream. + +Не мокает движок — ставит заглушку только на OpenCode-клиент. +""" + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +from src.rag.qmd import indexer as qmd_indexer +from src.rag.qmd import query as qmd_query + + +SAMPLE_MEETING = ( + "# Совещание 2026-06-10\n\n" + "Участники: Иванов, Петров, Сидорова.\n\n" + "## Повестка\n" + "1. Ход строительства 3-й очереди.\n" + "2. Авторизация подрядчиков в системе.\n" + "3. Сроки сдачи.\n\n" + "## Решения\n" + "- Завершить фундамент до 15 июля.\n" + "- Выдать JWT-токены подрядчикам.\n" + "- Срок сдачи — 30 ноября 2027.\n" +) + + +class E2ETestCase(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + from src.rag.qmd.collections import get_project_collection_dir + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.coll_dir = get_project_collection_dir("merakom", "2026", self.tmp) + self.coll_dir.mkdir(parents=True, exist_ok=True) + self.engines = [] + + async def asyncTearDown(self): + from src.rag.engine import invalidate_engine + for eng in self.engines: + eng.close() + invalidate_engine(self.coll_dir) + import time + time.sleep(0.05) + try: + self._tmp.cleanup() + except (PermissionError, OSError): + pass + + async def test_e2e_meeting_index_search_chat(self): + # 1. Ingest через Engine.index_file напрямую (в self.coll_dir). + from src.rag.engine import get_or_create_engine + eng = get_or_create_engine(self.coll_dir) + eng.warmup() + self.engines.append(eng) + + body = self.tmp / "meeting.txt" + summary = self.tmp / "meeting_summary.md" + body.write_text(SAMPLE_MEETING, encoding="utf-8") + summary.write_text("# Краткое\nОбсуждали строительство и JWT.", encoding="utf-8") + r1 = eng.index_file(body) + r2 = eng.index_file(summary) + self.assertFalse(r1.skipped) + self.assertFalse(r2.skipped) + + # 2. Search через тот же engine. + hits = eng.query("сдача объекта", limit=3, use_rerank=False) + self.assertGreater(len(hits), 0) + # top hit должен относиться к meeting.txt + self.assertTrue(any("meeting.txt" in h.file_path for h in hits)) + + # 3. Chat-stream с подменой OpenCode. + fake_chunks = [ + MagicMock(choices=[MagicMock(delta=MagicMock(content="Сдача "))]), + MagicMock(choices=[MagicMock(delta=MagicMock(content="30 ноября "))]), + MagicMock(choices=[MagicMock(delta=MagicMock(content="2027."))]), + ] + + class _FakeStream: + def __init__(self): + self._i = 0 + + def __aiter__(self): + return self + + async def __anext__(self): + if self._i >= len(fake_chunks): + raise StopAsyncIteration + item = fake_chunks[self._i] + self._i += 1 + return item + + async def _fake_create(*args, **kwargs): + if kwargs.get("stream"): + return _FakeStream() + return MagicMock(choices=[MagicMock(message=MagicMock(content="final"))]) + + with patch("src.rag.qmd.query.AsyncOpenAI") as fake_cls: + fake_instance = MagicMock() + fake_instance.chat = MagicMock() + fake_instance.chat.completions = MagicMock() + fake_instance.chat.completions.create = AsyncMock(side_effect=_fake_create) + fake_cls.return_value = fake_instance + + events = [] + async for ev in qmd_query.qmd_chat_stream( + question="Когда сдача?", + org_slug="merakom", + history=[], + project_slug="2026", + api_key="test-key", + use_rerank=False, + ): + events.append(ev) + + types = [e["type"] for e in events] + self.assertEqual(types[0], "context") + self.assertIn("chunk", types) + self.assertEqual(types[-1], "done") + self.assertEqual(events[-1]["answer"], "Сдача 30 ноября 2027.") + # Контекст непустой и содержит source-annotation + self.assertIn("[source:", events[0]["context"]) diff --git a/tests/test_qmd_cache.py b/tests/test_qmd_cache.py new file mode 100644 index 0000000..22895da --- /dev/null +++ b/tests/test_qmd_cache.py @@ -0,0 +1,69 @@ +"""Tests for qmd search-result cache (TTL + mtime invalidation).""" + +import asyncio +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from src.rag.qmd.cache import SearchCache + + +class SearchCacheTestCase(unittest.IsolatedAsyncioTestCase): + async def test_get_set_roundtrip(self): + cache = SearchCache(ttl_seconds=60) + args = ("merakom", "2026", "bm25", True) + self.assertIsNone(cache.get("hello", args)) + cache.set("hello", args, "ctx-blob") + self.assertEqual(cache.get("hello", args), "ctx-blob") + + async def test_question_normalization(self): + cache = SearchCache(ttl_seconds=60) + args = ("merakom", "2026", "bm25", True) + cache.set("Hello World", args, "ctx") + self.assertEqual(cache.get(" hello world ", args), "ctx") + + async def test_different_args_yield_different_keys(self): + cache = SearchCache(ttl_seconds=60) + cache.set("q", ("o", "p1", "bm25", True), "ctx1") + cache.set("q", ("o", "p2", "bm25", True), "ctx2") + self.assertEqual(cache.get("q", ("o", "p1", "bm25", True)), "ctx1") + self.assertEqual(cache.get("q", ("o", "p2", "bm25", True)), "ctx2") + + async def test_set_skips_empty_value(self): + cache = SearchCache(ttl_seconds=60) + cache.set("q", ("o", "p", "bm25", True), "") + self.assertIsNone(cache.get("q", ("o", "p", "bm25", True))) + + async def test_ttl_expiry(self): + cache = SearchCache(ttl_seconds=0) + args = ("o", "p", "bm25", True) + with patch("src.rag.qmd.cache._index_mtime", return_value=0.0): + cache.set("q", args, "ctx") + await asyncio.sleep(0.05) + self.assertIsNone(cache.get("q", args)) + + async def test_mtime_invalidation(self): + cache = SearchCache(ttl_seconds=60) + args = ("o", "p", "bm25", True) + with patch("src.rag.qmd.cache._index_mtime", return_value=10.0): + cache.set("q", args, "ctx") + with patch("src.rag.qmd.cache._index_mtime", return_value=20.0): + self.assertIsNone(cache.get("q", args)) + + async def test_clear(self): + cache = SearchCache(ttl_seconds=60) + cache.set("q", ("o", "p", "bm25", True), "ctx") + cache.clear() + self.assertIsNone(cache.get("q", ("o", "p", "bm25", True))) + + async def test_stats(self): + cache = SearchCache(ttl_seconds=42) + stats = cache.stats() + self.assertEqual(stats["entries"], 0) + self.assertEqual(stats["ttl_seconds"], 42) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_rag_stream_ws.py b/tests/test_rag_stream_ws.py new file mode 100644 index 0000000..bc05bd0 --- /dev/null +++ b/tests/test_rag_stream_ws.py @@ -0,0 +1,59 @@ +"""Smoke-test WebSocket RAG streaming against running service.""" + +import asyncio +import json +import sys +import urllib.error +import urllib.request + +import websockets + + +async def main() -> int: + login_body = json.dumps({"username": "admin", "password": "admin123"}).encode() + login_req = urllib.request.Request( + "http://127.0.0.1:8000/api/auth/login", + data=login_body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(login_req, timeout=10) as resp: + token = json.loads(resp.read().decode())["access_token"] + except urllib.error.HTTPError as e: + print(f"Login failed: {e.read().decode()}", file=sys.stderr) + return 1 + + uri = f"ws://127.0.0.1:8000/ws?token={token}" + types: list[str] = [] + chunks = 0 + + async with websockets.connect(uri, open_timeout=10) as ws: + await ws.send(json.dumps({ + "action": "rag_query_global", + "question": "Кратко: что есть в базе знаний?", + "history": [], + "chat_mode": "hybrid", + "retrieval_mode": "naive", + })) + + while len(types) < 200: + raw = await asyncio.wait_for(ws.recv(), timeout=180) + msg = json.loads(raw) + msg_type = msg.get("type") + types.append(msg_type) + if msg_type == "rag_chunk": + chunks += 1 + if msg_type in ("rag_response", "rag_error"): + if msg_type == "rag_error": + print(f"RAG error: {msg.get('error')}", file=sys.stderr) + return 1 + print(f"OK: types={types[:5]}... chunks={chunks} answer_len={len(msg.get('answer', ''))}") + return 0 + + print("No terminal message received", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/tests/test_stub_writer.py b/tests/test_stub_writer.py new file mode 100644 index 0000000..ae379fa --- /dev/null +++ b/tests/test_stub_writer.py @@ -0,0 +1,66 @@ +"""Tests for stub_writer (binary file -> .md frontmatter stub).""" + +import tempfile +import unittest +from pathlib import Path + +from src.ingest.stub_writer import write_stub, _infer_kind + + +class StubWriterTestCase(unittest.TestCase): + def test_infer_kind_video(self): + self.assertEqual(_infer_kind(Path("meeting.mp4")), "video") + self.assertEqual(_infer_kind(Path("recording.MKV")), "video") + + def test_infer_kind_audio(self): + self.assertEqual(_infer_kind(Path("track.wav")), "audio") + + def test_infer_kind_image(self): + self.assertEqual(_infer_kind(Path("photo.png")), "image") + + def test_infer_kind_archive(self): + self.assertEqual(_infer_kind(Path("backup.zip")), "archive") + + def test_infer_kind_other(self): + self.assertEqual(_infer_kind(Path("unknown.xyz")), "other") + + def test_write_stub_creates_md(self): + with tempfile.TemporaryDirectory() as tmp: + mp4 = Path(tmp) / "video.mp4" + mp4.write_bytes(b"fake-mp4-content") + stub = write_stub(mp4, project="2026") + self.assertEqual(stub, mp4.with_suffix(".mp4.md")) + self.assertTrue(stub.exists()) + text = stub.read_text(encoding="utf-8") + self.assertIn("source: video.mp4", text) + self.assertIn("kind: video", text) + self.assertIn("project: 2026", text) + self.assertIn("size: 16", text) + self.assertIn("Бинарный файл", text) + + def test_write_stub_overwrites(self): + with tempfile.TemporaryDirectory() as tmp: + mp4 = Path(tmp) / "video.mp4" + mp4.write_bytes(b"a" * 10) + stub = write_stub(mp4, project="p1") + stub.write_text("OLD", encoding="utf-8") + stub2 = write_stub(mp4, project="p1") + self.assertEqual(stub, stub2) + self.assertNotIn("OLD", stub.read_text(encoding="utf-8")) + + def test_write_stub_missing_file_raises(self): + with tempfile.TemporaryDirectory() as tmp: + missing = Path(tmp) / "nope.mp4" + with self.assertRaises(FileNotFoundError): + write_stub(missing, project="x") + + def test_explicit_kind_overrides_inference(self): + with tempfile.TemporaryDirectory() as tmp: + f = Path(tmp) / "weird.bin" + f.write_bytes(b"x") + stub = write_stub(f, project="p", kind="custom") + self.assertIn("kind: custom", stub.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main()