eee8f4c8a4
- New: src/rag/engine/ — in-process hybrid search (FTS5 BM25 + sqlite-vec + LLM rerank) - New: src/rag/qmd/ — compatibility layer (qmd_query, qmd_chat, qmd_chat_stream, qmd_index_*) - New: src/ingest/stub_writer.py — .md stubs for binary files (videos, archives) - New: scripts/deploy.sh + scripts/pull_models.sh + Makefile + .env.example - Removed: LightRAG, sentence-transformers embedding via separate package, rag_standalone/ - Removed: @nousresearch/qmd npm dep (package not published); Node.js from Dockerfile - Updated: tests/ (46 passed), docker-compose, .dockerignore, config.yaml, README Engine: in-process Python (no daemon, no npm), sentence-transformers 384-dim, RRF fusion (k=60), BM25 + vector with numpy fallback. WebSocket API unchanged. Deploy: 'git clone' + 'make init' + 'make pull-models MODELS_SOURCE=...' + 'make up'. Models (5.83 GB) live outside git; pulled via rsync from dev host.
56 lines
2.4 KiB
Bash
56 lines
2.4 KiB
Bash
#!/usr/bin/env bash
|
|
# scripts/pull_models.sh — загрузка моделей (rsync из источника + fallback на download).
|
|
#
|
|
# Использование:
|
|
# MODELS_SOURCE=user@host:/opt/transcription/models/ bash scripts/pull_models.sh
|
|
# bash scripts/pull_models.sh # если MODELS_SOURCE не задан — попробует скачать из интернета
|
|
#
|
|
# Переменные окружения:
|
|
# MODELS_SOURCE — rsync-источник (user@host:path) — приоритетный способ
|
|
# MODELS_DIR — локальная папка (по умолчанию ./models)
|
|
# FORCE_DOWNLOAD — если "1", игнорирует кэш и качает заново
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
MODELS_DIR="${MODELS_DIR:-$PROJECT_ROOT/models}"
|
|
MARKER="$MODELS_DIR/huggingface/hub/models--Systran--faster-whisper-large-v3"
|
|
|
|
# Если модели уже есть — выходим
|
|
if [ -d "$MARKER" ] && [ "${FORCE_DOWNLOAD:-0}" != "1" ]; then
|
|
echo "[pull_models] Models already present at $MODELS_DIR — skipping"
|
|
exit 0
|
|
fi
|
|
|
|
# 1. Приоритет: rsync
|
|
if [ -n "${MODELS_SOURCE:-}" ]; then
|
|
echo "[pull_models] Pulling models from $MODELS_SOURCE ..."
|
|
mkdir -p "$MODELS_DIR/huggingface"
|
|
rsync -avz --progress "$MODELS_SOURCE/huggingface/" "$MODELS_DIR/huggingface/"
|
|
rsync -avz --progress "$MODELS_SOURCE/nltk_data/" "$MODELS_DIR/nltk_data/" 2>/dev/null || true
|
|
echo "[pull_models] Done."
|
|
exit 0
|
|
fi
|
|
|
|
# 2. Fallback: скачать из интернета (если rsync не задан)
|
|
echo "[pull_models] MODELS_SOURCE not set, falling back to download_models.py ..."
|
|
cd "$PROJECT_ROOT"
|
|
|
|
# Проверяем, что python доступен
|
|
if ! command -v python >/dev/null 2>&1; then
|
|
echo "[pull_models] ERROR: python not found in PATH" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Устанавливаем минимальный набор для download
|
|
pip install --quiet --no-cache-dir openai-whisper 2>/dev/null || true
|
|
pip install --quiet --no-cache-dir "whisperx @ git+https://github.com/m-bain/whisperx.git" 2>/dev/null || true
|
|
|
|
python scripts/download_models.py || {
|
|
echo "[pull_models] ERROR: download_models.py failed." >&2
|
|
echo "Hint: set MODELS_SOURCE=user@host:/opt/transcription/models/ and retry." >&2
|
|
exit 1
|
|
}
|
|
echo "[pull_models] Downloaded to $MODELS_DIR"
|