diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index 59d9c13..9d1d56c 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -2,7 +2,7 @@ > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`. -**Test Verification Baseline:** **1052 Automated Tests** (699 Pytest Backend + 303 Vitest Frontend + 50 Playwright E2E). +**Test Verification Baseline:** **1053 Automated Tests** (700 Pytest Backend + 303 Vitest Frontend + 50 Playwright E2E). --- @@ -586,12 +586,13 @@ classDiagram - `test_api_stats_reports_healthy_with_real_embedded_store` - _Verify /admin/api/stats health check against a real embedded Qdrant vector store without mocks._ - `test_main_app_favicon_and_root_redirect` - _Verify root app redirect, favicon routes, and static asset handling._ -#### `tests/backend/test_indexer_and_embeddings.py` (25 tests) +#### `tests/backend/test_indexer_and_embeddings.py` (26 tests) - `test_embeddings_generation` - `test_empty_embeddings_batches` - `test_api_embeddings_mode` - `test_sparse_embeddings_failures` - `test_init_embeddings_fallbacks` +- `test_offline_missing_assets_no_unexpected_outbound_requests` - _Asserts that when local assets are missing and fallback is disabled, no outbound requests occur and RuntimeError is raised on embedding attempts._ - `test_chunk_uuid_consistency` - `test_extract_keywords` - `test_ensure_collection` diff --git a/app/api/routers/settings.py b/app/api/routers/settings.py index 77b2fa2..2c89d10 100644 --- a/app/api/routers/settings.py +++ b/app/api/routers/settings.py @@ -436,12 +436,17 @@ async def api_save_embedding_settings(payload: EmbeddingSettingsRequest): litellm_api_key=payload.litellm_api_key, vision_ocr_model=payload.vision_ocr_model, chat_model=payload.chat_model, + allow_fallback=payload.allow_fallback, + fallback_dense_model=payload.fallback_dense_model, ) return { "status": "success", "message": "Embedding engine settings updated successfully.", "config": updated_cfg } + except ValueError as ve: + logger.warning(f"Validation error updating embedding settings: {ve}") + return JSONResponse(status_code=400, content={"status": "error", "error": str(ve), "message": str(ve)}) except Exception as e: logger.error(f"Error updating embedding settings: {e}") return JSONResponse(status_code=500, content={"status": "error", "error": "Failed to update embedding settings.", "message": "Failed to update embedding settings."}) diff --git a/app/models/schemas.py b/app/models/schemas.py index 7bb95b7..d3323df 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -186,6 +186,8 @@ class EmbeddingSettingsRequest(BaseModel): litellm_api_key: Optional[str] = None vision_ocr_model: Optional[str] = None chat_model: Optional[str] = None + allow_fallback: Optional[bool] = None + fallback_dense_model: Optional[str] = None # Topology Graph Models class TopologyNode(BaseModel): diff --git a/app/services/database/connection.py b/app/services/database/connection.py index 4158996..88d3f96 100644 --- a/app/services/database/connection.py +++ b/app/services/database/connection.py @@ -220,6 +220,10 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A vision_ocr_model = os.getenv("VISION_OCR_MODEL", "").strip() chat_model = os.getenv("CHAT_MODEL", "").strip() + allow_fallback_env = os.getenv("EMBEDDING_ALLOW_FALLBACK", "false").lower().strip() + allow_fallback = allow_fallback_env in ("1", "true", "yes") + fallback_dense_model = os.getenv("EMBEDDING_FALLBACK_DENSE_MODEL", "").strip() + return { "provider": provider, "dense_model": dense_model, @@ -230,6 +234,8 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A "litellm_api_key": litellm_api_key, "vision_ocr_model": vision_ocr_model, "chat_model": chat_model, + "allow_fallback": allow_fallback, + "fallback_dense_model": fallback_dense_model, "system_cpus": sys_res["cpus"], "system_memory_gb": sys_res["memory_gb"], } @@ -253,12 +259,19 @@ def get_embedding_db_config() -> Dict[str, Any]: or get_metadata("chat_model") or get_metadata("embedding_chat_model") ) + allow_fallback_str = get_metadata("embedding_allow_fallback") + fallback_dense_model = get_metadata("embedding_fallback_dense_model") default_cfg = _resolve_default_embedding_config() threads = int(threads_str) if threads_str and threads_str.isdigit() else default_cfg["threads"] batch_size = int(batch_size_str) if batch_size_str and batch_size_str.isdigit() else default_cfg["batch_size"] + if allow_fallback_str is not None: + allow_fallback = allow_fallback_str == "1" + else: + allow_fallback = default_cfg["allow_fallback"] + return { "provider": (provider or default_cfg["provider"]).lower().strip(), "dense_model": (dense_model or default_cfg["dense_model"]).strip(), @@ -269,6 +282,8 @@ def get_embedding_db_config() -> Dict[str, Any]: "litellm_api_key": litellm_api_key or default_cfg["litellm_api_key"], "vision_ocr_model": (vision_ocr_model or default_cfg["vision_ocr_model"]).strip(), "chat_model": (chat_model or default_cfg["chat_model"]).strip(), + "allow_fallback": allow_fallback, + "fallback_dense_model": (fallback_dense_model if fallback_dense_model is not None else default_cfg["fallback_dense_model"]).strip(), "system_cpus": default_cfg["system_cpus"], "system_memory_gb": default_cfg["system_memory_gb"], } @@ -284,6 +299,8 @@ def set_embedding_db_config( litellm_api_key: Optional[str] = None, vision_ocr_model: Optional[str] = None, chat_model: Optional[str] = None, + allow_fallback: Optional[bool] = None, + fallback_dense_model: Optional[str] = None, ): if provider is not None: set_metadata("embedding_provider", provider.lower().strip()) @@ -307,6 +324,10 @@ def set_embedding_db_config( set_metadata("openai_chat_model", chat_model.strip()) set_metadata("chat_model", chat_model.strip()) set_metadata("embedding_chat_model", chat_model.strip()) + if allow_fallback is not None: + set_metadata("embedding_allow_fallback", "1" if allow_fallback else "0") + if fallback_dense_model is not None: + set_metadata("embedding_fallback_dense_model", fallback_dense_model.strip()) def get_vision_ocr_model() -> str: diff --git a/app/services/embeddings.py b/app/services/embeddings.py index 9c6695f..9c559cc 100644 --- a/app/services/embeddings.py +++ b/app/services/embeddings.py @@ -16,7 +16,9 @@ def _get_db_emb_config() -> Dict[str, Any]: _initial_cfg = _get_db_emb_config() +REQUESTED_PROVIDER = _initial_cfg["provider"] EMBEDDING_PROVIDER = _initial_cfg["provider"] +REQUESTED_DENSE_MODEL = _initial_cfg["dense_model"] DENSE_MODEL_NAME = _initial_cfg["dense_model"] SPARSE_MODEL_NAME = _initial_cfg["sparse_model"] EMBEDDING_NUM_THREADS = _initial_cfg["threads"] @@ -25,6 +27,8 @@ def _get_db_emb_config() -> Dict[str, Any]: LITELLM_API_KEY = _initial_cfg.get("litellm_api_key", os.getenv("LITELLM_API_KEY", "dummy")) VISION_OCR_MODEL = _initial_cfg.get("vision_ocr_model", os.getenv("VISION_OCR_MODEL", "")) CHAT_MODEL = _initial_cfg.get("chat_model", os.getenv("CHAT_MODEL", "")) +ALLOW_FALLBACK = _initial_cfg.get("allow_fallback", False) +FALLBACK_DENSE_MODEL = _initial_cfg.get("fallback_dense_model", "") _dense_model = None _sparse_model = None @@ -42,16 +46,21 @@ def init_embeddings( litellm_api_key: Optional[str] = None, vision_ocr_model: Optional[str] = None, chat_model: Optional[str] = None, + allow_fallback: Optional[bool] = None, + fallback_dense_model: Optional[str] = None, ): global _dense_model, _sparse_model, _openai_client - global EMBEDDING_PROVIDER, DENSE_MODEL_NAME, SPARSE_MODEL_NAME + global REQUESTED_PROVIDER, EMBEDDING_PROVIDER + global REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, SPARSE_MODEL_NAME global EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, LITELLM_URL, LITELLM_API_KEY - global VISION_OCR_MODEL, CHAT_MODEL + global VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL cfg = _get_db_emb_config() - EMBEDDING_PROVIDER = (provider or cfg.get("provider") or "local").lower().strip() - DENSE_MODEL_NAME = (dense_model or cfg.get("dense_model") or "BAAI/bge-small-en-v1.5").strip() + REQUESTED_PROVIDER = (provider or cfg.get("provider") or "local").lower().strip() + EMBEDDING_PROVIDER = REQUESTED_PROVIDER + REQUESTED_DENSE_MODEL = (dense_model or cfg.get("dense_model") or "BAAI/bge-small-en-v1.5").strip() + DENSE_MODEL_NAME = REQUESTED_DENSE_MODEL SPARSE_MODEL_NAME = (sparse_model or cfg.get("sparse_model") or "Qdrant/bm25").strip() EMBEDDING_NUM_THREADS = int(threads if threads is not None else cfg.get("threads", 2)) EMBEDDING_BATCH_SIZE = int(batch_size if batch_size is not None else cfg.get("batch_size", 32)) @@ -59,11 +68,19 @@ def init_embeddings( LITELLM_API_KEY = (litellm_api_key or cfg.get("litellm_api_key") or "dummy").strip() VISION_OCR_MODEL = (vision_ocr_model if vision_ocr_model is not None else (cfg.get("vision_ocr_model") or os.getenv("VISION_OCR_MODEL", ""))).strip() CHAT_MODEL = (chat_model if chat_model is not None else (cfg.get("chat_model") or os.getenv("CHAT_MODEL", ""))).strip() + ALLOW_FALLBACK = allow_fallback if allow_fallback is not None else cfg.get("allow_fallback", False) + FALLBACK_DENSE_MODEL = (fallback_dense_model if fallback_dense_model is not None else cfg.get("fallback_dense_model", "")).strip() # Set underlying OpenMP / BLAS thread guard os.environ["OMP_NUM_THREADS"] = str(EMBEDDING_NUM_THREADS) - if EMBEDDING_PROVIDER == "local": + with _dense_lock: + _dense_model = None + with _sparse_lock: + _sparse_model = None + _openai_client = None + + if REQUESTED_PROVIDER == "local": try: from fastembed import TextEmbedding, SparseTextEmbedding logger.info(f"Initializing FastEmbed dense model: {DENSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}") @@ -74,9 +91,29 @@ def init_embeddings( _sparse_model = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS) logger.info("FastEmbed dense & sparse models initialized successfully with resource limits.") except Exception as e: - logger.error(f"Failed to initialize local FastEmbed models: {e}. Falling back to API.") - EMBEDDING_PROVIDER = "api" - + logger.error(f"Failed to initialize local FastEmbed models: {e}") + with _dense_lock: + _dense_model = None + with _sparse_lock: + _sparse_model = None + + if not ALLOW_FALLBACK: + EMBEDDING_PROVIDER = "local" + raise RuntimeError( + f"Local FastEmbed initialization failed for dense model '{DENSE_MODEL_NAME}': {e}. " + "Fallback to API is disabled (allow_fallback=False)." + ) from e + else: + if not FALLBACK_DENSE_MODEL: + EMBEDDING_PROVIDER = "local" + raise ValueError( + f"Local FastEmbed initialization failed for '{DENSE_MODEL_NAME}': {e}, " + "and allow_fallback=True but no valid fallback_dense_model was configured." + ) from e + logger.warning(f"Local FastEmbed initialization failed: {e}. Falling back to API with model '{FALLBACK_DENSE_MODEL}'.") + EMBEDDING_PROVIDER = "api" + DENSE_MODEL_NAME = FALLBACK_DENSE_MODEL + if EMBEDDING_PROVIDER == "api": try: from openai import OpenAI @@ -96,9 +133,14 @@ def init_embeddings( _sparse_model = None except Exception as oe: logger.error(f"Failed to initialize API client: {oe}") + if REQUESTED_PROVIDER == "local" and ALLOW_FALLBACK: + raise RuntimeError(f"Fallback to API failed: {oe}") from oe -# Auto-initialize on import -init_embeddings() +# Auto-initialize on import safely +try: + init_embeddings() +except Exception as _e: + logger.warning(f"Initial embedding auto-init failed: {_e}") def get_embedding_config() -> Dict[str, Any]: """Returns current active embedding configuration along with system hardware stats.""" @@ -110,13 +152,19 @@ def get_embedding_config() -> Dict[str, Any]: return { "provider": EMBEDDING_PROVIDER, + "requested_provider": REQUESTED_PROVIDER, + "effective_provider": EMBEDDING_PROVIDER, "dense_model": DENSE_MODEL_NAME, + "requested_dense_model": REQUESTED_DENSE_MODEL, + "effective_dense_model": DENSE_MODEL_NAME, "sparse_model": SPARSE_MODEL_NAME, "threads": EMBEDDING_NUM_THREADS, "batch_size": EMBEDDING_BATCH_SIZE, "litellm_url": LITELLM_URL, "vision_ocr_model": VISION_OCR_MODEL, "chat_model": CHAT_MODEL, + "allow_fallback": ALLOW_FALLBACK, + "fallback_dense_model": FALLBACK_DENSE_MODEL, "system_cpus": sys_res["cpus"], "system_memory_gb": sys_res["memory_gb"], } @@ -131,6 +179,8 @@ def update_embedding_config( litellm_api_key: Optional[str] = None, vision_ocr_model: Optional[str] = None, chat_model: Optional[str] = None, + allow_fallback: Optional[bool] = None, + fallback_dense_model: Optional[str] = None, ) -> Dict[str, Any]: """Updates embedding configuration in SQLite and hot-reloads models in memory.""" from app.services.database import set_embedding_db_config @@ -144,6 +194,8 @@ def update_embedding_config( litellm_api_key=litellm_api_key, vision_ocr_model=vision_ocr_model, chat_model=chat_model, + allow_fallback=allow_fallback, + fallback_dense_model=fallback_dense_model, ) init_embeddings( provider=provider, @@ -155,6 +207,8 @@ def update_embedding_config( litellm_api_key=litellm_api_key, vision_ocr_model=vision_ocr_model, chat_model=chat_model, + allow_fallback=allow_fallback, + fallback_dense_model=fallback_dense_model, ) return get_embedding_config() @@ -168,11 +222,13 @@ def get_dense_embeddings_batch(texts: List[str], batch_size: Optional[int] = Non if not texts: return [] bs = batch_size or EMBEDDING_BATCH_SIZE - if EMBEDDING_PROVIDER == "local" and _dense_model is not None: + if EMBEDDING_PROVIDER == "local": + if _dense_model is None: + raise RuntimeError("Local dense embedding model is not initialized.") with _dense_lock: embeddings = list(_dense_model.embed(texts, batch_size=bs)) return [e.tolist() for e in embeddings] - else: + elif EMBEDDING_PROVIDER == "api": if _openai_client is None: raise RuntimeError("API Embedding client not initialized.") response = _openai_client.embeddings.create( @@ -180,6 +236,8 @@ def get_dense_embeddings_batch(texts: List[str], batch_size: Optional[int] = Non model=DENSE_MODEL_NAME ) return [d.embedding for d in response.data] + else: + raise RuntimeError(f"Unknown embedding provider: {EMBEDDING_PROVIDER}") def get_sparse_embedding(text: str) -> Optional[qmodels.SparseVector]: """Generates BM25 sparse vector embedding for a single text.""" diff --git a/pytest.ini b/pytest.ini index 00b9ce4..a021144 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,7 +1,10 @@ [pytest] pythonpath = . +asyncio_default_fixture_loop_scope = function filterwarnings = error + ignore:.*asyncio_default_fixture_loop_scope.*:pytest.PytestDeprecationWarning + ignore:.*asyncio_default_fixture_loop_scope.*:DeprecationWarning ignore:.*Field 'lifespan' has an incomplete definition.*:UserWarning ignore:.*Field 'lifespan' has an incomplete definition.*:Warning ignore:.*unauthenticated requests to the HF Hub.*:UserWarning diff --git a/tests/backend/test_indexer_and_embeddings.py b/tests/backend/test_indexer_and_embeddings.py index cc09b1b..f0d807c 100644 --- a/tests/backend/test_indexer_and_embeddings.py +++ b/tests/backend/test_indexer_and_embeddings.py @@ -91,28 +91,81 @@ def test_sparse_embeddings_failures(): assert res == [None, None] def test_init_embeddings_fallbacks(): - # Local fastembed failure falls back to API - with patch.object(emb_module, "EMBEDDING_PROVIDER", "local"), \ - patch("fastembed.TextEmbedding", side_effect=Exception("Model weight missing")), \ - patch("openai.OpenAI"): - init_embeddings() - assert emb_module.EMBEDDING_PROVIDER == "api" - - # API mode with fastembed sparse failure - with patch.object(emb_module, "EMBEDDING_PROVIDER", "api"), \ - patch("openai.OpenAI"), \ + # 1. Local mode without fallback opt-in fails visibly when FastEmbed init fails + with patch("fastembed.TextEmbedding", side_effect=Exception("Model weight missing")): + with pytest.raises(RuntimeError, match="Fallback to API is disabled"): + init_embeddings(provider="local", dense_model="BAAI/bge-small-en-v1.5", allow_fallback=False) + cfg = emb_module.get_embedding_config() + assert cfg["requested_provider"] == "local" + assert cfg["effective_provider"] == "local" + + # 2. Local mode with fallback opt-in but missing fallback_dense_model raises ValueError + with patch("fastembed.TextEmbedding", side_effect=Exception("Model weight missing")): + with pytest.raises(ValueError, match="no valid fallback_dense_model was configured"): + init_embeddings( + provider="local", + dense_model="BAAI/bge-small-en-v1.5", + allow_fallback=True, + fallback_dense_model="" + ) + + # 3. Local mode with fallback opt-in and valid fallback_dense_model switches to API with fallback model + with patch("fastembed.TextEmbedding", side_effect=Exception("Model weight missing")), \ + patch("openai.OpenAI") as mock_openai: + init_embeddings( + provider="local", + dense_model="BAAI/bge-small-en-v1.5", + allow_fallback=True, + fallback_dense_model="text-embedding-3-small" + ) + cfg = emb_module.get_embedding_config() + assert cfg["requested_provider"] == "local" + assert cfg["effective_provider"] == "api" + assert cfg["requested_dense_model"] == "BAAI/bge-small-en-v1.5" + assert cfg["effective_dense_model"] == "text-embedding-3-small" + mock_openai.assert_called_once() + + # 4. API mode with fastembed sparse failure + with patch("openai.OpenAI"), \ patch("fastembed.SparseTextEmbedding", side_effect=Exception("Sparse model missing")): - init_embeddings() + init_embeddings(provider="api", dense_model="text-embedding-3-small") assert emb_module._sparse_model is None - # API mode with OpenAI client init failure - with patch.object(emb_module, "EMBEDDING_PROVIDER", "api"), \ - patch("openai.OpenAI", side_effect=Exception("OpenAI init failure")): - init_embeddings() + # 5. API mode with OpenAI client init failure + with patch("openai.OpenAI", side_effect=Exception("OpenAI init failure")): + init_embeddings(provider="api", dense_model="text-embedding-3-small") - # Restore local for other tests - emb_module.EMBEDDING_PROVIDER = "local" - init_embeddings() + # Restore local embeddings for subsequent tests + try: + init_embeddings(provider="local", dense_model="BAAI/bge-small-en-v1.5", allow_fallback=False) + except Exception: + pass + + +def test_offline_missing_assets_no_unexpected_outbound_requests(): + """Asserts that when local assets are missing and fallback is disabled, no outbound requests occur and RuntimeError is raised on embedding attempts.""" + with patch("fastembed.TextEmbedding", side_effect=ValueError("injected missing approved local model asset")), \ + patch("openai.OpenAI") as mock_openai: + with pytest.raises(RuntimeError, match="Fallback to API is disabled"): + init_embeddings(provider="local", dense_model="BAAI/bge-small-en-v1.5", allow_fallback=False) + + cfg = emb_module.get_embedding_config() + assert cfg["requested_provider"] == "local" + assert cfg["effective_provider"] == "local" + assert cfg["dense_model"] == "BAAI/bge-small-en-v1.5" + + # Ensure OpenAI client was never instantiated + mock_openai.assert_not_called() + + # Attempting to generate dense embeddings must raise RuntimeError and NOT send API request + with pytest.raises(RuntimeError, match="Local dense embedding model is not initialized."): + get_dense_embeddings_batch(["test string"]) + + # Restore local embeddings + try: + init_embeddings(provider="local", dense_model="BAAI/bge-small-en-v1.5", allow_fallback=False) + except Exception: + pass def test_chunk_uuid_consistency(): u1 = get_chunk_uuid("repo_a", "src/main.py", 0)