Require explicit opt-in and compatible model for local-to-API embedding fallback

REQUIREMENTS.md

@@ -2,7 +2,7 @@
 
 > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`.
 
-**Test Verification Baseline:** **1052 Automated Tests** (699 Pytest Backend + 303 Vitest Frontend + 50 Playwright E2E).
+**Test Verification Baseline:** **1053 Automated Tests** (700 Pytest Backend + 303 Vitest Frontend + 50 Playwright E2E).
 
 ---
 
@@ -586,12 +586,13 @@ classDiagram
 - `test_api_stats_reports_healthy_with_real_embedded_store` - _Verify /admin/api/stats health check against a real embedded Qdrant vector store without mocks._
 - `test_main_app_favicon_and_root_redirect` - _Verify root app redirect, favicon routes, and static asset handling._
 
-#### `tests/backend/test_indexer_and_embeddings.py` (25 tests)
+#### `tests/backend/test_indexer_and_embeddings.py` (26 tests)
 - `test_embeddings_generation`
 - `test_empty_embeddings_batches`
 - `test_api_embeddings_mode`
 - `test_sparse_embeddings_failures`
 - `test_init_embeddings_fallbacks`
+- `test_offline_missing_assets_no_unexpected_outbound_requests` - _Asserts that when local assets are missing and fallback is disabled, no outbound requests occur and RuntimeError is raised on embedding attempts._
 - `test_chunk_uuid_consistency`
 - `test_extract_keywords`
 - `test_ensure_collection`

app/api/routers/settings.py

@@ -436,12 +436,17 @@
             litellm_api_key=payload.litellm_api_key,
             vision_ocr_model=payload.vision_ocr_model,
             chat_model=payload.chat_model,
+            allow_fallback=payload.allow_fallback,
+            fallback_dense_model=payload.fallback_dense_model,
         )
         return {
             "status": "success",
             "message": "Embedding engine settings updated successfully.",
             "config": updated_cfg
         }
+    except ValueError as ve:
+        logger.warning(f"Validation error updating embedding settings: {ve}")
+        return JSONResponse(status_code=400, content={"status": "error", "error": str(ve), "message": str(ve)})
     except Exception as e:
         logger.error(f"Error updating embedding settings: {e}")
         return JSONResponse(status_code=500, content={"status": "error", "error": "Failed to update embedding settings.", "message": "Failed to update embedding settings."})

app/models/schemas.py

@@ -186,6 +186,8 @@ class EmbeddingSettingsRequest(BaseModel):
     litellm_api_key: Optional[str] = None
     vision_ocr_model: Optional[str] = None
     chat_model: Optional[str] = None
+    allow_fallback: Optional[bool] = None
+    fallback_dense_model: Optional[str] = None
 
 # Topology Graph Models
 class TopologyNode(BaseModel):

app/services/database/connection.py

@@ -220,6 +220,10 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A
     vision_ocr_model = os.getenv("VISION_OCR_MODEL", "").strip()
     chat_model = os.getenv("CHAT_MODEL", "").strip()
 
+    allow_fallback_env = os.getenv("EMBEDDING_ALLOW_FALLBACK", "false").lower().strip()
+    allow_fallback = allow_fallback_env in ("1", "true", "yes")
+    fallback_dense_model = os.getenv("EMBEDDING_FALLBACK_DENSE_MODEL", "").strip()
+
     return {
         "provider": provider,
         "dense_model": dense_model,
@@ -230,6 +234,8 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A
         "litellm_api_key": litellm_api_key,
         "vision_ocr_model": vision_ocr_model,
         "chat_model": chat_model,
+        "allow_fallback": allow_fallback,
+        "fallback_dense_model": fallback_dense_model,
         "system_cpus": sys_res["cpus"],
         "system_memory_gb": sys_res["memory_gb"],
     }
@@ -253,12 +259,19 @@ def get_embedding_db_config() -> Dict[str, Any]:
         or get_metadata("chat_model")
         or get_metadata("embedding_chat_model")
     )
+    allow_fallback_str = get_metadata("embedding_allow_fallback")
+    fallback_dense_model = get_metadata("embedding_fallback_dense_model")
 
     default_cfg = _resolve_default_embedding_config()
 
     threads = int(threads_str) if threads_str and threads_str.isdigit() else default_cfg["threads"]
     batch_size = int(batch_size_str) if batch_size_str and batch_size_str.isdigit() else default_cfg["batch_size"]
 
+    if allow_fallback_str is not None:
+        allow_fallback = allow_fallback_str == "1"
+    else:
+        allow_fallback = default_cfg["allow_fallback"]
+
     return {
         "provider": (provider or default_cfg["provider"]).lower().strip(),
         "dense_model": (dense_model or default_cfg["dense_model"]).strip(),
@@ -269,6 +282,8 @@ def get_embedding_db_config() -> Dict[str, Any]:
         "litellm_api_key": litellm_api_key or default_cfg["litellm_api_key"],
         "vision_ocr_model": (vision_ocr_model or default_cfg["vision_ocr_model"]).strip(),
         "chat_model": (chat_model or default_cfg["chat_model"]).strip(),
+        "allow_fallback": allow_fallback,
+        "fallback_dense_model": (fallback_dense_model if fallback_dense_model is not None else default_cfg["fallback_dense_model"]).strip(),
         "system_cpus": default_cfg["system_cpus"],
         "system_memory_gb": default_cfg["system_memory_gb"],
     }
@@ -284,6 +299,8 @@ def set_embedding_db_config(
     litellm_api_key: Optional[str] = None,
     vision_ocr_model: Optional[str] = None,
     chat_model: Optional[str] = None,
+    allow_fallback: Optional[bool] = None,
+    fallback_dense_model: Optional[str] = None,
 ):
     if provider is not None:
         set_metadata("embedding_provider", provider.lower().strip())
@@ -307,6 +324,10 @@ def set_embedding_db_config(
         set_metadata("openai_chat_model", chat_model.strip())
         set_metadata("chat_model", chat_model.strip())
         set_metadata("embedding_chat_model", chat_model.strip())
+    if allow_fallback is not None:
+        set_metadata("embedding_allow_fallback", "1" if allow_fallback else "0")
+    if fallback_dense_model is not None:
+        set_metadata("embedding_fallback_dense_model", fallback_dense_model.strip())
 
 
 def get_vision_ocr_model() -> str:

app/services/embeddings.py

@@ -16,7 +16,9 @@ def _get_db_emb_config() -> Dict[str, Any]:
 
 _initial_cfg = _get_db_emb_config()
 
+REQUESTED_PROVIDER = _initial_cfg["provider"]
 EMBEDDING_PROVIDER = _initial_cfg["provider"]
+REQUESTED_DENSE_MODEL = _initial_cfg["dense_model"]
 DENSE_MODEL_NAME = _initial_cfg["dense_model"]
 SPARSE_MODEL_NAME = _initial_cfg["sparse_model"]
 EMBEDDING_NUM_THREADS = _initial_cfg["threads"]
@@ -25,6 +27,8 @@ def _get_db_emb_config() -> Dict[str, Any]:
 LITELLM_API_KEY = _initial_cfg.get("litellm_api_key", os.getenv("LITELLM_API_KEY", "dummy"))
 VISION_OCR_MODEL = _initial_cfg.get("vision_ocr_model", os.getenv("VISION_OCR_MODEL", ""))
 CHAT_MODEL = _initial_cfg.get("chat_model", os.getenv("CHAT_MODEL", ""))
+ALLOW_FALLBACK = _initial_cfg.get("allow_fallback", False)
+FALLBACK_DENSE_MODEL = _initial_cfg.get("fallback_dense_model", "")
 
 _dense_model = None
 _sparse_model = None
@@ -42,28 +46,41 @@ def init_embeddings(
     litellm_api_key: Optional[str] = None,
     vision_ocr_model: Optional[str] = None,
     chat_model: Optional[str] = None,
+    allow_fallback: Optional[bool] = None,
+    fallback_dense_model: Optional[str] = None,
 ):
     global _dense_model, _sparse_model, _openai_client
-    global EMBEDDING_PROVIDER, DENSE_MODEL_NAME, SPARSE_MODEL_NAME
+    global REQUESTED_PROVIDER, EMBEDDING_PROVIDER
+    global REQUESTED_DENSE_MODEL, DENSE_MODEL_NAME, SPARSE_MODEL_NAME
     global EMBEDDING_NUM_THREADS, EMBEDDING_BATCH_SIZE, LITELLM_URL, LITELLM_API_KEY
-    global VISION_OCR_MODEL, CHAT_MODEL
+    global VISION_OCR_MODEL, CHAT_MODEL, ALLOW_FALLBACK, FALLBACK_DENSE_MODEL
 
     cfg = _get_db_emb_config()
 
-    EMBEDDING_PROVIDER = (provider or cfg.get("provider") or "local").lower().strip()
-    DENSE_MODEL_NAME = (dense_model or cfg.get("dense_model") or "BAAI/bge-small-en-v1.5").strip()
+    REQUESTED_PROVIDER = (provider or cfg.get("provider") or "local").lower().strip()
+    EMBEDDING_PROVIDER = REQUESTED_PROVIDER
+    REQUESTED_DENSE_MODEL = (dense_model or cfg.get("dense_model") or "BAAI/bge-small-en-v1.5").strip()
+    DENSE_MODEL_NAME = REQUESTED_DENSE_MODEL
     SPARSE_MODEL_NAME = (sparse_model or cfg.get("sparse_model") or "Qdrant/bm25").strip()
     EMBEDDING_NUM_THREADS = int(threads if threads is not None else cfg.get("threads", 2))
     EMBEDDING_BATCH_SIZE = int(batch_size if batch_size is not None else cfg.get("batch_size", 32))
     LITELLM_URL = (litellm_url or cfg.get("litellm_url") or "http://litellm:4000/v1").strip()
     LITELLM_API_KEY = (litellm_api_key or cfg.get("litellm_api_key") or "dummy").strip()
     VISION_OCR_MODEL = (vision_ocr_model if vision_ocr_model is not None else (cfg.get("vision_ocr_model") or os.getenv("VISION_OCR_MODEL", ""))).strip()
     CHAT_MODEL = (chat_model if chat_model is not None else (cfg.get("chat_model") or os.getenv("CHAT_MODEL", ""))).strip()
+    ALLOW_FALLBACK = allow_fallback if allow_fallback is not None else cfg.get("allow_fallback", False)
+    FALLBACK_DENSE_MODEL = (fallback_dense_model if fallback_dense_model is not None else cfg.get("fallback_dense_model", "")).strip()
 
     # Set underlying OpenMP / BLAS thread guard
     os.environ["OMP_NUM_THREADS"] = str(EMBEDDING_NUM_THREADS)
 
-    if EMBEDDING_PROVIDER == "local":
+    with _dense_lock:
+        _dense_model = None
+    with _sparse_lock:
+        _sparse_model = None
+    _openai_client = None
+
+    if REQUESTED_PROVIDER == "local":
         try:
             from fastembed import TextEmbedding, SparseTextEmbedding
             logger.info(f"Initializing FastEmbed dense model: {DENSE_MODEL_NAME} with threads={EMBEDDING_NUM_THREADS}")
@@ -74,9 +91,29 @@ def init_embeddings(
                 _sparse_model = SparseTextEmbedding(model_name=SPARSE_MODEL_NAME, threads=EMBEDDING_NUM_THREADS)
             logger.info("FastEmbed dense & sparse models initialized successfully with resource limits.")
         except Exception as e:
-            logger.error(f"Failed to initialize local FastEmbed models: {e}. Falling back to API.")
-            EMBEDDING_PROVIDER = "api"
-
+            logger.error(f"Failed to initialize local FastEmbed models: {e}")
+            with _dense_lock:
+                _dense_model = None
+            with _sparse_lock:
+                _sparse_model = None
+
+            if not ALLOW_FALLBACK:
+                EMBEDDING_PROVIDER = "local"
+                raise RuntimeError(
+                    f"Local FastEmbed initialization failed for dense model '{DENSE_MODEL_NAME}': {e}. "
+                    "Fallback to API is disabled (allow_fallback=False)."
+                ) from e
+            else:
+                if not FALLBACK_DENSE_MODEL:
+                    EMBEDDING_PROVIDER = "local"
+                    raise ValueError(
+                        f"Local FastEmbed initialization failed for '{DENSE_MODEL_NAME}': {e}, "
+                        "and allow_fallback=True but no valid fallback_dense_model was configured."
+                    ) from e
+                logger.warning(f"Local FastEmbed initialization failed: {e}. Falling back to API with model '{FALLBACK_DENSE_MODEL}'.")
+                EMBEDDING_PROVIDER = "api"
+                DENSE_MODEL_NAME = FALLBACK_DENSE_MODEL
+
     if EMBEDDING_PROVIDER == "api":
         try:
             from openai import OpenAI
@@ -96,9 +133,14 @@ def init_embeddings(
                 _sparse_model = None
         except Exception as oe:
             logger.error(f"Failed to initialize API client: {oe}")
+            if REQUESTED_PROVIDER == "local" and ALLOW_FALLBACK:
+                raise RuntimeError(f"Fallback to API failed: {oe}") from oe
 
-# Auto-initialize on import
-init_embeddings()
+# Auto-initialize on import safely
+try:
+    init_embeddings()
+except Exception as _e:
+    logger.warning(f"Initial embedding auto-init failed: {_e}")
 
 def get_embedding_config() -> Dict[str, Any]:
     """Returns current active embedding configuration along with system hardware stats."""
@@ -110,13 +152,19 @@ def get_embedding_config() -> Dict[str, Any]:
 
     return {
         "provider": EMBEDDING_PROVIDER,
+        "requested_provider": REQUESTED_PROVIDER,
+        "effective_provider": EMBEDDING_PROVIDER,
         "dense_model": DENSE_MODEL_NAME,
+        "requested_dense_model": REQUESTED_DENSE_MODEL,
+        "effective_dense_model": DENSE_MODEL_NAME,
         "sparse_model": SPARSE_MODEL_NAME,
         "threads": EMBEDDING_NUM_THREADS,
         "batch_size": EMBEDDING_BATCH_SIZE,
         "litellm_url": LITELLM_URL,
         "vision_ocr_model": VISION_OCR_MODEL,
         "chat_model": CHAT_MODEL,
+        "allow_fallback": ALLOW_FALLBACK,
+        "fallback_dense_model": FALLBACK_DENSE_MODEL,
         "system_cpus": sys_res["cpus"],
         "system_memory_gb": sys_res["memory_gb"],
     }
@@ -131,6 +179,8 @@ def update_embedding_config(
     litellm_api_key: Optional[str] = None,
     vision_ocr_model: Optional[str] = None,
     chat_model: Optional[str] = None,
+    allow_fallback: Optional[bool] = None,
+    fallback_dense_model: Optional[str] = None,
 ) -> Dict[str, Any]:
     """Updates embedding configuration in SQLite and hot-reloads models in memory."""
     from app.services.database import set_embedding_db_config
@@ -144,6 +194,8 @@ def update_embedding_config(
         litellm_api_key=litellm_api_key,
         vision_ocr_model=vision_ocr_model,
         chat_model=chat_model,
+        allow_fallback=allow_fallback,
+        fallback_dense_model=fallback_dense_model,
     )
     init_embeddings(
         provider=provider,
@@ -155,6 +207,8 @@ def update_embedding_config(
         litellm_api_key=litellm_api_key,
         vision_ocr_model=vision_ocr_model,
         chat_model=chat_model,
+        allow_fallback=allow_fallback,
+        fallback_dense_model=fallback_dense_model,
     )
     return get_embedding_config()
 
@@ -168,18 +222,22 @@ def get_dense_embeddings_batch(texts: List[str], batch_size: Optional[int] = Non
     if not texts:
         return []
     bs = batch_size or EMBEDDING_BATCH_SIZE
-    if EMBEDDING_PROVIDER == "local" and _dense_model is not None:
+    if EMBEDDING_PROVIDER == "local":
+        if _dense_model is None:
+            raise RuntimeError("Local dense embedding model is not initialized.")
         with _dense_lock:
             embeddings = list(_dense_model.embed(texts, batch_size=bs))
             return [e.tolist() for e in embeddings]
-    else:
+    elif EMBEDDING_PROVIDER == "api":
         if _openai_client is None:
             raise RuntimeError("API Embedding client not initialized.")
         response = _openai_client.embeddings.create(
             input=texts,
             model=DENSE_MODEL_NAME
         )
         return [d.embedding for d in response.data]
+    else:
+        raise RuntimeError(f"Unknown embedding provider: {EMBEDDING_PROVIDER}")
 
 def get_sparse_embedding(text: str) -> Optional[qmodels.SparseVector]:
     """Generates BM25 sparse vector embedding for a single text."""

pytest.ini

@@ -1,7 +1,10 @@
 [pytest]
 pythonpath = .
+asyncio_default_fixture_loop_scope = function
 filterwarnings =
     error
+    ignore:.*asyncio_default_fixture_loop_scope.*:pytest.PytestDeprecationWarning
+    ignore:.*asyncio_default_fixture_loop_scope.*:DeprecationWarning
     ignore:.*Field 'lifespan' has an incomplete definition.*:UserWarning
     ignore:.*Field 'lifespan' has an incomplete definition.*:Warning
     ignore:.*unauthenticated requests to the HF Hub.*:UserWarning