expose readiness failures when application metadata is unavailable

README.md

@@ -322,7 +322,7 @@ The server starts on port `3000`:
 - **MCP SSE Transport**: `http://localhost:3000/sse`
 - **MCP Streamable HTTP Transport**: `http://localhost:3000/mcp`
 - **OAuth Protected Resource Metadata**: `http://localhost:3000/.well-known/oauth-protected-resource`
-- **Health Check**: `http://localhost:3000/health`
+- **Health Check Probes**: Liveness at `http://localhost:3000/health`, Readiness at `http://localhost:3000/ready`
 
 ---
 

REQUIREMENTS.md

@@ -2,7 +2,7 @@
 
 > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`.
 
-**Test Verification Baseline:** **1052 Automated Tests** (699 Pytest Backend + 303 Vitest Frontend + 50 Playwright E2E).
+**Test Verification Baseline:** **1057 Automated Tests** (704 Pytest Backend + 303 Vitest Frontend + 50 Playwright E2E).
 
 ---
 
@@ -580,11 +580,16 @@ classDiagram
 - `test_git_syncer_nonexistent_repo`
 - `test_git_progress_tracker_pending_cancellation`
 
-#### `tests/backend/test_health_check_status.py` (4 tests)
+#### `tests/backend/test_health_check_status.py` (9 tests)
 - `test_api_stats_reports_healthy_when_store_is_healthy`
 - `test_api_stats_reports_unhealthy_when_store_fails`
 - `test_api_stats_reports_healthy_with_real_embedded_store` - _Verify /admin/api/stats health check against a real embedded Qdrant vector store without mocks._
 - `test_main_app_favicon_and_root_redirect` - _Verify root app redirect, favicon routes, and static asset handling._
+- `test_health_endpoints_cheap_liveness` - _Verify GET /health and /healthz return cheap liveness HTTP 200 responses._
+- `test_readiness_endpoint_success` - _Verify GET /ready and aliases return HTTP 200 when database schema is initialized._
+- `test_readiness_endpoint_missing_schema` - _Verify GET /ready returns HTTP 503 when required schema tables are missing._
+- `test_readiness_split_database_failure` - _Verify split-database scenario (#45) where direct connection targets empty DB and fails readiness._
+- `test_readiness_sanitized_public_response` - _Verify public readiness response never leaks credentials, host details, or raw stack traces._
 
 #### `tests/backend/test_indexer_and_embeddings.py` (25 tests)
 - `test_embeddings_generation`

app/services/database/__init__.py

@@ -43,6 +43,7 @@
     get_file_settings,
     set_file_settings,
     ensure_file_summaries_columns,
+    check_db_readiness,
 )
 from app.services.database.credentials import (
     list_git_host_credentials,

app/services/database/connection.py

@@ -2,9 +2,9 @@
 import sys
 import sqlite3
 import logging
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, Tuple
 
-from sqlalchemy import select
+from sqlalchemy import select, func
 
 from app.services.database.schema import TABLES
 from app.services.database.engine import (
@@ -430,6 +430,46 @@ def ensure_file_summaries_columns(conn=None):
         logger.debug(f"Migration error for file_summaries: {e}")
 
 
+def check_db_readiness() -> Tuple[bool, Dict[str, Any]]:
+    """
+    Verifies that the application database engine and core connection pathways are reachable
+    and that required relational schema tables exist and are readable.
+    Returns (is_ready, sanitized_info).
+    """
+    db_type = "postgresql" if is_postgres() else "sqlite"
+    required_tables = [
+        "system_metadata",
+        "git_repositories",
+        "indexed_paths",
+        "indexed_files",
+        "ast_symbols",
+        "file_summaries",
+    ]
+
+    try:
+        # 1. Verify primary SQLAlchemy engine connectivity & schema access
+        with get_connection() as conn:
+            for table_name in required_tables:
+                if table_name in TABLES:
+                    conn.execute(select(func.count()).select_from(TABLES[table_name])).scalar()
+
+        # 2. Verify direct connection pathway used by legacy/subrouter queries
+        if is_postgres():
+            with get_connection() as raw_conn:
+                for table_name in required_tables:
+                    raw_conn.execute(text(f"SELECT 1 FROM {table_name} LIMIT 1"))
+        else:
+            with get_db_connection() as raw_conn:
+                cursor = raw_conn.cursor()
+                for table_name in required_tables:
+                    cursor.execute(f"SELECT 1 FROM {table_name} LIMIT 1")
+
+        return True, {"status": "ok", "type": db_type}
+    except Exception as e:
+        logger.warning(f"Database readiness check failed ({db_type}): {e}")
+        return False, {"status": "error", "type": db_type}
+
+
 
 def get_file_settings() -> Dict[str, Any]:
     """Retrieves file reading and large file summarization settings."""

docs/guide/getting-started.md

@@ -86,13 +86,16 @@ Follow these steps for manual installation:
    - Web Admin Dashboard: `http://localhost:3000/admin/`
    - MCP Server-Sent Events: `http://localhost:3000/sse`
    - MCP Streamable HTTP: `http://localhost:3000/mcp`
-   - Health Check Probes: `http://localhost:3000/health` and `http://localhost:3000/healthz`
+   - Health Check Probes: Liveness (`http://localhost:3000/health`, `/healthz`) and Readiness (`http://localhost:3000/ready`, `/readiness`, `/health/ready`)
 
 3. Verify server health and MCP JSON-RPC `ping` readiness from a terminal:
    ```bash
-   # HTTP health probe
+   # HTTP process liveness probe
    curl -s http://localhost:3000/healthz
 
+   # HTTP application readiness probe (verifies relational database & schema accessibility)
+   curl -s http://localhost:3000/ready
+
    # Lightweight JSON-RPC ping (zero session allocation)
    curl -s -X POST http://localhost:3000/mcp \
      -H "Content-Type: application/json" \

docs/guide/user-guide/diagnostics.md

@@ -23,22 +23,36 @@ ContextCortex maintains an in-memory ring buffer of the latest 500 system events
 
 ---
 
-## Health Check & MCP Liveness Endpoints
+## Health Check Probes & MCP Liveness Endpoints
 
-ContextCortex exposes standard HTTP health probes and a zero-allocation MCP JSON-RPC `ping` handler for container orchestrators and MCP gateways (such as Docker, Kubernetes, Uptime Kuma, and Model Context Gateway):
+ContextCortex distinguishes process liveness from application readiness for deployment probes in container orchestrators (such as Docker Compose, Kubernetes, Uptime Kuma, and Model Context Gateway):
+
+### 1. Process Liveness Probes (`/health`, `/healthz`)
+- **Purpose**: Cheap, low-overhead HTTP process check returning `{"status": "healthy", ...}` (HTTP 200).
+- **Scope**: Verifies that the FastAPI process is running and accepting incoming HTTP requests. Process liveness does **not** verify database schema or backend storage readiness.
+
+### 2. Application Readiness Probes (`/ready`, `/readiness`, `/health/ready`, `/healthz/ready`)
+- **Purpose**: Deep application readiness signal returning `{"status": "ready", ...}` (HTTP 200) when ready, or `{"status": "not_ready", ...}` (HTTP 503) when required dependencies are unavailable.
+- **Dependencies Covered**: Verifies active connectivity to the configured database engine (PostgreSQL or SQLite) and asserts that required relational schema tables (`system_metadata`, `git_repositories`, `indexed_paths`, `indexed_files`, `ast_symbols`, `file_summaries`) exist and are accessible through the selected database pathway used by core operations.
+- **Sanitized Response**: Public diagnostic responses are sanitized to ensure database credentials, connection strings, and host secrets are never exposed.
+- **Ingestion Note**: Gateway process liveness or readiness verifies schema and API availability, but does not equate to complete background repository ingestion or vector store indexing, which run asynchronously in background workers.
 
 ```bash
-# 1. HTTP liveness & readiness probe (/health or /healthz)
+# 1. Cheap HTTP process liveness probe
 curl -s http://localhost:8021/healthz
-# Response: {"status":"healthy","service":"ContextCortex","version":"2.19.0","protocolVersion":"2025-06-18"}
+# Response: {"status":"healthy","protocolVersion":"2026-07-28","server":"contextcortex"}
+
+# 2. Deep application readiness probe
+curl -s http://localhost:8021/ready
+# Response: {"status":"ready","protocolVersion":"2026-07-28","server":"contextcortex","database":{"status":"ok","type":"sqlite"}}
 
-# 2. Lightweight MCP JSON-RPC ping (bypasses SSE stream/session allocation)
+# 3. Lightweight MCP JSON-RPC ping (bypasses SSE stream/session allocation)
 curl -s -X POST http://localhost:8021/mcp \
   -H "Content-Type: application/json" \
   -d '{"jsonrpc":"2.0","id":1,"method":"ping"}'
 # Response: {"jsonrpc":"2.0","id":1,"result":{}}
 
-# 3. Detailed index and system statistics
+# 4. Detailed index and system statistics
 curl -s http://localhost:8021/admin/api/stats
 ```
 

main.py

@@ -24,7 +24,7 @@
     get_auth_service,
     set_current_auth_context,
 )
-from app.services.database import init_db
+from app.services.database import init_db, check_db_readiness
 import app.services.indexing as indexer
 from app.services.indexing import run_full_indexing, VAULT_PATH
 from app.services.poller import start_poller_daemon, stop_poller_daemon
@@ -201,6 +201,10 @@ async def __call__(self, scope, receive, send):
             path == "/.well-known/oauth-protected-resource"
             or path == "/health"
             or path == "/healthz"
+            or path == "/ready"
+            or path == "/readiness"
+            or path == "/health/ready"
+            or path == "/healthz/ready"
             or path == "/"
             or path == "/favicon.svg"
             or path == "/favicon.ico"
@@ -333,5 +337,42 @@ async def health():
     )
 
 
+@app.get("/ready")
+@app.get("/readiness")
+@app.get("/health/ready")
+@app.get("/healthz/ready")
+async def readiness():
+    try:
+        is_ready, db_info = check_db_readiness()
+    except Exception as e:
+        logger.warning(f"Readiness check unhandled exception: {e}")
+        is_ready = False
+        db_info = {"status": "error", "type": "unknown"}
+
+    if is_ready:
+        return JSONResponse(
+            status_code=200,
+            content={
+                "status": "ready",
+                "protocolVersion": "2026-07-28",
+                "server": "contextcortex",
+                "database": db_info,
+            },
+            headers={"Cache-Control": "no-cache, no-transform"},
+        )
+    else:
+        return JSONResponse(
+            status_code=503,
+            content={
+                "status": "not_ready",
+                "protocolVersion": "2026-07-28",
+                "server": "contextcortex",
+                "database": db_info,
+                "error": "Required database schema or metadata unavailable",
+            },
+            headers={"Cache-Control": "no-cache, no-transform"},
+        )
+
+
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=3000)

pytest.ini

@@ -1,5 +1,6 @@
 [pytest]
 pythonpath = .
+asyncio_default_fixture_loop_scope = function
 filterwarnings =
     error
     ignore:.*Field 'lifespan' has an incomplete definition.*:UserWarning

tests/backend/test_health_check_status.py

@@ -100,3 +100,87 @@ def test_main_app_favicon_and_root_redirect():
     res_ico = main_client.get("/favicon.ico")
     assert res_ico.status_code == 200
 
+
+def test_health_endpoints_cheap_liveness():
+    """Verify GET /health and /healthz return cheap liveness HTTP 200 responses."""
+    from main import app as main_app
+    main_client = TestClient(main_app)
+
+    for endpoint in ["/health", "/healthz"]:
+        res = main_client.get(endpoint)
+        assert res.status_code == 200
+        data = res.json()
+        assert data["status"] == "healthy"
+        assert data["server"] == "contextcortex"
+
+
+def test_readiness_endpoint_success(tmp_path):
+    """Verify GET /ready and aliases return HTTP 200 when database schema is initialized."""
+    from app.services.database import init_db
+    from main import app as main_app
+
+    db_path = str(tmp_path / "readiness_init.db")
+    with patch("app.services.database.engine.get_default_db_path", return_value=db_path), \
+         patch("app.services.database.connection.CACHE_DB_PATH", db_path):
+        init_db(vault_path=str(tmp_path))
+        main_client = TestClient(main_app)
+
+        for endpoint in ["/ready", "/readiness", "/health/ready", "/healthz/ready"]:
+            res = main_client.get(endpoint)
+            assert res.status_code == 200
+            data = res.json()
+            assert data["status"] == "ready"
+            assert data["database"]["status"] == "ok"
+
+
+def test_readiness_endpoint_missing_schema():
+    """Verify GET /ready returns HTTP 503 when required schema tables are missing."""
+    from main import app as main_app
+    main_client = TestClient(main_app)
+
+    with patch("main.check_db_readiness", return_value=(False, {"status": "error", "type": "sqlite"})):
+        res = main_client.get("/ready")
+        assert res.status_code == 503
+        data = res.json()
+        assert data["status"] == "not_ready"
+        assert data["database"]["status"] == "error"
+        assert "error" in data
+
+
+def test_readiness_split_database_failure(tmp_path):
+    """Verify split-database scenario (#45) where direct connection targets empty DB and fails readiness."""
+    from main import app as main_app
+    empty_db = str(tmp_path / "empty_cache.db")
+    # Touch empty file without schema
+    sqlite3.connect(empty_db).close()
+
+    main_client = TestClient(main_app)
+
+    with patch("app.services.database.connection.get_db_connection") as mock_direct:
+        def get_empty_conn():
+            c = sqlite3.connect(empty_db)
+            c.row_factory = sqlite3.Row
+            return c
+        mock_direct.side_effect = get_empty_conn
+
+        res = main_client.get("/ready")
+        assert res.status_code == 503
+        data = res.json()
+        assert data["status"] == "not_ready"
+        assert data["database"]["status"] == "error"
+
+
+def test_readiness_sanitized_public_response():
+    """Verify public readiness response never leaks credentials, host details, or raw stack traces."""
+    from main import app as main_app
+    main_client = TestClient(main_app)
+
+    with patch("main.check_db_readiness", side_effect=Exception("postgresql://user:secret_pass@db.internal:5432/cortex")):
+        res = main_client.get("/ready")
+        assert res.status_code == 503
+        data = res.json()
+        assert data["status"] == "not_ready"
+        body_str = res.text
+        assert "secret_pass" not in body_str
+        assert "db.internal" not in body_str
+        assert "postgresql://" not in body_str