From 25c7c31f83889e078eddbc12717951f8df1654c7 Mon Sep 17 00:00:00 2001 From: spelech <28486500+spelech@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:21:40 +0000 Subject: [PATCH 1/3] feat(health): expose database readiness signal and schema verification Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- README.md | 2 +- REQUIREMENTS.md | 9 ++- app/services/database/__init__.py | 1 + app/services/database/connection.py | 39 ++++++++++- docs/guide/getting-started.md | 7 +- docs/guide/user-guide/diagnostics.md | 26 +++++-- main.py | 43 +++++++++++- pytest.ini | 1 + tests/backend/test_health_check_status.py | 84 +++++++++++++++++++++++ 9 files changed, 198 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 111ea50..94582aa 100644 --- a/README.md +++ b/README.md @@ -322,7 +322,7 @@ The server starts on port `3000`: - **MCP SSE Transport**: `http://localhost:3000/sse` - **MCP Streamable HTTP Transport**: `http://localhost:3000/mcp` - **OAuth Protected Resource Metadata**: `http://localhost:3000/.well-known/oauth-protected-resource` -- **Health Check**: `http://localhost:3000/health` +- **Health Check Probes**: Liveness at `http://localhost:3000/health`, Readiness at `http://localhost:3000/ready` --- diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index 59d9c13..545ff51 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -2,7 +2,7 @@ > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`. -**Test Verification Baseline:** **1052 Automated Tests** (699 Pytest Backend + 303 Vitest Frontend + 50 Playwright E2E). +**Test Verification Baseline:** **1057 Automated Tests** (704 Pytest Backend + 303 Vitest Frontend + 50 Playwright E2E). --- @@ -580,11 +580,16 @@ classDiagram - `test_git_syncer_nonexistent_repo` - `test_git_progress_tracker_pending_cancellation` -#### `tests/backend/test_health_check_status.py` (4 tests) +#### `tests/backend/test_health_check_status.py` (9 tests) - `test_api_stats_reports_healthy_when_store_is_healthy` - `test_api_stats_reports_unhealthy_when_store_fails` - `test_api_stats_reports_healthy_with_real_embedded_store` - _Verify /admin/api/stats health check against a real embedded Qdrant vector store without mocks._ - `test_main_app_favicon_and_root_redirect` - _Verify root app redirect, favicon routes, and static asset handling._ +- `test_health_endpoints_cheap_liveness` - _Verify GET /health and /healthz return cheap liveness HTTP 200 responses._ +- `test_readiness_endpoint_success` - _Verify GET /ready and aliases return HTTP 200 when database schema is initialized._ +- `test_readiness_endpoint_missing_schema` - _Verify GET /ready returns HTTP 503 when required schema tables are missing._ +- `test_readiness_split_database_failure` - _Verify split-database scenario (#45) where direct connection targets empty DB and fails readiness._ +- `test_readiness_sanitized_public_response` - _Verify public readiness response never leaks credentials, host details, or raw stack traces._ #### `tests/backend/test_indexer_and_embeddings.py` (25 tests) - `test_embeddings_generation` diff --git a/app/services/database/__init__.py b/app/services/database/__init__.py index 20b6735..1efdf8a 100644 --- a/app/services/database/__init__.py +++ b/app/services/database/__init__.py @@ -43,6 +43,7 @@ get_file_settings, set_file_settings, ensure_file_summaries_columns, + check_db_readiness, ) from app.services.database.credentials import ( list_git_host_credentials, diff --git a/app/services/database/connection.py b/app/services/database/connection.py index 4158996..150ed5c 100644 --- a/app/services/database/connection.py +++ b/app/services/database/connection.py @@ -2,9 +2,9 @@ import sys import sqlite3 import logging -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, Tuple -from sqlalchemy import select +from sqlalchemy import select, func from app.services.database.schema import TABLES from app.services.database.engine import ( @@ -430,6 +430,41 @@ def ensure_file_summaries_columns(conn=None): logger.debug(f"Migration error for file_summaries: {e}") +def check_db_readiness() -> Tuple[bool, Dict[str, Any]]: + """ + Verifies that the application database engine and core connection pathways are reachable + and that required relational schema tables exist and are readable. + Returns (is_ready, sanitized_info). + """ + db_type = "postgresql" if is_postgres() else "sqlite" + required_tables = [ + "system_metadata", + "git_repositories", + "indexed_paths", + "indexed_files", + "ast_symbols", + "file_summaries", + ] + + try: + # 1. Verify primary SQLAlchemy engine connectivity & schema access + with get_connection() as conn: + for table_name in required_tables: + if table_name in TABLES: + conn.execute(select(func.count()).select_from(TABLES[table_name])).scalar() + + # 2. Verify direct connection pathway used by legacy/subrouter queries + with get_db_connection() as raw_conn: + cursor = raw_conn.cursor() + for table_name in required_tables: + cursor.execute(f"SELECT 1 FROM {table_name} LIMIT 1") + + return True, {"status": "ok", "type": db_type} + except Exception as e: + logger.warning(f"Database readiness check failed ({db_type}): {e}") + return False, {"status": "error", "type": db_type} + + def get_file_settings() -> Dict[str, Any]: """Retrieves file reading and large file summarization settings.""" diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 5494df0..375791a 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -86,13 +86,16 @@ Follow these steps for manual installation: - Web Admin Dashboard: `http://localhost:3000/admin/` - MCP Server-Sent Events: `http://localhost:3000/sse` - MCP Streamable HTTP: `http://localhost:3000/mcp` - - Health Check Probes: `http://localhost:3000/health` and `http://localhost:3000/healthz` + - Health Check Probes: Liveness (`http://localhost:3000/health`, `/healthz`) and Readiness (`http://localhost:3000/ready`, `/readiness`, `/health/ready`) 3. Verify server health and MCP JSON-RPC `ping` readiness from a terminal: ```bash - # HTTP health probe + # HTTP process liveness probe curl -s http://localhost:3000/healthz + # HTTP application readiness probe (verifies relational database & schema accessibility) + curl -s http://localhost:3000/ready + # Lightweight JSON-RPC ping (zero session allocation) curl -s -X POST http://localhost:3000/mcp \ -H "Content-Type: application/json" \ diff --git a/docs/guide/user-guide/diagnostics.md b/docs/guide/user-guide/diagnostics.md index 895b255..fca6f55 100644 --- a/docs/guide/user-guide/diagnostics.md +++ b/docs/guide/user-guide/diagnostics.md @@ -23,22 +23,36 @@ ContextCortex maintains an in-memory ring buffer of the latest 500 system events --- -## Health Check & MCP Liveness Endpoints +## Health Check Probes & MCP Liveness Endpoints -ContextCortex exposes standard HTTP health probes and a zero-allocation MCP JSON-RPC `ping` handler for container orchestrators and MCP gateways (such as Docker, Kubernetes, Uptime Kuma, and Model Context Gateway): +ContextCortex distinguishes process liveness from application readiness for deployment probes in container orchestrators (such as Docker Compose, Kubernetes, Uptime Kuma, and Model Context Gateway): + +### 1. Process Liveness Probes (`/health`, `/healthz`) +- **Purpose**: Cheap, low-overhead HTTP process check returning `{"status": "healthy", ...}` (HTTP 200). +- **Scope**: Verifies that the FastAPI process is running and accepting incoming HTTP requests. Process liveness does **not** verify database schema or backend storage readiness. + +### 2. Application Readiness Probes (`/ready`, `/readiness`, `/health/ready`, `/healthz/ready`) +- **Purpose**: Deep application readiness signal returning `{"status": "ready", ...}` (HTTP 200) when ready, or `{"status": "not_ready", ...}` (HTTP 503) when required dependencies are unavailable. +- **Dependencies Covered**: Verifies active connectivity to the configured database engine (PostgreSQL or SQLite) and asserts that required relational schema tables (`system_metadata`, `git_repositories`, `indexed_paths`, `indexed_files`, `ast_symbols`, `file_summaries`) exist and are accessible through the selected database pathway used by core operations. +- **Sanitized Response**: Public diagnostic responses are sanitized to ensure database credentials, connection strings, and host secrets are never exposed. +- **Ingestion Note**: Gateway process liveness or readiness verifies schema and API availability, but does not equate to complete background repository ingestion or vector store indexing, which run asynchronously in background workers. ```bash -# 1. HTTP liveness & readiness probe (/health or /healthz) +# 1. Cheap HTTP process liveness probe curl -s http://localhost:8021/healthz -# Response: {"status":"healthy","service":"ContextCortex","version":"2.19.0","protocolVersion":"2025-06-18"} +# Response: {"status":"healthy","protocolVersion":"2026-07-28","server":"contextcortex"} + +# 2. Deep application readiness probe +curl -s http://localhost:8021/ready +# Response: {"status":"ready","protocolVersion":"2026-07-28","server":"contextcortex","database":{"status":"ok","type":"sqlite"}} -# 2. Lightweight MCP JSON-RPC ping (bypasses SSE stream/session allocation) +# 3. Lightweight MCP JSON-RPC ping (bypasses SSE stream/session allocation) curl -s -X POST http://localhost:8021/mcp \ -H "Content-Type: application/json" \ -d '{"jsonrpc":"2.0","id":1,"method":"ping"}' # Response: {"jsonrpc":"2.0","id":1,"result":{}} -# 3. Detailed index and system statistics +# 4. Detailed index and system statistics curl -s http://localhost:8021/admin/api/stats ``` diff --git a/main.py b/main.py index 1d73134..8db0f52 100644 --- a/main.py +++ b/main.py @@ -24,7 +24,7 @@ get_auth_service, set_current_auth_context, ) -from app.services.database import init_db +from app.services.database import init_db, check_db_readiness import app.services.indexing as indexer from app.services.indexing import run_full_indexing, VAULT_PATH from app.services.poller import start_poller_daemon, stop_poller_daemon @@ -201,6 +201,10 @@ async def __call__(self, scope, receive, send): path == "/.well-known/oauth-protected-resource" or path == "/health" or path == "/healthz" + or path == "/ready" + or path == "/readiness" + or path == "/health/ready" + or path == "/healthz/ready" or path == "/" or path == "/favicon.svg" or path == "/favicon.ico" @@ -333,5 +337,42 @@ async def health(): ) +@app.get("/ready") +@app.get("/readiness") +@app.get("/health/ready") +@app.get("/healthz/ready") +async def readiness(): + try: + is_ready, db_info = check_db_readiness() + except Exception as e: + logger.warning(f"Readiness check unhandled exception: {e}") + is_ready = False + db_info = {"status": "error", "type": "unknown"} + + if is_ready: + return JSONResponse( + status_code=200, + content={ + "status": "ready", + "protocolVersion": "2026-07-28", + "server": "contextcortex", + "database": db_info, + }, + headers={"Cache-Control": "no-cache, no-transform"}, + ) + else: + return JSONResponse( + status_code=503, + content={ + "status": "not_ready", + "protocolVersion": "2026-07-28", + "server": "contextcortex", + "database": db_info, + "error": "Required database schema or metadata unavailable", + }, + headers={"Cache-Control": "no-cache, no-transform"}, + ) + + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=3000) diff --git a/pytest.ini b/pytest.ini index 00b9ce4..4bc8c42 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ [pytest] pythonpath = . +asyncio_default_fixture_loop_scope = function filterwarnings = error ignore:.*Field 'lifespan' has an incomplete definition.*:UserWarning diff --git a/tests/backend/test_health_check_status.py b/tests/backend/test_health_check_status.py index 95beee6..ee3b3c7 100644 --- a/tests/backend/test_health_check_status.py +++ b/tests/backend/test_health_check_status.py @@ -100,3 +100,87 @@ def test_main_app_favicon_and_root_redirect(): res_ico = main_client.get("/favicon.ico") assert res_ico.status_code == 200 + +def test_health_endpoints_cheap_liveness(): + """Verify GET /health and /healthz return cheap liveness HTTP 200 responses.""" + from main import app as main_app + main_client = TestClient(main_app) + + for endpoint in ["/health", "/healthz"]: + res = main_client.get(endpoint) + assert res.status_code == 200 + data = res.json() + assert data["status"] == "healthy" + assert data["server"] == "contextcortex" + + +def test_readiness_endpoint_success(tmp_path): + """Verify GET /ready and aliases return HTTP 200 when database schema is initialized.""" + from app.services.database import init_db + from main import app as main_app + + db_path = str(tmp_path / "readiness_init.db") + with patch("app.services.database.engine.get_default_db_path", return_value=db_path), \ + patch("app.services.database.connection.CACHE_DB_PATH", db_path): + init_db(vault_path=str(tmp_path)) + main_client = TestClient(main_app) + + for endpoint in ["/ready", "/readiness", "/health/ready", "/healthz/ready"]: + res = main_client.get(endpoint) + assert res.status_code == 200 + data = res.json() + assert data["status"] == "ready" + assert data["database"]["status"] == "ok" + + +def test_readiness_endpoint_missing_schema(): + """Verify GET /ready returns HTTP 503 when required schema tables are missing.""" + from main import app as main_app + main_client = TestClient(main_app) + + with patch("main.check_db_readiness", return_value=(False, {"status": "error", "type": "sqlite"})): + res = main_client.get("/ready") + assert res.status_code == 503 + data = res.json() + assert data["status"] == "not_ready" + assert data["database"]["status"] == "error" + assert "error" in data + + +def test_readiness_split_database_failure(tmp_path): + """Verify split-database scenario (#45) where direct connection targets empty DB and fails readiness.""" + from main import app as main_app + empty_db = str(tmp_path / "empty_cache.db") + # Touch empty file without schema + sqlite3.connect(empty_db).close() + + main_client = TestClient(main_app) + + with patch("app.services.database.connection.get_db_connection") as mock_direct: + def get_empty_conn(): + c = sqlite3.connect(empty_db) + c.row_factory = sqlite3.Row + return c + mock_direct.side_effect = get_empty_conn + + res = main_client.get("/ready") + assert res.status_code == 503 + data = res.json() + assert data["status"] == "not_ready" + assert data["database"]["status"] == "error" + + +def test_readiness_sanitized_public_response(): + """Verify public readiness response never leaks credentials, host details, or raw stack traces.""" + from main import app as main_app + main_client = TestClient(main_app) + + with patch("main.check_db_readiness", side_effect=Exception("postgresql://user:secret_pass@db.internal:5432/cortex")): + res = main_client.get("/ready") + assert res.status_code == 503 + data = res.json() + assert data["status"] == "not_ready" + body_str = res.text + assert "secret_pass" not in body_str + assert "db.internal" not in body_str + assert "postgresql://" not in body_str From eb9ccb3929c0eef080a4c20ad8d39e88970c4514 Mon Sep 17 00:00:00 2001 From: spelech <28486500+spelech@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:55:44 +0000 Subject: [PATCH 2/3] feat(health): expose database readiness signal and schema verification Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> From 5e892d8c7b3a2ad82752cfaa5d2c3df125d5ff1b Mon Sep 17 00:00:00 2001 From: Antigravity Agent Date: Tue, 27 Oct 2026 21:30:14 -0500 Subject: [PATCH 3/3] feat(health): use postgres connection branch for readiness check when postgres configured --- app/services/database/connection.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/app/services/database/connection.py b/app/services/database/connection.py index 150ed5c..a6273cd 100644 --- a/app/services/database/connection.py +++ b/app/services/database/connection.py @@ -454,10 +454,15 @@ def check_db_readiness() -> Tuple[bool, Dict[str, Any]]: conn.execute(select(func.count()).select_from(TABLES[table_name])).scalar() # 2. Verify direct connection pathway used by legacy/subrouter queries - with get_db_connection() as raw_conn: - cursor = raw_conn.cursor() - for table_name in required_tables: - cursor.execute(f"SELECT 1 FROM {table_name} LIMIT 1") + if is_postgres(): + with get_connection() as raw_conn: + for table_name in required_tables: + raw_conn.execute(text(f"SELECT 1 FROM {table_name} LIMIT 1")) + else: + with get_db_connection() as raw_conn: + cursor = raw_conn.cursor() + for table_name in required_tables: + cursor.execute(f"SELECT 1 FROM {table_name} LIMIT 1") return True, {"status": "ok", "type": db_type} except Exception as e: