From 11485432ff77a7bd0fa3ffaf830963c074a90b97 Mon Sep 17 00:00:00 2001 From: Endogen Date: Mon, 20 Jul 2026 23:26:40 +0200 Subject: [PATCH] Harden recipe integrations and contracts --- .github/workflows/ci.yml | 24 ++++++ .gitignore | 4 + catalog.yaml | 10 ++- pyproject.toml | 9 ++ recipes/allenai/recipe.yaml | 14 +++ recipes/allenai/scraper.py | 120 ++++++++++++++------------ recipes/brave-search/recipe.yaml | 7 ++ recipes/brave-search/scraper.py | 3 - recipes/deepl/scraper.py | 1 - recipes/nvidia/recipe.yaml | 15 +++- recipes/nvidia/scraper.py | 54 +++++++----- recipes/openstreetmap/README.md | 20 +++-- recipes/openstreetmap/plugin.yaml | 6 ++ recipes/openstreetmap/recipe.yaml | 22 +++++ recipes/openstreetmap/scraper.py | 138 ++++++++++++++++++++---------- recipes/web-reader/scraper.py | 15 +++- recipes/wikipedia/recipe.yaml | 7 ++ recipes/wikipedia/scraper.py | 4 +- recipes/x/plugin.yaml | 2 +- recipes/x/recipe.yaml | 7 ++ recipes/x/scraper.py | 25 ++++-- tests/test_recipe_contracts.py | 35 ++++++++ 22 files changed, 398 insertions(+), 144 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 pyproject.toml create mode 100644 recipes/openstreetmap/plugin.yaml create mode 100644 tests/test_recipe_contracts.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e405118 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,24 @@ +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: Endogen/web2api + path: web2api-core + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -e './web2api-core[dev]' + - run: ruff check recipes tests + - run: pytest -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c903b31 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ diff --git a/catalog.yaml b/catalog.yaml index 38fd0a9..b24db86 100644 --- a/catalog.yaml +++ b/catalog.yaml @@ -13,6 +13,9 @@ recipes: description: "X/Twitter recipe (requires bird CLI and auth env vars)." source: "./recipes/x" trusted: true + requires_env: + - BIRD_AUTH_TOKEN + - BIRD_CT0 brave-search: description: "Brave Search recipe." @@ -35,10 +38,15 @@ recipes: trusted: true openstreetmap: - description: "Geocoding, reverse geocoding, routing, and place search via OpenStreetMap." + description: "Geocoding, reverse geocoding, routing, and place search via configured Nominatim and OSRM providers." source: "./recipes/openstreetmap" trusted: true + requires_env: + - NOMINATIM_BASE_URL + - OSRM_BASE_URL nvidia: description: "NVIDIA NIM recipe using the direct integrate.api.nvidia.com API." source: "./recipes/nvidia" trusted: true + requires_env: + - NVIDIA_API_KEY diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4a9761a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/recipes/allenai/recipe.yaml b/recipes/allenai/recipe.yaml index 8404496..041b145 100644 --- a/recipes/allenai/recipe.yaml +++ b/recipes/allenai/recipe.yaml @@ -11,6 +11,8 @@ endpoints: tools_url: description: "MCP HTTP bridge URL for tool calling (GET /tools, POST /tools/{name})" required: false + pattern: "^https?://" + max_length: 2048 example: "http://localhost:8100" items: container: "body" @@ -30,6 +32,8 @@ endpoints: tools_url: description: "MCP HTTP bridge URL for tool calling (GET /tools, POST /tools/{name})" required: false + pattern: "^https?://" + max_length: 2048 example: "http://localhost:8100" items: container: "body" @@ -49,6 +53,8 @@ endpoints: tools_url: description: "MCP HTTP bridge URL for tool calling (GET /tools, POST /tools/{name})" required: false + pattern: "^https?://" + max_length: 2048 example: "http://localhost:8100" items: container: "body" @@ -68,6 +74,8 @@ endpoints: tools_url: description: "MCP HTTP bridge URL for tool calling (GET /tools, POST /tools/{name})" required: false + pattern: "^https?://" + max_length: 2048 example: "http://localhost:8100" items: container: "body" @@ -87,6 +95,8 @@ endpoints: tools_url: description: "MCP HTTP bridge URL for tool calling (GET /tools, POST /tools/{name})" required: false + pattern: "^https?://" + max_length: 2048 example: "http://localhost:8100" items: container: "body" @@ -106,6 +116,8 @@ endpoints: tools_url: description: "MCP HTTP bridge URL for tool calling (GET /tools, POST /tools/{name})" required: false + pattern: "^https?://" + max_length: 2048 example: "http://localhost:8100" items: container: "body" @@ -135,6 +147,7 @@ endpoints: description: "Molmo 2 8B vision model — analyze images and video" tool_name: "molmo2_vision" requires_query: true + accepts_files: true url: "https://playground.allenai.org" items: container: "body" @@ -150,6 +163,7 @@ endpoints: description: "Molmo 2 8B Tracking — 8fps video tracking" tool_name: "molmo2_tracking" requires_query: true + accepts_files: true url: "https://playground.allenai.org" items: container: "body" diff --git a/recipes/allenai/scraper.py b/recipes/allenai/scraper.py index 6618b54..0e2753b 100644 --- a/recipes/allenai/scraper.py +++ b/recipes/allenai/scraper.py @@ -1,6 +1,6 @@ """Allen AI Playground scraper — chat with OLMo, Tülu, and Molmo models. -All API calls use Python's urllib (no browser needed). Supports generic +All API calls use async HTTP (no browser needed). Supports generic tool calling: pass `tools_url` pointing to any MCP HTTP bridge (GET /tools, POST /tools/{name}) and the model will use those tools. @@ -10,16 +10,15 @@ from __future__ import annotations -import asyncio import json import logging -import subprocess +import mimetypes import uuid from typing import Any -from urllib.request import Request, urlopen +import httpx from playwright.async_api import Page - +from web2api.network_security import validate_httpx_request from web2api.scraper import BaseScraper, ScrapeResult logger = logging.getLogger(__name__) @@ -46,25 +45,35 @@ # ── HTTP helpers ──────────────────────────────────────────────────── -def _http_get_json(url: str) -> Any: - """Synchronous GET returning parsed JSON.""" - req = Request(url, headers={"Accept": "application/json"}) - with urlopen(req, timeout=30) as resp: - return json.loads(resp.read()) - - -def _http_post_json(url: str, body: dict) -> Any: - """Synchronous POST with JSON body returning parsed JSON.""" - data = json.dumps(body).encode() - req = Request(url, data=data, headers={ - "Content-Type": "application/json", - "Accept": "application/json", - }) - with urlopen(req, timeout=30) as resp: - return json.loads(resp.read()) +async def _http_get_json(url: str) -> Any: + """GET JSON with redirect and private-network validation.""" + async with httpx.AsyncClient( + follow_redirects=True, + timeout=30, + event_hooks={"request": [validate_httpx_request]}, + ) as client: + response = await client.get(url, headers={"Accept": "application/json"}) + response.raise_for_status() + return response.json() + + +async def _http_post_json(url: str, body: dict) -> Any: + """POST JSON with redirect and private-network validation.""" + async with httpx.AsyncClient( + follow_redirects=True, + timeout=30, + event_hooks={"request": [validate_httpx_request]}, + ) as client: + response = await client.post( + url, + json=body, + headers={"Accept": "application/json"}, + ) + response.raise_for_status() + return response.json() -def _chat_api( +async def _chat_api( anon_id: str, model_id: str, content: str, @@ -87,25 +96,32 @@ def _chat_api( "enableToolCalling": "true" if enable_tool_calling else None, } - # Use curl with -F flags for the streaming NDJSON response. - # --http1.1 avoids HTTP/2 stream errors with Allen AI's server. - cmd = [ - "curl", "-s", "--http1.1", "--max-time", "90", - "-X", "POST", f"{API_BASE}/v4/threads/", - "-H", f"X-Anonymous-User-ID: {anon_id}", - ] - for name, value in fields.items(): - if value is not None: - cmd.extend(["-F", f"{name}={value}"]) - # Attach files for vision models (Molmo 2) - if file_paths: - import mimetypes - for fpath in file_paths: - mime = mimetypes.guess_type(fpath)[0] or "application/octet-stream" - fname = fpath.rsplit("/", 1)[-1] - cmd.extend(["-F", f"files=@{fpath};type={mime};filename={fname}"]) - proc = subprocess.run(cmd, capture_output=True, timeout=95) - raw = proc.stdout.decode("utf-8") + upload_handles = [] + files: list[tuple[str, tuple[str, Any, str]]] = [] + try: + for file_path in file_paths or []: + handle = open(file_path, "rb") # noqa: SIM115 - closed together below + upload_handles.append(handle) + mime = mimetypes.guess_type(file_path)[0] or "application/octet-stream" + filename = file_path.rsplit("/", 1)[-1] + files.append(("files", (filename, handle, mime))) + + async with httpx.AsyncClient( + follow_redirects=True, + timeout=95, + event_hooks={"request": [validate_httpx_request]}, + ) as client: + response = await client.post( + f"{API_BASE}/v4/threads/", + data={name: value for name, value in fields.items() if value is not None}, + files=files or None, + headers={"X-Anonymous-User-ID": anon_id}, + ) + response.raise_for_status() + raw = response.text + finally: + for handle in upload_handles: + handle.close() # Parse NDJSON stream (robust: handles newlines inside JSON values) response_content = "" @@ -182,15 +198,13 @@ def _clean_schema(schema: dict) -> dict: async def _fetch_tools(tools_url: str) -> list[dict]: """Fetch tool definitions from an MCP HTTP bridge.""" - return await asyncio.to_thread(_http_get_json, f"{tools_url}/tools") + return await _http_get_json(f"{tools_url}/tools") async def _call_tool(tools_url: str, tool_name: str, args: dict) -> str: """Call a tool and return the result as a string.""" try: - result = await asyncio.to_thread( - _http_post_json, f"{tools_url}/tools/{tool_name}", args - ) + result = await _http_post_json(f"{tools_url}/tools/{tool_name}", args) payload = result.get("result", result) return json.dumps(payload) if not isinstance(payload, str) else payload except Exception as ex: @@ -202,11 +216,13 @@ async def _call_tool(tools_url: str, tool_name: str, args: dict) -> str: class Scraper(BaseScraper): """Query Allen AI models with optional tool calling.""" + requires_browser = False + def supports(self, endpoint: str) -> bool: return endpoint in _SUPPORTED async def scrape( - self, endpoint: str, page: Page, params: dict[str, Any] + self, endpoint: str, page: Page | None, params: dict[str, Any] ) -> ScrapeResult: if endpoint == "models": return await self._list_models() @@ -217,8 +233,6 @@ async def scrape( return ScrapeResult( items=[{"prompt": "", "response": "", "model": model_id}] ) - - file_paths = params.get("file_paths") or [] tools_url = (params.get("tools_url") or "").strip().rstrip("/") return await self._chat(model_id, query, tools_url, file_paths=file_paths) @@ -250,8 +264,7 @@ async def _chat( tool_defs_json = json.dumps(allen_tools) # Initial request - result = await asyncio.to_thread( - _chat_api, + result = await _chat_api( anon_id, model_id, prompt, None, None, None, tool_defs_json, @@ -301,8 +314,7 @@ async def _chat( }) # Send tool result back to model - result = await asyncio.to_thread( - _chat_api, + result = await _chat_api( anon_id, model_id, tool_result, parent_id, @@ -337,9 +349,7 @@ async def _chat( async def _list_models(self) -> ScrapeResult: """Fetch available models from the API.""" - result = await asyncio.to_thread( - _http_get_json, f"{API_BASE}/v4/models/" - ) + result = await _http_get_json(f"{API_BASE}/v4/models/") items = [] for model in result: diff --git a/recipes/brave-search/recipe.yaml b/recipes/brave-search/recipe.yaml index b2787fa..45a9e40 100644 --- a/recipes/brave-search/recipe.yaml +++ b/recipes/brave-search/recipe.yaml @@ -7,6 +7,13 @@ endpoints: description: "Web search (q=query, count=N, page=N)" requires_query: true url: "https://search.brave.com/search?q={query}" + params: + count: + description: "Maximum number of organic results" + type: "integer" + minimum: 1 + maximum: 50 + example: 20 items: container: "#results .snippet" fields: diff --git a/recipes/brave-search/scraper.py b/recipes/brave-search/scraper.py index 6300c65..a68e265 100644 --- a/recipes/brave-search/scraper.py +++ b/recipes/brave-search/scraper.py @@ -3,13 +3,10 @@ from __future__ import annotations import asyncio -import json -import re from typing import Any from urllib.parse import quote from playwright.async_api import Page - from web2api.scraper import BaseScraper, ScrapeResult diff --git a/recipes/deepl/scraper.py b/recipes/deepl/scraper.py index 3c66361..847d7d5 100644 --- a/recipes/deepl/scraper.py +++ b/recipes/deepl/scraper.py @@ -6,7 +6,6 @@ from typing import Any from playwright.async_api import Page - from web2api.scraper import BaseScraper, ScrapeResult # Map endpoint names to (source_lang, target_lang) pairs diff --git a/recipes/nvidia/recipe.yaml b/recipes/nvidia/recipe.yaml index b53acb3..2db47d2 100644 --- a/recipes/nvidia/recipe.yaml +++ b/recipes/nvidia/recipe.yaml @@ -41,16 +41,25 @@ endpoints: example: "You are a concise assistant." temperature: description: "Sampling temperature" + type: "number" + minimum: 0 + maximum: 2 required: false - example: "0.2" + example: 0.2 top_p: description: "Top-p nucleus sampling" + type: "number" + minimum: 0 + maximum: 1 required: false - example: "0.7" + example: 0.7 max_tokens: description: "Maximum output tokens" + type: "integer" + minimum: 1 + maximum: 32768 required: false - example: "512" + example: 512 items: container: "body" fields: diff --git a/recipes/nvidia/scraper.py b/recipes/nvidia/scraper.py index faac832..74c59a6 100644 --- a/recipes/nvidia/scraper.py +++ b/recipes/nvidia/scraper.py @@ -2,15 +2,13 @@ from __future__ import annotations -import asyncio import json import os from typing import Any -from urllib.error import HTTPError, URLError -from urllib.request import Request, urlopen +import httpx from playwright.async_api import Page - +from web2api.network_security import validate_httpx_request from web2api.scraper import BaseScraper, ScrapeResult API_BASE = "https://integrate.api.nvidia.com/v1" @@ -18,28 +16,38 @@ API_KEY_ENV = "NVIDIA_API_KEY" -def _http_json(url: str, *, method: str = "GET", body: dict[str, Any] | None = None) -> Any: - data = None +async def _http_json( + url: str, + *, + method: str = "GET", + body: dict[str, Any] | None = None, +) -> Any: headers = {"Accept": "application/json"} api_key = os.environ.get(API_KEY_ENV, "").strip() if body is not None: - data = json.dumps(body).encode("utf-8") headers["Content-Type"] = "application/json" if api_key: headers["Authorization"] = f"Bearer {api_key}" - req = Request(url, data=data, headers=headers, method=method) try: - with urlopen(req, timeout=60) as resp: - return json.loads(resp.read()) - except HTTPError as exc: - detail = exc.read().decode("utf-8", errors="replace").strip() - if exc.code == 401: + async with httpx.AsyncClient( + follow_redirects=True, + timeout=60, + event_hooks={"request": [validate_httpx_request]}, + ) as client: + response = await client.request(method, url, json=body, headers=headers) + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as exc: + detail = exc.response.text.strip() + if exc.response.status_code == 401: raise RuntimeError( f"NVIDIA API authorization failed. Set {API_KEY_ENV} and try again." ) from None - raise RuntimeError(f"NVIDIA API request failed ({exc.code}): {detail or exc.reason}") from None - except URLError as exc: - raise RuntimeError(f"NVIDIA API network error: {exc.reason}") from None + raise RuntimeError( + f"NVIDIA API request failed ({exc.response.status_code}): {detail}" + ) from None + except httpx.HTTPError as exc: + raise RuntimeError(f"NVIDIA API network error: {exc}") from None def _as_float(value: Any, *, default: float | None = None) -> float | None: @@ -55,10 +63,17 @@ def _as_int(value: Any, *, default: int | None = None) -> int | None: class Scraper(BaseScraper): + requires_browser = False + def supports(self, endpoint: str) -> bool: return endpoint in {"models", "chat"} - async def scrape(self, endpoint: str, page: Page, params: dict[str, Any]) -> ScrapeResult: + async def scrape( + self, + endpoint: str, + page: Page | None, + params: dict[str, Any], + ) -> ScrapeResult: if endpoint == "models": return await self._models(params) if endpoint == "chat": @@ -66,7 +81,7 @@ async def scrape(self, endpoint: str, page: Page, params: dict[str, Any]) -> Scr raise RuntimeError(f"Unsupported endpoint: {endpoint}") async def _models(self, params: dict[str, Any]) -> ScrapeResult: - payload = await asyncio.to_thread(_http_json, f"{API_BASE}/models") + payload = await _http_json(f"{API_BASE}/models") owner = (params.get("owner") or "").strip().lower() prefix = (params.get("prefix") or "").strip().lower() @@ -115,8 +130,7 @@ async def _chat(self, params: dict[str, Any]) -> ScrapeResult: if max_tokens is not None: body["max_tokens"] = max_tokens - payload = await asyncio.to_thread( - _http_json, + payload = await _http_json( f"{API_BASE}/chat/completions", method="POST", body=body, diff --git a/recipes/openstreetmap/README.md b/recipes/openstreetmap/README.md index 42ede63..b6ec8e0 100644 --- a/recipes/openstreetmap/README.md +++ b/recipes/openstreetmap/README.md @@ -54,11 +54,21 @@ GET /openstreetmap/search?q=pharmacy&lat=52.52&lon=13.405&radius=2000 ## Requirements -- No API key needed -- No environment variables required -- Uses Nominatim and OSRM public APIs (please respect usage policies) +Configure provider endpoints before enabling the recipe: + +```bash +export NOMINATIM_BASE_URL="https://nominatim.your-provider.example" +export OSRM_BASE_URL="https://osrm.your-provider.example" +export OPENSTREETMAP_USER_AGENT="your-app/1.0 (ops@example.com)" +``` + +Use a self-hosted deployment or a provider whose terms permit your workload. +Private-network endpoints are blocked unless the Web2API operator explicitly sets +`WEB2API_ALLOW_PRIVATE_NETWORK=true`. ## Notes -- Nominatim has a [usage policy](https://operations.osmfoundation.org/policies/nominatim/) limiting to 1 request per second -- OSRM demo server is for evaluation — consider self-hosting for production use +- The recipe enforces at most one Nominatim request per second per process. +- The public Nominatim service's [usage policy](https://operations.osmfoundation.org/policies/nominatim/) + does not permit generic proxy/no-code access, so it is no longer the default. +- The OSRM demo server is for evaluation and is no longer the default. diff --git a/recipes/openstreetmap/plugin.yaml b/recipes/openstreetmap/plugin.yaml new file mode 100644 index 0000000..98ffc72 --- /dev/null +++ b/recipes/openstreetmap/plugin.yaml @@ -0,0 +1,6 @@ +version: "2.0.0" +web2api: + min: "0.5.0" +requires_env: + - NOMINATIM_BASE_URL + - OSRM_BASE_URL diff --git a/recipes/openstreetmap/recipe.yaml b/recipes/openstreetmap/recipe.yaml index d46a846..735ffad 100644 --- a/recipes/openstreetmap/recipe.yaml +++ b/recipes/openstreetmap/recipe.yaml @@ -38,6 +38,11 @@ endpoints: tool_name: "calculate_route" requires_query: true url: "https://router.project-osrm.org" + params: + profile: + description: "Routing profile" + enum: ["driving", "walking", "cycling"] + example: "driving" items: container: "body" fields: @@ -53,6 +58,23 @@ endpoints: tool_name: "search_places" requires_query: true url: "https://nominatim.openstreetmap.org" + params: + lat: + description: "Center latitude for a bounded nearby search" + type: "number" + minimum: -90 + maximum: 90 + lon: + description: "Center longitude for a bounded nearby search" + type: "number" + minimum: -180 + maximum: 180 + radius: + description: "Search radius in meters" + type: "number" + minimum: 1 + maximum: 50000 + example: 5000 items: container: "body" fields: diff --git a/recipes/openstreetmap/scraper.py b/recipes/openstreetmap/scraper.py index e62b9e4..8ed648a 100644 --- a/recipes/openstreetmap/scraper.py +++ b/recipes/openstreetmap/scraper.py @@ -1,48 +1,82 @@ """OpenStreetMap scraper — geocoding, reverse geocoding, routing, and place search. -Uses Nominatim (geocoding/search) and OSRM (routing). All API calls use -Python's urllib — no browser needed. +Uses operator-configured Nominatim (geocoding/search) and OSRM (routing) +services. No browser is needed. """ from __future__ import annotations -import json +import asyncio +import os import re +from time import monotonic from typing import Any -from urllib.parse import quote, urlencode -from urllib.request import Request, urlopen +from urllib.parse import urlencode +import httpx from playwright.async_api import Page - +from web2api.network_security import validate_httpx_request from web2api.scraper import BaseScraper, ScrapeResult -NOMINATIM_BASE = "https://nominatim.openstreetmap.org" -OSRM_BASE = "https://router.project-osrm.org" -USER_AGENT = "web2api/1.0 (https://github.com/Endogen/web2api)" +NOMINATIM_BASE_ENV = "NOMINATIM_BASE_URL" +OSRM_BASE_ENV = "OSRM_BASE_URL" +USER_AGENT = os.environ.get( + "OPENSTREETMAP_USER_AGENT", + "web2api/1.0 (+https://github.com/Endogen/web2api)", +) HTTP_TIMEOUT = 15 - - -def _get_json(url: str) -> Any: - """GET request returning parsed JSON.""" - req = Request(url, headers={ - "Accept": "application/json", - "User-Agent": USER_AGENT, - }) - with urlopen(req, timeout=HTTP_TIMEOUT) as resp: - return json.loads(resp.read()) - - -def _geocode(query: str) -> list[dict[str, Any]]: +_NOMINATIM_LOCK = asyncio.Lock() +_LAST_NOMINATIM_REQUEST = 0.0 + + +def _configured_base(env_name: str) -> str: + value = os.environ.get(env_name, "").strip().rstrip("/") + if not value: + raise RuntimeError( + f"{env_name} is required; configure a self-hosted or contracted provider endpoint" + ) + return value + + +async def _get_json(url: str, *, nominatim: bool = False) -> Any: + """GET JSON with outbound validation and a one-request-per-second limiter.""" + global _LAST_NOMINATIM_REQUEST + if nominatim: + async with _NOMINATIM_LOCK: + delay = 1.0 - (monotonic() - _LAST_NOMINATIM_REQUEST) + if delay > 0: + await asyncio.sleep(delay) + _LAST_NOMINATIM_REQUEST = monotonic() + + async with httpx.AsyncClient( + follow_redirects=True, + timeout=HTTP_TIMEOUT, + event_hooks={"request": [validate_httpx_request]}, + ) as client: + response = await client.get( + url, + headers={"Accept": "application/json", "User-Agent": USER_AGENT}, + ) + response.raise_for_status() + return response.json() + + +async def _geocode(query: str) -> list[dict[str, Any]]: """Forward geocode: address/place → coordinates.""" - url = f"{NOMINATIM_BASE}/search?{urlencode({'q': query, 'format': 'jsonv2', 'addressdetails': 1, 'limit': 5})}" - results = _get_json(url) + base = _configured_base(NOMINATIM_BASE_ENV) + query_params = {"q": query, "format": "jsonv2", "addressdetails": 1, "limit": 5} + url = f"{base}/search?{urlencode(query_params)}" + results = await _get_json(url, nominatim=True) items = [] for r in results: address = r.get("address", {}) items.append({ "title": r.get("display_name", ""), - "url": f"https://www.openstreetmap.org/{r.get('osm_type', 'node')}/{r.get('osm_id', '')}", + "url": ( + f"https://www.openstreetmap.org/{r.get('osm_type', 'node')}" + f"/{r.get('osm_id', '')}" + ), "latitude": r.get("lat", ""), "longitude": r.get("lon", ""), "type": r.get("type", ""), @@ -55,10 +89,12 @@ def _geocode(query: str) -> list[dict[str, Any]]: return items -def _reverse_geocode(lat: str, lon: str) -> list[dict[str, Any]]: +async def _reverse_geocode(lat: str, lon: str) -> list[dict[str, Any]]: """Reverse geocode: coordinates → address.""" - url = f"{NOMINATIM_BASE}/reverse?{urlencode({'lat': lat, 'lon': lon, 'format': 'jsonv2', 'addressdetails': 1})}" - r = _get_json(url) + base = _configured_base(NOMINATIM_BASE_ENV) + query_params = {"lat": lat, "lon": lon, "format": "jsonv2", "addressdetails": 1} + url = f"{base}/reverse?{urlencode(query_params)}" + r = await _get_json(url, nominatim=True) if "error" in r: raise RuntimeError(f"Reverse geocode failed: {r['error']}") @@ -80,7 +116,10 @@ def _reverse_geocode(lat: str, lon: str) -> list[dict[str, Any]]: }] -def _route(waypoints: list[tuple[str, str]], profile: str = "driving") -> list[dict[str, Any]]: +async def _route( + waypoints: list[tuple[str, str]], + profile: str = "driving", +) -> list[dict[str, Any]]: """Calculate route between waypoints using OSRM. waypoints: list of (lat, lon) tuples @@ -88,8 +127,9 @@ def _route(waypoints: list[tuple[str, str]], profile: str = "driving") -> list[d """ # OSRM uses lon,lat order coords = ";".join(f"{lon},{lat}" for lat, lon in waypoints) - url = f"{OSRM_BASE}/route/v1/{profile}/{coords}?overview=full&geometries=geojson&steps=true" - data = _get_json(url) + base = _configured_base(OSRM_BASE_ENV) + url = f"{base}/route/v1/{profile}/{coords}?overview=full&geometries=geojson&steps=true" + data = await _get_json(url) if data.get("code") != "Ok": raise RuntimeError(f"Routing failed: {data.get('message', data.get('code', 'unknown'))}") @@ -129,7 +169,12 @@ def _route(waypoints: list[tuple[str, str]], profile: str = "driving") -> list[d return items -def _search_places(query: str, lat: str = "", lon: str = "", radius: str = "") -> list[dict[str, Any]]: +async def _search_places( + query: str, + lat: float | str = "", + lon: float | str = "", + radius: float | str = "", +) -> list[dict[str, Any]]: """Search for places/POIs, optionally near a location.""" params: dict[str, str] = { "q": query, @@ -142,18 +187,25 @@ def _search_places(query: str, lat: str = "", lon: str = "", radius: str = "") - params["lon"] = lon # viewbox for nearby search r_deg = float(radius) / 111000 if radius else 0.05 # ~5km default - params["viewbox"] = f"{float(lon)-r_deg},{float(lat)+r_deg},{float(lon)+r_deg},{float(lat)-r_deg}" + params["viewbox"] = ( + f"{float(lon) - r_deg},{float(lat) + r_deg}," + f"{float(lon) + r_deg},{float(lat) - r_deg}" + ) params["bounded"] = "1" - url = f"{NOMINATIM_BASE}/search?{urlencode(params)}" - results = _get_json(url) + base = _configured_base(NOMINATIM_BASE_ENV) + url = f"{base}/search?{urlencode(params)}" + results = await _get_json(url, nominatim=True) items = [] for r in results: address = r.get("address", {}) items.append({ "title": r.get("display_name", ""), - "url": f"https://www.openstreetmap.org/{r.get('osm_type', 'node')}/{r.get('osm_id', '')}", + "url": ( + f"https://www.openstreetmap.org/{r.get('osm_type', 'node')}" + f"/{r.get('osm_id', '')}" + ), "latitude": r.get("lat", ""), "longitude": r.get("lon", ""), "type": r.get("type", ""), @@ -181,20 +233,20 @@ def _parse_coords(text: str) -> list[tuple[str, str]]: class Scraper(BaseScraper): """OpenStreetMap geocoding, routing, and search.""" + requires_browser = False + def supports(self, endpoint: str) -> bool: return endpoint in {"geocode", "reverse", "route", "search"} async def scrape( - self, endpoint: str, page: Page, params: dict[str, Any] + self, endpoint: str, page: Page | None, params: dict[str, Any] ) -> ScrapeResult: query = (params.get("query") or "").strip() if not query: raise RuntimeError("Missing query — pass q=") - import asyncio - if endpoint == "geocode": - items = await asyncio.to_thread(_geocode, query) + items = await _geocode(query) elif endpoint == "reverse": coords = _parse_coords(query) @@ -203,7 +255,7 @@ async def scrape( "Invalid coordinates. Use format: q=52.52,13.405" ) lat, lon = coords[0] - items = await asyncio.to_thread(_reverse_geocode, lat, lon) + items = await _reverse_geocode(lat, lon) elif endpoint == "route": coords = _parse_coords(query) @@ -214,13 +266,13 @@ async def scrape( profile = params.get("profile", "driving") if profile not in ("driving", "walking", "cycling"): profile = "driving" - items = await asyncio.to_thread(_route, coords, profile) + items = await _route(coords, profile) elif endpoint == "search": lat = params.get("lat", "") lon = params.get("lon", "") radius = params.get("radius", "") - items = await asyncio.to_thread(_search_places, query, lat, lon, radius) + items = await _search_places(query, lat, lon, radius) else: raise RuntimeError(f"Unknown endpoint: {endpoint}") diff --git a/recipes/web-reader/scraper.py b/recipes/web-reader/scraper.py index f1ba828..47d0377 100644 --- a/recipes/web-reader/scraper.py +++ b/recipes/web-reader/scraper.py @@ -8,7 +8,10 @@ from urllib.parse import urlparse from playwright.async_api import Page - +from web2api.network_security import ( + private_network_access_enabled, + validate_public_http_url, +) from web2api.scraper import BaseScraper, ScrapeResult # Max characters to return (prevents huge pages from overwhelming the model) @@ -46,6 +49,11 @@ async def scrape( parsed = urlparse(url) if not parsed.hostname: raise RuntimeError(f"Invalid URL: {url}") + await asyncio.to_thread( + validate_public_http_url, + url, + allow_private_network=private_network_access_enabled(), + ) # Navigate with anti-detection await page.add_init_script( @@ -117,6 +125,11 @@ async def scrape( # Get the final URL (after redirects) final_url = page.url + await asyncio.to_thread( + validate_public_http_url, + final_url, + allow_private_network=private_network_access_enabled(), + ) item: dict[str, Any] = { "title": title or final_url, diff --git a/recipes/wikipedia/recipe.yaml b/recipes/wikipedia/recipe.yaml index 635fe58..3c7d611 100644 --- a/recipes/wikipedia/recipe.yaml +++ b/recipes/wikipedia/recipe.yaml @@ -7,6 +7,13 @@ endpoints: description: "Search articles (q=query, count=N, page=N)" requires_query: true url: "https://en.wikipedia.org/w/index.php?search={query}&ns0=1" + params: + count: + description: "Maximum number of search results" + type: "integer" + minimum: 1 + maximum: 50 + example: 20 items: container: ".mw-search-result" fields: diff --git a/recipes/wikipedia/scraper.py b/recipes/wikipedia/scraper.py index 6637747..e3de40b 100644 --- a/recipes/wikipedia/scraper.py +++ b/recipes/wikipedia/scraper.py @@ -8,7 +8,6 @@ from urllib.parse import quote from playwright.async_api import Page - from web2api.scraper import BaseScraper, ScrapeResult @@ -247,7 +246,8 @@ async def _extract_sections(page: Page) -> list[dict[str, str]]: # Skip edit links text and meta sections title = re.sub(r"\[edit\]$", "", title).strip() - if not title or title.lower() in ("references", "external links", "notes", "further reading"): + skipped_sections = ("references", "external links", "notes", "further reading") + if not title or title.lower() in skipped_sections: continue # Collect text from sibling elements until the next heading diff --git a/recipes/x/plugin.yaml b/recipes/x/plugin.yaml index 93416fc..3a456b0 100644 --- a/recipes/x/plugin.yaml +++ b/recipes/x/plugin.yaml @@ -10,7 +10,7 @@ dependencies: apt: - nodejs npm: - - "@steipete/bird" + - "@steipete/bird@0.8.0" healthcheck: command: - bird diff --git a/recipes/x/recipe.yaml b/recipes/x/recipe.yaml index 93743fd..f9fc21f 100644 --- a/recipes/x/recipe.yaml +++ b/recipes/x/recipe.yaml @@ -7,6 +7,13 @@ endpoints: description: "Get recent posts by username (q=username, count=N)" requires_query: true url: "https://x.com/{query}" + params: + count: + description: "Maximum number of posts" + type: "integer" + minimum: 1 + maximum: 50 + example: 10 items: container: "[data-testid='tweet']" fields: diff --git a/recipes/x/scraper.py b/recipes/x/scraper.py index f73e624..9edcabc 100644 --- a/recipes/x/scraper.py +++ b/recipes/x/scraper.py @@ -8,18 +8,13 @@ from typing import Any from playwright.async_api import Page - from web2api.scraper import BaseScraper, ScrapeResult -# Auth tokens read from ~/.bird_auth or environment -_AUTH_TOKEN = os.environ.get("BIRD_AUTH_TOKEN", "") -_CT0 = os.environ.get("BIRD_CT0", "") - def _load_auth() -> tuple[str, str]: """Load bird auth tokens from env or ~/.bird_auth file.""" - auth_token = _AUTH_TOKEN - ct0 = _CT0 + auth_token = os.environ.get("BIRD_AUTH_TOKEN", "") + ct0 = os.environ.get("BIRD_CT0", "") if auth_token and ct0: return auth_token, ct0 @@ -44,10 +39,17 @@ def _load_auth() -> tuple[str, str]: class Scraper(BaseScraper): """Fetch user tweets via the bird CLI.""" + requires_browser = False + def supports(self, endpoint: str) -> bool: return endpoint == "posts" - async def scrape(self, endpoint: str, page: Page, params: dict[str, Any]) -> ScrapeResult: + async def scrape( + self, + endpoint: str, + page: Page | None, + params: dict[str, Any], + ) -> ScrapeResult: username = (params.get("query") or "").strip().lstrip("@") if not username: raise RuntimeError("Missing username — pass q=") @@ -69,7 +71,12 @@ async def scrape(self, endpoint: str, page: Page, params: dict[str, Any]) -> Scr stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) + except (TimeoutError, asyncio.CancelledError): + proc.kill() + await proc.wait() + raise if proc.returncode != 0: error_msg = stderr.decode().strip() diff --git a/tests/test_recipe_contracts.py b/tests/test_recipe_contracts.py new file mode 100644 index 0000000..c2e90ac --- /dev/null +++ b/tests/test_recipe_contracts.py @@ -0,0 +1,35 @@ +"""Catalog-wide recipe contract tests.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml +from web2api.recipe_manager import load_catalog, validate_source_recipe_dir + +ROOT = Path(__file__).resolve().parents[1] +RECIPES_DIR = ROOT / "recipes" + + +def test_every_recipe_validates_with_its_custom_scraper() -> None: + recipe_dirs = sorted(path for path in RECIPES_DIR.iterdir() if path.is_dir()) + + assert recipe_dirs + for recipe_dir in recipe_dirs: + assert validate_source_recipe_dir(recipe_dir, trusted=True) == recipe_dir.name + + +def test_catalog_covers_every_recipe_and_declares_explicit_trust() -> None: + catalog = load_catalog(ROOT / "catalog.yaml") + recipe_slugs = {path.name for path in RECIPES_DIR.iterdir() if path.is_dir()} + + assert set(catalog) == recipe_slugs + assert all(isinstance(entry.get("trusted"), bool) for entry in catalog.values()) + + +def test_custom_scraper_params_are_declared() -> None: + for recipe_path in sorted(RECIPES_DIR.glob("*/recipe.yaml")): + payload = yaml.safe_load(recipe_path.read_text(encoding="utf-8")) + for endpoint in payload["endpoints"].values(): + assert "page" not in endpoint.get("params", {}) + assert "q" not in endpoint.get("params", {})