diff --git a/Cargo.lock b/Cargo.lock index f04e0152..160549de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,7 +158,6 @@ dependencies = [ name = "asset-discovery" version = "0.1.0" dependencies = [ - "base64", "clickhouse", "lambda_runtime 0.13.0", "prices-clickhouse", @@ -166,7 +165,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "stellar-strkey", + "stellar-strkey 0.0.13", "stellar-xdr", "thiserror 2.0.18", "tokio", @@ -1022,6 +1021,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" @@ -1260,7 +1265,7 @@ dependencies = [ "prices-clickhouse", "serde", "serde_json", - "stellar-strkey", + "stellar-strkey 0.0.13", "thiserror 2.0.18", "tokio", "toml", @@ -1491,7 +1496,7 @@ dependencies = [ [[package]] name = "domain" version = "0.1.0" -source = "git+https://github.com/rumblefishdev/soroban-block-explorer.git?branch=develop#31be5f747abadb2e0cc0d2ae4f59702604519bd5" +source = "git+https://github.com/rumblefishdev/soroban-block-explorer.git?branch=develop#d1251f30d1a9258bbe6643f8f5e35feafc7f6c4a" dependencies = [ "chrono", "serde", @@ -1883,6 +1888,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1909,6 +1923,17 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32", + "stable_deref_trait", + "zeroize", +] + [[package]] name = "heck" version = "0.5.0" @@ -2695,7 +2720,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "stellar-strkey", + "stellar-strkey 0.0.13", "stellar-xdr", "thiserror 2.0.18", "tokio", @@ -2881,7 +2906,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "stellar-strkey", + "stellar-strkey 0.0.13", "thiserror 2.0.18", "tokio", "tower 0.5.3", @@ -2918,6 +2943,7 @@ name = "prices-ingest-core" version = "0.1.0" dependencies = [ "aquarius-extractor", + "base64", "clickhouse", "comet-extractor", "extractors-core", @@ -2925,17 +2951,19 @@ dependencies = [ "ledger-processor", "phoenix-extractor", "prices-clickhouse", + "reqwest", "rust_decimal", "serde", "serde_json", "sha2 0.10.9", "soroswap-extractor", - "stellar-strkey", + "stellar-strkey 0.0.13", "stellar-xdr", "thiserror 2.0.18", "tokio", "tracing", "xdr-parser", + "zstd", ] [[package]] @@ -3574,7 +3602,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "stellar-strkey", + "stellar-strkey 0.0.13", "stellar-xdr", "thiserror 2.0.18", "tokio", @@ -3929,18 +3957,30 @@ dependencies = [ "data-encoding", ] +[[package]] +name = "stellar-strkey" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5038ed2bf4e11842210d118f65791d8ca4747622daddd222338b3d06b6051bc1" +dependencies = [ + "crate-git-revision 0.0.9", + "data-encoding", + "heapless", + "zeroize", +] + [[package]] name = "stellar-xdr" -version = "28.0.0" +version = "30.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93d09ff8b9f919b084f664003c4c546ac66a76affd5429460dbe29f4b326f8e" +checksum = "56f2fb2008b42fc746c0f6b46e375600068dd91f07748e9f1737c6341a2255f2" dependencies = [ "crate-git-revision 0.0.9", "escape-bytes", "ethnum", "hex", "sha2 0.10.9", - "stellar-strkey", + "stellar-strkey 1.0.0", ] [[package]] @@ -4839,7 +4879,7 @@ dependencies = [ [[package]] name = "xdr-parser" version = "0.1.0" -source = "git+https://github.com/rumblefishdev/soroban-block-explorer.git?branch=develop#31be5f747abadb2e0cc0d2ae4f59702604519bd5" +source = "git+https://github.com/rumblefishdev/soroban-block-explorer.git?branch=develop#d1251f30d1a9258bbe6643f8f5e35feafc7f6c4a" dependencies = [ "base64", "domain", @@ -4927,6 +4967,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index d3f7ac9c..558cfaca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ members = [ ] [workspace.dependencies] -stellar-xdr = { version = "=28.0.0" } +stellar-xdr = { version = "=30.0.0" } stellar-strkey = "0.0.13" sha2 = "0.10" xdr-parser = { git = "https://github.com/rumblefishdev/soroban-block-explorer.git", branch = "develop" } diff --git a/README.md b/README.md index a1774abb..f2019e5b 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,10 @@ Stellar DEX (SDEX order book) and the Soroban AMMs (Soroswap, Phoenix, Aquarius, SushiSwap V3), with a public REST API and a self-service portal for API keys. -- **API:** `https://prices-api.sorobanscan.rumblefish.dev` (key-gated `/v1/*`; - the OpenAPI document is public at `/api-docs-json`) -- **Portal (key sign-up):** `https://sorobanscan.rumblefish.dev/prices-api/` +- **API:** `https://prices-api.sorobanscan.com` (key-gated `/v1/*`; the + OpenAPI document is public at `/api-docs-json`). The old host + `https://prices-api.sorobanscan.rumblefish.dev` still answers. +- **Portal (key sign-up):** `https://sorobanscan.com/api/prices-api/` - **Endpoint reference:** [`docs/scf/api-endpoints.md`](docs/scf/api-endpoints.md) - **Architecture overview:** [`docs/prices-api-general-overview.md`](docs/prices-api-general-overview.md) diff --git a/docs/database-schema/amm-trades-schema.md b/docs/database-schema/amm-trades-schema.md index 822fa346..f1aa4e09 100644 --- a/docs/database-schema/amm-trades-schema.md +++ b/docs/database-schema/amm-trades-schema.md @@ -466,12 +466,11 @@ references. The remaining item is still pending BE input. `Symbol("swap")` emitters outside the three venues are dropped via a strict allowlist (no `venue: unknown` bucket) — see lore 0005. -3. **Decimals normalisation.** `amount_in` and `amount_out` are stored - as `NUMERIC(28,14)`. Confirm whether the BE Ledger Processor will - divide raw on-chain amounts by each token's `decimals()` before - insertion, or write the raw integer scalar. The Prices API expects - the **decimal-normalised** value (matching how it stores quantities - throughout `price_ohlcv` and `current_prices`). _Still open._ +3. **~~Decimals normalisation.~~** **Resolved** by lore 0329. The Prices + ingest normalises raw on-chain amounts itself: classic assets and their + SACs at 7, a pure Soroban token at its `decimals()`, read once over RPC + and kept in `prices.asset_decimals`. Until then it assumed 7 for every + token, which priced non-7-decimal tokens off by 10^(7 − decimals). --- diff --git a/docs/database-schema/database-schema-overview.md b/docs/database-schema/database-schema-overview.md index d5738acd..957a23bb 100644 --- a/docs/database-schema/database-schema-overview.md +++ b/docs/database-schema/database-schema-overview.md @@ -11,6 +11,7 @@ | Date | Sections | Driver | Summary | | ---------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-06 | §2, §3.0 note, §3.10b (new), §5, §13, App. A | [Task 0329](../../lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md) | **Added §3.10b `asset_decimals`**: each pure Soroban token's `decimals()`, by which the ingest now scales that token's AMM swap legs. Before it, every leg was scaled at a fixed 7 and every non-7-decimal token priced off by 10^(7 − decimals). | | 2026-10-01 | §3.1, §3.2, §3.3, §3.4, §3.9, §3.10, App. A | [Task 0139](../../lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md) | **`asset_id` is derived by ClickHouse from the identity** — `xxh3(concat(asset_code, ':', issuer_address, ':', contract_address))`, `UInt64`, rendered once by `prices_clickhouse::asset_id`. `assets.asset_id` is `MATERIALIZED` (a writer naming it is refused); candle tiers and `oracle_prices` take the identity in `EPHEMERAL` columns and derive the ids as `DEFAULT` (the rollup MVs and enrichment still copy ids with `INSERT … SELECT`); `current_prices`, `asset_supply` and `asset_metadata` hold plain `UInt64`. A `CHECK` refuses 0 and the id of a blank identity; on `oracle_prices` a blank identity is the REDSTONE sentinel 0. Two identities can no longer share an id, which closes the fan-out of every `asset_id` join. | | 2026-09-21 | §3.0 note, §3.6, §3.8 (removed), §13, App. A | [Task 0256](../../lore/1-tasks/active/0256_BUG_asset-discovery-ledger-scan-never-runs.md) | **`prices.discovery_state` removed** together with asset-discovery's ledger scan, which never ran on production. §3.8 is kept as a numbered tombstone so §3.9 onwards do not shift; the table leaves the storage-engine list (§3.0), the §13 at-a-glance table, and App. A's index and ER diagram. `init.sql` no longer creates it — a database created earlier keeps the (always empty) table until an operator drops it. The Asset Discovery Lambda is no longer listed as a `pool_registry` writer (§3.6, §13): the live Ledger Processor maintains that table (task 0291). | | 2026-09-17 | §3.6 (`pool_registry`), App. A | [Task 0291](../../lore/1-tasks/active/0291_BUG_pool-registry-is-stale-since-the-backfill-ended.md) | **`pool_registry` has two new writers.** The live Ledger Processor persists pools it learns from factory events (new rows only), and `events-backfill --discover-pools` fills a range's missing pools. Corrected the Asset Discovery claim: its scan has never run on production (task 0256). | @@ -149,16 +150,16 @@ are pushed to the Hetzner cluster via separate post-backfill tools. ## 2. Database Tech Stack -| Component | Technology | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Database engine | **ClickHouse** on BE's shared Hetzner cluster (separate `prices` database, ADR 0007) | -| Storage engines | `ReplacingMergeTree(version)` for OHLCV and `unresolved_pools`; `ReplacingMergeTree(updated_at)` for `current_prices` / `assets` / `asset_metadata` / `backfill_progress` / `pool_registry`; `ReplacingMergeTree(fetched_at)` for `asset_supply` / `asset_symbol`; `ReplacingMergeTree(ledger)` for `ingest_cursor` (highest-ledger-wins, §3.12); bare `ReplacingMergeTree` for `oracle_prices` / `backfill_sdex_ledgers`; `ReplacingMergeTree(version)` for `usd_rate` | -| Rollups | Chain of CH materialised views: `price_ohlcv_1m → _15m → _1h → _4h → _1d → _1w → _1M` (replaces the OHLCV Rollup Lambda) | -| Partitioning | `PARTITION BY toYYYYMM(timestamp)` on every OHLCV/oracle table; cleanup via `ALTER TABLE … DROP PARTITION` | -| Database client (Rust) | [`clickhouse`](https://crates.io/crates/clickhouse) — async, native protocol over HTTPS-mTLS | -| Schema tooling | Plain SQL DDL applied by the prices-api schema applier on first deploy; prices-api owns `prices.*` migrations unilaterally (ADR 0007 §3.7) | -| Hosting | BE-managed Hetzner box behind Caddy:443; cross-cloud (AWS → Hetzner) hop, ~80–130 ms RTT mitigated by warm connection reuse and batched per-ledger writes | -| Credentials | AWS Secrets Manager — per-env client `{cert,key,ca}` as a single JSON bundle secret per identity (one secret per identity per env, named by `MTLS_SECRET_NAME`; ADR 0007 / task 0063) | +| Component | Technology | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Database engine | **ClickHouse** on BE's shared Hetzner cluster (separate `prices` database, ADR 0007) | +| Storage engines | `ReplacingMergeTree(version)` for OHLCV and `unresolved_pools`; `ReplacingMergeTree(updated_at)` for `current_prices` / `assets` / `asset_metadata` / `backfill_progress` / `pool_registry`; `ReplacingMergeTree(fetched_at)` for `asset_supply` / `asset_symbol` / `asset_decimals`; `ReplacingMergeTree(ledger)` for `ingest_cursor` (highest-ledger-wins, §3.12); bare `ReplacingMergeTree` for `oracle_prices` / `backfill_sdex_ledgers`; `ReplacingMergeTree(version)` for `usd_rate` | +| Rollups | Chain of CH materialised views: `price_ohlcv_1m → _15m → _1h → _4h → _1d → _1w → _1M` (replaces the OHLCV Rollup Lambda) | +| Partitioning | `PARTITION BY toYYYYMM(timestamp)` on every OHLCV/oracle table; cleanup via `ALTER TABLE … DROP PARTITION` | +| Database client (Rust) | [`clickhouse`](https://crates.io/crates/clickhouse) — async, native protocol over HTTPS-mTLS | +| Schema tooling | Plain SQL DDL applied by the prices-api schema applier on first deploy; prices-api owns `prices.*` migrations unilaterally (ADR 0007 §3.7) | +| Hosting | BE-managed Hetzner box behind Caddy:443; cross-cloud (AWS → Hetzner) hop, ~80–130 ms RTT mitigated by warm connection reuse and batched per-ledger writes | +| Credentials | AWS Secrets Manager — per-env client `{cert,key,ca}` as a single JSON bundle secret per identity (one secret per identity per env, named by `MTLS_SECRET_NAME`; ADR 0007 / task 0063) | **Why ClickHouse on a BE-shared cluster (ADR 0007):** @@ -313,7 +314,7 @@ erDiagram > The registry, bookkeeping, and enrichment side tables — `pool_registry` > (§3.6), `unresolved_pools` (§3.7), `asset_metadata` > (§3.9), `asset_supply` (§3.10), `asset_symbol` (§3.10a), -> `backfill_sdex_ledgers` (§3.11), and +> `asset_decimals` (§3.10b), `backfill_sdex_ledgers` (§3.11), and > `ingest_cursor` (§3.12) — are omitted here to keep the price path legible. > **Appendix A** carries every `prices.*` table. @@ -1481,6 +1482,47 @@ each of which rules out an easier home: --- +### 3.10b `prices.asset_decimals` — Soroban token decimals (task 0329) + +A swap event carries each leg as a raw i128 in its token's own decimals. A +classic identity is 7-decimal by protocol and needs no row here; a pure Soroban +token's decimals are whatever its SEP-41 `decimals()` returns. A SAC the ingest +does not yet know as one (its classic asset not seen yet, task 0242) is held as +a contract and gets a row too, with 7. + +```sql +CREATE TABLE prices.asset_decimals ( + contract_address String, + decimals UInt8, + fetched_at DateTime DEFAULT now() -- RMT version +) +ENGINE = ReplacingMergeTree(fetched_at) +ORDER BY (contract_address) +SETTINGS index_granularity = 8192; +``` + +**Written by:** the ingest itself: the live ledger processor, `events-backfill`, +and `sdex-backfill` in `combined` mode. The first time a swap names a token with +no row, the decode drops that trade rather than guessing a scale. The caller +asks `decimals()` over Soroban RPC, writes the row, and decodes the ledger +again. **Read by:** the same three, in full, at startup. + +> **Only answers are stored.** A token whose call got no answer has no row. The +> live processor asks again after ten minutes; a backfill retries on the spot +> (1 s, 5 s, 20 s) and then on the token's next trade. A token that answered +> with no usable scale is not asked again until the process restarts, unless +> the node was still behind the ledger being decoded, in which case it is asked +> again on the next trade. Until it resolves, its trades are +> not priced. The live processor publishes them as `TradesMissingDecimals` +> (alarmed), and both backfills print them as `trades dropped (decimals)`. +> Rows are never refreshed, because decimals are fixed at deploy. + +> ⚠️ **Deploy order.** The live processor's cold start reads this table and +> fails init if it is absent, so apply `init.sql` before deploying an ingest +> built with task 0329. + +--- + ### 3.11 `prices.backfill_sdex_ledgers` — Per-ledger done-marks One row per processed ledger sequence — the resume source for the SDEX stream. @@ -1658,6 +1700,7 @@ data by orders of magnitude without the per-row cost of B-tree indexes. | `prices.asset_metadata` | `(asset_id)` | — (small table, no partitioning) | One row per asset; enrichment `LEFT JOIN` target | | `prices.asset_supply` | `(asset_id)` | — (small table, no partitioning) | One row per asset; supply `LEFT JOIN` for `market_cap_usd` | | `prices.asset_symbol` | `(contract_address)` | — (small table, no partitioning) | One row per Soroban contract; symbol `LEFT JOIN` for the displayed `asset_code` | +| `prices.asset_decimals` | `(contract_address)` | — (small table, no partitioning) | One row per Soroban token; read in full at ingest startup | | `prices.backfill_sdex_ledgers` | `(sequence)` | — | Set-membership probe: "is this ledger already done?" | | `prices.ingest_cursor` | `(id)` | — | One row per consumer; live resume point | @@ -2362,6 +2405,7 @@ criteria from the delivery plan, restated against the canonical | `prices.asset_metadata` | `ReplacingMergeTree(updated_at)` | none | `(asset_id)` | Discovery/enrichment worker (sole writer) | `GET /assets`, `GET /assets/{id}` — API queries `LEFT JOIN` it on `asset_id` (no view reads it) | | `prices.asset_supply` | `ReplacingMergeTree(fetched_at)` | none | `(asset_id)` | Supply worker (sole writer, hourly) | `current_prices` path, via `LEFT JOIN` for `market_cap_usd` | | `prices.asset_symbol` | `ReplacingMergeTree(fetched_at)` | none | `(contract_address)` | Asset Discovery worker, symbol stage (sole writer, hourly) | `GET /assets`, `GET /assets/{id}` — composed into `asset_code` at read time (§3.10a) | +| `prices.asset_decimals` | `ReplacingMergeTree(fetched_at)` | none | `(contract_address)` | the ingest (live processor, `events-backfill`, `sdex-backfill` combined) on first sight of a token | the same writers, at startup — scales AMM legs (§3.10b) | | `prices.backfill_sdex_ledgers` | `ReplacingMergeTree()` | none | `(sequence)` | SDEX backfill stream — one row per processed ledger | SDEX backfill at startup (skip already-done ledgers) | | `prices.ingest_cursor` | `ReplacingMergeTree(ledger)` | none | `(id)` | Ledger Processor reconcile loop (written last each run) | Ledger Processor at cold start | | `prices.price_usd_series` / `_1h`, `usd_reference` / `_1h`, `identity_by_contract` | `VIEW` (plain, derived) | none (read-through) | n/a (defined over `price_ohlcv_1d` / `_1h`) | n/a — derived at read time from `close_usd` / `close` on the OHLCV tables (task 0061) | BE historical USD close series (BE task 0199); `price_usd_at` endpoint (task 0040) | @@ -2399,6 +2443,7 @@ erDiagram assets ||--o| asset_metadata : "asset_id (logical, 1:1 enrichment)" assets ||--o| asset_supply : "asset_id (logical, 1:1 supply)" assets ||--o| asset_symbol : "contract_address (logical, 1:1 soroban symbol)" + assets ||--o| asset_decimals : "contract_address (logical, 1:1 soroban decimals)" price_ohlcv_1m ||--o{ price_ohlcv_15m : "MV: 1m → 15m" price_ohlcv_15m ||--o{ price_ohlcv_1h : "MV: 15m → 1h" price_ohlcv_1h ||--o{ price_ohlcv_4h : "MV: 1h → 4h" @@ -2571,6 +2616,14 @@ erDiagram ORDER_BY sort_key "(contract_address)" } + asset_decimals { + String contract_address PK "logical FK to assets.contract_address" + UInt8 decimals "SEP-41 decimals(); only answers are stored" + DateTime fetched_at "DEFAULT now() — RMT version column" + ENGINE engine "ReplacingMergeTree(fetched_at)" + ORDER_BY sort_key "(contract_address)" + } + backfill_sdex_ledgers { UInt32 sequence PK "processed ledger; set-membership marker" ENGINE engine "ReplacingMergeTree()" diff --git a/docs/prices-api-general-overview.md b/docs/prices-api-general-overview.md index e365e765..569ed628 100644 --- a/docs/prices-api-general-overview.md +++ b/docs/prices-api-general-overview.md @@ -15,6 +15,7 @@ the API surface, or the cost / budget framing. | Date | Sections touched | Driver | Summary | | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-05 | §4.1, §4.3 | [Task 0242](../lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md) | **A SAC address is an alias of its classic asset.** A classic asset's Stellar Asset Contract address, XLM's included, used to answer `404` or, for a SAC we had minted as a second identity, a near-empty listing of the same token. It now answers as the classic asset on every identifier-taking endpoint, named by `{code}:{issuer}` or `native`, and the classic wins over a not-yet-healed second identity. In a batch the found entry carries the classic's name while `not_found` echoes what was asked. | | 2026-10-01 | §3.1, §3.2, §3.3, §3.4 | [Task 0139](../lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md) | **`asset_id` is derived by ClickHouse from the identity**, `xxh3` of `code:issuer:contract` as `UInt64`, instead of a `UInt32` counter in each writer. Two writers' counters had handed one id to two assets 3,315 times, so every join on `asset_id` fanned out. | | 2026-09-24 | §4 (intro, §4.1, §4.2, §4.3) | [Task 0306](../lore/1-tasks/active/0306_DOCS_published-docs-drift-from-production.md) · [Task 0178](../lore/1-tasks/archive/0178_BUG_current-prices-cannot-publish-the-quote-asset.md) | **§4 brought to what production answers**, next to the published API reference. The base URL is the production host, and a new note separates the two error bodies: the API's `{code, message}` and API Gateway's `{message}` for `403` and `429`, with the handler's own `401` not armed in production. The `/assets` and `/price` examples are the 2026-09-23 production snapshot (AQUA; XLM with its five venues) and carry `method`; `GET /assets/{asset_identifier}` and `POST /prices/batch` gain their response bodies. `volume_24h_usd` is defined as both sides of each trade (task 0178) against the base side in `sources` — the old example's venues summed exactly to the total, which production never shows. Recorded as not populated yet: `home_domain`, a classic asset's SAC in `contract_address`, and a Soroban token's `asset_code`. | | 2026-09-16 | §3.2, §4.2 (`GET /ohlcv`) | [Task 0286](../lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md) · [ADR 0287](../lore/2-adrs/0287_candle-prices-come-from-price-forming-fills-and-a-windowed-close.md) | **A candle's prices come only from the price-forming trades of its own bucket** — `open`/`close` the first and last of them, `high`/`low` their extremes, nothing carried from a neighbouring bucket. A fill's price is the ratio of two integer stroop amounts, so a few-stroop fill prints an exact small fraction that is arithmetically right and can sit hundreds of percent off the market; such fills no longer set a price. The task-0116 guidance to filter dust **on the client** is replaced by the rule and by three new `/ohlcv` fields — `pf_trade_count` (`0` means the bucket has no price and every price field is `null`), `pf_vwap` (the price-forming mean) and `close_divergent` (`close` more than 1% from it). Added the three `pf_*` columns to the §3.2 DDL and re-cut the rollup sketch to the shipped generator's pf-gated form, with the month rolled from the day. | @@ -169,7 +170,7 @@ to their own infrastructure at any time if needed. | **EventBridge Scheduler** | Scheduled triggers | Cron/rate rules for all periodic Lambda workers | | **Secrets Manager** | Credentials & mTLS material | Per-env client `{cert,key,ca}` for Caddy:443 mTLS (single JSON bundle secret per identity, named by `MTLS_SECRET_NAME`); Soroswap/Aquarius API keys; oracle contract address | | **CloudWatch + X-Ray** | Observability | API latency, error rates, ingestion lag, Lambda duration/concurrency, backfill progress; mTLS cert NotAfter alarm | -| **S3** (API docs) | Documentation hosting | self-service onboarding portal + API reference, served from the block explorer's bucket and CloudFront distribution at `sorobanscan.rumblefish.dev/prices-api/`; the OpenAPI document is served by the API itself | +| **S3** (API docs) | Documentation hosting | self-service onboarding portal + API reference, served from the block explorer's bucket and CloudFront distribution at `sorobanscan.com/api/prices-api/`; the OpenAPI document is served by the API itself | **Components no longer in the Prices API budget** (eliminated by ADR 0007): @@ -522,7 +523,7 @@ PARTITION` statements over HTTPS-mTLS to Caddy:443. ## 4. API Endpoints Design -**Base URL:** `https://prices-api.sorobanscan.rumblefish.dev/v1` +**Base URL:** `https://prices-api.sorobanscan.com/v1` **Authentication and errors.** Every `/v1` request carries an `x-api-key` header, which API Gateway checks before the request reaches the handler (§1.1, §7). An error therefore has one of two bodies, @@ -634,7 +635,12 @@ Get single asset details. `asset_identifier` can be: - `{contract_address}` for Soroban tokens (e.g. `CABC...DEF`) - `native` for XLM -A classic asset's SAC address is not an identifier yet: it answers `404`. +A classic asset's Stellar Asset Contract address, XLM's included, is accepted as an alias of that +asset, and the answer names it by `{code}:{issuer}` or `native`. The same holds on `/price`, +`/ohlcv`, `GET /oracles/{asset_identifier}` and `POST /prices/batch`. For example, +`GET /assets/CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA` returns XLM with +`"asset": "native"`. Any other `C…` address is looked up as a contract: a Soroban token we hold +answers as itself, and an address we do not hold answers `404`. **Response:** `code` is the token's symbol for a Soroban token and `XLM` for `native`. @@ -863,9 +869,12 @@ Fetch current prices for multiple assets in one call. ``` **Response:** `prices` holds the `GET /assets/{asset_identifier}/price` object of every identifier -that has a current price, in the order of the request; `not_found` lists the rest, in canonical -form. A request names 1 to 100 identifiers, a repeated one is answered each time, and a malformed -body, an empty or over-long list or a single invalid identifier fails the whole request with `400`. +that has a current price, in the order of the request with the not-found ones left out; +`not_found` lists the rest as they were asked, in canonical form. Each entry's `asset` is the +canonical identifier of the asset found, so a classic asset's SAC address comes back as +`{code}:{issuer}` or `native` (§4.1), and a request naming both forms gets two identical entries. +A request names 1 to 100 identifiers, a repeated one is answered each time, and a malformed body, +an empty or over-long list or a single invalid identifier fails the whole request with `400`. ```json { diff --git a/docs/runbooks/0100-coverage-sweep-triage.md b/docs/runbooks/0100-coverage-sweep-triage.md index be556602..c792eb29 100644 --- a/docs/runbooks/0100-coverage-sweep-triage.md +++ b/docs/runbooks/0100-coverage-sweep-triage.md @@ -128,6 +128,11 @@ three outcomes: permanent family-wide ignore would hide the next pool of a real venue. - **(c) A router, aggregator or other non-AMM contract.** Add a permanent `[[contract]]` entry with `reason` and `task`. Never register it. + If the same non-AMM wasm keeps deploying new instances, each new one pages + on its own. Add one `[[wasm]]` entry with `not_a_venue = true` in place of + `until` instead (task 0336: the securities-financing ledger `f3dd3edf…`, + four new instances in one week). Do this only when the code itself rules out + a market, not because the instances seen so far are small. Also look at: @@ -147,7 +152,8 @@ Also look at: `packages/coverage-sweep-probe/allowlist.toml`: - `cargo test -p coverage-sweep-probe` validates it: reason and task on every - entry, `until` on every `[[wasm]]` entry, valid `C…` strkeys, 64-lowercase-hex + entry, `until` on every `[[wasm]]` entry except a `not_a_venue` one (which + must not have it), valid `C…` strkeys, 64-lowercase-hex hashes, no duplicates, no unknown keys. - The list is compiled into the binary, so an edit **ships only with an EventBridge deploy** (§4.4). @@ -370,7 +376,7 @@ This shows that the sweep would have caught SushiSwap V3 in April 2026. ```bash curl --fail-with-body -sS --cert "$READ_CERT" --key "$READ_KEY" --cacert "$PRICES_CA" \ - "https://ch.sorobanscan.rumblefish.dev/?param_lo=61926675¶m_hi=62147853&default_format=TSVWithNames" \ + "https://ch.sorobanscan.com/?param_lo=61926675¶m_hi=62147853&default_format=TSVWithNames" \ --data-binary @query.sql ``` diff --git a/docs/runbooks/0139-asset-id-migration.md b/docs/runbooks/0139-asset-id-migration.md index e245e5b9..4bfac2d0 100644 --- a/docs/runbooks/0139-asset-id-migration.md +++ b/docs/runbooks/0139-asset-id-migration.md @@ -49,10 +49,10 @@ Set up the CH shell once, in `tmux`: tmux new -s rekey0139 mkdir -p ~/rekey-0139 set -o pipefail -CH=https://ch.sorobanscan.rumblefish.dev/ +CH=https://ch.sorobanscan.com/ WCERT=(--cert ~/.certs/adamkot-write.crt --key ~/.certs/adamkot-write.key --cacert ~/prices-mtls/ca.crt) rk() { - ~/rekey-0139/prices-clickhouse-rekey "$@" --mtls-domain ch.sorobanscan.rumblefish.dev \ + ~/rekey-0139/prices-clickhouse-rekey "$@" --mtls-domain ch.sorobanscan.com \ --mtls-cert ~/.certs/adamkot-write.crt --mtls-key ~/.certs/adamkot-write.key \ --mtls-ca ~/prices-mtls/ca.crt 2>&1 | tee -a ~/rekey-0139/window.log } @@ -733,7 +733,7 @@ B=window # next-day | period-close | month sed -n "//,//p" \ docs/runbooks/0139-asset-id-migration.md | grep '^SELECT ' | while IFS= read -r q; do r=$(printf '%s' "$q" | curl --fail-with-body -sS --cert "$READ_CERT" --key "$READ_KEY" \ - --cacert "$PRICES_CA" "https://ch.sorobanscan.rumblefish.dev/${M:+?param_m=$M}" --data-binary @-) + --cacert "$PRICES_CA" "https://ch.sorobanscan.com/${M:+?param_m=$M}" --data-binary @-) printf '%s\t%s\n' "$r" "$q" done ``` diff --git a/docs/runbooks/0142-rollup-mv-reapply.md b/docs/runbooks/0142-rollup-mv-reapply.md index 75d1aa84..40a3b4fa 100644 --- a/docs/runbooks/0142-rollup-mv-reapply.md +++ b/docs/runbooks/0142-rollup-mv-reapply.md @@ -50,7 +50,7 @@ this is an operator procedure rather than something the apply path does. **local dev ClickHouse** and exits 0 — a clean result that says nothing whatever about ch-prod-01. Every command below is written for the Hetzner host. -Prod's HTTP endpoint (`ch.sorobanscan.rumblefish.dev`) is mTLS-only behind Caddy +Prod's HTTP endpoint (`ch.sorobanscan.com`) is mTLS-only behind Caddy and `prices_clickhouse::client()` builds a plaintext client, so there is no path from a local machine to prod for this binary. Run it **on the host, against the loopback port** — the same pattern as diff --git a/docs/runbooks/0151-zero-invariant-probe-rollout.md b/docs/runbooks/0151-zero-invariant-probe-rollout.md index 8e2f8be3..caf2e92a 100644 --- a/docs/runbooks/0151-zero-invariant-probe-rollout.md +++ b/docs/runbooks/0151-zero-invariant-probe-rollout.md @@ -153,7 +153,7 @@ A client pointed at `localhost:8123` reads the **local dev ClickHouse** and returns a clean result that says nothing whatever about production — check the host in the command before you read the answer. -Prod's HTTP endpoint (`ch.sorobanscan.rumblefish.dev`) is mTLS-only behind Caddy +Prod's HTTP endpoint (`ch.sorobanscan.com`) is mTLS-only behind Caddy (task 0276). Run the reads from a workstation with your **personal read certificate**, which maps to the read-only ClickHouse user `dev_read`: @@ -162,7 +162,7 @@ curl --fail-with-body -sS \ --cert "$READ_CERT" \ --key "$READ_KEY" \ --cacert "$PRICES_CA" \ - https://ch.sorobanscan.rumblefish.dev/ \ + https://ch.sorobanscan.com/ \ --data-binary @query.sql ``` diff --git a/docs/runbooks/0242-sac-identity-heal.md b/docs/runbooks/0242-sac-identity-heal.md new file mode 100644 index 00000000..0436caac --- /dev/null +++ b/docs/runbooks/0242-sac-identity-heal.md @@ -0,0 +1,906 @@ +# Runbook — healing SACs held as contract identities (task 0242) + +A Stellar Asset Contract (SAC) is a facet of its classic asset, never an asset of +its own. Production holds 36 SACs as `Contract` identities in `prices.assets`, +each with its own `asset_id`. This runbook removes them without a SQL rekey: +0286 phase 3 rebuilds the history under the classic ids, and a small cleanup +deletes what is left. + +**Applies to:** `prices.assets`, `prices.asset_symbol` and the seven +`prices.price_ohlcv_*` tiers on ch-prod-01; the Compute, EventBridge and +Observability stacks. + +**Read first:** [`0286-reingest-history.md`](0286-reingest-history.md) (phase 3, +its stages and its orchestrator) and lore task 0242 (decisions D1–D9, PC3, PC6, +PC9). + +--- + +## 0. What and why + +Measured 2026-10-05 (task 0242 §"Re-measured on prod, 2026-10-05"): + +| Group | Contract rows | What their candles are | +| ----- | ------------- | ----------------------------------------------------------------------------------------------------------- | +| A | 13 | Split pairs: the classic row exists. 145 of 151 SAC-keyed `1m` rows are byte-identical to a classic row | +| B | 23 | SACs whose classic was never in `assets`. Their candles are the **only copy** of those trades | +| — | 6 latent | Pool-leg SACs with no identity yet (SUSHI, HYPE, STELLA, TESTTTT, TEST77, TEST12). They mint on first trade | + +Group A's 13: POINTS, USDM1, SODA, VCHF, USDP, VEUR, WHLAQUA, USD, USDT0, XCR, +ESP, EUR, FrogST. + +The heal is the 0286 phase-3 re-ingest, not a SQL rekey (D3). A rekey would need +four kinds of row operation (delete the duplicates, move group B, merge three +mixed coarse buckets, re-enrich), plus a base/quote flip for Z/Q and XLD/SODA, +whose order changes when `Contract` becomes `Credit`. Phase 3 drops and rebuilds +every month anyway; with the classic identities in `assets` it writes them under +the classic ids by construction. + +The code of this task (one PR) stops new mints: a SAC leg resolves to its classic +only on a sha256-verified proof, and a SAC BE flags `is_sac` that nothing proves +is skipped and counted, never minted (D1, D2). The code protects the future; this +runbook heals the past. + +--- + +## 1. Order of events + +| # | Step | When | Section | +| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | ------- | +| 1 | Seed the 29 classic identities. Works with the `events-backfill` already on ch-prod-01 (D4) | After phase-3 stage B (202402–202404), before stage C | §3 | +| 2 | Re-plan phase 3 to 202609, run stage C with `--to-month 202609` (D4; the plan ended at 202608) | After the seed | §4a–§4d | +| 3 | Deploy the PR: Compute (live ledger processor + API), then EventBridge (probe). Not Observability (D8). Swap `events-backfill` on ch-prod-01 only between phase-3 stages, never while a month is at its `amm` step (D7) | After stage D (`finish`) and before §5 (D9: the API alias needs the classic history in place) | §4e | +| 4 | Residual cleanup (D3) | After stage D (`finish`) AND after the live slice is deployed with at least one cold start | §5 | +| 5 | Deploy Observability with the `SacContractIdentities` alarm actions on (D6, D8) | After §5's post-checks are green | §6 | + +Step 4 waits for the live deploy because until then a warm container that predates +the seed, or a SAC the seed does not cover, can still mint a contract identity +and write candles under it. + +--- + +## 2. Operator identity and tooling + +- **CH shell:** `dev_shared` over mTLS, with the `chq` function, set up exactly + as in [`0139-asset-id-migration.md` → Who runs it, and where](0139-asset-id-migration.md#who-runs-it-and-where). + `echo "SELECT currentUser()" | chq` must print `dev_shared`. +- **Orchestrator shell:** `fishuser-hero`, the checkout `reingest_0286.py` runs + from, as in 0286 §3. +- **AWS shell:** production credentials, a checkout of the merged PR. +- **API:** `API=https://prices-api.sorobanscan.rumblefish.dev/v1` and an operator + `PRICES_API_KEY`. + +```bash +mkdir -p ~/heal-0242 +Q=~/Projects/stellar-prices-api/.planning/quick/261005-htu-0242-sac-minted-as-a-second-identity-res +XCR=XCR:GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7 +XCR_SAC=CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI +XLM_SAC=CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA +``` + +`$Q` is in Adam's checkout, under the gitignored `.planning/`. The seed and its +checks stay there, outside the repo. + +Every `chq` step stops at the first non-zero exit. Read the message before going on. + +--- + +## 3. Seed (after stage B, before stage C) + +`$Q/seed-0242.sql` inserts 29 classic identities: the 23 of group B plus the 6 +latent SACs. Every `sac_address` in it was re-derived from code + issuer by +sha256 (`$Q/measurements/sac_check.py`: ok 29, bad 0), and none of the 29 was in +`assets` on 2026-10-05. Adam runs it. + +**Why `sac_address` must be filled.** The ingest registry would resolve a seeded +classic's SAC even with the column empty, because `from_existing` derives it. +But `identity_by_contract`, the API alias (D5), the `events-backfill` preload +filter and the probe all read the stored column (0242 research, pitfall 8). + +**Pre-check.** + +- `reingest_0286.py status` shows 202404 done and no month in progress. +- ```bash + chq < $Q/seed-0242-check.sql | tee ~/heal-0242/seed-check-before.tsv + ``` + Query 1 prints `N N`, query 2 prints `N`. Record query 3 (`count() - uniqExact(asset_id)`) + as `COLLISIONS_BEFORE`. + - `N = 0` when the seed comes before the §4e deploy, the order of §1. + - `N > 0` when the 0242 build went live first: the live processor proved some of + the 29 in-band and interned their classics itself, with `sac_address`. Those + rows belong to live; the run below inserts only the `29 - N` others and keeps + only those for the seed rollback. + - The two numbers of query 1 differ: one of the 29 is held without + `sac_address`. Stop and bring it to Adam. + +**Run.** Stage the 29 rows, keep the ones `assets` does not hold yet, then +insert exactly those: + +```bash +echo "CREATE TABLE prices.bak_0242_seed_stage AS prices.assets" | chq +sed 's/^INSERT INTO prices\.assets /INSERT INTO prices.bak_0242_seed_stage /' $Q/seed-0242.sql | chq +chq <<'SQL' +CREATE TABLE prices.bak_0242_seed ENGINE = MergeTree ORDER BY asset_id AS +SELECT asset_id, asset_code, issuer_address, sac_address FROM prices.bak_0242_seed_stage +WHERE (asset_code, issuer_address) NOT IN (SELECT asset_code, issuer_address FROM prices.assets FINAL); +INSERT INTO prices.assets (asset_code, asset_type, issuer_address, contract_address, sac_address, is_active) +SELECT asset_code, asset_type, issuer_address, contract_address, sac_address, is_active +FROM prices.bak_0242_seed_stage +WHERE (asset_code, issuer_address) IN (SELECT asset_code, issuer_address FROM prices.bak_0242_seed); +SELECT count() FROM prices.bak_0242_seed_stage; +SELECT count() FROM prices.bak_0242_seed +SQL +``` + +The two counts print `29` and `29 - N`. `bak_0242_seed` holds the rows this seed +added and nothing live wrote; the seed rollback and §7 read it. + +**Post-check.** + +```bash +chq < $Q/seed-0242-check.sql | tee ~/heal-0242/seed-check-after.tsv +``` + +Query 1 prints `29 29` (29 rows, all with `sac_address`), whatever `N` was. +Query 2 prints `29` (each seeded SAC is a `credit` row of +`identity_by_contract`). Query 3 equals `COLLISIONS_BEFORE` (AC4). + +**When it takes effect.** Phase 3 reads `assets` at the start of every +`events-backfill` run, so stage C sees the seed at once. The live processor sees +it at its next cold start (`resumed cursor from ClickHouse` in +`/aws/lambda/prices-production-ledger-processor` after the seed time). Until +then a warm container can still mint one of the 29. + +**Seed rollback.** Only while no candle names a seeded id, i.e. before stage C +writes any: + +```bash +for T in 1m 15m 1h 4h 1d 1w 1M; do + echo "SELECT '$T', count() FROM prices.price_ohlcv_$T WHERE asset_id IN (SELECT asset_id FROM prices.bak_0242_seed) OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_seed)" | chq +done # every line 0 +echo "ALTER TABLE prices.assets DELETE WHERE (asset_code, issuer_address) IN (SELECT asset_code, issuer_address FROM prices.bak_0242_seed) SETTINGS mutations_sync = 2" | chq +echo "SELECT mutation_id, is_done, latest_fail_reason FROM system.mutations WHERE database = 'prices' AND table = 'assets' AND NOT is_done" | chq # no row +chq < $Q/seed-0242-check.sql # query 1 prints the `N N` of the pre-check again (`0 0` in the order of §1) +``` + +A mutation is fine on a ~211k-row table. `mutations_sync = 2` makes the +`ALTER` return only once the mutation is done on every replica, and fail if it +fails (`dev_shared` is not `readonly`, so SETTINGS are allowed). Do not re-seed +or start stage C before query 1 reads the pre-check's numbers again. Once stage C +has written candles under the seeded ids there is no seed rollback: those ids +are the right identities. + +--- + +## 4. Stage C and the deploy + +### 4a. Extend the plan to 202609 + +The phase-3 plan ends at 202608 (`plan` took the last complete month on the day +it ran). Stage C must reach 202609: it holds the only copies of the USDT0 and +SODA trades and part of 0139's hole (D4). + +Re-planning is safe: `cmd_plan` rewrites only the state's month map +(`st.d["plan"]`), and the progress of every month lives under a separate key +(`st.d["months"]`), which `plan` does not touch (`tools/scripts/reingest_0286.py` +`cmd_plan`). + +**Pre-check** (orchestrator shell, between stages, no `run` holding the lock): + +```bash +cp -p ~/reingest-0286/state.json ~/reingest-0286/state.json.pre0242 +python3 -c "import json,os; p=json.load(open(os.path.expanduser('~/reingest-0286/state.json')))['plan']; print(min(p), max(p), len(p))" +``` + +It prints ` 202608 `. Pass `` back explicitly, so the re-plan +cannot start from a different first month. + +**Run.** + +```bash +python3 tools/scripts/reingest_0286.py plan --from-month --to-month 202609 +``` + +**Post-check.** The last line reads ` months, ..202609, archive tip …`. +`reingest_0286.py status` still shows every finished month as done. + +**Rollback.** `cp -p ~/reingest-0286/state.json.pre0242 ~/reingest-0286/state.json`. + +### 4b. Run stage C + +**Pre-check: the seed is in** (§3 post-check: query 1 prints `29 29`). It is +mandatory, not just ordered first. The `events-backfill` on ch-prod-01 until the +§4e swap resolves a SAC only when its classic is in `assets`, so stage C without +the seed rebuilds group B's history, BLTA/BLTB/BLTC/PPRIME/LumenJoule (U3, §8) +included, under the SAC ids again: the second identities this task removes. + +The flags are stage B's (task 0286, "Stage B command"), plus `--ack-0285` (202607 +on) and `--ack-0139-binaries` (0286 §10a). Use the restart wrapper of 0286 §3: + +```bash +python3 tools/scripts/reingest_0286.py run --to-month 202609 --ack-0285 --ack-0139-binaries --yes +``` + +**Which `events-backfill` is on the host.** A one-ledger dry run tells (CH host, +as `default`): + +```bash +read -rs CH_PW +CLICKHOUSE_PASSWORD="$CH_PW" ~/events-backfill --start 51500460 --end 51500460 \ + --clickhouse-url http://localhost:8123 --dry-run | grep 'unproven sac swaps:' +``` + +A 0242 binary prints `unproven sac swaps:` followed by a count; a binary built +before 0242 prints nothing. Both are fine for stage C once the seed is in (PC6). +A month run on a pre-0242 binary carries the note `printed no unproven sac swaps: +line`. Once a month of the state has printed the line, a later month without it +is a STOP: the host binary went back to a pre-0242 build (review of 0242). + +**Post-check: U3's candles are back under the classic ids.** Once 202609 is done +(CH shell): + +```bash +chq <<'SQL' | tee ~/heal-0242/u3-post-stage-c.tsv +WITH ['CCUYL75XNUTAHFE3RZAN7NEFQWZ7OIRJZNN4KUXXPA34RAHVUGLTBGT5', -- BLTA + 'CAANCS7TKC7IFWLRACKL4T4HMNMDUWSWNOL5EV244TVQCU3O7BM7N4VD', -- BLTB + 'CB2J5FHCJJBQE5UPQ5FPZAWIEQD5KKP4GWLZZ3QT5EFVIT3FW5CI2OGK', -- BLTC + 'CBVWPBYEDJ7GYIUHL2HITMEEWM75WAMFINIQCR4ZAFZ62ISDFBVERQCX', -- LumenJoule + 'CBI2NXIZQ33L3K5RMQW53OGV52HDHZU2AUISCUFXDYTDR345VKPHAQEP' -- PPRIME + ] AS sacs, + (SELECT groupArray(asset_id) FROM prices.assets FINAL + WHERE contract_address = '' AND has(sacs, sac_address)) AS classic, + (SELECT groupArray(asset_id) FROM prices.assets FINAL + WHERE has(sacs, contract_address)) AS contract +SELECT toYYYYMM(timestamp) AS month, length(classic) AS classics, + countIf(has(classic, asset_id) OR has(classic, quote_asset_id)) AS classic_rows, + sumIf(trade_count, has(classic, asset_id) OR has(classic, quote_asset_id)) AS classic_trades, + sumIf(pf_trade_count, has(classic, asset_id) OR has(classic, quote_asset_id)) AS classic_pf_trades, + countIf(has(contract, asset_id) OR has(contract, quote_asset_id)) AS sac_id_rows +FROM prices.price_ohlcv_1h FINAL +WHERE source = 'soroswap' AND timestamp >= '2026-03-01 00:00:00' AND timestamp < '2026-10-01 00:00:00' + AND (has(classic, asset_id) OR has(classic, quote_asset_id) + OR has(contract, asset_id) OR has(contract, quote_asset_id)) +GROUP BY month ORDER BY month +FORMAT TSVWithNames +SQL +``` + +`classics` = 5 and `sac_id_rows` = 0 on every line. `classic_trades` equals BE's +swap count on the four registry pools of these SACs (two BLT pools, +LumenJoule/USDC, PPRIME/USDC; measured 2026-10-05, `$Q/U3-root-cause.md`): + +| month | 202603 | 202604 | 202605 | 202606 | 202607 | 202608 | 202609 | total | +| ------------------- | ------ | ------ | ------ | ------ | ------ | ------ | ------ | ------ | +| `classic_trades` | 24 | 7,037 | 3,301 | 23,422 | 10,189 | 6,958 | 24,649 | 75,580 | +| `classic_pf_trades` | 24 | 0 | 0 | 6 | 0 | 0 | 0 | 30 | + +The query counts candle rows, not assets, so a BLTB/BLTA trade, which names two +of the five, counts once. `classic_pf_trades` is LumenJoule's 24 and PPRIME's 6: every BLT fill is +dust (about 0.0001 units each), so BLTA, BLTB and BLTC come back with volume and +`trade_count` but `pf_trade_count = 0` and no price (ADR 0287 §1). That is the +rule working, not a gap. Any `sac_id_rows` above 0, a month short of the table, +or `classics` below 5: stop before §4e and bring the output to Adam. + +### 4c. A STOP on `unproven sac swaps:` + +A 0242 binary prints, after `negative apply order:`, + +``` +unproven sac swaps: 0 +``` + +Anything but `0` means swaps were skipped because a leg is a contract BE flags +`is_sac` and no proof resolved it (D2). The orchestrator STOPs on it +(`amm_summary`), after either pass, and the STOP names the pass. It reads the +line after the `--dry-run` pass first, so a STOP there comes after `drop_1m` and +`sdex` and before anything AMM is written: the month's `1m` holds SDEX-only +candles, its coarse tiers are untouched, and the month stays at its `amm` step. +A STOP on the write pass is step 4's last case. + +1. **Find the contracts.** In `~/reingest-0286//events-backfill.log`: + - the WARN `swaps skipped on SACs with no proof (task 0242 D2); the month is NOT complete`, + whose `sample` lists up to 20 contracts; + - the start-of-run INFO `preloaded SAC proofs (task 0242)` (`sacs_verified`, + `sacs_rejected`) and, when `sacs_rejected > 0`, the WARN + `SAC proofs that did not verify`. +2. **Check each contract** (CH shell; `C1,C2` = the contracts, quoted): + + ```bash + chq <<'SQL' + SELECT contract_id, groupUniqArray(is_sac) AS is_sac_versions FROM default.soroban_contracts + WHERE contract_id IN ('C1', 'C2') GROUP BY contract_id; + SELECT sc.addr AS contract, x.signature, x.ledger_sequence, + JSONExtractString(x.topics_xdr, JSONLength(x.topics_xdr), 'value') AS sep11 + FROM (SELECT contract_id, signature, ledger_sequence, topics_xdr FROM default.soroban_events + WHERE contract_id IN (SELECT id FROM default.soroban_contracts WHERE contract_id IN ('C1', 'C2')) + AND signature IN ('transfer', 'mint', 'burn', 'clawback') + LIMIT 3 BY contract_id) AS x + INNER JOIN (SELECT id, any(contract_id) AS addr FROM default.soroban_contracts + WHERE contract_id IN ('C1', 'C2') GROUP BY id) AS sc ON sc.id = x.contract_id + SQL + ``` + + Then derive the SAC of each `sep11` (`CODE:ISSUER`, or `native`) with + `$Q/measurements/sac_check.py` and compare it with the contract. + +3. **Fix, then `run` again.** + - **The SEP-11 derives to the contract:** the SAC is real and its proof did not + reach the run. Seed its classic as §3 seeded the 29: one row + `(asset_code, 'classic', issuer, '', sac_address = the contract, 1)`, with + the §3 checks narrowed to it. Then `run` with the same flags. It resumes at + the `amm` step, and its `--dry-run` pass is the proof: the binary derives + the SAC itself, so a wrong seed STOPs again with the same count and writes + nothing. Then remove the wrong row (§3's seed rollback, narrowed to it) + before anything else: the API alias would answer the contract as it. + - **BE has corrected `is_sac` to false:** the next run treats it as a genuine + token only once no version of its row says `true`. The code reads BE's + `soroban_contracts` (a ReplacingMergeTree) without `FINAL`, on purpose, so an + unmerged old version keeps the contract a candidate and its swaps skipped. + `run` again only when `is_sac_versions` above is `[false]`. While it holds + both values, BE's parts have not merged: wait for them (never `OPTIMIZE` + BE's table), or take step 4. +4. **`rollback` the month instead** (`reingest_0286.py rollback `, which + restores `1m` and the four fine tiers from the month's snapshot and forgets + the month, so the next `run` starts it at `gates`): + - when no SAC event exists for the contract, or its SEP-11 does not derive to + it. No data fix can prove it. Stop phase 3 there and bring it to Adam; + - when the fix cannot land the same day: the month serves SDEX-only `1m` + candles while it waits; + - when the non-zero line came from the **write** pass (the log shows that + attempt's dry-run summary reading `0`, then the write's non-zero): the + month's AMM candles are in without those swaps. + +### 4d. `--amm wait` + +`amm-done` records only `--fallbacks`. It does not read `unproven sac swaps:` +(nor `swaps dropped (unresolved):`). On this path the operator is the gate: + +1. Run the host command the orchestrator prints with `--dry-run` first, and read + `unproven sac swaps:`. Non-zero: do not run the write, do not `amm-done`; + follow §4c. +2. Run the write. Read the line again. Non-zero: do not `amm-done`; `rollback` + the month (§4c step 4). +3. Only on `0` (or no such line, a pre-0242 binary): `amm-done --fallbacks `. + +### 4e. Deploy the PR + +**When: after stage D (`finish`), before §5 (D9).** The API alias answers a SAC +address as its classic. Until stage C has rebuilt a SAC's months under the +classic id, that classic holds only part of the history, while the `Contract` +row still holds the rest. Deployed earlier, `GET /v1/assets/{C…}/ohlcv` would +lose the pre-heal AMM series. Group B is the visible case: its classics were +seeded on 2026-10-06 and fill month by month as stage C runs. §5 still needs the +live slice deployed with one cold start, so the deploy goes between the two. + +**Pre-check before the live deploy: `prices_writer` can read BE's SAC set.** The +live Lambda loads `default.soroban_contracts` at cold start as `prices_writer`. +When it cannot, it does not fail Init (PC3, reversed in review): it ingests with +no SAC candidates, so an unproven SAC mints a `Contract` identity, and every run +publishes `SacCandidatesUnavailable`. Deploy with the read working. + +```bash +chq <<'SQL' +SELECT user_name, role_name, access_type, database, table FROM system.grants +WHERE (user_name = 'prices_writer' + OR role_name IN (SELECT granted_role_name FROM system.role_grants WHERE user_name = 'prices_writer')) + AND access_type IN ('SELECT', 'ALL') + AND (database = 'default' OR database IS NULL) + AND (table = 'soroban_contracts' OR table IS NULL) +SQL +``` + +At least one row (on 2026-10-05: `prices_writer SELECT default soroban_contracts`). +Then the read itself, as that identity (fishuser-hero, the orchestrator's writer +bundle): + +```bash +curl -sS --fail-with-body --cert ~/prices-mtls/prices_writer.crt --key ~/prices-mtls/prices_writer.key \ + --cacert ~/prices-mtls/ca.crt https://ch.sorobanscan.com/ \ + --data-binary "SELECT count() FROM (SELECT DISTINCT contract_id FROM default.soroban_contracts WHERE is_sac)" +``` + +About 4,042 (M13b). An error or `0`: **do not deploy Compute**; fix the grant +first. The deploy would not stop ingest, but it would run without candidates. + +**Record what runs now** (AWS shell), for the rollback: + +```bash +for f in ledger-processor api-handler rollup-freshness-probe; do + aws lambda get-function-configuration --function-name "prices-production-$f" \ + --query '[FunctionName,CodeSha256,LastModified]' --output text +done | tee ~/heal-0242/deployed-before.tsv +``` + +**Deploy** (AWS shell, `cd infra`): + +| Order | Target | Ships | +| ----- | ------------------------------------ | ---------------------------------------------------------------------------------------- | +| 1 | `make deploy-production-compute` | the live ledger processor (proof pre-pass, skip, `SacUnprovenSkipped`) and the API alias | +| 2 | `make deploy-production-eventbridge` | `rollup-freshness-probe` with `SacContractIdentities` | + +**Observability waits for §6 (D8).** Deployed now, `sac-contract-identities` +would sit in ALARM, red on the dashboard's Row-0 alarm strip, through stage C, +stage D and §5. That strip is the SCF evidence of task 0294 (all alarms OK). +Its two alarms come in §6 instead. Every Observability deploy from a commit +that holds this PR creates them, so tell the team: no +`make deploy-production-observability` from `develop` until §6. A deploy that +cannot wait shows the red tile; its description says it is expected. + +Until §6 no alarm watches `SacUnprovenSkipped` or `SacCandidatesUnavailable`. +Read both by hand after the cold start and at least daily until §6: + +```bash +for M in SacUnprovenSkipped SacCandidatesUnavailable; do + echo "== $M" + aws cloudwatch get-metric-statistics --namespace Prices/Ingest --metric-name "$M" \ + --dimensions Name=Environment,Value=production \ + --start-time "$(date -u -d '-1 day' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" \ + --period 3600 --statistics Sum +done +``` + +No datapoints is healthy: both are published only when something is wrong. +Any datapoint: §9. + +**Post-checks.** + +- The live Lambda cold-started on the new build: `/aws/lambda/prices-production-ledger-processor` + shows `loaded the is_sac contract set` with `sac_contracts` ≈ 4,042, then + `resumed cursor from ClickHouse`. No Init error, and no WARN + `is_sac contract set unreadable`. +- The probe: `aws lambda invoke --function-name prices-production-rollup-freshness-probe ~/heal-0242/probe.json`, + then `jq '.asset_id_uniqueness | {sac_contract_rows, sac_contract_scanned, be_sac_contracts}' ~/heal-0242/probe.json`. + `sac_contract_rows` = 36 (the baseline until §5), `be_sac_contracts` ≈ 4,042. + If the file holds an `errorMessage`, read the metric instead: + `aws cloudwatch get-metric-statistics --namespace Prices/Rollup --metric-name SacContractIdentities --dimensions Name=Environment,Value=production --start-time "$(date -u -d '-30 min' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" --period 900 --statistics Maximum`. +- `curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XLM_SAC" | jq '{asset, code}'` → `"native"`, `"XLM"`. + +**`events-backfill` on ch-prod-01.** Build it from the merge commit as in 0286 +§10a, keep the current one as `~/events-backfill.pre0242` (never overwrite it), +and swap only between phase-3 stages, never while a month is at its `amm` step +(D7). Confirm with the one-ledger dry run of §4b. + +**Rollback.** Redeploy the previous commit with the same two `make` targets (the +CodeSha256 in `deployed-before.tsv` is what must come back). On the host, +`cp -p ~/events-backfill.pre0242 ~/events-backfill`. Nothing in the PR writes a +schema change. + +--- + +## 5. Residual cleanup (after stage D and the live cold start) + +What phase 3 does not rewrite stays under the SAC ids: 202404 (stage B ran before +the seed, so Z/Q was re-minted under its SAC ids, U2), any month after 202609, +and the 1w/1M rows `finish` rolled from those. Then 72 metadata rows: 36 in +`assets`, 36 in `asset_symbol`. + +**Precondition.** `reingest_0286.py finish` done (stage D); §4e post-checks green +(the live slice has cold-started). + +### 5a. Capture the SAC ids + +The probe's predicate, with each contract's classic beside it: + +```bash +chq <<'SQL' +CREATE TABLE prices.bak_0242_sac_ids +ENGINE = MergeTree ORDER BY contract_address AS +SELECT c.contract_address AS contract_address, + c.asset_id AS asset_id, + cl.asset_id AS classic_id, + cl.asset_code AS classic_code, + cl.issuer_address AS classic_issuer, + now() AS captured_at +FROM (SELECT contract_address, asset_id FROM prices.assets FINAL + WHERE contract_address != '' + AND (contract_address IN (SELECT sac_address FROM prices.assets WHERE sac_address != '') + OR contract_address IN (SELECT contract_id FROM default.soroban_contracts WHERE is_sac))) AS c +LEFT JOIN (SELECT sac_address, asset_id, asset_code, issuer_address FROM prices.assets FINAL + WHERE sac_address != '') AS cl ON cl.sac_address = c.contract_address; +SELECT count(), countIf(classic_id = 0) FROM prices.bak_0242_sac_ids; +SELECT count() AS identities, uniqExact(asset_id) AS ids, identities - ids AS collisions FROM prices.assets FINAL +SQL +echo "SELECT * FROM prices.bak_0242_sac_ids ORDER BY classic_code FORMAT TSVWithNames" | chq > ~/heal-0242/sac_ids.tsv +``` + +**Check.** 36 rows, 0 without a classic (the seed gave group B theirs). Group A +is the 13 codes of §0. More than 36: the probe has seen new mints. They are in +the capture and go through the same gates; name them on the task. A row with +`classic_id = 0` has no classic to re-key to: every candle under it is an only +copy (5b). Record `identities`, `ids` and `collisions` as the AC4 "before". + +### 5b. Count the residual, and gate it + +**The window after the last phase-3 month.** Measured 2026-10-05: 0 rows since +2026-10-01 on any tier keyed on the 36 SAC ids. The window closes only once the +seed is in AND live has cold-started since, or the 0242 build is deployed; until +then live can still write under a SAC id. Re-count it now, for every month after +202609 (`1w`/`1M` roll from `1d`, so `1d` decides): + +```bash +for T in 1m 15m 1h 4h 1d; do + chq <= '2026-10-01 00:00:00' + AND (asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids) + OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids)) +GROUP BY month ORDER BY month +SQL +done | tee ~/heal-0242/window.tsv +``` + +- Empty → go on. +- Any row → it is one of the residual rows below, and the PC9 gate decides it + like any other. A month after 202609 can only be re-run once it is complete + (`plan` refuses the current month). + +**Every residual row is an only copy until proven otherwise.** A residual row +exists only where phase 3 did not rebuild its bucket under the classic id after +the seed, so it may hold trades that no classic row holds. A trade count cannot +tell: a classic row with more trades can hold different ones (for example a SAC +hour written before the live cold start beside a classic hour written after it), +and by §5 the `1m`/`15m` retention has removed the finer rows that would show it. + +The only proof that releases a row is a **byte-level duplicate**: a classic row +at the same pair after re-keying (both orientations, because Z/Q and XLD/SODA +flip), the same source and bucket, the same `trade_count` and the same volume on +each leg. That is the double count of group A (145 of 151 SAC-keyed `1m` rows on +2026-10-05). Every other row is an _only copy_, whatever the classic row holds. +Per tier and partition: + +```bash +for T in 1m 15m 1h 4h 1d 1w 1M; do + chq < t.1, sc) AS s, arrayMap(t -> t.2, sc) AS c +SELECT '$T' AS tier, x.p AS partition, count() AS residual_rows, + countIf(y.hit) AS byte_identical, residual_rows - byte_identical AS only_copy, + sumIf(x.trade_count, NOT y.hit) AS only_copy_trades, + groupUniqArrayIf(x.sac, NOT y.hit) AS only_copy_sacs +FROM (SELECT _partition_id AS p, if(has(s, asset_id), asset_id, quote_asset_id) AS sac, + transform(asset_id, s, c, asset_id) AS a, transform(quote_asset_id, s, c, quote_asset_id) AS q, + least(a, q) AS lo, greatest(a, q) AS hi, source, timestamp, trade_count, + if(a = lo, volume_base, volume_quote) AS vol_lo, if(a = lo, volume_quote, volume_base) AS vol_hi + FROM prices.price_ohlcv_$T FINAL + WHERE has(s, asset_id) OR has(s, quote_asset_id)) AS x +ANY LEFT JOIN (SELECT least(asset_id, quote_asset_id) AS lo, greatest(asset_id, quote_asset_id) AS hi, + source, timestamp, trade_count, + if(asset_id = lo, volume_base, volume_quote) AS vol_lo, + if(asset_id = lo, volume_quote, volume_base) AS vol_hi, true AS hit + FROM prices.price_ohlcv_$T FINAL + WHERE has(c, asset_id) OR has(c, quote_asset_id)) AS y + USING (lo, hi, source, timestamp, trade_count, vol_lo, vol_hi) +GROUP BY partition ORDER BY partition +SQL +done | tee ~/heal-0242/residual.tsv +``` + +`s` and `c` come from one read of pairs, so `s[i]` and `c[i]` are always the same +capture row. Two separate `groupArray` reads need not return the same order once +`bak_0242_sac_ids` has more than one part, and would re-key a SAC to the wrong +classic. + +`vol_lo`/`vol_hi` are each leg's volume, whichever side it is stored on, so a +flipped pair compares leg by leg. A row with `classic_id = 0` re-keys to `0`, +matches nothing and counts as an only copy. On a scratch fixture a SAC hour of 1 +trade beside a classic hour of 2 different trades reads `only_copy = 1`. + +`quote_asset_id` is not a key prefix, so this scans each tier: run it off-peak. + +**The PC9 gate. It STOPs §5 on any partition with `only_copy > 0`**, in any tier, +`1w` and `1M` included. A coarse bucket or a month phase 3 did not re-ingest after +the seed is no exception: only the byte-level match above releases a row. An +only-copy row is never deleted silently. For each such partition Adam picks one +of the two, and it is recorded on task 0242 before 5c: + +- **Re-run the month through phase 3.** In a fresh state dir, as the 0139 second + pass does (0286 §10d), so a finished month can run again: + + ```bash + mkdir -p ~/reingest-0286-0242 && cp -r ~/reingest-0286/ledger-cache ~/reingest-0286-0242/ + python3 tools/scripts/reingest_0286.py plan --state-dir ~/reingest-0286-0242 --from-month --to-month + python3 tools/scripts/reingest_0286.py run --state-dir ~/reingest-0286-0242 --from-month --to-month --yes + ``` + + That rewrites `1m` and the four fine tiers. The month's `1w`/`1M` rows are only + rebuilt by `finish`, which TRUNCATEs both tables whole (hours, then the + whole-history re-enrichment). Until it runs again they still read as only + copies: run `finish` again, or record them as below. Then repeat 5b; the gate + passes only when every partition reads `only_copy = 0` or carries a recorded + loss. + +- **Record the loss** on task 0242, with Adam's ack: tier, partition, + `only_copy_trades`, `only_copy_sacs`. For `1w`/`1M` rows of a re-run month, say + that the classic's `1w`/`1M` lack those trades until the next `finish`. + +Either way the rows are in the 5c copy before 5d deletes them. **Expected here:** +202404's Z/Q rows, in every tier stage B wrote and in the `1w`/`1M` rows `finish` +rolled from them (U2). + +### 5c. Rollback copies, before any delete + +Every residual candle row and the 72 metadata rows. The column lists come from +`system.columns` without MATERIALIZED, ALIAS and EPHEMERAL columns: +`assets.asset_id` is MATERIALIZED (re-derived on re-insert), and the candles' +`base_*`/`quote_*` identity columns are EPHEMERAL (their ids are stored). + +**Pre-check.** `echo "SHOW TABLES FROM prices LIKE 'bak_0242_%'" | chq` lists only +`bak_0242_sac_ids`, `bak_0242_seed` and `bak_0242_seed_stage`. `CREATE TABLE` below has no `IF NOT EXISTS`, +so a second run fails instead of overwriting the first copy. + +```bash +chq <<'SQL' > ~/heal-0242/bak.sql +SELECT concat( + 'CREATE TABLE prices.bak_0242_', table, ' AS prices.', table, ';\n', + 'INSERT INTO prices.bak_0242_', table, ' (', cols, ') SELECT ', cols, ' FROM prices.', table, ' FINAL WHERE ', + if(table IN ('assets', 'asset_symbol'), + 'contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)', + 'asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids) OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids)'), + ';') +FROM (SELECT table, arrayStringConcat(groupArray(name), ', ') AS cols + FROM system.columns + WHERE database = 'prices' + AND table IN ('assets', 'asset_symbol', 'price_ohlcv_1m', 'price_ohlcv_15m', 'price_ohlcv_1h', + 'price_ohlcv_4h', 'price_ohlcv_1d', 'price_ohlcv_1w', 'price_ohlcv_1M') + AND default_kind NOT IN ('MATERIALIZED', 'ALIAS', 'EPHEMERAL') + GROUP BY table) +ORDER BY table +FORMAT TSVRaw +SQL +cat ~/heal-0242/bak.sql && chq < ~/heal-0242/bak.sql +``` + +This creates `prices.bak_0242_assets`, `prices.bak_0242_asset_symbol` and +`prices.bak_0242_price_ohlcv_` for the seven tiers. + +**Post-check.** + +```bash +for t in assets asset_symbol price_ohlcv_1m price_ohlcv_15m price_ohlcv_1h price_ohlcv_4h \ + price_ohlcv_1d price_ohlcv_1w price_ohlcv_1M; do + echo "SELECT '$t', count() FROM prices.bak_0242_$t" | chq +done | tee ~/heal-0242/bak-counts.tsv +``` + +`assets` and `asset_symbol` equal the 5a count (36). Each tier equals the sum of +its `residual_rows` in `residual.tsv`. + +### 5d. Delete the residual candles, fine tiers first + +One lightweight DELETE per tier and partition, over the partitions 5c copied. +Fine before coarse, so a re-roll of a coarse tier between two statements reads an +already cleaned finer tier. + +```bash +for T in 1m 15m 1h 4h 1d 1w 1M; do + for P in $(echo "SELECT DISTINCT _partition_id FROM prices.bak_0242_price_ohlcv_$T ORDER BY 1 FORMAT TSVRaw" | chq); do + echo "DELETE FROM prices.price_ohlcv_$T IN PARTITION ID '$P' WHERE asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids) OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids);" + done +done | tee ~/heal-0242/delete-candles.sql +cat ~/heal-0242/delete-candles.sql && chq < ~/heal-0242/delete-candles.sql +``` + +**Post-check (M4).** Every line `0`: + +```bash +for T in 1m 15m 1h 4h 1d 1w 1M; do + echo "SELECT '$T', count() FROM prices.price_ohlcv_$T FINAL WHERE asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids) OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids)" | chq +done +``` + +### 5e. Delete the 36 `assets` rows + +**Pre-check.** `echo "SELECT count() FROM prices.assets FINAL WHERE contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)" | chq` +equals the 5a count. + +```bash +echo "DELETE FROM prices.assets WHERE contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)" | chq +``` + +**Post-check.** The same count reads `0`. + +### 5f. Delete the 36 `asset_symbol` rows + +After `assets`, because the symbol queue is driven by the `assets` contract rows +(`asset-discovery` `symbols.rs`): deleted first, they would be fetched again. + +**Pre-check.** `echo "SELECT count() FROM prices.asset_symbol FINAL WHERE contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)" | chq` +equals the 5a count. + +```bash +echo "DELETE FROM prices.asset_symbol WHERE contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)" | chq +``` + +**Post-check.** The same count reads `0`. + +### 5g. Post-checks + +```bash +chq <<'SQL' +SELECT count() FROM prices.assets FINAL +WHERE contract_address != '' + AND (contract_address IN (SELECT sac_address FROM prices.assets WHERE sac_address != '') + OR contract_address IN (SELECT contract_id FROM default.soroban_contracts WHERE is_sac)); +SELECT count() FROM prices.assets FINAL WHERE contract_address != ''; +SELECT uniqExact(contract), countIf(asset_kind = 'contract'), max(n) +FROM (SELECT contract, any(asset_kind) AS asset_kind, count() AS n FROM prices.identity_by_contract + WHERE contract IN (SELECT contract_address FROM prices.bak_0242_sac_ids) GROUP BY contract); +SELECT count() AS identities, uniqExact(asset_id) AS ids, identities - ids AS collisions FROM prices.assets FINAL +SQL +``` + +| Check | Expected | +| ----------------------------------- | ------------------------------------------------------------------------------ | +| SAC contract rows (M1) | `0` | +| Contract rows left | the genuine tokens only: 23 on 2026-10-05, plus any minted since | +| `identity_by_contract` | the 5a count, `0` contract rows, `max(n)` = `1`: one classic row per SAC | +| identities / ids / collisions (AC4) | identities and ids each down by exactly the 5a count; collisions equal to 5a's | +| SAC-keyed candles (M4) | `0` in every tier (5d post-check) | + +The probe and the alarm metric: + +```bash +aws lambda invoke --function-name prices-production-rollup-freshness-probe ~/heal-0242/probe.json +jq '.asset_id_uniqueness | {sac_contract_rows, collisions: (.identities - .ids)}' ~/heal-0242/probe.json +``` + +`sac_contract_rows` = `0`, and `SacContractIdentities` reads `0` in CloudWatch +(§4e's `get-metric-statistics`). `AssetIdCollisions` unchanged (AC4). + +The API (D5, AC2, AC3): + +```bash +curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR_SAC" | jq '{asset, asset_kind, contract}' +curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR_SAC/price" | jq -S '{asset, price_usd, volume_24h_usd}' +curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR/price" | jq -S '{asset, price_usd, volume_24h_usd}' +W="granularity=1d&start=2026-09-01T00:00:00Z&end=2026-10-01T00:00:00Z" +diff <(curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR_SAC/ohlcv?$W" | jq -S .data) \ + <(curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR/ohlcv?$W" | jq -S .data) && echo SAME +curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XLM_SAC" | jq '{asset, asset_kind, code}' +``` + +- XCR by its SAC: `asset` = `XCR:GBLJ…`, `asset_kind` = `credit`, `contract` = `""`. +- Both `/price` calls print the same object: one asset, one volume. `SAME`. +- The XLM SAC answers 200 as XLM: `asset` = `native`, `code` = `XLM`. A 404 + means prod's XLM row lost its `sac_address`: read it first + (`SELECT sac_address FROM prices.assets FINAL WHERE asset_code = 'XLM' AND issuer_address = ''`). + On 2026-10-05 it held `CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA`, + and no classic row lacked one. + +### 5h. Rollback + +Re-insert from the copies with the same column lists, metadata first: + +```bash +chq <<'SQL' > ~/heal-0242/restore.sql +SELECT concat('INSERT INTO prices.', table, ' (', cols, ') SELECT ', cols, ' FROM prices.bak_0242_', table, ';') +FROM (SELECT table, arrayStringConcat(groupArray(name), ', ') AS cols + FROM system.columns + WHERE database = 'prices' + AND table IN ('assets', 'asset_symbol', 'price_ohlcv_1m', 'price_ohlcv_15m', 'price_ohlcv_1h', + 'price_ohlcv_4h', 'price_ohlcv_1d', 'price_ohlcv_1w', 'price_ohlcv_1M') + AND default_kind NOT IN ('MATERIALIZED', 'ALIAS', 'EPHEMERAL') + GROUP BY table) +ORDER BY table IN ('assets', 'asset_symbol') DESC, table +FORMAT TSVRaw +SQL +cat ~/heal-0242/restore.sql && chq < ~/heal-0242/restore.sql +``` + +Check: §5a's capture predicate counts the 5a count again, and each tier's +SAC-keyed count equals `bak-counts.tsv`. `asset_id` re-derives to the same value +from the same identity. Keep the `bak_0242_*` tables until 0242 is closed; then +`DROP TABLE prices.bak_0242_ SYNC`, one by one. + +--- + +## 6. Deploy Observability with the alarm on (D6, D8) + +Observability was held back in §4e (D8), so this deploy creates the three 0242 +alarms: `prices-production-ledger-processor-sac-unproven`, +`prices-production-ledger-processor-sac-candidates-unavailable` and +`prices-production-sac-contract-identities`, the last with its actions on. + +**Pre-check.** §5g is green and `SacContractIdentities` reads `0` for at least +two periods (30 min): + +```bash +aws cloudwatch get-metric-statistics --namespace Prices/Rollup --metric-name SacContractIdentities \ + --dimensions Name=Environment,Value=production \ + --start-time "$(date -u -d '-45 min' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" \ + --period 900 --statistics Maximum +``` + +Every `Maximum` is `0`. Created with its actions on while the metric read 36, +the alarm would fire at once. + +**Run.** Set `opsAlarms.sacContractIdentitiesActionsEnabled` to `true` in +`infra/envs/production.json` through a PR; once merged: + +```bash +cd infra && make deploy-production-observability +``` + +**Post-check.** + +```bash +aws cloudwatch describe-alarms \ + --alarm-names prices-production-sac-contract-identities prices-production-ledger-processor-sac-unproven \ + prices-production-ledger-processor-sac-candidates-unavailable \ + --query 'MetricAlarms[].[AlarmName,ActionsEnabled,StateValue]' --output text +``` + +All three print `True`, then `OK` (`INSUFFICIENT_DATA` until the first +evaluation). The `INSUFFICIENT_DATA` → `OK` transitions send OK notifications: +expected. + +**Rollback.** The same key back to `false` and the same deploy turns the +actions off. Redeploying Observability from a commit without this PR removes +all three alarms. + +--- + +## 7. Acceptance evidence + +| AC | Evidence | +| -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| AC1 a SAC no longer mints a second identity, both orders | `packages/prices-ledger-processor/tests/sac_identity.rs` (T2 SAC first, T3 both orders, T4 the SAC event after the trade), through the real `process_ledger`; `events-backfill` `a_swap_on_an_unproven_sac_is_skipped_and_counted_and_a_proven_one_prices` | +| AC2 the existing pairs are resolved | §5g: 0 SAC contract rows, one `identity_by_contract` row per SAC, 0 SAC-keyed candles. Group A by phase 3 plus 5d; group B by the seed plus phase 3 | +| AC3 one asset, one volume (XCR) | §5g: XCR by SAC and by `CODE:ISSUER` print the same price, volume and candles. For BLTA/BLTB/BLTC/PPRIME/LumenJoule, §4b's U3 post-check: their volume under the classic ids; BLTA/BLTB/BLTC carry no price by design (dust) | +| AC4 no new duplicate `asset_id` | §3 query 3 and §5a/§5g: `count() - uniqExact(asset_id)` unchanged at every step; `AssetIdCollisions` unchanged | + +Paste the outputs of §3, §5a, §5b, §5g and §6 into task 0242, then tell BE once +the heal lands: they read `current_price_usd` and `identity_by_contract`, and +each SAC now resolves to one classic row. + +--- + +## 8. Open items + +- **U3, settled 2026-10-05** (`$Q/U3-root-cause.md`). The 75,580 Soroswap swaps + on BLTA, BLTB, BLTC, PPRIME and LumenJoule had candles until 2026-10-02. The + task-0139 re-key left them in `price_ohlcv_*__pre0139`: each of the five SAC + contract identities shared its old id with an unrelated classic asset, so all + five were `colliding`, and `fill` copies only `mapped`/`sentinel` ids. It is a + sub-case of 0139's hole (colliding ids have no candles 2024-02..2026-09), not + an ingest or dust-rule bug, and needs no code. Stage C after the seed rebuilds + them from BE's events under the classic ids; §4b's post-check proves it, and + BLTA/BLTB/BLTC come back with volume and no price (dust). Without the seed + stage C would rebuild them under the SAC ids, which is why §4b requires it. +- **U5.** USDT0's 17:26 SAC trade came after its classic's 17:00-hour SDEX + candle: consistent with the warm-container race of 0139, not traced to a + container. +- **Known residual risk.** The live processor reads BE's `is_sac` set once per + cold start. A SAC deployed after it, traded with no SAC event in the same + transaction, is not a candidate yet and still mints a `Contract` identity; so + does a SAC BE does not flag, and so does every unproven SAC of a container + whose cold start could not read the set (`SacCandidatesUnavailable`). The + in-band proof missed 0 of 338,601 sampled legs, so this should stay rare. + `SacContractIdentities` counts it, §9. + +--- + +## 9. When an alarm fires + +**`prices-production-ledger-processor-sac-unproven`** (live, actions on). The +live processor skipped trades on a contract BE flags `is_sac` that no SAC event +in the same transaction proved. Those trades have no candle. The WARN +`skipped trades on is_sac contracts with no SAC proof (task 0242)` in +`/aws/lambda/prices-production-ledger-processor` names the contracts. Check each +as in §4c step 2. A real SAC: seed its classic as §3 did; the next cold start +resolves it. The skipped minutes stay without those trades until a re-ingest of +their month. + +**`prices-production-ledger-processor-sac-candidates-unavailable`** (live, +after §6). A container cold-started without BE's `is_sac` set. The WARN +`is_sac contract set unreadable` carries the error: usually the grant of §4e's +pre-check, a renamed table or a BE migration. Ingest is running. Fix the read, +then force a cold start (any configuration change of +`prices-production-ledger-processor`, or wait for the container to recycle). +Any SAC minted meanwhile shows in `SacContractIdentities`: treat it as below. + +**`prices-production-sac-contract-identities`** (after §6). A contract row of +`assets` is a SAC again. The alarm's description holds the query that names it. +Treat it as a one-row §5: capture, gate its candles, copy, delete. Find which +path minted it first (a pre-0242 binary somewhere, or a SAC BE does not flag). diff --git a/docs/runbooks/0286-candle-definitions-rollout.md b/docs/runbooks/0286-candle-definitions-rollout.md index 74094435..829ad597 100644 --- a/docs/runbooks/0286-candle-definitions-rollout.md +++ b/docs/runbooks/0286-candle-definitions-rollout.md @@ -40,7 +40,7 @@ tmux new -s rollout0286 cd ~/stellar-prices-api && git checkout develop && git pull chadmin() { curl -sS --cert ~/prices-mtls/prices-admin-production.crt \ --key ~/prices-mtls/prices-admin-production.key --cacert ~/prices-mtls/ca.crt \ - "https://ch.sorobanscan.rumblefish.dev/" --data-binary "$1"; } + "https://ch.sorobanscan.com/" --data-binary "$1"; } chadmin "SELECT currentUser(), version() FORMAT TSVWithNames" ``` @@ -437,7 +437,7 @@ statement rather than editing the SQL, which is rendered from ```bash curl -sS --cert ~/prices-mtls/prices-admin-production.crt \ --key ~/prices-mtls/prices-admin-production.key --cacert ~/prices-mtls/ca.crt \ - "https://ch.sorobanscan.rumblefish.dev/?max_partitions_per_insert_block=1000" \ + "https://ch.sorobanscan.com/?max_partitions_per_insert_block=1000" \ --data-binary "$(sed -n '/^INSERT INTO prices.price_ohlcv_1M/,$p' \ packages/prices-clickhouse/schema/preroll.sql)" ``` diff --git a/docs/runbooks/0286-reingest-history.md b/docs/runbooks/0286-reingest-history.md index e7757d01..636029a0 100644 --- a/docs/runbooks/0286-reingest-history.md +++ b/docs/runbooks/0286-reingest-history.md @@ -160,6 +160,10 @@ Four of those steps exist because of one fact and nothing else: wrapper exists to prevent. Watch `restarts.log` and answer the prompt, or run those stages without the wrapper. + Before stage C: the task-0242 seed, then `plan` and `run` with + `--to-month 202609` ([`0242-sac-identity-heal.md`](0242-sac-identity-heal.md) + §1, §3, §4). + - `events-backfill` reads `default.*` AND writes `prices.*`, so it runs **on the CH host as the `default` user** against `localhost:8123` — the prices mTLS user cannot read `default.*` @@ -255,13 +259,26 @@ the DROP and keep the output: SELECT source, count() AS candles, sum(trade_count) AS trades, - sum(volume_base) AS volume_base, - sum(volume_quote) AS volume_quote + sumIf(volume_base, + asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '') + AND quote_asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '')) + AS volume_base, + sumIf(volume_quote, + asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '') + AND quote_asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '')) + AS volume_quote FROM prices.price_ohlcv_1m FINAL WHERE toYYYYMM(timestamp) = GROUP BY source ORDER BY source FORMAT TabSeparated; ``` +Volumes leave out every candle that touches a pure Soroban token (task 0329). +The ingest now scales such a token's legs by its own `decimals()`, so its +volume moves by design: an 18-decimal token's base volume falls by 10^11, a +6-decimal one's rises tenfold. Trades are still counted on every candle. +`reingest_0286.py` runs exactly this, and a month whose "before" was read +without it gets volumes left uncompared, as a FINDING. + (If you later need a row-level diff rather than these sums, attach the frozen parts to a scratch table with the `ATTACH PARTITION … FROM '/var/lib/clickhouse/ shadow/…'` idiom in `repair-coarse-usd-values.md` → "Rollback".) @@ -300,7 +317,7 @@ Then, **one writer per source** (see the warning below): ```bash tmux new -s reingest -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt @@ -341,7 +358,9 @@ Re-run the step-4c query and compare, per source: | any source | `count()` may differ: a minute that traded only dust still exists, so candles are not lost — but a minute whose every fill was dropped upstream for a zero amount never existed in either shape. | Differences in `open`/`high`/`low`/`close` are the POINT of the exercise and are -not reconciled. Differences in volume are a defect. Record both. +not reconciled. Differences in volume (on the candles step 4c compares) are a +defect. Record both. So is a non-zero `trades dropped (decimals)` in §6: the +script reads it and marks the month DEFECT. ### 4g. DROP the overlapping coarse partitions and pre-roll the month @@ -447,7 +466,7 @@ month cannot be silently ordered the old way any more. What survives is the value check. `application_order` is `Int16`, and a negative value is not a position — `resolve_transaction_index` degrades it to 0 and counts -it. The run's last summary line reports it: +it. The run's summary reports it: ``` negative apply order: 0 @@ -457,6 +476,30 @@ negative apply order: 0 repaired** — do not record it as done. The run also emits one WARN, once, naming the first ledger and `application_order` it saw. +Since task 0242 the summary also prints `unproven sac swaps: N`: swaps skipped +because a leg is a contract BE flags `is_sac` and no proof resolved it. Anything +but `0` means the month is NOT complete, and the orchestrator STOPs on it after +either pass. After the `--dry-run` pass no AMM candle is written, but the month's +`1m` is already dropped and holds SDEX-only candles; after the write pass its AMM +candles lack those swaps. The STOP names the pass. A binary built before 0242 prints +no such line and is not checked. What to do: +[`0242-sac-identity-heal.md`](0242-sac-identity-heal.md) §4c, and §4d for +`--amm wait`, where `amm-done` does not read the line. + +Since task 0329 the summary ends with one more line: + +``` +trades dropped (decimals): 0 +``` + +A pure Soroban token's legs are scaled by its `decimals()`, read over Soroban +RPC the first time the run meets it and kept in `prices.asset_decimals`. +**Anything but `0` means a token did not resolve and its trades are NOT in the +month's candles.** The run WARNs naming the contract. Re-run the month once it +resolves. With `--amm wait`, pass the line to `amm-done … --decimals N`: the +script refuses an `amm.done` without it and marks a non-zero count DEFECT. The binary needs outbound HTTPS to `SOROBAN_RPC_URL` (default +`https://mainnet.sorobanrpc.com`), and `prices.asset_decimals` must exist. + --- ## 7. After the last month: 1w, 1M, and the re-enrichment @@ -477,8 +520,52 @@ source). The MV `mv_ohlcv_1d_to_1M` keeps them current afterwards. `close_usd = 0` and `volume_quote_usd = 0`, so the enrichment worker will re-price the entire estate — this is the cost the ledger estimate in precondition 8 does NOT include. For scale: task 0268's campaign re-priced 9.94 M candles in ~24 min; -0228 estimated ~190 M candles at ~4 h 40 m. This is all of them, and it runs on -the same worker under the same concurrency. +0228 estimated ~190 M candles at ~4 h 40 m. Those were one-shot runs. + +**Do not leave this to the hourly Lambda.** Its historical drain is bounded: +one month per run, at most `BATCH_SIZE × MAX_BATCHES` = 200 000 rows, and +`ENRICHMENT_ONE_SHOT` does not lift that (`frontier.rs` sets `one_shot = false` +for every historical month). On 2026-10-12 the re-ingest left ~900 M priceable +`1m` rows (2022-05 → 2026-08). At 200 k an hour that is ~190 days; the Lambda +works ~75 s of each hour. + +Drain `1m` from fishuser-hero with `coarse-repair`, which takes `price_ohlcv_1m` +in its fill-the-zeros mode only (no `--reset-*`, `--skip-snapshot` required). +It runs the same pass as the Lambda, one month at a time to the end, and writes +only rows still at zero, so the Lambda can keep running beside it. Same `tmux` +and env as §7b-2 ①: + +```bash +for M in $(seq 202205 202212) $(seq 202301 202312) $(seq 202401 202412) \ + $(seq 202501 202512) $(seq 202601 "$END"); do + ./target/release/coarse-repair --transport hetzner --table price_ohlcv_1m \ + --start-month "$M" --end-month "$M" --skip-snapshot --batch-size 50000 \ + || { echo "STOPPED at $M"; break; } +done 2>&1 | tee -a /tmp/0286_drain_1m.log +``` + +One month per command keeps each month's enumeration to a single partition, and +a re-run skips what is already priced. Each batch scans the whole partition, so +a bigger batch costs little more than a small one: measured on 202205, a 10 k +batch took ~3.8 s, which models the range at ~2 days with 50 k and ~6–8 days +with the default 10 k. Time the first month and re-plan from it. + +Memory follows the rows a statement reads, not the batch: ~70 MB per million +rows read, ~2 GiB on 202205 (2026-10-12, `system.query_log`). The 2022-12 … +2023-05 partitions are 2.5× that, and each priced row adds a version that +`FINAL` reads until the merge, so they can reach the writer's ~7.5 GiB cap. On +`Code: 241` (`MEMORY_LIMIT_EXCEEDED`), collapse the month's versions as +`prices_admin`, then re-run the loop from that month: + +```bash +chadmin "OPTIMIZE TABLE prices.price_ohlcv_1m PARTITION $M FINAL" +``` + +`--batch-size 20000` trims only the written part of the statement. + +The cluster is shared with the block explorer, and the loop keeps it busy +around the clock. Check the explorer's latency while it runs. `Ctrl-C` is a +safe pause at any point. **7b-2. Re-price the coarse tiers. The drain above never reaches them.** The enrichment worker prices `price_ohlcv_1m` and nothing else @@ -497,8 +584,11 @@ over the whole history on all six tables. It does not read `1m`. It prices each coarse row from that table's own reference candles, with the same tiers as the worker, including the measured USDC rate on the pivot. -- **Start only when the `1m` drain is finished.** Both passes load the same - shared cluster, and 7c/7e measure the finished state. **The frontier gauge +- **Do not run it at the same time as the `1m` drain.** It does not read `1m`, + so it may go first, and on a re-ingest it should: clients read the coarse + tiers. But both passes load the same shared cluster, and 7c/7e measure the + finished state, so run them one after the other and measure after both. + Before 7c/7e, confirm the `1m` drain is finished. **The frontier gauge alone cannot tell you this.** `EnrichmentFrontierMonthsPending` counts only months that are NOT marked `exhausted` in `prices.enrichment_frontier` (`frontier.rs`, `months_to_sweep`). Almost every historical month was marked @@ -599,10 +689,10 @@ cargo build --release -p enrichment-worker --features aws-mtls --bin coarse-repa chadmin() { curl -sS --fail-with-body --cacert ~/prices-mtls/ca.crt \ --cert ~/prices-mtls/prices-admin-production.crt \ --key ~/prices-mtls/prices-admin-production.key \ - "https://ch.sorobanscan.rumblefish.dev/" --data-binary "$1"; } + "https://ch.sorobanscan.com/" --data-binary "$1"; } chadmin "SELECT currentUser() FORMAT TSV" # must print prices_admin -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt diff --git a/docs/runbooks/backfill-handoff-covering-operator.md b/docs/runbooks/backfill-handoff-covering-operator.md index 22d707c5..88f6586f 100644 --- a/docs/runbooks/backfill-handoff-covering-operator.md +++ b/docs/runbooks/backfill-handoff-covering-operator.md @@ -440,7 +440,7 @@ session: ```bash # certs + CH endpoint (the write path) -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt diff --git a/docs/runbooks/continue-soroban-backfill.md b/docs/runbooks/continue-soroban-backfill.md index 0576c308..ab9a5c28 100644 --- a/docs/runbooks/continue-soroban-backfill.md +++ b/docs/runbooks/continue-soroban-backfill.md @@ -185,7 +185,7 @@ the run — see §11 on why finishing matters): ```bash tmux new -s backfill # detach later with Ctrl-b then d; reattach: tmux attach -t backfill -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt @@ -431,7 +431,7 @@ let it run to `=== sdex-backfill complete ===`. | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------- | | Ledger bucket | `s3://aws-public-blockchain/v1.1/stellar/ledgers/pubnet/` — region **us-east-2**, anonymous (`--no-sign-request`) | | Writer secret | `prices/production/clickhouse-mtls-prices-ingestion-production` (region `eu-central-1`, acct `750702271865`) | -| CH domain | `ch.sorobanscan.rumblefish.dev` | +| CH domain | `ch.sorobanscan.com` | | Prod CH shell | `ssh -i ~/.ssh/sorban-prod_ed25519 deploy@168.119.73.161` → `docker exec -i app-clickhouse-1 clickhouse-client --query='…'` | | Activation ledger | `50457424` (split between `combined` and `sdex-only`) | | Combined floor (`--end` max) | `63352611` (live starts at `63352612`) | diff --git a/docs/runbooks/fix-backfill-history-loss-and-rerun.md b/docs/runbooks/fix-backfill-history-loss-and-rerun.md index 563deec2..07236e67 100644 --- a/docs/runbooks/fix-backfill-history-loss-and-rerun.md +++ b/docs/runbooks/fix-backfill-history-loss-and-rerun.md @@ -192,7 +192,7 @@ cat > ~/run-full-backfill.sh <<'SCRIPT' #!/usr/bin/env bash set -uo pipefail -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt @@ -312,7 +312,7 @@ Set `START = min(current_ledger over the combined streams) + 1` (≈ **62,592,00 ### 6b.2 Run the bounded gap-fill (tmux, same certs as 6.1) ```bash -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt diff --git a/docs/runbooks/portal-oauth-deploy-prep.md b/docs/runbooks/portal-oauth-deploy-prep.md index f4b7f3e4..956a59f2 100644 --- a/docs/runbooks/portal-oauth-deploy-prep.md +++ b/docs/runbooks/portal-oauth-deploy-prep.md @@ -51,9 +51,10 @@ for this. - `export AWS_PROFILE=soroban-admin`, `export AWS_REGION=eu-central-1`. - The two hostnames, both from `infra/envs/production.json` since [0194] (2026-08-31): the **page** is served at `portalWebOrigin` - (`https://sorobanscan.rumblefish.dev`, the block explorer's distribution) - and the **backend** — where the callback lands and the session cookie is - set — at `apiDomain.domainName` (`https://prices-api.sorobanscan.rumblefish.dev`). + (`https://sorobanscan.com`, the block explorer's distribution) and the + **backend** — where the callback lands and the session cookie is set — at + `apiBaseUrl` (`https://prices-api.sorobanscan.com`, the + `sorobanscanApiDomain` since task 0340). The redirect URI below names the API host. (Our own CloudFront distribution and its `/prices/production/portal-distribution-domain` parameter were retired by [0195]; nothing reads that parameter any more.) @@ -74,12 +75,13 @@ In the [Discord Developer Portal](https://discord.com/developers/applications): `` is **the host the backend answers on**, not the host the page is served from — since task 0194 those are two hostnames: - `prices-api.sorobanscan.rumblefish.dev` runs the callback and sets the - session cookie, `sorobanscan.rumblefish.dev/prices-api/` is the page it sends the - visitor back to (`PORTAL_WEB_ORIGIN`). Register the former. In production: + `prices-api.sorobanscan.com` runs the callback and sets the + session cookie, `sorobanscan.com/api/prices-api/` is the page it + sends the visitor back to (`PORTAL_WEB_ORIGIN`). Register the former. In + production: ``` - https://prices-api.sorobanscan.rumblefish.dev/api/auth/callback + https://prices-api.sorobanscan.com/api/auth/callback ``` ⚠️ **Assume the match is character-exact.** Discord's documentation states @@ -297,12 +299,9 @@ finishing a slice. Before it happens, both must be true: which takes out `/v1` as well — the read is skipped entirely while the portal is closed, so this is a failure mode that only appears at the moment of opening). -- The gateway maps the portal prefix as a greedy `{proxy+}`. The four sign-in - routes are at **depth 3** (`auth/login`), and the deployed mapping at the time - of writing is the intermediate `{proxy}` + `{proxy}/{sub}` pair, which answers - `403 Missing Authentication Token` at that depth (`404 {"code": "not_found"}` - since task 0309). Task 0205 ships the committed shape. (Task 0187's `/key` is - at depth 3 as well, so it is the same deploy.) +- The gateway maps the portal prefix as a greedy `{proxy+}`, because the four + sign-in routes and task 0187's `/key` are at **depth 3** (`auth/login`). + Measured live on 2026-10-06 (task 0205). - **`ApiGatewayStack` has deployed at least once since task 0187 merged**, so that `/prices/production/pricing-api-free-plan-id` exists. See §7 — if it does not, opening the portal fails Lambda init and takes `/v1` down with it. @@ -342,10 +341,11 @@ from a scratch guild. ## 6. Moving the callback to another host — ordering -Done once already: on 2026-08-31 ([0194]) the callback moved from our -CloudFront distribution to `prices-api.sorobanscan.rumblefish.dev`, in the -order below, and the old distribution was retired afterwards ([0195]). Kept -as the procedure for the next move, whenever the API host changes again. +Done twice, in the order below: on 2026-08-31 ([0194]) the callback moved +from our CloudFront distribution to `prices-api.sorobanscan.rumblefish.dev`, +and the old distribution was retired afterwards ([0195]); on 2026-11-06 +(task 0340) it moved to `prices-api.sorobanscan.com`, and the old URI stays +registered beside it. Kept as the procedure for the next move. While a move is in progress the backend is reachable at **two** hostnames. The redirect URI is registered per-URL, and our stored `redirect_uri` is a diff --git a/docs/runbooks/repair-coarse-usd-values.md b/docs/runbooks/repair-coarse-usd-values.md index 936d7f95..06d34f1c 100644 --- a/docs/runbooks/repair-coarse-usd-values.md +++ b/docs/runbooks/repair-coarse-usd-values.md @@ -83,7 +83,7 @@ Binary lands at `./target/release/coarse-repair`. Point at the writer certs (same env as sdex-backfill): ```bash -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt diff --git a/docs/runbooks/running-ingestion-components.md b/docs/runbooks/running-ingestion-components.md index 0081977c..2f976fa9 100644 --- a/docs/runbooks/running-ingestion-components.md +++ b/docs/runbooks/running-ingestion-components.md @@ -45,16 +45,16 @@ seam section. ## Component summary -| Component | Kind | Writes | How it runs | Built? | -| ------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | ------------ | -| `sdex-backfill` | operator CLI | `price_ohlcv_1m` (`sdex` + AMM sources), `assets`, `oracle_prices`, `backfill_progress`, `pool_registry`, `unresolved_pools`, `backfill_sdex_ledgers` | manual, `--mode`/`--start`/`--end` | ✅ | -| `prices-ledger-processor` | Lambda / fixture CLI | `price_ohlcv_1m` (all sources), `oracle_prices`, `assets` | SQS doorbell (prod) or `--cursor` (local) | ✅ | -| `enrichment-worker` | Lambda / CLI | `price_ohlcv_1m` (`_usd` cols) | scheduled (prod) or CLI (local) | ✅ prototype | -| `asset-discovery` | scheduled Lambda | `assets`, `asset_symbol` | EventBridge `rate(1h)` | ✅ | -| `oracle-worker` | scheduled Lambda | `oracle_prices` | EventBridge schedule | ✅ | -| `supply-worker` | scheduled Lambda | `asset_supply` | EventBridge schedule | ✅ | -| `cleanup-worker` | scheduled Lambda | TTL/partition maintenance | EventBridge schedule | ✅ | -| `prices-api` | HTTP Lambda | reads only | API Gateway (prod) or `serve` bin (local) | ✅ | +| Component | Kind | Writes | How it runs | Built? | +| ------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | ------------ | +| `sdex-backfill` | operator CLI | `price_ohlcv_1m` (`sdex` + AMM sources), `assets`, `asset_decimals`, `oracle_prices`, `backfill_progress`, `pool_registry`, `unresolved_pools`, `backfill_sdex_ledgers` | manual, `--mode`/`--start`/`--end` | ✅ | +| `prices-ledger-processor` | Lambda / fixture CLI | `price_ohlcv_1m` (all sources), `oracle_prices`, `assets`, `asset_decimals` | SQS doorbell (prod) or `--cursor` (local) | ✅ | +| `enrichment-worker` | Lambda / CLI | `price_ohlcv_1m` (`_usd` cols) | scheduled (prod) or CLI (local) | ✅ prototype | +| `asset-discovery` | scheduled Lambda | `assets`, `asset_symbol` | EventBridge `rate(1h)` | ✅ | +| `oracle-worker` | scheduled Lambda | `oracle_prices` | EventBridge schedule | ✅ | +| `supply-worker` | scheduled Lambda | `asset_supply` | EventBridge schedule | ✅ | +| `cleanup-worker` | scheduled Lambda | TTL/partition maintenance | EventBridge schedule | ✅ | +| `prices-api` | HTTP Lambda | reads only | API Gateway (prod) or `serve` bin (local) | ✅ | --- @@ -192,7 +192,7 @@ no separate push step — so `/backfill/status` updates in real time. Needs the `aws-mtls` build and the operator's client bundle: ```bash -CH_DOMAIN=ch.sorobanscan.rumblefish.dev \ +CH_DOMAIN=ch.sorobanscan.com \ MTLS_CERT_PATH=… MTLS_KEY_PATH=… MTLS_CA_PATH=… \ target/release/sdex-backfill --transport hetzner \ --mode combined --start 50457424 --end --verbose diff --git a/docs/runbooks/seed-pool-registry.md b/docs/runbooks/seed-pool-registry.md index ed75991d..3d324ab6 100644 --- a/docs/runbooks/seed-pool-registry.md +++ b/docs/runbooks/seed-pool-registry.md @@ -73,7 +73,7 @@ set -a; . ./.env.local; set +a cargo run --release -p pool-registry-seed -- \ --network mainnet \ - --ch-domain ch.sorobanscan.rumblefish.dev \ + --ch-domain ch.sorobanscan.com \ --mtls-cert-path /path/to/prices_writer.crt \ --mtls-key-path /path/to/prices_writer.key \ --mtls-ca-path /path/to/ca.crt @@ -234,9 +234,18 @@ Confirm with the same `FINAL` count below, then drop `--dry-run` to write. Some pools have no factory event, so neither `learn_factory` nor this tool can find them. They are registered by a committed list, `STATIC_POOLS` in -`packages/prices-ingest-core/src/static_pools.rs`. Today it holds one pool: -Comet BLND/USDC `CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM` -(venue `comet`). +`packages/prices-ingest-core/src/static_pools.rs`. Today it holds five pools, +all venue `comet`: + +- Comet BLND/USDC `CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM`, + Blend v1/v2's backstop pool (task 0300); +- Comet v1.1 BLND/USDC `CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE`, + Blend v2.1's backstop pool, first swap 2026-10-07 (task 0334); +- Comet v1.1 BLNT/USDC `CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3` + and BLNT/XLM `CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC`, + deployed for the BLNT launch, first swaps 2026-10-25 (task 0337); +- Comet v1.1 BLND/USDC `CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ`, + a small pool, first swap 2026-10-23 (task 0337). - The live processor and the events-backfill reprice route it from the list, with or without a table row. @@ -249,9 +258,11 @@ Comet BLND/USDC `CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM` --clickhouse-url http://localhost:8123 --dry-run ``` - Expected until written: a `change="new"` line for `CAS3FL6T…` with - `venue="comet"`, then `to_write=1 per_venue={"comet": 1}` (plus any - factory pools created in that ledger). + Expected until written: a `change="new"` line per missing pool with + `venue="comet"`, then `to_write=N per_venue={"comet": N}` (plus any + factory pools created in that ledger). On production after task 0337 that + is `CDQMIKIH…`, `CA5HX3D4…` and `CCNDNFA7…`, `N = 3`: the two Blend pools + have been in the table since 0300 and 0334. - sdex-backfill's end-of-run registry write also persists it — the identical row, idempotent under the table's `ReplacingMergeTree`. diff --git a/docs/runbooks/xdr-protocol-watch.md b/docs/runbooks/xdr-protocol-watch.md index 48c54ebf..157c46a3 100644 --- a/docs/runbooks/xdr-protocol-watch.md +++ b/docs/runbooks/xdr-protocol-watch.md @@ -35,12 +35,12 @@ after 9 min, `rollup-freshness-1m` after 22 min). Four readings, compared every run: -| Reading | Source | -| ------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| **our pin** | `stellar-xdr` in `[workspace.dependencies]` of `Cargo.toml` (and `Cargo.lock` must agree, with only one major resolved) | -| **mainnet current** | `current_protocol_version` from `https://horizon.stellar.org/` | -| **core supports** | `core_supported_protocol_version` from the same page — rises **weeks before** the vote, once validators run the new stellar-core | -| **published crate** | `max_stable_version` from `https://crates.io/api/v1/crates/stellar-xdr` — only read when our pin lags `core supports` | +| Reading | Source | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| **our pin** | `stellar-xdr` in `[workspace.dependencies]` of `Cargo.toml` (and `Cargo.lock` must agree, with only one major resolved) | +| **mainnet current** | `current_protocol_version` from `https://horizon.stellar.org/` | +| **core supports** | `core_supported_protocol_version` from the same page — rises **weeks before** the vote, once validators run the new stellar-core | +| **published crate** | `max_stable_version` and the `versions` list from `https://crates.io/api/v1/crates/stellar-xdr` — only read when our pin lags `core supports` | Why the crate is read separately: stellar-core builds its XDR straight from the `stellar/stellar-xdr` git repo, so SDF runs a new protocol **before** the Rust @@ -79,6 +79,12 @@ Rules that are easy to get wrong: so; there is no bump to ask for, so the issue is left alone. A watch that silently checks nothing is the failure it exists to prevent, which is why the run still fails. +- **A later major is not a crate for this protocol** (task 0330). A bump + target for protocol N is a stable major above our pin and not above N. + Upstream skipped 29 (crates.io went 28.0.1 → 30.0.0), and 30.0.0 carries + protocol 30's XDR, so on mainnet 29 the watch is WAITING. Each protocol is + checked on its own: mainnet on 29 with core on 30 and 30.0.0 published is + LAGGING. - **A pre-release does not count.** `29.0.0-rc.1` is named in the report, but the tier stays WAITING until a stable `29.x` exists. - **Unchanged tier = silent.** The body is rewritten every run (with the fresh diff --git a/docs/scf/api-endpoints.md b/docs/scf/api-endpoints.md index 702d9c08..4c25c9e9 100644 --- a/docs/scf/api-endpoints.md +++ b/docs/scf/api-endpoints.md @@ -5,9 +5,11 @@ posture, and its cache TTL. Kept current as M2 tasks land so the Milestone 2 package (task 0128) can cite it instead of re-deriving the surface from the CDK source at submission time. -**Production base:** `https://prices-api.sorobanscan.rumblefish.dev` -(since 2026-08-31, task 0194 — the API's own hostname, a REGIONAL custom -domain on the REST API mapped at the root, so there is no stage path). +**Production base:** `https://prices-api.sorobanscan.com` (since 2026-11-06, +task 0340 — the API's own hostname, a REGIONAL custom domain on the REST API +mapped at the root, so there is no stage path, proxied by Cloudflare). The +first host, `https://prices-api.sorobanscan.rumblefish.dev` (task 0194, +2026-08-31), is the same API and still answers, with the same keys. ⚠️ **The execute-api origin is OFF as of 2026-09-02** (task 0126, `disableExecuteApiEndpoint: true`). `https://02mabge71l.execute-api.eu-central-1.amazonaws.com/production` @@ -93,21 +95,24 @@ missing or wrong `x-api-key` is a different answer and stays `403 {"message": ## Portal hosting — two hosts, no distribution of ours (tasks 0194, 0195) -**Portal:** `https://sorobanscan.rumblefish.dev/prices-api/` -**API reference:** `https://sorobanscan.rumblefish.dev/prices-api/docs` +**Portal:** `https://sorobanscan.com/api/prices-api/` +**API reference:** `https://sorobanscan.com/api/prices-api/docs` -Until task 0326 (2026-10) the portal lived at `/api/`, and for a few hours -on 2026-10-02 at `/pricing-api/` before the name settled; the explorer -answers every `/api…` and `/pricing-api…` URL with a `301` to the same path -under `/prices-api`, query string kept, so links shared before the move still -land on it. +Until task 0326 (2026-10) the portal lived at `/api/`, for a few hours on +2026-10-02 at `/pricing-api/` before the name settled, then at `/prices-api/` +until task 0339 (2026-11), on `sorobanscan.rumblefish.dev` until task 0340 +(2026-11-06). The explorer (its task 0660) answers every +`/prices-api…`, `/pricing-api…` and other `/api/…` URL with one `301` to the +same path under `/api/prices-api`, query string kept, so links shared before +any of the moves still land on it. `/api/` itself is the explorer's hub of +product portals. The portal is a page on one host that calls an API on another: -| What | Where | Owned by | -| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | -| the page (bundle) | `s3://production-soroban-explorer-api-spa/prices-api/`, behaviour `/prices-api/*` of the block explorer's distribution `EA2TLS5SS5M87` (alias `sorobanscan.rumblefish.dev`) | `soroban-block-explorer` (their stack); this repo syncs the bundle with `make -C infra sync-portal-explorer` | -| the backend + spec | `https://prices-api.sorobanscan.rumblefish.dev` — the custom domain above, no CloudFront in front of it | `ApiGatewayStack` here | +| What | Where | Owned by | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------ | +| the page (bundle) | `s3://production-soroban-explorer-api-spa/api/prices-api/`, behaviour `/api/prices-api/*` of the block explorer's distribution `EA2TLS5SS5M87` (alias `sorobanscan.com`) | `soroban-block-explorer` (their stack); this repo syncs the bundle with `make -C infra sync-portal-explorer` | +| the backend + spec | `https://prices-api.sorobanscan.com` — the custom domain above, proxied by Cloudflare, no CloudFront in front of it | `ApiGatewayStack` here | Decided 2026-08-31 (task 0194, "decision A"). The bundle is built with `VITE_PORTAL_API_ORIGIN` set to the API hostname, so every backend call is @@ -129,23 +134,27 @@ now answers `403`. ### The path convention -The bundle lives under **`/prices-api/`** on the explorer's host -(`/prices-api/login`, `/prices-api/docs` are pages); the backend under +The bundle lives under **`/api/prices-api/`** on the explorer's host +(`/api/prices-api/login`, `/api/prices-api/docs` are pages); the backend under **`/api/`** on the API's own host (`/api/auth/login`, `/api/api-docs-json`). -On the explorer's host the root belongs to the block explorer, so nothing of -ours may assume it. +On the explorer's host the root and `/api/` belong to the block explorer, so +nothing of ours may assume them. Both hosts have an `/api/`, and nothing +routes between them: the session cookie is host-only, and the bundle calls +the API host by name. From 2026-08-31 to task 0326 both shared one prefix, `/api/`, which read on the explorer's host as the explorer's own API; the move gave the bundle a -name of its own and left the backend where it was. The shared prefix +name of its own and left the backend where it was. Task 0339 put the bundle +back under `/api/`, now as one named portal among the product portals the +explorer's `/api/` hub lists. The shared prefix replaced task 0161's `/*` + `/api/*` convention on 2026-08-31, which for an app called "api" produced `/api/api/…` (task 0235's record of the three days that layout lived). The rule that decides which side of the split is enumerated: -- **the bundle's paths are a short, fixed list** — `/prices-api/`, - `/prices-api/index.html`, `/prices-api/favicon.ico`, - `/prices-api/assets/*` and the app's routes — and +- **the bundle's paths are a short, fixed list** — `/api/prices-api/`, + `/api/prices-api/index.html`, `/api/prices-api/favicon.ico`, + `/api/prices-api/assets/*` and the app's routes — and - **the backend is the open-ended side** (five slices added routes; none touched infrastructure), so it gets the catch-all. @@ -156,12 +165,13 @@ error in a browser (`web/portal/src/api/portal.ts` names the URL when it happens). On the explorer's host the split needs no routing table at all: their -`/prices-api/*` behaviour is S3, and their viewer-request function rewrites -**every path whose last segment has no `.`** to `/prices-api/index.html`. -That is what makes a hard refresh on `/prices-api/dashboard` — or on -`/prices-api/docs` — boot the portal, and it is also why the API reference is -a **route of the portal** rather than a static `docs/` folder in the bundle: a -folder would have been reachable only as `/prices-api/docs/index.html`. +`/api/prices-api/*` behaviour is S3, and their viewer-request function +rewrites **every path whose last segment has no `.`** to +`/api/prices-api/index.html`. That is what makes a hard refresh on +`/api/prices-api/dashboard` — or on `/api/prices-api/docs` — boot the portal, +and it is also why the API reference is a **route of the portal** rather than +a static `docs/` folder in the bundle: a folder would have been reachable only +as `/api/prices-api/docs/index.html`. Adding a page to the app therefore needs nothing on the hosting side; adding a backend route needs nothing either, because the gateway maps `/api/{proxy+}` and the bundle calls the API host directly. @@ -181,19 +191,19 @@ default release set (Compute + SPA) does not deploy, but a `-all` or ## OpenAPI specification (task 0124) ``` -GET https://prices-api.sorobanscan.rumblefish.dev/api-docs-json +GET https://prices-api.sorobanscan.com/api-docs-json ``` Anonymous — no API key needed. Verify with: ```bash -curl -sS https://prices-api.sorobanscan.rumblefish.dev/api-docs-json \ +curl -sS https://prices-api.sorobanscan.com/api-docs-json \ | jq '{openapi, servers, paths: (.paths | keys)}' ``` The same bytes are served at `/api/api-docs-json` (the portal's alias, task 0194), and rendered as the portal's API reference at -`https://sorobanscan.rumblefish.dev/prices-api/docs` (task 0195) — Swagger UI's +`https://sorobanscan.com/api/prices-api/docs` (task 0195) — Swagger UI's shape (tags, collapsible operations, parameters, responses, schemas) in the portal's own design system, reading the live document on every visit. @@ -205,7 +215,7 @@ portal's own design system, reading the live document on every visit. document contains nothing key-specific, which is what makes it safe to cache for all callers. - **CORS:** `Access-Control-Allow-Origin: *`, on both copies (task 0195). The - portal's API reference is a page on `sorobanscan.rumblefish.dev` that + portal's API reference is a page on `sorobanscan.com` that `fetch`es the document from the API host, and so may a partner's browser-side tooling. `*` rather than the portal's origin because the document is public and @@ -250,8 +260,10 @@ npm run openapi:extract # → target/openapi.json, servers stamped from config sends a request until the data routes answer CORS (task 0126). - **Onboarding portal** — open (`PORTAL_ENABLED=true` since task 0194) and public since 2026-09-23 (task 0305) at - `https://sorobanscan.rumblefish.dev/prices-api/` (`/api/` until task - 0326): the block explorer's basic auth is off for it. The move from the + `https://sorobanscan.com/api/prices-api/` (`/api/` until task 0326, + `/prices-api/` until task 0339, on `sorobanscan.rumblefish.dev` until task + 0340): the block explorer's basic auth is + off for it. The move from the test guild to the Stellar guild (task 0179) still precedes advertising the URL. - **CORS on `/v1`** (task 0126) — no browser can call the data routes yet; diff --git a/docs/scf/milestone-3-form-answers.md b/docs/scf/milestone-3-form-answers.md index 0cbadd34..3d0e7db9 100644 --- a/docs/scf/milestone-3-form-answers.md +++ b/docs/scf/milestone-3-form-answers.md @@ -1,11 +1,9 @@ # SCF Milestone 3 — Form Answers (stellar-prices-api) -> ⚠️ **DRAFT — first full draft 2026-10-02.** Mirrors the Milestone 2 form -> ([`milestone-2-form-answers.md`](milestone-2-form-answers.md)). Every figure is -> taken from [`milestone-3-evidence.md`](milestone-3-evidence.md); none may -> outrun it. Anything in `` waits for the release run on `master`, -> the recording, or the asset-id migration (task 0139); anything in -> `` is a link to place on submission day. +> Mirrors the Milestone 2 form +> ([`milestone-2-form-answers.md`](milestone-2-form-answers.md)), shortened +> before submission. Every figure is taken from +> [`milestone-3-evidence.md`](milestone-3-evidence.md); none may outrun it. > > Copy the text inside each blockquote into the matching field of the Stellar > Community Fund **Deliverable Verification** form. @@ -14,187 +12,111 @@ ## Field 1 — Tranche Deliverables -> **Deliverable 3 — Production Launch & Validation** (as originally approved). -> -> Milestone 3 opens the Stellar Prices API to developers. The service went -> public on **2026-09-23 at 09:40 CEST**, when the staging password came off -> the onboarding portal; the API itself had been serving on its custom domain -> since Milestone 2. Everything below is the production deployment. -> -> What is live and verifiable today: -> -> 1. **A self-service onboarding portal.** At -> `https://sorobanscan.rumblefish.dev/prices-api/` (at `/api/` until -> 2026-10-02; every old link answers 301 to the same page). Sign-in is -> through Discord, and eligibility is membership in the official Stellar -> Developers server. A key is issued at once on the free plan, one request a -> second and a hundred thousand a month, and the dashboard shows it with its -> plan, limits and usage. The quick start matches the published contract, -> and the portal carries its own privacy policy. The video records the walk -> on production on 2026-10-05: sign-in at 13:00:40 UTC, a `/v1` call with the -> key at 13:01:28 (200), the key revoked at 13:06:01 and refused with a 403 -> from 13:07. -> 2. **A published contract.** The API serves its own OpenAPI 3.1.0 document -> at `/api-docs-json`, generated from the handler code, with 74 -> production-shaped examples. CI lints it on every Rust change with -> Redocly's strict ruleset, and the portal renders it as an API reference. -> The release run on `master` (2026-10-05) lints it with no errors or -> warnings: `https://github.com/rumblefishdev/stellar-prices-api/actions/runs/37298469682` -> 3. **Integration tests in CI against a real database.** Since 2026-09-22, -> CI starts ClickHouse, applies the schema, starts the mTLS proxy and runs -> the integration suite against it, covering each of the seven `/v1` -> routes. On the first run on `master` after the release (2026-10-05) -> 1,407 unit and 306 ClickHouse integration tests pass, 0 failed: -> `https://github.com/rumblefishdev/stellar-prices-api/actions/runs/37298469682` -> 4. **Load-tested at the approved target.** 100 requests per second for five -> minutes: **p95 49.0 ms** against a 100 ms bar, **0 errors in 30,001 -> requests**, on the purpose-built plan the criterion asks us to name. The -> report also carries the miss-only row and the 500 and 1000 req/s rows. -> 5. **A security review.** ClickHouse is reachable only over mutual TLS; the -> client certificates live in Secrets Manager, not in environment variables; -> every invalid input gets a 400. No IAM statement grants every action or a -> whole service, and every statement with a wildcard resource is listed -> with what limits it: 22 in the release's templates, each for an action -> that has no resource ARN. -> 6. **Observability.** X-Ray tracing is active on the API handler, the ledger -> processor, the oracle and the enrichment worker, and on the gateway stage. -> The `prices-production-overview` dashboard covers API latency and errors, -> ingestion lag, ClickHouse write latency, the workers and an alarm strip with -> every `prices-production-*` alarm; all 83 were OK on 2026-10-05. -> 7. **Seven days of post-launch monitoring.** From 2026-09-23 09:40 to -> 2026-09-30 09:40 CEST: **uptime 100.000 %, 0 × 5XX in 65,806 requests, -> gateway p95 145.2 ms.** The report recomputes from a raw CloudWatch export -> kept in the repository, so a reviewer can rerun it without AWS access. -> 8. **A public repository.** Public since 2026-09-16, with a root README -> leading to the architecture documents, the operator runbooks and a -> fresh-account deployment runbook. -> -> The tranche's backfill target, SDEX history back to January 2018, was -> reached early, in Milestone 2: the archive completed on 2026-07-27 and -> reaches back to **2015-11-18**. -> -> **Declared deviations.** Four places where the delivery departs from the -> wording, each with its reason in the deviations document: the backfill that -> finished early (so liveness is graded on the ingestion signals), the two -> monitoring-report metrics it made flat, Redocly and the portal's own renderer -> in place of `openapi-validator` and Swagger UI, and the scope of the -> load-test figure. -> -> **Known issues and limitations are listed, each with the task that owns it:** -> the history recomputation that finishes after delivery, the history of the -> assets that shared an id until the asset-id fix (task 0139), still being -> re-ingested, a slow path we measured but did not instrument, and the ceiling -> of the database server we share with the Soroban Block Explorer. -> -> **Full evidence — acceptance-criteria mapping with runnable commands, the -> deviation rationale, known issues and limitations:** -> `https://drive.google.com/file/d/1AktIqEMocAW_DtaHiISdZTfptMu58Xg2/view?usp=sharing` +> Deliverable 3 — Production Launch & Validation (as originally approved). +> +> The Stellar Prices API went public on 2026-09-23 at 09:40 CEST. Everything +> below is the production deployment: +> +> 1. Self-service onboarding portal: https://sorobanscan.rumblefish.dev/prices-api/ +> — Discord sign-in for members of the Stellar Developers server, a +> free-plan key issued at once, usage on the dashboard. The full key flow was +> recorded on production on 2026-10-05. +> 2. Published contract: an OpenAPI 3.1.0 document generated from the handler +> code, linted clean in CI with Redocly and rendered in the portal. +> 3. Integration tests in CI against a real ClickHouse: 1,407 unit and 306 +> integration tests, 0 failed, on the release run on master (2026-10-05). +> 4. Load test: 100 req/s for five minutes, p95 49.0 ms against a 100 ms bar, +> 0 errors in 30,001 requests. +> 5. Security: ClickHouse over mutual TLS only, secrets in Secrets Manager, +> every input validated; the 22 IAM statements with a wildcard resource are +> each listed with what limits them. +> 6. Observability: X-Ray tracing end to end, the prices-production-overview +> dashboard, all 83 alarms OK on 2026-10-05. +> 7. Seven days of post-launch monitoring (2026-09-23 to 09-30): uptime +> 100.000 %, 0 × 5XX in 65,806 requests, gateway p95 145.2 ms. +> 8. A public repository with a fresh-account deployment runbook. +> +> SDEX history reaches back to 2015-11-18, past the January 2018 target. Four +> deviations from the wording are declared with their reasons; known issues and +> limitations are listed, each with the task that owns it. +> +> Full evidence: https://drive.google.com/file/d/1AktIqEMocAW_DtaHiISdZTfptMu58Xg2/view?usp=sharing --- ## Field 2 — Deliverable Verification - Video -> `https://drive.google.com/file/d/1KTFv9A58sU1FLtnqa4xBYYE8oaCI3DgW/view?usp=sharing` +> https://drive.google.com/file/d/1KTFv9A58sU1FLtnqa4xBYYE8oaCI3DgW/view?usp=sharing > -> About eight minutes, all on production: (1) scope; (2) the portal, end to -> end — sign-in, a key, a `/v1` call with it, revocation; (3) the OpenAPI -> document, the rendered reference and a 404 in the error envelope; (4) the -> integration tests in CI and the load test; (5) the security checklist; -> (6) operations — backfill status against Horizon, the dashboard and its -> alarms, the monitoring report; (7) the public repository and the -> fresh-account runbook; (8) the deviations and limitations. +> About eight minutes, all on production: the portal and the key flow, the +> OpenAPI reference, CI and the load test, security, operations (backfill, +> dashboard, alarms, monitoring report), the repository and the deviations. --- ## Field 3 — Additional Deliverable Verification -> **Evidence package (Google Drive):** `https://drive.google.com/drive/folders/1XnHTsg1hI7e7XR9z5G1MZ2zuyjVmO2K0?usp=sharing` -> -> - Evidence PDF: `https://drive.google.com/file/d/1AktIqEMocAW_DtaHiISdZTfptMu58Xg2/view?usp=sharing` -> - Demo video: `https://drive.google.com/file/d/1KTFv9A58sU1FLtnqa4xBYYE8oaCI3DgW/view?usp=sharing` -> -> The nine acceptance criteria, in order. Deviations are set out once, in -> `.../blob/master/docs/scf/milestone-3-rfp-deviations.md`, and referred to -> here by number. -> -> 1. **Backfill status: depth met by two years.** `earliest_data_available` -> is 2015-11-18 against a 2018-01-01 bar; liveness is graded on the -> ingestion alarms and `realtime_tip_ledger` (deviation 1). -> `GET https://prices-api.sorobanscan.rumblefish.dev/v1/backfill/status` -> 2. **OpenAPI lints clean; reference deployed.** Met: the release run on -> `master` (2026-10-05) lints it with no errors or warnings. Reference: -> `https://sorobanscan.rumblefish.dev/prices-api/docs` (deviation 3). -> Reproduce: `npm run openapi:lint` -> 3. **Portal accessible; self-service key flow works.** -> `https://sorobanscan.rumblefish.dev/prices-api/`, anonymous since -> 2026-09-23; Discord sign-in for keys. Met: recorded on 2026-10-05, -> sign-in 13:00:40 UTC, `/v1` call 13:01:28 (200), revoked 13:06:01, 403 -> from 13:07 (evidence AC 3). -> 4. **Integration suite passes on CI.** Met: first run on `master` after the -> release, `https://github.com/rumblefishdev/stellar-prices-api/actions/runs/37298469682` -> (2026-10-05): 1,407 unit and 306 ClickHouse integration tests, 0 failed. -> Reproduce locally: -> `tools/scripts/ignored-tests.sh` -> 5. **Load test: met.** p95 49.0 ms at 100 req/s for five minutes, 0 errors -> in 30,001 requests, plan `prices-production-loadtest-plan`; scope in -> deviation 4. Report: -> `.../blob/master/docs/prices-api-load-test-100rps.md` -> 6. **Security checklist: met.** mTLS-only ClickHouse, secrets in Secrets -> Manager, validated inputs, no `Action: "*"` or `service:*`; every -> `Resource: "*"` statement inventoried in evidence §5, AC 6. -> 7. **Repository public; deployable from the README: met on the -> fresh-account runbook.** `https://github.com/rumblefishdev/stellar-prices-api`, -> `README.md` → `infra/README.md` §"Fresh-account deployment". Not run in an -> empty AWS account; the claim rests on the runbook, a credential-free synth -> in CI and a name-by-name check of the runbook against the code. -> 8. **Dashboard and alarms: met.** -> `prices-production-overview`, `eu-central-1`; all 83 alarms OK on -> 2026-10-05. Read-only access for a named reviewer on request -> (see Field 4). -> 9. **7-day post-launch report: met.** Uptime 100.000 %, 0 × 5XX in 65,806 -> requests, gateway p95 145.2 ms; two named metrics replaced by live -> signals (deviation 2). Reproduce offline: -> `python3 docs/scf/milestone-3-monitoring/report.py` -> -> **Live & anonymous:** -> -> - OpenAPI document: `https://prices-api.sorobanscan.rumblefish.dev/api-docs-json` -> - Rendered API reference: `https://sorobanscan.rumblefish.dev/prices-api/docs` -> - Health probe: `https://prices-api.sorobanscan.rumblefish.dev/health` -> -> **Key-gated** (`x-api-key`; the Milestone 2 reviewer key works, or issue -> one in the portal): the seven `/v1` route groups listed in evidence §9. +> Evidence package: https://drive.google.com/drive/folders/1XnHTsg1hI7e7XR9z5G1MZ2zuyjVmO2K0?usp=sharing +> +> Deviations: https://github.com/rumblefishdev/stellar-prices-api/blob/master/docs/scf/milestone-3-rfp-deviations.md +> +> 1. Backfill status — depth met: earliest_data_available is 2015-11-18 against +> 2018-01-01; liveness graded on the ingestion signals (deviation 1). +> GET https://prices-api.sorobanscan.rumblefish.dev/v1/backfill/status +> 2. OpenAPI lints clean, reference deployed — met on the release run on master +> (2026-10-05): https://sorobanscan.rumblefish.dev/prices-api/docs +> (deviation 3) +> 3. Portal and self-service keys — met, recorded on 2026-10-05: +> https://sorobanscan.rumblefish.dev/prices-api/ +> 4. Integration suite on CI — met: +> https://github.com/rumblefishdev/stellar-prices-api/actions/runs/37298469682 +> 5. Load test — met: p95 49.0 ms at 100 req/s (deviation 4) +> 6. Security checklist — met: evidence §5, AC 6 +> 7. Public repository, deployable from the README — met on the fresh-account +> runbook, not run in an empty AWS account: +> https://github.com/rumblefishdev/stellar-prices-api +> 8. Dashboard and alarms — met: all 83 alarms OK on 2026-10-05; read-only +> access for a named reviewer on request +> 9. 7-day post-launch report — met: uptime 100.000 %, 0 × 5XX in 65,806 +> requests (deviation 2) +> +> No key needed: https://prices-api.sorobanscan.rumblefish.dev/api-docs-json and +> https://prices-api.sorobanscan.rumblefish.dev/health. The seven /v1 route +> groups need an x-api-key: the Milestone 2 reviewer key works, or issue one in +> the portal. --- ## Field 4 — Support Needed -> — No support is needed to complete or verify this tranche: nothing in it -> depends on the Stellar side. SDF settled the Discord integration by asking -> the project to run it itself. -> -> To open the CloudWatch dashboard and the alarms, ask us with your name, -> surname, e-mail, purpose and the end date of access; a read-only IAM Identity -> Center user is issued for the review window. +> None. --- ## Pre-submission checklist -- [ ] Every figure in Fields 1 and 3 appears, with the same value and date, in +Checked 2026-10-06. + +- [x] Every figure in Fields 1 and 3 appears, with the same value and date, in the evidence package -- [ ] Every deviation the package declares is mentioned in Field 3 by number -- [ ] Every `` and `` placeholder replaced -- [ ] The reviewer key still works and is on the free plan -- [ ] `develop` released to `master`: the repository's default branch shows +- [x] Every deviation the package declares is mentioned in Field 3 by number + → deviations 1–4, all four the deviations document declares +- [x] Every `` and `` placeholder replaced +- [x] The reviewer key still works and is on the free plan + → `prices-production-scf-reviewer-key-20260909T120021Z`: enabled, on + `pricing-api-free-production` +- [x] `develop` released to `master`: the repository's default branch shows the root `README.md` and the fresh-account runbook (AC 7), and every - `.../blob/master/...` link above resolves. On 2026-09-28 `master` still - had neither -- [ ] The first CI run on `master` after the release is green in every job, + `.../blob/master/...` link above resolves + → release #381 (2026-10-05), the package in #391 +- [x] The first CI run on `master` after the release is green in every job, `XDR protocol lag` included (fixed by task 0325, PR #383) -- [ ] The 0139 window is closed: alarms back to OK, the alarm count updated + → run 37298469682, all four jobs green +- [x] The 0139 window is closed: alarms back to OK, the alarm count updated wherever "65" appears (evidence, video script, Field 1 item 6) -- [ ] Portal links use `/prices-api/`, and an old `/api/` link still answers + → 83 of 83 `prices-production-*` alarms OK; no alarm count of 65 left +- [x] Portal links use `/prices-api/`, and an old `/api/` link still answers `301` on submission day -- [ ] The video was recorded against the public deployment, no secrets on + → `/api/` and `/api/dashboard` answer `301` to `/prices-api/…` +- [x] The video was recorded against the public deployment, no secrets on screen (see the scenario's ⚠️ section) diff --git a/infra/Makefile b/infra/Makefile index 4c936edd..5369fe99 100644 --- a/infra/Makefile +++ b/infra/Makefile @@ -131,13 +131,14 @@ destroy-production-apigateway: build-production npx cdk --app "$(PRODUCTION_APP)" destroy Prices-production-ApiGateway --force # The portal (task 0194; the only deploy path since task 0195). -# `https://sorobanscan.rumblefish.dev/prices-api/` is served by the block explorer's +# `https://sorobanscan.com/api/prices-api/` is served by the block explorer's # distribution (EA2TLS5SS5M87) from ITS bucket, which nothing in this repo # deploys — this sync is the only way a build reaches it. Built with the API's # own hostname baked in (`VITE_PORTAL_API_ORIGIN`, see -# `web/portal/src/api-origin.ts`): on that host `/prices-api/*` is a static SPA, so -# the bundle must call the backend on `apiDomain.domainName` rather than -# relatively. `--skip-nx-cache` because the variable is not an Nx input — a +# `web/portal/src/api-origin.ts`): on that host `/api/prices-api/*` is a static SPA, so +# the bundle must call the backend on its own host rather than relatively: +# `sorobanscanApiDomain.domainName` since task 0340 (same-site with the +# portal), else `apiDomain.domainName`. `--skip-nx-cache` because the variable is not an Nx input — a # cached build would hand back whichever variant ran last, and the relative # one is indistinguishable from this one until a visitor opens the page. # @@ -154,7 +155,7 @@ EXPLORER_DISTRIBUTION_ID ?= EA2TLS5SS5M87 # `// empty`, so a renamed key yields `https://` rather than `https://null`. # Either is refused by `check-portal-api-origin` below before a build can bake # it in; a missing `jq` produces the same empty value and the same refusal. -PORTAL_API_ORIGIN ?= https://$(shell jq -r '.apiDomain.domainName // empty' envs/production.json) +PORTAL_API_ORIGIN ?= https://$(shell jq -r '(.sorobanscanApiDomain.domainName // .apiDomain.domainName) // empty' envs/production.json) # `$(shell …)` swallows a failed `jq` — binary missing, key renamed — into the # value, and the value is baked into the bundle: every backend call then goes @@ -193,13 +194,13 @@ verify-portal-guild: cd .. && node tools/scripts/verify-discord-guild.mjs --ssm --env production sync-portal-explorer: build-portal-explorer verify-portal-guild - aws s3 sync ../web/portal/dist s3://$(EXPLORER_PORTAL_BUCKET)/prices-api/ \ + aws s3 sync ../web/portal/dist s3://$(EXPLORER_PORTAL_BUCKET)/api/prices-api/ \ --exclude '*' --include 'assets/*' \ --cache-control 'public, max-age=31536000, immutable' - aws s3 sync ../web/portal/dist s3://$(EXPLORER_PORTAL_BUCKET)/prices-api/ \ + aws s3 sync ../web/portal/dist s3://$(EXPLORER_PORTAL_BUCKET)/api/prices-api/ \ --exclude 'assets/*' \ --cache-control 'public, max-age=0, must-revalidate' - aws cloudfront create-invalidation --distribution-id $(EXPLORER_DISTRIBUTION_ID) --paths '/prices-api/*' + aws cloudfront create-invalidation --distribution-id $(EXPLORER_DISTRIBUTION_ID) --paths '/api/prices-api/*' deploy-production-eventbridge: build-production build-lambdas npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-EventBridge --exclusively --require-approval broadening diff --git a/infra/envs/production.json b/infra/envs/production.json index 2263d683..bd9ab99d 100644 --- a/infra/envs/production.json +++ b/infra/envs/production.json @@ -14,13 +14,17 @@ }, "apiGatewayCacheEnabled": true, "coverageSweepEnabled": true, - "apiBaseUrl": "https://prices-api.sorobanscan.rumblefish.dev", + "apiBaseUrl": "https://prices-api.sorobanscan.com", "apiDomain": { "domainName": "prices-api.sorobanscan.rumblefish.dev", "hostedZoneId": "Z10396861CRMUIWWA8TL9", "hostedZoneName": "sorobanscan.rumblefish.dev" }, - "portalWebOrigin": "https://sorobanscan.rumblefish.dev", + "sorobanscanApiDomain": { + "domainName": "prices-api.sorobanscan.com", + "certificateArn": "arn:aws:acm:eu-central-1:750702271865:certificate/c937aeef-20b3-4e94-ba90-ade97c7d00ec" + }, + "portalWebOrigin": "https://sorobanscan.com", "apiHandler": { "memoryMb": 512, "timeoutSeconds": 15 @@ -56,6 +60,7 @@ "chDiskFreePercent": 20, "dlqEscalationDepths": [10, 50], "usdSanityEscalationCounts": [1, 100, 10000], + "sacContractIdentitiesActionsEnabled": false, "slack": { "workspaceIdSsmParam": "/prices/production/slack-workspace-id", "channelIdSsmParam": "/prices/production/slack-channel-id" diff --git a/infra/src/lib/app.ts b/infra/src/lib/app.ts index f2c9918d..ce069314 100644 --- a/infra/src/lib/app.ts +++ b/infra/src/lib/app.ts @@ -45,7 +45,7 @@ export function createApp({ config }: CreateAppOptions): void { // private bucket and a CloudFront distribution fronting both the bundle // and this API — was retired by task 0195 on 2026-09-01: since task 0194 // the page is served from the block explorer's distribution at - // `https://sorobanscan.rumblefish.dev/prices-api/` (its bucket, synced by + // `https://sorobanscan.com/api/prices-api/` (its bucket, synced by // `make -C infra sync-portal-explorer`) and calls this API on // `config.apiDomain` directly, so the distribution had become a second, // ungated front door to the same portal. Constructing `ApiGatewayStack` is diff --git a/infra/src/lib/mtls.ts b/infra/src/lib/mtls.ts index a5e709cb..3669d8d8 100644 --- a/infra/src/lib/mtls.ts +++ b/infra/src/lib/mtls.ts @@ -55,7 +55,9 @@ export function secretsManagerLayerArn(region: string): string { * `default.soroban_contracts` over this identity; those two SELECTs are * BE's to grant (request text: docs/runbooks/0100-coverage-sweep-triage.md * §4.1), and until they land the probe fails with Code 497. A dedicated - * read-only identity for it is task 0258's later clean-up. + * read-only identity for it is task 0258's later clean-up. Task 0242 also + * reads `default.soroban_contracts` over it, from the ledger processor's + * cold start and the rollup-freshness probe. * - `api` → CH user `prices_reader` (axum read handlers; * `SELECT ON prices.*`). */ diff --git a/infra/src/lib/stacks/api-gateway-stack.ts b/infra/src/lib/stacks/api-gateway-stack.ts index 3d144846..8fcd1ba5 100644 --- a/infra/src/lib/stacks/api-gateway-stack.ts +++ b/infra/src/lib/stacks/api-gateway-stack.ts @@ -145,14 +145,8 @@ const PORTAL_API_RESOURCE_PATH = '/api/{proxy+}'; * `/auth/logout`; `DELETE` for task 0192's revoke if it prefers that shape to a * `POST`. * - * ⚠️ Task 0186's four routes sit at **depth 3** (`auth/login`, not `login`), - * which the greedy `{proxy+}` above covers and the intermediate - * `{proxy}` + `{proxy}/{sub}` pair that is CURRENTLY DEPLOYED does not — see - * task 0205, which ships this file's committed shape. Until that deploy runs, - * `/api/auth/login` answers the gateway's own - * `403 Missing Authentication Token` rather than reaching the handler. That is a - * deployment gap, not a defect in either task: the flag keeps the handler dark - * regardless, and `/config` at depth 1 is unaffected. + * Task 0186's four routes sit at **depth 3** (`auth/login`, not `login`), which + * is why the proxy is greedy rather than a fixed number of `{proxy}` levels. * * ⚠️ A verb that is NOT listed here never reaches the handler: it gets the * gateway's own `404 {"code": "not_found", …}` (task 0309's gateway response) @@ -560,9 +554,9 @@ export class ApiGatewayStack extends cdk.Stack { // below — declared outside the `cacheEnabled` branch, because with the stage // cache off is exactly when an unbounded anonymous route costs the most. // `/api/{proxy+}` directly under the root: the bundle's paths share the - // prefix but never reach the gateway — the page lives on another host - // (task 0194) and only its backend calls come here, on the API's own - // hostname. The OpenAPI alias `/api/api-docs-json` rides on this same + // prefix (`/api/prices-api/`, task 0339) but never reach the gateway — the + // page lives on another host (task 0194) and only its backend calls come + // here, on the API's own hostname. The OpenAPI alias `/api/api-docs-json` rides on this same // proxy. const portalProxy = this.api.root .addResource('api') @@ -574,14 +568,15 @@ export class ApiGatewayStack extends cdk.Stack { // The preflight for the bundle's cross-origin calls (task 0194). // // The bundle is served from `config.portalWebOrigin` — another - // application's distribution, whose `/api/*` behaviour is a static SPA - // that rewrites every extensionless path to its `index.html`. A - // same-origin `fetch('/api/config')` from there gets the bundle back as - // `200 text/html`, so the bundle calls this API on `config.apiDomain` - // instead, and a browser asks before a non-simple request crosses an - // origin: the revoke's marker header (`PORTAL_REQUEST_HEADER`) is one, and - // so is any `DELETE`. Answered here by a MOCK integration rather than by - // the Lambda — no invocation, and the `max-age` has to be set by whoever + // application's distribution, whose `/api/*` behaviour is a `301` into a + // static SPA (`/api/prices-api/*`) that rewrites every extensionless path + // to its `index.html`. A same-origin `fetch('/api/config')` from there + // gets the bundle back as `200 text/html`, so the bundle calls this API on + // `config.apiDomain` instead, and a browser asks before a non-simple + // request crosses an origin: the revoke's marker header + // (`PORTAL_REQUEST_HEADER`) is one, and so is any `DELETE`. Answered here + // by a MOCK integration rather than by the Lambda — no invocation, and + // the `max-age` has to be set by whoever // answers the preflight, which is the gateway — with the handler adding // `Access-Control-Allow-Origin` on the actual responses (`portal::apply` // in the Rust crate names the same one origin). @@ -1092,6 +1087,35 @@ export class ApiGatewayStack extends cdk.Stack { target: alias, }); + // The API's host on the sorobanscan.com zone (task 0340): one more + // custom domain on the same stage and root base path, beside `apiDomain`, + // which keeps serving. No record here: that zone's DNS is Cloudflare's, in + // soroban-infrastructure's `cloudflare/`, which takes the regional target + // below. The certificate is referenced, not created: DNS validation in that + // zone is not this stack's to write. The `*.sorobanscan.com` one (explorer + // task 0661) is in use by the explorer's API domains, so ACM keeps renewing + // it, unlike the unattached wildcard the `apiDomain` note above avoids. + if (config.sorobanscanApiDomain) { + const sorobanscanApiDomain = this.api.addDomainName( + 'SorobanscanApiDomain', + { + domainName: config.sorobanscanApiDomain.domainName, + certificate: acm.Certificate.fromCertificateArn( + this, + 'SorobanscanApiCertificate', + config.sorobanscanApiDomain.certificateArn, + ), + endpointType: apigateway.EndpointType.REGIONAL, + securityPolicy: apigateway.SecurityPolicy.TLS_1_2, + }, + ); + new cdk.CfnOutput(this, 'SorobanscanApiRegionalTarget', { + value: sorobanscanApiDomain.domainNameAliasDomainName, + description: + 'CNAME target of the sorobanscan.com API host (soroban-infrastructure)', + }); + } + new cdk.CfnOutput(this, 'ApiCustomDomain', { value: `https://${config.apiDomain.domainName}`, description: 'The API on its own hostname (root base path → this stage)', diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index 19ed6799..09132d1f 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -339,6 +339,17 @@ export class ObservabilityStack extends cdk.Stack { public readonly ledgerProcessorForcedPartialFlushAlarm: cloudwatch.Alarm; /** Task 0291: live dropped trades from a pool missing from `pool_registry`. */ public readonly ledgerProcessorUnregisteredPoolAlarm: cloudwatch.Alarm; + public readonly ledgerProcessorTradesMissingDecimalsAlarm: cloudwatch.Alarm; + /** + * Task 0242 D2: live skipped trades on a token BE flags `is_sac` that no SAC + * event proved, rather than minting it as a second identity. + */ + public readonly ledgerProcessorSacUnprovenAlarm: cloudwatch.Alarm; + /** + * Task 0242: a live container started without BE's `is_sac` set, so it mints + * an unproven SAC as a `Contract` identity instead of skipping it. + */ + public readonly ledgerProcessorSacCandidatesAlarm: cloudwatch.Alarm; /** * Task 0100: the weekly coverage sweep found swap/trade emitters in neither * `pool_registry` nor the committed allow-list (layer 3). @@ -386,6 +397,21 @@ export class ObservabilityStack extends cdk.Stack { * 0139): a writer sent ids, not identities. Keyed by count. */ public readonly assetIdOrphanCandleAlarms: Record; + /** + * Contract identities in `assets FINAL` that are SACs (task 0242 D6). Actions + * behind `opsAlarms.sacContractIdentitiesActionsEnabled`: the baseline is 36 + * until the heal. + */ + public readonly sacContractIdentitiesAlarm: cloudwatch.Alarm; + + /** + * Stored candles on all seven `price_ohlcv_*` tiers that break the OHLC band + * or carry a price <= 0 (task 0236), keyed by count. Asserted by the probe at + * the source with zero tolerance, because task 0229's read-path clamp repairs + * a crossed candle on its way out of `/ohlcv` and so hides it from the API. + * The healthy reading is exactly 0. + */ + public readonly ohlcBandAlarms: Record; /** * A rollup MV that has lost `APPEND` (task 0204, gap 3) — history destroyed * on every refresh. Separate from {@link mvDriftAlarm} because this is the @@ -1468,6 +1494,98 @@ export class ObservabilityStack extends cdk.Stack { this.ledgerProcessorUnregisteredPoolAlarm.addAlarmAction(snsAction); this.ledgerProcessorUnregisteredPoolAlarm.addOkAction(snsAction); + // Task 0242 D2 — live skipped trades on an unproven SAC. A SAC leg resolves + // to its classic asset only when the same transaction carries the SAC's own + // transfer/mint/burn/clawback event; a token BE flags `is_sac` with no such + // proof is skipped rather than minted as a second identity. Same shape as + // the 0291 alarm above: `SacUnprovenSkipped` is published only when + // non-zero. + this.ledgerProcessorSacUnprovenAlarm = new cloudwatch.Alarm( + this, + 'LedgerProcessorSacUnprovenAlarm', + { + alarmName: `prices-${config.envName}-ledger-processor-sac-unproven`, + alarmDescription: + 'The ledger-processor skipped AMM trades with a leg that BE flags is_sac (default.soroban_contracts) but that no SAC transfer/mint/burn/clawback event in the same transaction proved (task 0242 D2). Those trades produce no candle; they are skipped rather than minted as a second, Contract identity of the asset. The WARN "skipped trades on is_sac contracts with no SAC proof (task 0242)" lists the contracts. Fix: docs/runbooks/0242-sac-identity-heal.md.', + metric: new cloudwatch.Metric({ + namespace: 'Prices/Ingest', + metricName: 'SacUnprovenSkipped', + dimensionsMap: { Environment: config.envName }, + statistic: 'Sum', + period: cdk.Duration.minutes(5), + }), + threshold: 1, + evaluationPeriods: 1, + datapointsToAlarm: 1, + comparisonOperator: + cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + // Emitted only when trades were skipped: "missing" is healthy. + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + }, + ); + this.ledgerProcessorSacUnprovenAlarm.addAlarmAction(snsAction); + this.ledgerProcessorSacUnprovenAlarm.addOkAction(snsAction); + + // Task 0242 — the cold start could not read BE's `is_sac` set and the + // processor runs on without it rather than failing Init (review of 0242). + // Each run of such a container publishes one `SacCandidatesUnavailable`; + // nothing is published while the set is loaded. + this.ledgerProcessorSacCandidatesAlarm = new cloudwatch.Alarm( + this, + 'LedgerProcessorSacCandidatesAlarm', + { + alarmName: `prices-${config.envName}-ledger-processor-sac-candidates-unavailable`, + alarmDescription: + 'The ledger-processor cold-started without BE\'s is_sac contract set (default.soroban_contracts unreadable as prices_writer: grant, rename or a BE migration). Ingest continues, but until a cold start reads the set again an unproven SAC is minted as a Contract identity instead of being skipped (task 0242). The WARN "is_sac contract set unreadable" carries the error. Fix the read, then force a cold start; SacContractIdentities shows any identity minted meanwhile. Runbook: docs/runbooks/0242-sac-identity-heal.md.', + metric: new cloudwatch.Metric({ + namespace: 'Prices/Ingest', + metricName: 'SacCandidatesUnavailable', + dimensionsMap: { Environment: config.envName }, + statistic: 'Sum', + period: cdk.Duration.minutes(5), + }), + threshold: 1, + evaluationPeriods: 1, + datapointsToAlarm: 1, + comparisonOperator: + cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + // Emitted only while the set is missing: "missing" is healthy. + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + }, + ); + this.ledgerProcessorSacCandidatesAlarm.addAlarmAction(snsAction); + this.ledgerProcessorSacCandidatesAlarm.addOkAction(snsAction); + + // Task 0329 — live dropped trades whose Soroban token decimals did not + // resolve. The ingest scales each AMM leg by its token's `decimals()`, read + // over RPC on first sight, and never guesses: until the call answers, the + // token's trades produce no candle. Published only when non-zero. + this.ledgerProcessorTradesMissingDecimalsAlarm = new cloudwatch.Alarm( + this, + 'LedgerProcessorTradesMissingDecimalsAlarm', + { + alarmName: `prices-${config.envName}-ledger-processor-trades-missing-decimals`, + alarmDescription: + 'The ledger-processor dropped AMM trades because a Soroban token\'s decimals() did not resolve over RPC (task 0329). Those trades produce no candle. Either the RPC (SOROBAN_RPC_URL, default mainnet.sorobanrpc.com) was failing, in which case the token is asked again after 10 minutes, or the token has no usable decimals(), in which case it is not asked again until a cold start. The WARN "dropped AMM trades whose token decimals did not resolve" lists the contracts. Fix: once decimals() answers, reprice the dropped minutes with events-backfill. If it never will, insert the row into prices.asset_decimals by hand AND force a cold start (any configuration update of the ledger-processor function, e.g. aws lambda update-function-configuration --description): a warm processor reads that table only at cold start and keeps dropping the token until then. Reprice after the cold start.', + metric: new cloudwatch.Metric({ + namespace: 'Prices/Ingest', + metricName: 'TradesMissingDecimals', + dimensionsMap: { Environment: config.envName }, + statistic: 'Sum', + period: cdk.Duration.minutes(5), + }), + threshold: 1, + evaluationPeriods: 1, + datapointsToAlarm: 1, + comparisonOperator: + cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + // Emitted only when trades were dropped: "missing" is healthy. + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + }, + ); + this.ledgerProcessorTradesMissingDecimalsAlarm.addAlarmAction(snsAction); + this.ledgerProcessorTradesMissingDecimalsAlarm.addOkAction(snsAction); + // Task 0100 — layer 3 of the coverage model. Layer 2 (the alarm above) // sees only pools shaped like a venue we already index; the weekly coverage // sweep looks at EVERY contract emitting swap/trade-shaped events over a @@ -1749,6 +1867,56 @@ export class ObservabilityStack extends cdk.Stack { `${count} or more price_ohlcv_1m candles of the last 2 h carry an asset_id or quote_asset_id that prices.assets does not hold. Since task 0139 a writer sends the identity and ClickHouse derives both ids; an orphan means a writer sent ids itself (a stale pre-0139 binary or Lambda version: RowBinary is width-exact, so an old u32 writer misaligns rather than failing) or wrote candles before their assets rows. Stop that writer first, then find it from the rows' source column. Runbook: docs/runbooks/0139-asset-id-migration.md. First rung is fixed at 1.`, zeroLadder, ); + // Stored candles outside the OHLC band, or with a price <= 0, on all seven + // tiers (task 0236). Same ladder, same reasoning as the zero invariants + // above: the healthy reading is exactly 0, so the first rung carries the + // meaning and the rest say how fast it is growing. Since task 0229 the + // /ohlcv clamp repairs a crossed candle on the way out, so this is the only + // place a corrupt stored row is seen. See + // packages/rollup-freshness-probe/src/ohlc_band.rs and ADR-0011 section 3. + this.ohlcBandAlarms = usdSanityRungs( + 'CandleBandViolations', + 'OhlcBandAlarmCount', + 'ohlc-band', + (count) => + `${count} or more stored candles on price_ohlcv_{1m,15m,1h,4h,1d,1w,1M} (FINAL) are internally inconsistent: priced (pf_trade_count > 0) yet low > least(open, close), high < greatest(open, close) or low > high, or any price <= 0. Healthy is exactly 0 (prod baseline 2026-09-28). A priced _1m row with close = 0 also fires zero-invariant (expected overlap); low = 0 beside a positive close fires only this. The window is on BUCKET time, 48 h widened by one bucket per tier: a backfill into older buckets is NOT covered. The value sums tier-rows, so one bad minute can count up to seven times; a violation on a coarse tier with a clean _1m means a rollup defect. Find the writer before repairing - /ohlcv clamps since task 0229, so the API will not show it. Per-tier counts: probe log field ohlc_band_detail. See packages/rollup-freshness-probe/src/ohlc_band.rs and ADR-0011 s3. First rung is fixed at 1.`, + // The zero ladder above: the first rung is pinned at 1. + zeroLadder, + ); + + // SAC contract identities (task 0242 D6). A SAC is a facet of its classic + // asset; a contract row of assets FINAL that is a SAC is the 0242 defect. + // One alarm, not the ladder: usdSanityRungs takes no actionsEnabled, and the + // actions must start off because production reads 36 until the heal + // (13 split pairs + 23 SACs whose classic was never held, 2026-10-05). + // Actions off does not stop evaluation: the alarm sits in ALARM, red on the + // Row-0 strip, until the heal. It stays on the strip, which shows every + // alarm (SCF Tranche 3 AC 8); the description says the red is expected. + this.sacContractIdentitiesAlarm = new cloudwatch.Alarm( + this, + 'SacContractIdentitiesAlarm', + { + alarmName: `prices-${config.envName}-sac-contract-identities`, + alarmDescription: + "Contract rows of prices.assets FINAL whose address is a SAC: the sac_address of a classic row, or a contract BE flags is_sac in default.soroban_contracts (task 0242 D6). Baseline 36 until the 0242 heal, which is why actions start disabled (opsAlarms.sacContractIdentitiesActionsEnabled) and the state reads ALARM until then: that red tile on the dashboard's alarm strip is expected, not an incident. After the heal it reads 0, and a non-zero value means a SAC was minted as a Contract identity: a second asset_id for an asset we hold, splitting its candles and volume. Find it: SELECT asset_id, contract_address FROM prices.assets FINAL WHERE contract_address IN (SELECT sac_address FROM prices.assets WHERE sac_address != '') OR contract_address IN (SELECT contract_id FROM default.soroban_contracts WHERE is_sac). Runbook: docs/runbooks/0242-sac-identity-heal.md.", + metric: new cloudwatch.Metric({ + namespace: 'Prices/Rollup', + metricName: 'SacContractIdentities', + dimensionsMap: { Environment: config.envName }, + statistic: 'Maximum', + period: cdk.Duration.minutes(15), + }), + threshold: 1, + evaluationPeriods: 2, + datapointsToAlarm: 1, + comparisonOperator: + cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + actionsEnabled: config.opsAlarms.sacContractIdentitiesActionsEnabled, + }, + ); + this.sacContractIdentitiesAlarm.addAlarmAction(snsAction); + this.sacContractIdentitiesAlarm.addOkAction(snsAction); // Materialized-view drift, on a schedule (task 0204, gap 3). Task 0142 built // `prices-clickhouse-drift` and NOTHING RAN IT — a check nobody runs is a diff --git a/infra/src/lib/types.ts b/infra/src/lib/types.ts index 31e32902..b49c0551 100644 --- a/infra/src/lib/types.ts +++ b/infra/src/lib/types.ts @@ -170,13 +170,14 @@ export interface EnvironmentConfig { * not a resource under it. * * Why the portal needs it at all: the bundle is served from another - * application's distribution (`portalWebOrigin`), whose `/prices-api/*` + * application's distribution (`portalWebOrigin`), whose `/api/prices-api/*` * behaviour is a static SPA — every extensionless path under it is rewritten - * to `/prices-api/index.html` at the edge and answered `200 text/html`, and - * `/api/config` there is a `301` into it (task 0326). There is nothing on - * that host for a same-origin call to reach, so the bundle calls this - * hostname directly, cross-origin and same-site — the pattern the explorer's - * own SPA uses for its API. + * to `/api/prices-api/index.html` at the edge and answered `200 text/html`, + * and `/api/config` there is a `301` into it (explorer task 0660; that host's + * `/api/` is not this API's). There is nothing on that host for a + * same-origin call to reach, so the bundle calls this hostname directly, + * cross-origin and same-site — the pattern the explorer's own SPA uses for + * its API. * * `hostedZoneName` must be a suffix of `domainName`, and the zone must live * in this account: the certificate's validation record and the alias records @@ -188,9 +189,22 @@ export interface EnvironmentConfig { readonly hostedZoneName: string; }; + /** + * A second hostname for the same API on the `sorobanscan.com` zone (task + * 0340), beside `apiDomain`, which keeps serving. That zone's DNS is + * Cloudflare's (soroban-infrastructure's `cloudflare/`), so this stack writes + * no record: its record takes the `SorobanscanApiRegionalTarget` output. + * `certificateArn` names an issued certificate in this region that covers + * `domainName`. Unset: no second hostname. + */ + readonly sorobanscanApiDomain?: { + readonly domainName: string; + readonly certificateArn: string; + }; + /** * The origin the portal's bundle is served from — - * `https://sorobanscan.rumblefish.dev`, scheme and host, no path. + * `https://sorobanscan.com`, scheme and host, no path. * * Two things hang off it and only these two: it is the one origin the * portal routes' CORS answer names (`allowOrigins` on the gateway's @@ -451,8 +465,13 @@ export interface EnvironmentConfig { * and the rationale below for `100` / `10000` is about the USDT metrics, not * about it. That ladder therefore pins its first rung at `1` in the stack * and borrows only the rungs above it from here, so raising the first rung - * to quiet a USDT ladder cannot hide a violation there. **Changing the - * higher rungs still re-tunes all three ladders.** + * to quiet a USDT ladder cannot hide a violation there. Task 0139 added + * two more on the same pinned ladder (`zeroLadder` in the stack): + * `AssetIdCollisions` and `AssetIdOrphanCandles`. Task 0236 adds the + * sixth, `CandleBandViolations`: all seven `price_ohlcv_*` tiers, `FINAL`, + * over a 48 h window widened by one bucket per tier, also healthy at + * exactly 0 and on `zeroLadder`. **Changing the higher rungs re-tunes all + * six ladders.** * * ⚠️ **Why a ladder and not a single `>= 1`.** A wrong `close_usd` is a * **standing condition** — it stays wrong until a person repairs it — so it @@ -477,6 +496,14 @@ export interface EnvironmentConfig { * Must be strictly increasing positive integers. */ readonly usdSanityEscalationCounts: readonly number[]; + /** + * Actions of the `SacContractIdentities` alarm (task 0242 D6). Starts + * `false`: production holds 36 SAC contract identities until the 0242 heal, + * so an enabled alarm would latch from its first datum. The last step of + * docs/runbooks/0242-sac-identity-heal.md sets it `true`. A config key, not + * a console toggle, because the next CDK deploy would undo the toggle. + */ + readonly sacContractIdentitiesActionsEnabled: boolean; /** * Optional AWS Chatbot → Slack routing for the ops-alarms topic (task 0056). * When set, `ObservabilityStack` subscribes `prices-{env}-ops-alarms` to a @@ -809,6 +836,26 @@ export function validateConfig(config: EnvironmentConfig): void { `apiDomain.hostedZoneId must be a Route 53 hosted zone id, got: "${hostedZoneId}"`, ); } + if (config.sorobanscanApiDomain) { + const second = config.sorobanscanApiDomain; + if ( + !hostname.test(second.domainName ?? '') || + second.domainName === domainName + ) { + errors.push( + `sorobanscanApiDomain.domainName must be a lowercase hostname other than apiDomain.domainName, got: "${second.domainName}"`, + ); + } + if ( + !/^arn:aws:acm:[a-z0-9-]+:\d{12}:certificate\/[0-9a-f-]{36}$/.test( + second.certificateArn ?? '', + ) + ) { + errors.push( + `sorobanscanApiDomain.certificateArn must be an ACM certificate ARN, got: "${second.certificateArn}"`, + ); + } + } if ( typeof config.portalWebOrigin !== 'string' || !/^https:\/\/[a-z0-9.-]+(:\d+)?$/.test(config.portalWebOrigin) @@ -1081,6 +1128,11 @@ export function validateConfig(config: EnvironmentConfig): void { `opsAlarms.currentPricesFreshnessSeconds must be an integer above the healthy peak of ${CURRENT_PRICES_HEALTHY_PEAK_SECONDS}s (60 s refresh interval + 40 s worst accepted refresh), or the alarm fires on a healthy table; got: ${ops.currentPricesFreshnessSeconds}`, ); } + if (typeof ops.sacContractIdentitiesActionsEnabled !== 'boolean') { + errors.push( + `opsAlarms.sacContractIdentitiesActionsEnabled must be a boolean, got: ${ops.sacContractIdentitiesActionsEnabled}`, + ); + } if ( !Number.isInteger(ops.ledgerProcessorLagSeconds) || ops.ledgerProcessorLagSeconds < 1 diff --git a/lore/0-session/team.yaml b/lore/0-session/team.yaml index 83b1b2c3..0953e2b6 100644 --- a/lore/0-session/team.yaml +++ b/lore/0-session/team.yaml @@ -12,3 +12,8 @@ stkrolikiewicz: name: 'Stanisław Królikiewicz' role: 'Developer' focus: 'Stellar prices API' + +mkowalski: + name: 'Marek Kowalski' + role: 'Developer' + focus: 'Stellar prices API' diff --git a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md index 5dbaab82..f809aa5e 100644 --- a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md +++ b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md @@ -202,14 +202,53 @@ Prior art: soroban-block-explorer ADR 0051 (a SAC is a facet of the classic asse - **One PR** with ordered commit slices: resolver + live, then `events-backfill`, then probe + alarm, then API alias. This overrides the ~400-line norm for this task. - The heal is a runbook section, with no window. - The new `events-backfill` binary is swapped on ch-prod-01 only between phase-3 stages, never during an `amm` step. +- **D8 Observability after the cleanup (Adam, 2026-10-06).** The PR deploys Compute and EventBridge only. Observability, with both 0242 alarms, is deployed in runbook §6, after §5, with the `SacContractIdentities` actions already on. + - Why: deployed earlier, that alarm would sit in ALARM on the dashboard's Row-0 alarm strip through stage C, stage D and the cleanup. That strip is [[0294]]'s SCF evidence (all alarms OK). + - Cost: until §6 no alarm watches `SacUnprovenSkipped`. The runbook has the operator read the metric by hand after the cold start and at least daily. + - Constraint on the team: no `make deploy-production-observability` from `develop` between the merge and §6. Such a deploy creates the alarms early, and the red tile's description says it is expected. +- **D9 Compute after stage D (Adam, 2026-10-06, from Oskar's review of PR #390).** Compute (live + API) deploys after phase-3 stage D and before the §5 cleanup. + - Why: the API alias answers a SAC address as its classic. Until stage C has rebuilt a SAC's months under the classic id, that classic holds only part of the history, so `GET /v1/assets/{C…}/ohlcv` would lose the pre-heal AMM series. Group B is the visible case: its classics were seeded on 2026-10-06. + - Cost: the new live code lands two or three days later. The seed already resolves every known SAC on the old live after its next cold start. Plan choices confirmed by Adam on 2026-10-05 (GSD plan `261005-htu`): - **PC3 Fail closed at the live cold start.** If the Lambda cannot read BE's `is_sac` set, Init fails, like the other cold reads. An empty set would mint `Contract` identities, which D2 forbids. The cost: if BE's `default.soroban_contracts` breaks, live ingest stops until it is fixed. - **PC6 A summary without the new line passes.** The orchestrator STOPs only when `unproven sac swaps:` is present and non-zero. A binary built before 0242 prints no such line, so phase 3 keeps running on it, which is why D4 holds. +Changed by Adam on 2026-10-06, after Oskar's review of PR #390: + +- **PC3 reversed: fail open.** An unreadable `is_sac` set no longer fails Init. The container ingests with no candidates, logs a WARN and publishes `SacCandidatesUnavailable` on every run, with an alarm on it (deployed in §6, D8). + - Why: the candidates guard only a SAC trade with no in-band proof, which happened 0 times in 338,601 legs. Failing closed stopped SDEX, oracle and AMM ingest over one BE table. + - The probe's `SacContractIdentities` counts any identity minted meanwhile. +- **PC6 amended.** A month without the `unproven sac swaps:` line gets a visible note. Once a month of the same state has printed the line, a later month without it is a STOP: the host binary went back to a pre-0242 build. Stage C on the pre-0242 binary is unaffected. - **PC9 Residual rows of a SAC with no classic.** Such a row is the only copy of its trades, so it is never deleted silently. The operator either re-runs that month through phase 3 or records the loss on this task. The rows are copied beside the 72 metadata rows for rollback. The 202404 Z/Q row, from stage B before the seed, is the expected case. Open: **U3**. About 75.6k Soroswap swaps on BLTA/BLTB/BLTC/PPRIME/LumenJoule produce no candle under any id, and the cause is not established. It must be settled before AC3 promises numbers for those assets. +Stays in scope of 0242 (Adam, 2026-10-05): this task closes everything that concerns SACs. + +**U3 settled (2026-10-05).** Neither the dust rule nor an ingest bug: those swaps were missing from the candle tables because of the 0139 re-key. +- Each of the five SAC contract identities shared its old UInt32 id with an unrelated classic asset: 123376 LumenJoule/BTC, 123377 BLTB/ETH, 123378 BLTA/AST, 123379 BLTC/ASC1148, 123380 PPRIME/KWD. +- `asset_id_map_0139` therefore marks all five `colliding`. The re-key copies only `mapped` and `sentinel` (`rekey.rs` `COPYABLE`), so their candles stayed in `price_ohlcv_*__pre0139`. +- Before 0139, the 1h counts matched BE's swaps exactly for 202603–202606. +- Jul–Sep were already short before 0139: the 07-09 Soroswap gap (0101) and burst minutes before the 0282 fix. +- Locally, five real transactions from 202603–202609 produce candles under the classic identity on the 0242 branch. + +No code fix is needed. Phase 3 stage C, run with the 29-row seed, rebuilds them under the classic ids. AC3 for these five then rests on stage C's post-check; the BLT assets will have volume but no price, because their fills are dust. + +The six latent SACs have other causes. SUSHI and HYPE traded before SushiSwap was indexed, and phase 3 covers that under 0290. STELLA, TESTTTT, TEST77 and TEST12 have pools with no swaps. + +**October window (measured 2026-10-05, `dev_read`).** Since 2026-10-01, no candle on any tier is keyed on the 36 SAC ids, and no contract row has been minted since BURNMKR. No decision is needed now. The runbook re-checks months after 202609 before the cleanup, and any non-zero result goes through PC9. + +## Heal log + +- **2026-10-06, seed (runbook §3).** Stage B finished 202404. The seed ran at 10:29:11 UTC as `dev_shared`, straight from `seed-0242.sql` rather than through the staged variant of §3. There is therefore no `bak_0242_seed_stage`, and `bak_0242_seed` was created afterwards from the 29 rows with that `created_at`. + - Check: `29 29 / 29 / 0`, with collisions at 0. No candle under any seeded id on any tier. + - Comet's `pool_registry` row is present (1). +- **2026-11-02, a new split (found in [[0338]]).** BLNT's SAC `CCBNWP5L…` traded on Comet from 2026-10-26 14:22 UTC, before SDEX interned its classic at 18:42, so 10 trades sit on the `soroban` id 14236029787030528679. The "no contract row since BURNMKR" of the October window no longer holds. + - The reprice in 0338 rewrites those trades under the classic id, so the rows left on that id are copies. Add it to the residual cleanup. It does not fall under PC9, because the classic exists. +- **2026-11-05, deploy (runbook §4e).** After stage D. Pre-check as `prices_writer` from fishuser-hero (Adam): 4,137 distinct `is_sac` contracts readable. `deploy-production-compute` (11:33 UTC) and `deploy-production-eventbridge` (11:35) from `develop` `43a28399`; the previous CodeSha256s are in `~/heal-0242/deployed-before.tsv` on stkrolikiewicz's laptop. Observability not deployed (§6). + - Cold start: `loaded the is_sac contract set` with `sac_contracts` 4137, then `resumed cursor from ClickHouse` at 65318468. No WARN. + - `SacUnprovenSkipped` and `SacCandidatesUnavailable`: no datapoints in the first hour. + - Probe: `sac_contract_rows` **40**, not the runbook's baseline 36 (`sac_contract_scanned` 74, `be_sac_contracts` 4137). The old build ran until this deploy, so contract rows minted since the October baseline (BLNT above among them) count too. §5 counts its residual from today's number, not from 36. ## Acceptance Criteria diff --git a/lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md b/lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md index a7e268df..1185ad71 100644 --- a/lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md +++ b/lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md @@ -197,6 +197,16 @@ history: finished" need two readings, because the frontier gauge skips months still marked exhausted from before the re-ingest. The criterion above now matches. + - date: "2026-10-16" + status: active + who: akot + note: > + 1m drain on fishuser-hero finished 2026-10-15 14:27 UTC (202608). + After-checks run read-only: 1 802 reachable 1m zeros, all USDT + pivot-window residue; post_run_0228_it 2/2; no USDT peg (0212); + no carried product on 5 tiers (0207); seven dust days within 5 % + of Bitstamp; 2 round-price XLM/USDC buckets on 15m/1h, not dust, + awaiting decision. The frontier gauge (49) is a recorded deviation. --- # Candles are built from dust fills in the wrong order @@ -911,6 +921,101 @@ stays DISABLED until M3 completes (decided 2026-09-14). **Rough end:** history rebuilt by mid-October, provided 0300 lands before the loop reaches 202405. +### Phase 3 — after-checks (2026-10-16) + +Runbook §7b-2 (a)(b), §7c, §7e, §7e-2, §7f, §8.3 and §8.4, run read-only on +prod on 2026-10-16 ~08:00–09:30 UTC. Asset ids by identity: XLM +`5209214538714742248`, USDC `1368471977071604712`, USDT +`11430142929638344312`. + +**The 1m drain on fishuser-hero is finished.** Its last batch ran on +2026-10-15 at 14:27:10 UTC, on 202608. That month is `END`, two months +back; 202609 on belongs to the live sweep. `system.query_log` shows +~415 M rows written from 10-15 02:00 with 0 exceptions. Since 14:27 only +the live ingest and the hourly Lambda write to `price_ohlcv_1m`. + +**§7b-2 (a)/(b) cannot pass as written, so the drain was measured +directly.** + +- `EnrichmentFrontierMonthsPending` reads 49. It went 52 → 49 over 48 h, + not one month per hour. +- 45 months, 202212–202608, have no row in `prices.enrichment_frontier`. + 202103 and 202209–202211 are `pending`. +- The Lambda takes 202103 first every hour and leaves it `pending`, with + 81 112 zeros, none of them reachable (below). Why is not investigated. + +In its place, runbook ③'s `coarse_zeros` columns were computed on +`price_ohlcv_1m FINAL`, one partition per query, over 201511–202608 +(130 months): + +| zero_usd | no_volume | no_price | other_quote | before_reference | reachable_left | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 245 112 743 | 0 | 27 658 212 | 212 555 564 | 4 897 165 | **1 802** | + +Every one of the 1 802 is USDT-quoted. Each sits more than 24 h after the +last USDT/USDC candle, which is the pivot-window residue of §7b-2 ⑤: + +| Month | Rows | When | USDT/USDC gap | +| --- | ---: | --- | --- | +| 202102 | 1 787 | 02-08 → 02-14 | silent from 02-09 15:35 to 02-12 01:09 | +| 202304 / 202305 / 202306 | 6 / 1 / 8 | one day each | 24 h 3 min to ~25 h | + +**§7c — `post_run_0228_it`: green, 2 of 2.** Run with +`POST_RUN_0228_VERSION_BEFORE_1W=45340987003` and `_1M=1407435801128`, +both captured on fishuser-hero on 2026-10-13 before `finish`. + +**§7e — USDT peg: none.** The runbook's `peg_written` (rate ≥ 0.9) cannot +tell the peg from the pivot before the depeg: USDT/USDC traded at +0.98–1.00 from 2021-02 to 2022-05. Peg was therefore counted as +`close_usd = close` exactly, per year: + +| Tier | Years | `peg_exact` | Rate < 0.5 | +| --- | --- | ---: | --- | +| 1m | 2023–2026 | 0 | every priced row (2023: 77 000 of 77 000) | +| 1m | 2021–2022 | 6 872 | — | +| 1h | 2023–2026 | 0 | every priced row (2023: 30 685 of 30 685) | +| 1h | 2021–2022 | 988 | — | + +All 988 on 1h fall in 2021-03 … 2022-05. In 960 of them, USDT/USDC's 1h +close in the same hour is exactly 1.0, so they are measured, not pegged. +Priced USDT-quoted rows start at 2021-02-07 19:42, the first USDT/USDC +candle. + +**§7e-2 — carried product: no rows** on 1h, 4h, 1d, 1w or 1M, XLM and USDT +legs. Before the re-ingest there were 1 757. + +**§7f — nothing to start.** All six `mv_reconcile_*` read `Scheduled` and +refreshed at 08:00 UTC. + +**§8.3 — the seven dust days are within 5 % of Bitstamp XLM/USD (1d +close).** + +| Day | Source | Close | Bitstamp | Deviation | +| --- | --- | ---: | ---: | ---: | +| 2021-09-05 | sdex | 0.39600 | 0.39803 | −0.51 % | +| 2021-12-28 | sdex | 0.27520 | 0.27500 | +0.07 % | +| 2022-01-11 | sdex | 0.26252 | 0.26314 | −0.23 % | +| 2023-03-11 | sdex | 0.08220 | 0.07936 | +3.58 % | +| 2023-03-15 | sdex | 0.08382 | 0.08399 | −0.21 % | +| 2026-02-09 | four sources | 0.15956–0.15987 | 0.1595 | +0.03 to +0.23 % | +| 2026-04-02 | five sources | 0.16254–0.16331 | 0.16295 | −0.25 to +0.22 % | + +**§8.4 — two buckets, not zero; neither is quantisation.** The check +applies 0278's definition to XLM/USDC: a close that is an exact `n/d` +with `d ≤ 200`, more than 5 % off the bucket VWAP. 4h, 1d and 1w hold 0; +15m and 1h hold one each: + +- **1h 2023-07-13 17:00, sdex: close 0.125 (1/8).** VWAP 0.1188. The bucket + holds 4 544 price-forming fills and 4.4 M XLM, and Bitstamp's range that + hour was 0.1137–0.1425. It is a real price in a fast hour that happens to + be round. +- **15m 2026-03-09 10:00, aquarius: close 0.128 (16/125).** VWAP 0.1499. The + bucket holds 11 fills and 48 k XLM. Bitstamp was 0.1496–0.1501, so this + is an off-market pool price, 14.6 % below, not stroop dust. Whether + anything was priced from it is not checked. + +Whether these two count as explained is Adam's decision. + ## Out of scope - D9, the cross-source rule on the read path (median instead of largest @@ -1019,6 +1124,9 @@ Phase 3: 85 699 / 30 064 / 10 938 on 15m / 1h / 4h / 1d / 1w) is zero after the re-ingest and pre-roll. [[0266]] closed into this criterion (2026-09-24, [[0314]]) — it is that task's only remaining check. + 2026-10-16: the seven days are within 5 % (worst +3.58 %). Quantised + buckets: 1 on 15m, 1 on 1h, 0 on 4h/1d/1w. Neither is dust; awaiting + Adam's decision ("Phase 3 — after-checks"). - [ ] The whole history re-enriched (`close_usd > 0` wherever a reference exists) and `post_run_0228_it` green on the repaired reference; 0228's reset-mode campaign recorded as superseded, not run. @@ -1032,6 +1140,11 @@ Phase 3: non-`exhausted` or swept before `T7` (the last §4e re-ingest's end). The gauge alone reads 0 for a re-ingested month still marked `exhausted`. + 2026-10-16: the drain finished 10-15 14:27 UTC, after the coarse + re-price, which the runbook allows. Neither reading passes: the + gauge is 49 and 45 months have no frontier row. A direct count on + 1m leaves 1 802 reachable zeros, all explained ("Phase 3 — + after-checks"). Recorded as a known deviation. - [ ] Dry run recorded for all six coarse tables. Every table lists months up to END (two months back), and none reports 0 months. - [ ] `coarse-repair` run in plain mode (no `--reset-*`) on `price_ohlcv_15m`, @@ -1042,18 +1155,22 @@ Phase 3: `reachable_left` ~0 (any remainder explained through the drill-down); `before_reference`, `no_price` and `no_volume` unchanged; `other_quote` not higher than the baseline. - - [ ] `post_run_0228_it` green after this step. -- [ ] No USDT-quoted `price_ohlcv_1m` row carries the $1 peg after the + - [x] `post_run_0228_it` green after this step. ✅ 2026-10-16, 2 of 2. +- [x] No USDT-quoted `price_ohlcv_1m` row carries the $1 peg after the re-enrichment ([[0212]]'s query: `peg_written = 0`, `pivot_written > 0`, measured on 1m and on one coarse tier) — the re-ingest replaces the 1.56 M rows 0172/0182 never reached, so 0212 closes here (re-ingest runbook §7e). 0212 is archived pending this check ([[0314]]). -- [ ] No coarse row whose `close_usd` contradicts its own `close` after §7a + ✅ 2026-10-16: `close_usd = close` on 0 rows from 2023 on, on 1m + and 1h. The 988 at ×1.0 on 1h (2021-03 … 2022-05) are measured + USDT/USDC closes of exactly 1.0 before the depeg. +- [x] No coarse row whose `close_usd` contradicts its own `close` after §7a and §7b-2 ([[0207]]'s query, re-ingest runbook §7e-2: no row on 1h, 4h, 1d, 1w, 1M more than 10x off its quote leg's median rate in the bucket, XLM and USDT legs). Before: 1 757 rows, all XLM-quoted, 2022-01 … 2022-04 — the pre-0286 carried product, which the rate form cannot write. 0207 is archived pending this check. + ✅ 2026-10-16: no rows on any of the five tiers. ## Implementation Notes diff --git a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md new file mode 100644 index 00000000..bafa78b2 --- /dev/null +++ b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md @@ -0,0 +1,323 @@ +--- +id: "0329" +title: "Soroban AMM amounts are scaled as if every token had 7 decimals — every contract token's price is off by 10^(7 − decimals)" +type: BUG +status: active +assignee: stkrolikiewicz +related_adr: [] +related_tasks: ["0286", "0242", "0210", "0097", "0060", "0048", "0018"] +tags: [layer-backend, priority-high, effort-medium, ingest, soroban, data-correctness, defect] +links: + - "../../../packages/prices-ingest-core/src/soroban.rs" + - "../../../packages/asset-discovery/src/symbols.rs" + - "../../../docs/database-schema/amm-trades-schema.md" +history: + - date: "2026-10-07" + status: active + who: stkrolikiewicz + note: > + Observability stack deployed (alarm OK). First post-deploy prices are + right: SolvBTC 82 512 $, 1.2 % under CoinGecko; deJTRSY 1.027 $. Seeded + prices.asset_decimals for all 46 historical contract tokens (7 → 53). + Swapped the Hetzner `~/events-backfill` for the #395 build, so 0286 phase + 3 picks it up at its next AMM step. Affected tokens trade only from + 2025-07, and phase 3 stands at 2024-09, so no month it already did needs + a redo; Oskar must restart on the new reingest_0286.py before 2025-07. + Recent-window repair (2026-09-01 → deploy) under way. + - date: "2026-10-07" + status: active + who: stkrolikiewicz + note: > + PR #395 merged (24e07cc0). prices.asset_decimals created on prod and + seeded with the 7 tokens. Only the ledger-processor was deployed, through + `update-function-code` rather than `make deploy-production-compute`: the + Compute stack would also have shipped #373's prices-api (0274, + `price_basis`), whose schema is not on prod, and that breaks every + price/list/batch request. Live and healthy. See "Rollout log". Still + open: the first post-deploy price, the observability stack, the history + repair, and the reply to SBE. + - date: "2026-10-06" + status: active + who: stkrolikiewicz + note: > + Renumbered from 0328. This task was committed as 0328 but its push did + not land, and another 0328 (board page assignee) was pushed and merged + meanwhile. An unpushed task reserves no ID. + - date: "2026-10-06" + status: active + who: stkrolikiewicz + note: > + Activated for implementation with the proposed design accepted: + SEP-41 decimals() over RPC, resolved by the ingest on a cache miss, + persisted in prices.asset_decimals, never defaulting to 7. + - date: "2026-10-06" + status: backlog + who: stkrolikiewicz + note: > + Reported by Karol (SBE) on 2026-10-05: SolvBTC, XAUM and XRP priced at + 10^(7 − decimals) of market. SBE holds off pricing pools that contain + these tokens until we fix it. Root cause confirmed in code; decimals of + all seven contract tokens the API lists were read on mainnet, and none + is 7. Design for where decimals come from proposed below, not yet + decided. +--- + +# Soroban AMM amounts are scaled as if every token had 7 decimals + +## Summary + +`amm_trade_to_tick` turns both raw i128 legs of every Soroban swap into +`Decimal` with one fixed scale, `AMM_AMOUNT_SCALE = 7` +([soroban.rs:37](../../../packages/prices-ingest-core/src/soroban.rs), used at +:859–860). Seven is right for a SAC (classic assets are 7-decimal by protocol) +and wrong for a pure contract token, which has its own `decimals()`. Against a +7-decimal counter-leg the stored price is `true × 10^(7 − d)`. The constant came +in with 0060's sizing backfill as a "documented sizing-measurement +approximation" and never got replaced, though 0018 and 0048 §7.2 had both +specified reading `decimals()`. `amm-trades-schema.md` §3 still lists decimals +normalisation as "_Still open._" + +## Evidence (2026-10-05) + +`/v1/assets?type=soroban&min_volume_usd=0` lists 7 assets. Decimals were read +with `stellar contract invoke --network mainnet --send no -- decimals`. + +| Token | Contract | decimals | Our price | × 10^(d−7) | Corrected | +|---|---|---|---|---|---| +| SolvBTC | `CBIJBDNZ…M6VN` | 8 | 8 580.10 | ×10 | ~85 800 | +| XAUM | `CC2RBGYN…VAGO` | 9 | 41.92 | ×100 | ~4 192 | +| XRP | `CB7OOP3V…37XP` | 6 | 14.78 | ÷10 | ~1.48 | +| deJTRSY | `CBI7UCH5…IHRV` | 18 | 1.026e-11 | ×10¹¹ | ~1.03 | +| deJAAA | `CC64WBDG…YGSL` | 18 | 1.05e-11 | ×10¹¹ | ~1.05 | +| BnUSD | `CCT4ZYIY…X7KP` | 18 | 9.98e-12 | ×10¹¹ | ~0.998 | +| xSolvBTC | `CAUP7NFA…772J` | 8 | unpriced | — | — | + +**Not proven:** that the other leg in every priced pool is 7-decimal. This is +inferred only from the corrected prices matching the market. + +## Impact + +- OHLC, `close_usd` and therefore `price_usd_series*` (views over the candle + tables) are wrong for every contract-token asset, over its whole history. +- Base volume in token units is inflated by 10^(d − 7). USD volume is right, + because the price and quantity errors cancel. +- An asset quoted *in* a contract token gets a correct USD price (the two + errors multiply out), but its price in that token's units is wrong. +- 18-decimal tokens: a price near 1e-11 keeps only 3–4 significant digits in + `Decimal(38,14)`. A token under about $0.001 falls below 1e-14 and its fills + are classified non-price-forming (`price_survives_column_scale`). Rescaling + stored rows can recover neither case, so history needs a re-ingest, not an + `UPDATE`. +- `price_forming_i128` classifies the raw integer legs before scaling, so it + is unaffected. + +## Design: where decimals come from (accepted 2026-10-06) + +**Source: the token's SEP-41 `decimals()`, read once via RPC `simulateTransaction`.** +This is the only universal source. Swap events carry raw i128 only. Instance +storage layout depends on the implementation (soroban-token-sdk `METADATA`, +OpenZeppelin, custom tokens). The live ingest also never sees an old token's +instance entry in ledger meta. A SAC is 7 without a call. A SAC we still hold +as a Contract identity ([[0242]]) answers 7 over RPC anyway, so it needs no +special case. Decimals are fixed at deploy for every real token, which is the +same "absence, not staleness" rule [[0210]] applies to `prices.asset_symbol`: +fetch once, never refresh. + +**Store:** a new `prices.asset_decimals (contract_address, decimals, fetched_at)`, +ReplacingMergeTree. Live ingest and backfills read the same value. + +**Resolve in the ingest on a cache miss, not in asset-discovery.** The price is +fixed at write time, and asset-discovery runs hourly, so a new token's first +swaps would land before its decimals do. That leaves a guessed price or a hole +to re-price by hand for every new token. Load the table at run start (it holds +tens of rows). A miss costs one RPC call with the existing 5 s timeout and is +persisted for every later run. Reuse `build_simulate_envelope` from +asset-discovery, which is already generic over the function name, by moving it +into a shared crate. Lambdas have egress (no VPC), and `SOROBAN_RPC_URL` is +already the convention. + +**On failure, or decimals > 28 (rust_decimal's maximum scale): never fall back +to 7.** Drop the tick, record it next to `prices.unresolved_pools`, and repair +with `events-backfill` once the token resolves. A wrong price is worse than a +missing one. + +Rejected: +- **Hard-coded map of the 7 tokens.** Every new token is mispriced or unpriced + until a PR lands. Since the RPC code exists, the map is barely smaller. +- **Asset-discovery only.** Leaves an hourly gap and a manual re-price per new + token. +- **Instance storage from ledger meta.** Layout varies by implementation, and + old tokens never appear in the meta. + +## Plan + +1. Shared simulate helper and a `decimals()` resolver with the three-way + outcome from `symbols.rs`. +2. `prices.asset_decimals` DDL. +3. `amm_trade_to_tick` scales each leg by its own decimals: SAC = 7, unresolved + → drop and record. This is the only production call site of the scale. +4. Backfills (`events-backfill` and the soroban path of `sdex-backfill`) share + the same resolver. +5. **History rides [[0286]] phase 3 if timing allows.** Stages B/C re-ingest the + AMM era through `events-backfill`. If this fix is on `fishuser-hero` before + stage C reaches the months these tokens traded, nothing extra runs. + Otherwise, a targeted `events-backfill` plus pre-roll over those pools' + range. Coordinate with the phase-3 operator. +6. Tell Karol (SBE) when prices are correct. + +## Implementation (branch `fix/0329_soroban-token-amounts-assume-seven-decimals`) + +- `prices_ingest_core::soroban_rpc`: the `simulateTransaction` envelope and the + Absent/Transient boundary, moved from asset-discovery's `symbols.rs`, which now + delegates to it and re-exports the old names. +- `AssetRegistry::decimals_of`: classic identities and SACs → 7, a `Contract` + → its resolved decimals or `None`. `amm_trade_to_tick` scales each leg by its + own decimals. An unknown leg yields no tick, interns nothing, and lands in + `LedgerSoroban::missing_decimals`. +- `prices.asset_decimals` + `OhlcvWriter::{load,write}_decimals` + + `DecimalsResolver` (`decimals()` must be a `U32` ≤ 28). +- Callers decode, resolve what was missing, persist it, record it, and decode + that ledger again: the live reconcile loop, `events-backfill`, and + `sdex-backfill` in `combined` mode. + +### Design decisions + +#### Emerged + +1. **Dropped trades are counted, not recorded in a table.** `unresolved_pools` + is keyed by pool, so there is no row for a token. Instead the live processor + counts them on `RunStats`, WARNs with the contracts, and publishes + `TradesMissingDecimals`, alarmed like `UnregisteredPoolEvents`. Both + backfills print `trades dropped (decimals)`, and `reingest_0286.py` marks a + month with a non-zero count DEFECT. Added after review (PR #395). +2. **Absent is terminal per process; Transient depends on the caller.** A + contract that answered with no usable scale is not asked again until a cold + start, and is never persisted as a sentinel. Exception: a node behind the + ledger being decoded (`latestLedger` from the simulate response) gives + `Behind`, which is asked again on the next trade. A contract that gave no + answer is parked for 10 minutes live; a backfill retries it inline + (1/5/20 s) and then on the next trade. Calls run concurrently, so one ledger + costs one RPC timeout at worst. Refined after Adam's review (PR #395). +3. **Decode again rather than a two-phase tick.** Decoding is deterministic and + the registry inserts are idempotent map writes, so the first result is + discarded whole. +4. **`CandleSink::write_decimals` defaults to a no-op.** Only `ClickHouseSink` + persists. The test sinks need no change. +5. **`sdex-backfill` reads the table only in `combined` mode.** The 0286 phase-3 + `sdex-only` runs therefore do not need the table to exist. +6. **An `events-backfill` dry-run resolves but does not persist.** +7. **Phase-3 volume reconcile leaves out contract-token candles.** Their volume + moves by design (an 18-decimal token's base volume falls by 10^11), so + `reingest_0286.py` compares volumes on the other candles only and trades on + all of them. A "before" read without that scope compares trades only, as a + FINDING. Added after review (PR #395). +8. **A price past `Decimal::MAX` drops the trade.** Per-token scales can carry + a factor of up to 10^28, so the quotient uses `checked_div`. Before the + review a plain `/` would panic and stall ingest on that ledger for good. +9. **One `decode_resolving` for all three callers**, generic over a + `ResolveDecimals` trait so its persist-then-record order is tested with a + fake. + +### Deploy order + +1. Apply `init.sql` on the box: it creates `prices.asset_decimals`. The live + processor's init fails without it. +2. Optional pre-seed, so the first cold start needs no RPC call (the Lambda has + a 60 s budget): + `INSERT INTO prices.asset_decimals (contract_address, decimals) VALUES ('CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN', 8), ('CBI7UCH5KGSVQRO5H4SUCZUTZABCITZLRHQQZTWL2TK4RZ72TAR6IHRV', 18), ('CB7OOP3VSAWBZOOTOG2YEFANVU45GVWYUUM5HI32DKLHVKUDOFVQ37XP', 6), ('CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO', 9), ('CC64WBDGS6QQP22QTTIACYIXT3WF7BBQEYOQPLTP7GTKYY7PZ74QYGSL', 18), ('CCT4ZYIYZ3TUO2AWQFEOFGBZ6HQP3GW5TA37CK7CRZVFRDXYTHTYX7KP', 18), ('CAUP7NFABXE5TJRL3FKTPMWRLC7IAXYDCTHQRFSCLR5TMGKHOOQO772J', 8)` +3. Deploy the ledger processor. +4. Repair history right away. Until it is repaired, `vwap_24h` blends old and + new candles for 24 h, and a `carried` price keeps the old candle. Run + `events-backfill` + pre-roll at least over a recent window, and the full + range via 0286 phase 3 if stage C has not passed these months yet. + +### Rollout log (2026-10-07) + +- **Schema and seed.** Done before the merge (07:40 UTC) by the operator on + `deploy@ch-prod-01`, applying only the `CREATE TABLE` from init.sql. Before: + `EXISTS TABLE` returned 0. After: 7 rows (6, 8, 8, 9, 18, 18, 18), matching + the mainnet reads. `prices_writer` holds `SELECT, INSERT … ON prices.*`. +- **Why not the Compute stack.** `develop` carries #373 (0274) besides this + task. Its rollout order is binding: ALTER `current_prices.price_basis`, then + views.sql, then current.sql, then prices-api. On prod all three schema + checks returned 0 (the column, `current_price_usd`, `mv_current_prices`). + `cdk diff Prices-production-Compute` showed both the ledger-processor and + the api-handler code changing. +- **Ledger-processor only.** Built with `make build-lambdas` from `develop @ + 24e07cc0`, 12 bootstraps verified, rustc 1.97.1 / cargo-lambda 1.9.1. + Bootstrap sha256 (base64) `b+M4AVx6wL5PZAqZ/oMc3OoUNry3PZ/6SCoyn7u6Mlc=`. + It carries `prices.asset_decimals`, `TradesMissingDecimals` and + `simulateTransaction`. Zipped (bootstrap only) and uploaded with `aws lambda + update-function-code` at 07:54:51 UTC. `CodeSha256 + aMzmQuGWzyLcyExHr57QA2sB2AB7eReo/X//uYTaZIU=`, arm64, Successful/Active. The + SQS mapping targets `$LATEST` and there are no aliases. +- **Verified.** Cold start at 07:54:55 logged `loaded soroban token decimals + {tokens: 7}`. 12 invocations a minute before and after, 0 errors, 0 + throttles, max duration about 2.4 s. The first run wrote 113 rows (cursor + 64814771 → 64814783). No resolver warnings, no panics. +- **Not yet verified.** A correct price in the API needs a swap after the + deploy: SolvBTC still read 8 326 $ with `as_of 06:11Z`. +- **Drift, until the next Compute deploy.** The CloudFormation template still + names the old ledger-processor asset. The next `make + deploy-production-compute` reconciles it, but it must come after 0274's + schema rollout; Adam has been told. api-handler is untouched (pre-#373). +- **Rollback.** `aws lambda update-function-code --function-name + prices-production-ledger-processor --s3-bucket + cdk-hnb659fds-assets-750702271865-eu-central-1 --s3-key + 8df901778d1c4aaf4384269fc0d085ea512d0e30e6c9934e201f95afdd2c7cd4.zip` (the + package deployed on 2026-10-05). +- **Observability.** `make deploy-production-observability` ran at 08:06 UTC: + the new alarm `prices-production-ledger-processor-trades-missing-decimals` + and the dashboard strip (76 → 77). The diff also listed 39 alarm description + changes. They were mangled-character repairs in the diff's view only: + CloudFormation updated just the alarm, the dashboard and the metadata. The + alarm went to OK, and `TradesMissingDecimals` has never been published. +- **First prices after the deploy.** deJTRSY (18 decimals) read 1.027 $ at + 08:43 UTC, where it had read 1.03e-11 $. SolvBTC (8 decimals) read 82 512 $ + at 09:42 UTC, where it had read 8 326 $; CoinGecko said 83 538 $ (−1.2 %). +- **Seed of the whole history.** 46 contract tokens had a 1d candle and no + decimals row. 45 were read on mainnet; the public RPC answered 13 of them + with 429 at first, and they succeeded on retry. The 46th, + `CDPV3H7C…QTMW` (Axelar USD Coin), fails every call with "Missing Entry + __InterchainTokenMigrationData", so its 6 comes from SBE's + `default.soroban_contract_metadata`. The table went from 7 to 53 rows, by + decimals: 6 → 5, 7 → 39, 8 → 2, 9 → 1, 18 → 6. Besides the first seven, + five more tokens are not 7-decimal: two USDT variants and Axelar USDC at 6, + so their prices were 10× too high; STBL_USST and two test tokens at 18. +- **events-backfill on Hetzner.** Built from `develop @ 63b7661c` for x86_64 + (cargo-lambda/zig; needs glibc ≥ 2.30, the box has 2.39), sha256 + `10872b2393961c056c5dd959900ca509766f1c82c2ea0afe835c825170d70ca8`. + Uploaded as `~/events-backfill-0329`, smoke-tested with `--help`, then put + in place as `~/events-backfill` while no events-backfill process ran. The + old binary (`ee992dd9…`) is kept as `~/events-backfill.pre0329`; rollback + is `mv ~/events-backfill.pre0329 ~/events-backfill`. The box reaches Soroban + RPC (`getHealth` healthy). +- **0286 phase 3.** It stands at 2024-09: the SDEX step ended 2026-10-06 + 23:09 UTC and the AMM step has not run since. Candles of tokens with + decimals other than 7 exist only from 2025-07, so no month phase 3 has done + needs a redo. From 2024-09 on it uses the new binary. The running + orchestrator still holds the old reingest_0286.py in memory, so Oskar must + restart it on the new one before 2025-07. Otherwise months with 18-decimal + tokens stop as DEFECT on the volume drop, and nothing reads `trades dropped + (decimals)`. +- **Recent-window repair.** Scripts are in a local `repair-0329/`, not + committed. The window runs from ledger 64216185 (2026-09-01 00:00) to + 64814771 (minute 07:53, the last ledger the old code processed), with t_e + `2026-10-07 07:54:00`. Only rows of tokens with decimals other than 7 are + replaced, and only after a backup: delete, then events-backfill, then + enrichment, then the affected-only pre-roll. Lightweight `DELETE FROM` is + used because the corrected rows tie the stale ones on `version`. + +## Acceptance Criteria + +- [x] Unit test: an 8-, 18- and 6-decimal leg against a 7-decimal leg gives the + true price and the true base volume + (`each_leg_is_scaled_by_its_own_tokens_decimals`). +- [x] A token whose decimals are unresolved never produces a price + (`a_leg_with_unknown_decimals_prices_nothing_and_is_reported`). +- [ ] After deploy, all six priced tokens are within a few percent of an + external reference. +- [ ] History of the affected assets re-ingested; `price_usd_series*` right over + the full range, with significant digits restored for the 18-decimal + tokens. +- [x] `amm-trades-schema.md` §3 closed; the `AMM_AMOUNT_SCALE` constant removed. diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md new file mode 100644 index 00000000..68696858 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -0,0 +1,281 @@ +--- +id: "0331" +title: "No USD price before 2021-02 — price that history from an imported hourly XLM/USD rate" +type: FEATURE +status: active +assignee: akot +related_adr: [] +related_tasks: ["0286", "0201", "0172", "0212", "0267", "0294"] +tags: [layer-backend, priority-high, effort-medium, enrichment, clickhouse, data-correctness] +links: + - "../../../../docs/runbooks/0286-reingest-history.md" + - "../../../../packages/prices-ingest-core/src/canonical.rs" + - "../../../../packages/prices-clickhouse/schema/current.sql" + - "../../../../docs/scf/milestone-3-evidence.md" +history: + - date: "2026-10-16" + status: active + who: akot + note: > + History run done (Plan step 4). Backup repair_0331_bak_* of + 201802–202101, source = backup on six tiers. coarse-repair + --pre-usdc-xlm-rate over seven tiers wrote exactly what each dry + run listed, with reset 0 everywhere (1h 794,672; 1m 4,971,301; 1d + 114,117). No insert into 202102 or later. 97 % of what stays + unpriced is quoted in other assets with no two-hop path. The + Bitstamp tolerance is still open. + - date: "2026-10-15" + status: active + who: akot + note: > + Decisions amended (5–11): a third-party hourly XLM/USD series replaces + par and the on-chain reference; dollar-token quotes are converted + through their own XLM market; the API builds native XLM's pre-2021 USD + series from the rate at request time. Implemented in PR #403 (PR #401 + closed). Measured read-only on prod before deploy: native XLM 1d + priced on 1081/1081 days, close vs Binance median 0.43 %; 794,672 of + 814,269 in-scope 1h rows priced, 0 set back to 0. + - date: "2026-10-14" + status: active + who: akot + note: > + Reassigned from stkrolikiewicz to Adam, who takes the task over + (PR #401 included). + - date: "2026-10-13" + status: active + who: stkrolikiewicz + note: > + Pre-run report in notes/: 11 dollar tokens traded against XLM before + 2021-02; no usable reference before 2018-08; par agrees with Binance to + 0.3–1.9 % median by year; an off-peg USDT sets the reference on 19 days, + which the median filter cannot stop. Implementation in draft PR #401 + (coarse-repair --par-*), integration test green on CH 26.3.10.60. + - date: "2026-10-13" + status: active + who: stkrolikiewicz + note: > + Activated to implement the pre-2021 par mode in coarse-repair on a + branch, for Adam to review and run after 0286 §7b (~2026-10-16). + - date: "2026-10-13" + status: backlog + who: stkrolikiewicz + note: > + Opened after the M3 reviewer found XLM daily USD prices null over + 2018–2025 and asked for priced candles from 2018 and 2022. 2021-02 + onward is repaired by 0286 §7b-2/§7a. Before 2021-02 no row has a USD + reference. Decided with Marek: before 2021-02 every dollar token counts + as $1, on-chain, with the safeguards below. Pre-run report goes to notes/. +--- + +# No USD price before 2021-02 — price that history from an imported hourly XLM/USD rate + +## Summary + +Every USD value we publish is anchored on canonical USDC (`GA5Z…`), priced with a +measured USDC/USD rate. USDC trades on Stellar from 2021-02-01, so nothing earlier has +a USD reference: candles from 2015-11 to 2021-01 carry volumes and no USD price, on +every tier. This task prices that period from an imported, measured hourly XLM/USD +rate: candles quoted in XLM directly, candles quoted in a dollar token through that +token's own XLM market, and native XLM's own USD series in the API. The first plan, +dollar tokens at $1 (Decision 2026-10-13, PR #401), was replaced on 2026-10-14 +(Decisions 5–11). + +## Context + +- **The gap.** The 0286 runbook calls these rows `before_reference` ("Nothing can price + them"). [[0201]] measured the first XLM/USDC candle at 2021-02-01 21:00. Adam puts + them at ~1.8 % of rows (2026-10-13, not measured here). They were unpriced before the + 0286 re-ingest too. +- **Why there is a USDC-only rule.** Stellar USDT (`GCQTGZ…`) was valued at $1 from + 2018-05 to 2026-08. It lost its peg in June 2022 and is worth ~$0.13 now, so its + quoted candles came out ~7.4× too high ([[0172]], [[0212]]). `current.sql` and + `views.sql` since fence the USD reference to USDC by name. +- **The depeg is one dated event.** The pivot prices Stellar USDT at monthly closes of + 0.977–1.005 from 2021-02 to 2022-05, then 0.363 in 2022-06. A par rule that ends at + 2021-02-01 never reaches it. +- **Trigger.** M3 review feedback (via Marek, 2026-10-13): XLM daily USD null over + 2018–2025; asked for priced 2018 and 2022 candles. 2022 is repaired; 2018 is this + task. + +## Decision (amended 2026-10-14 and 2026-10-15) + +Decided by Adam. Supersedes Decisions 1, 2 and 4 and Safeguards 1, 2 and 4 below; +Decision 3 is widened by Decision 9. + +5. **A third-party hourly XLM/USD series replaces par and the on-chain reference.** + 25,551 hourly rows, 2018-02-16 23:00 → 2021-01-31 23:00 UTC, in `prices.usd_rate` + as `('native','XLM','','')`, staged `external-candidate` with + `reference_asset = 'third-party'`. Code reads only `method = 'external'` rows of + that source, so nothing changes until the rows are promoted. The source is not + named in project documents. It is not Bitstamp, so Safeguard 5 stays an + independent check. PR #401 (par) is closed. +6. **Dollar-token quotes are measured, not assumed.** A candle quoted in a dollar + token (`USD`, `USDT`, `USDC` of any issuer, classic, except canonical USDC) is + converted through that token's own XLM market within `--pivot-window-s` (24 h), + then through the rate. A token that lost its peg is priced at what it traded for, + so the write side has no exclusion list. No market in the window: the row stays + unpriced. +7. **Wide tiers roll up from 1h.** 4h, 1d, 1w and 1M take the sum of their priced + hours' USD volume and the last priced hour's `close_usd / close`, as `preroll.sql` + does. `price_ohlcv_1h` is repaired first. +8. **Label: the existing `external`.** No new word, so Safeguard 4 is dropped. On a + quote leg the API labels every priced non-USDC row before 2021-02-01 `external`. + That holds while `coarse-repair --pre-usdc-xlm-rate` is the only writer of such + rows; on 2026-10-15 production had none from any other source. +9. **Scope: history plus native XLM's series in the API.** In USD mode + `GET /assets/native/ohlcv` builds pre-2021 buckets from the promoted rate at + request time: `open`/`close` are the first and last hourly rate, `high`/`low` the + extremes of the hourly averages, volume is XLM traded against dollar tokens minus + the four off-peg ones (`OFF_PEG_DOLLAR_TOKENS`), and `method = external`, + `derived = true`, `source = third-party`. It goes live when the rate is promoted, + independently of the `coarse-repair` run. This changes the `/ohlcv` contract and + `openapi.json`. Nothing from 2021-02-01 on changes. +10. **XLM's own pre-2021 candles stay priced** through their XLM/USDT legs, as the + M3 reviewer asked. +11. **The 1m/15m native series reads its partitions outside the table key**, because + a bucket appears wherever XLM traded and most XLM markets have XLM as the quote. + Accepted in review on 2026-10-15: at most 1.15 M rows and 0.53 s per request on + production, and pre-2021 partitions no longer grow. + +## Decision (2026-10-13, superseded in part) + +1. Before 2021-02-01, **every dollar token** counts as $1. Not a list of issuers. +2. **On-chain only**, chosen over an external XLM/USD series. The trade-off is accepted: + a bucket stays unpriced when no dollar-token market traded within the pivot window. +3. **History only.** The change lives in the repricing pass run from fishuser-hero. The + live path and the USDC-only rule for 2021-02 onward stay as they are. +4. **Amended the same day after the pre-run report (Stanisław):** every dollar token + except four off-peg ones, excluded by identity (`--par-exclude`, its default): + USDT `GC2PJ6J3`, USD `GD3LHBHY`, USDT `GC3YRTJQ`, USD `GA5O7NTU`. USDT `GC2PJ6J3` + holds most of the volume on 19 days and is then the median itself, which no + threshold can fix. With the four out, 1 day of 2018–2020 is more than 10 % off + Binance, against ~20 with all of them. Threshold `--par-max-dev 0.25`. + **Accepted by Marek the same day.** + +## Safeguards + +Anyone can issue a token called `USD`, so "every dollar token" needs these: + +1. ~~The XLM/USD reference is volume-weighted across all dollar-token markets in the + bucket.~~ Superseded by Decision 5: the rate is imported. +2. ~~A market whose implied XLM/USD deviates from the bucket's median by more than a + threshold is left out.~~ Superseded by Decision 5. +3. **Pre-run report** (`notes/`): every dollar token traded before 2021-02, with + volume, implied XLM/USD and deviation. Done. +4. ~~A distinct `method` on these candles.~~ Dropped by Decision 8. +5. A spot-check against Bitstamp XLM/USD where Bitstamp has data. + +## Open questions + +- ~~Which codes make a dollar token~~: `USD`, `USDT` and `USDC`, classic, any issuer + (`DOLLAR_TOKEN_PRED` in `prices-clickhouse`). +- ~~`method` label~~: the existing `external`, Decision 8. +- ~~The outlier threshold~~ and ~~per-token exclusion~~: no longer apply (Decision 5). + The four off-peg tokens are still left out of native XLM's volume (Decision 9). +- ~~Coverage~~: priced history starts where the rate does, 2018-02-16 23:00. Before + that nothing is priced (task 0333). +- ~~Coverage ceiling~~: native XLM's series is built from the rate (Decision 9), so it + is no longer bound by the 21, 63 and 257 days of its XLM/USDT candles. +- The tolerance for the Bitstamp spot-check (Safeguard 5), to agree with Marek. + +## Plan + +1. Pre-run report → `notes/` (done 2026-10-13, data in `data/`). +2. Decide codes, source and label; record them here (done, Decisions 5–11). +3. Implement `coarse-repair --pre-usdc-xlm-rate` and native XLM's series, with tests + (PR #403). +4. Deploy per PR #403: API, promotion of the rate, then after the 0286 drain and A6 a + dry run and a real run over 2018-02 → 2021-01, tier by tier (1h first). One pass at + a time on the cluster shared with the block explorer. +5. Spot-check against Bitstamp. Then the M3 reply and evidence. + +Steps 1–4 are done. The API went out and the rate was promoted on 2026-10-15 +(version 1791990002, 25,551 `external` rows). The history run was 2026-10-16. + +## Rollout of the history run (2026-10-16) + +**Order and preconditions.** The 0286 1m drain finished on 2026-10-15 at +14:27 UTC, and 0286's after-checks (A6) passed on 2026-10-16, both recorded +on 0286. + +**Backup.** Partitions 201802–202101 of 15m, 1h, 4h, 1d, 1w and 1M were +copied into `prices.repair_0331_bak_` as `dev_shared`, with +`ATTACH PARTITION … FROM`. Each table holds 36 partitions, with source +equal to backup on every tier: 2,302,390 / 1,043,707 / 454,131 / 143,771 / +36,674 / 13,563 rows. Rollback is +`REPLACE PARTITION

FROM prices.repair_0331_bak_`. + +**Run.** `coarse-repair --pre-usdc-xlm-rate --start-month 201802 +--end-month 202101`, from Adam's laptop as `prices_writer`, 08:34–~09:05 +UTC. Each tier had a dry run first, and a gate stopped the chain on any +`reset` > 0 or any month ≥ 202102. Tier order: 1h → 15m → 1m (`--skip-snapshot`) +→ 4h → 1d → 1w → 1M (`--skip-snapshot --snapshots-verified`). + +| Tier | Written | Left unpriced | Reset | +| --- | ---: | ---: | ---: | +| 1h | 794,672 | 3,351 | 0 | +| 15m | 1,655,500 | 8,545 | 0 | +| 1m | 4,971,301 | 25,553 | 0 | +| 4h | 357,495 | 1,284 | 0 | +| 1d | 114,117 | 383 | 0 | +| 1w | 27,770 | 55 | 0 | +| 1M | 9,303 | 16 | 0 | + +- Every tier wrote exactly what its dry run listed. +- On 1h, 3,005 of the unpriced rows are 201802 hours before the rate starts + (task 0333). The other 346 are spread over 2018–2021. +- `system.part_log` shows new parts only in partitions 201802–202101, with + row counts equal to the table above, and none in 202102–202508. + +**What is still unpriced, and why.** Measured on 1d rows with volume and a +price, 2018-01-01 → 2021-01-31, after the run. + +| Year | Rows priced | +| --- | ---: | +| 2018 | 74.8 % | +| 2019 | 77.7 % | +| 2020 | 81.9 % | +| 2021-01 | 81.8 % | + +Of the 30,054 rows still unpriced: + +| Cause | Rows | +| --- | ---: | +| Quoted in an asset other than XLM or a dollar token (330 quotes, 275 bases) | 29,328 | +| XLM-quoted before the rate (task 0333) | 579 | +| Dollar-token quote with no XLM market within 24 h | 147 | + +- **No two-hop pricing.** The largest group needs a second step, quote → + XLM → USD, which the pipeline does not have on any date. The same pairs + are unpriced after 2021 too. +- **The top quotes have XLM markets.** Those quotes are XCN, HKDT, CMS, + NGNT, LTC, ETH, BTC, XRP and CNY. Most of them have an XLM market on + 450–1,070 of the 1,080 days. + +**XLM-quoted rows, implied XLM/USD.** Median per year 0.229 / 0.083 / 0.073 +(2018 / 2019 / 2020). The 2020 minimum is 0.031, in March 2020. + +**Bitstamp, after the run.** The OHLC endpoint returned data for one of +five dates tried; the other four returned 404. That date was 2020-11-24, +where XLM/USDT 1d `close_usd` is 0.19589 and Bitstamp's close is 0.19589. +Agreement over the whole range was measured on the rate before deploy (AC +below). + +## Acceptance Criteria + +- [x] Pre-run report in `notes/`, read by Marek or Adam +- [x] Codes, source and label decided and recorded here (Decisions 5–11) +- [x] Native XLM's daily USD candles priced on every day the rate covers + (2018-02-16 → 2021-01-31). Candles quoted in XLM priced wherever the rate + covers them; candles quoted in a dollar token also need the token's XLM market + within 24 h. The remaining rows explained. Measured on prod before deploy + (2026-10-15): 1081 of 1081 days; 794,672 of 814,269 in-scope 1h rows, the rest + before the rate (task 0333) or 358 dollar-quoted rows with no token market + ✅ 2026-10-16. The native series has been live since 2026-10-15 on 1081 of + 1081 days. The history run priced 794,672 1h rows, as measured. The rows + left are explained in "Rollout of the history run". +- [ ] Spot-check against Bitstamp XLM/USD within an agreed tolerance (measured before + deploy: median 0.30 %, p95 1.76 %, without 2020-05-28 → 07-15 where Bitstamp + printed a flat stale price). The tolerance is still to agree with Marek. +- [x] Rows from 2021-02-01 on unchanged. ✅ 2026-10-16: the run listed no month + ≥ 202102, and `system.part_log` shows no insert into 202102–202508. diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/collect.py b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/collect.py new file mode 100644 index 00000000..c163ec53 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/collect.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""0331 pre-run report: every dollar token (USD/USDT/USDC, any issuer) traded +against XLM before 2021-02-01, its implied XLM/USD at par, and how it compares +with the day's other dollar markets and with Binance XLM/USDT. + +Resumable: every fetch is cached under OUT. Usage: KEY=... collect.py OUT +""" +import csv, json, os, ssl, statistics, sys, time, urllib.error, urllib.request + +CTX = ssl.create_default_context(cafile="/etc/ssl/cert.pem") + +OUT = sys.argv[1] +API = "https://prices-api.sorobanscan.rumblefish.dev/v1" +KEY = os.environ["KEY"] +UA = {"User-Agent": "stellar-prices-skill/1"} +CODES = ["USD", "USDT", "USDC"] +CANON = { + ("USDC", "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"), + ("USDT", "GCQTGZQQ5G4PTM2GL7CDIFKUBIPEC52BROAQIAPW53XBRJVN6ZJVTG6V"), +} +START, END = "2015-11-01", "2021-01-31" +os.makedirs(f"{OUT}/daily", exist_ok=True) + + +def get(url, headers, pace=0.0): + for i in range(6): + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=90, context=CTX) as r: + data = json.load(r) + time.sleep(pace) + return data + except urllib.error.HTTPError as e: + time.sleep(max(pace, 1)) + if e.code == 404: + return None + if e.code == 400: + return {"_err": 400, "body": e.read().decode()[:300]} + if e.code in (429, 500, 502, 503, 504): + time.sleep(2 * (i + 1)) + continue + raise + except Exception: + time.sleep(3 * (i + 1)) + return {"_err": "retries"} + + +def api(path): + return get(API + path, {**UA, "x-api-key": KEY}, pace=1.05) + + +def cached(path, fn): + if os.path.exists(path): + with open(path) as f: + return json.load(f) + data = fn() + with open(path, "w") as f: + json.dump(data, f) + return data + + +# 1. Every issuer of a dollar code, from Horizon. +def enumerate_assets(): + out = [] + for code in CODES: + url = f"https://horizon.stellar.org/assets?asset_code={code}&limit=200" + while url: + page = get(url, UA) + recs = page["_embedded"]["records"] + out += [{"code": r["asset_code"], "issuer": r["asset_issuer"], + "accounts": r.get("accounts", {}).get("authorized")} for r in recs] + url = page["_links"]["next"]["href"] if len(recs) == 200 else None + return out + + +assets = cached(f"{OUT}/assets.json", enumerate_assets) +print(f"assets: {len(assets)}", flush=True) + +# 2. Monthly scan: which of them traded against XLM (token = base) before 2021-02. +mpath = f"{OUT}/monthly.jsonl" +done = set() +if os.path.exists(mpath): + with open(mpath) as f: + done = {json.loads(l)["id"] for l in f} +with open(mpath, "a") as f: + for i, a in enumerate(assets): + aid = f"{a['code']}:{a['issuer']}" + if aid in done or (a["code"], a["issuer"]) in CANON: + continue + r = api(f"/assets/{aid}/ohlcv?start={START}&end={END}&granularity=1M&base_currency=XLM") + months = [] if not r or "data" not in r else [ + {"m": d["timestamp"][:7], "vol": d["volume_base"], "n": d["trade_count"]} for d in r["data"]] + f.write(json.dumps({"id": aid, "months": months, "err": (r or {}).get("_err")}) + "\n") + f.flush() + if i % 50 == 0: + print(f"monthly {i}/{len(assets)}", flush=True) + +with open(mpath) as f: + monthly = [json.loads(l) for l in f] +traded = [m for m in monthly if m["months"]] +print(f"traded vs XLM before 2021-02: {len(traded)}", flush=True) + +# 3. Daily series for the traded ones. +daily = {} +for m in traded: + aid = m["id"] + p = f"{OUT}/daily/{aid.replace(':', '_')}.json" + daily[aid] = cached(p, lambda: api(f"/assets/{aid}/ohlcv?start={START}&end={END}&granularity=1d&base_currency=XLM")) + +# 4. External reference: Binance XLM/USDT daily (from 2018-05-31). +def binance(): + rows, t = [], 1514764800000 + while t < 1612137600000: + k = get(f"https://api.binance.com/api/v3/klines?symbol=XLMUSDT&interval=1d&startTime={t}&limit=1000", UA) + if not k: + break + rows += k + t = k[-1][0] + 86400000 + return {time.strftime("%Y-%m-%d", time.gmtime(r[0] / 1000)): float(r[7]) / float(r[5]) + for r in rows if float(r[5]) > 0} + +bn = cached(f"{OUT}/binance.json", binance) + +# 5. XLM's own daily series (XLM as base = the canonical XLM/USDT market before 2021). +xlm = cached(f"{OUT}/xlm.json", lambda: api(f"/assets/native/ohlcv?start={START}&end={END}&granularity=1d")) + +# 6. Analysis. +per_day = {} # date -> list of (aid, implied_xlm_usd, vol_usd) +for aid, r in daily.items(): + for d in (r or {}).get("data", []): + px = d.get("pf_vwap") or d.get("vwap") + if not px or float(px) <= 0: + continue + per_day.setdefault(d["timestamp"][:10], []).append((aid, 1 / float(px), float(d["volume_base"]))) + + +def wmedian(pairs): + pairs = sorted(pairs) + tot, acc = sum(w for _, w in pairs), 0.0 + for v, w in pairs: + acc += w + if acc >= tot / 2: + return v + return None + + +day_rows, tok = [], {} +for day in sorted(per_day): + mk = per_day[day] + vol = sum(v for _, _, v in mk) + med = wmedian([(x, v) for _, x, v in mk if v > 0]) if vol > 0 else statistics.median(x for _, x, _ in mk) + ext = bn.get(day) + day_rows.append({"date": day, "markets": len(mk), "vol_usd_at_par": round(vol, 2), + "ref_wmedian": round(med, 6), "binance_vwap": round(ext, 6) if ext else "", + "ref_vs_binance_pct": round((med / ext - 1) * 100, 2) if ext else ""}) + for aid, x, v in mk: + t = tok.setdefault(aid, {"days": 0, "vol": 0.0, "first": day, "last": day, "dev_med": [], "dev_bn": []}) + t["days"] += 1; t["vol"] += v; t["last"] = day + t["dev_med"].append(x / med - 1) + if ext: + t["dev_bn"].append(x / ext - 1) + +tok_rows = [] +for aid, t in sorted(tok.items(), key=lambda kv: -kv[1]["vol"]): + dm = [abs(x) for x in t["dev_med"]] + db = [abs(x) for x in t["dev_bn"]] + tok_rows.append({"asset": aid, "days": t["days"], "vol_usd_at_par": round(t["vol"], 2), + "first": t["first"], "last": t["last"], + "median_abs_dev_vs_day_pct": round(statistics.median(dm) * 100, 2), + "days_dev_gt_10pct": sum(x > 0.10 for x in dm), "days_dev_gt_25pct": sum(x > 0.25 for x in dm), + "median_abs_dev_vs_binance_pct": round(statistics.median(db) * 100, 2) if db else "", + "binance_days": len(db)}) + +for name, rows in (("tokens.csv", tok_rows), ("daily.csv", day_rows)): + with open(f"{OUT}/{name}", "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader(); w.writerows(rows) + +xlm_days = {} +for d in (xlm or {}).get("data", []): + y = d["timestamp"][:4] + xlm_days.setdefault(y, [0, 0.0]); xlm_days[y][0] += 1; xlm_days[y][1] += float(d["volume_base"]) +years = {} +for r in day_rows: + y = r["date"][:4] + years.setdefault(y, {"days_with_ref": 0, "vol_usd": 0.0}) + years[y]["days_with_ref"] += 1; years[y]["vol_usd"] += r["vol_usd_at_par"] +summary = {"assets_enumerated": len(assets), "scanned": len(monthly), + "errors": sum(1 for m in monthly if m["err"]), "traded_vs_xlm": len(traded), + "days_with_ref": len(day_rows), "years": years, + "xlm_own_series": {y: {"days": v[0], "vol_xlm": round(v[1], 2)} for y, v in xlm_days.items()}, + "binance_days": len(bn)} +with open(f"{OUT}/summary.json", "w") as f: + json.dump(summary, f, indent=1) +print(json.dumps(summary, indent=1)) diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/daily.csv b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/daily.csv new file mode 100644 index 00000000..92c1b794 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/daily.csv @@ -0,0 +1,1006 @@ +date,markets,vol_usd_at_par,ref_wmedian,binance_vwap,ref_vs_binance_pct +2016-10-18,1,0.02,0.0022,, +2017-06-17,1,0.73,0.036349,, +2017-06-18,1,19.22,0.038442,, +2017-06-19,1,1.01,0.037268,, +2017-06-20,1,3.02,0.040926,, +2017-06-24,1,0.22,0.037282,, +2017-07-09,1,0.02,0.021424,, +2017-07-14,1,0.02,0.0158,, +2017-07-18,1,1.01,0.018351,, +2017-07-22,1,1.01,0.019067,, +2017-07-29,1,1.01,0.015702,, +2017-08-01,1,1.01,0.015604,, +2017-08-04,1,1.01,0.018927,, +2017-08-07,1,2.08,0.021879,, +2017-08-10,1,0.02,0.022661,, +2018-04-11,1,0.01,0.18517,, +2018-05-03,1,0.2,0.438748,, +2018-05-04,1,2.05,0.429616,, +2018-05-07,1,2.0,0.409855,, +2018-05-11,1,2.95,0.340176,, +2018-05-13,1,202.76,0.350851,, +2018-05-14,1,0.36,0.357143,, +2018-05-16,1,0.3,0.337079,, +2018-05-17,1,3.02,0.334727,, +2018-05-18,1,0.8,0.314041,, +2018-05-19,1,0.3,0.316799,, +2018-05-20,1,70.52,0.327812,, +2018-05-21,1,34.66,0.308774,, +2018-05-22,1,56.6,0.313171,, +2018-05-23,1,10.49,0.290863,, +2018-05-24,1,0.21,0.282643,, +2018-05-25,1,148.63,0.287691,, +2018-05-26,1,40.43,0.294442,, +2018-05-27,1,78.69,0.280542,, +2018-05-28,1,65.62,0.265922,, +2018-05-29,1,438.81,0.266866,, +2018-05-30,1,355.04,0.279823,, +2018-05-31,1,120.53,0.291356,0.296407,-1.7 +2018-06-01,1,104.6,0.293346,0.289636,1.28 +2018-06-02,1,0.72,0.295582,0.296764,-0.4 +2018-06-03,1,0.18,0.298814,0.303709,-1.61 +2018-06-04,1,1.76,0.292247,0.295668,-1.16 +2018-06-05,1,1.58,0.291045,0.288565,0.86 +2018-06-06,1,0.3,0.298733,0.293302,1.85 +2018-06-10,1,187.83,0.255169,0.259165,-1.54 +2018-06-11,1,246.0,0.253174,0.24264,4.34 +2018-06-12,1,151.83,0.252875,0.239271,5.69 +2018-06-13,1,42.4,0.222766,0.220336,1.1 +2018-06-15,1,92.72,0.2337,0.236426,-1.15 +2018-06-17,1,0.02,0.245461,0.230318,6.58 +2018-06-18,1,0.24,0.241383,0.228649,5.57 +2018-06-19,1,0.31,0.220244,0.233785,-5.79 +2018-06-20,1,9.0,0.23438,0.229937,1.93 +2018-06-21,1,1.11,0.221578,0.230041,-3.68 +2018-06-22,1,43.12,0.210292,0.211786,-0.71 +2018-06-23,1,0.58,0.193873,0.199784,-2.96 +2018-06-24,1,0.38,0.190922,0.188255,1.42 +2018-06-25,1,2.07,0.185676,0.194254,-4.42 +2018-06-26,1,0.19,0.18677,0.190926,-2.18 +2018-06-27,1,1.78,0.178057,0.186617,-4.59 +2018-06-28,1,3.58,0.184384,0.187925,-1.88 +2018-06-29,1,0.51,0.17023,0.176768,-3.7 +2018-06-30,1,0.51,0.173896,0.194774,-10.72 +2018-07-01,1,2.68,0.178014,0.193928,-8.21 +2018-07-02,1,3.27,0.192152,0.205923,-6.69 +2018-07-03,1,0.95,0.206018,0.212632,-3.11 +2018-07-04,1,1.39,0.198702,0.212211,-6.37 +2018-07-05,1,0.4,0.202454,0.206962,-2.18 +2018-07-06,1,0.98,0.195272,0.20353,-4.06 +2018-07-07,1,0.59,0.197101,0.204032,-3.4 +2018-07-08,1,0.81,0.201661,0.210481,-4.19 +2018-07-09,1,0.8,0.199782,0.210498,-5.09 +2018-07-11,1,0.73,0.183357,0.187798,-2.36 +2018-07-12,1,0.52,0.174852,0.182888,-4.39 +2018-07-13,1,2.03,0.184522,0.193622,-4.7 +2018-07-14,1,1.36,0.194612,0.204012,-4.61 +2018-07-15,1,1.83,0.203226,0.216784,-6.25 +2018-07-16,1,1.94,0.215588,0.228765,-5.76 +2018-07-17,1,29.25,0.2378,0.241381,-1.48 +2018-07-18,1,976.54,0.268476,0.292679,-8.27 +2018-07-19,1,2.55,0.283245,0.303386,-6.64 +2018-07-20,1,0.56,0.280405,0.285046,-1.63 +2018-07-21,1,2.43,0.269667,0.282497,-4.54 +2018-07-22,1,1.69,0.282476,0.29488,-4.21 +2018-07-23,1,1.65,0.275831,0.292111,-5.57 +2018-07-24,1,2.24,0.279532,0.295907,-5.53 +2018-07-25,1,890.17,0.30888,0.31924,-3.25 +2018-07-26,1,129.23,0.322042,0.323974,-0.6 +2018-07-27,1,9.0,0.319914,0.310411,3.06 +2018-07-28,1,6.0,0.333926,0.316758,5.42 +2018-07-29,1,1.0,0.322581,0.310443,3.91 +2018-07-30,1,504.38,0.304056,0.292679,3.89 +2018-07-31,1,305.0,0.292042,0.276633,5.57 +2018-08-01,1,5.0,0.283782,0.273162,3.89 +2018-08-02,1,16.14,0.280436,0.268901,4.29 +2018-08-03,1,255.0,0.266531,0.254916,4.56 +2018-08-04,1,208.0,0.256676,0.24692,3.95 +2018-08-05,2,25.92,0.233852,0.240488,-2.76 +2018-08-06,1,4.0,0.251668,0.237617,5.91 +2018-08-07,2,9.0,0.245844,0.236674,3.87 +2018-08-08,2,358.0,0.220866,0.208841,5.76 +2018-08-10,2,13521.77,0.231133,0.223561,3.39 +2018-08-11,2,6349.63,0.216612,0.213527,1.45 +2018-08-12,1,0.2,0.223265,0.223,0.12 +2018-08-14,2,3788.94,0.205177,0.210822,-2.68 +2018-08-15,2,0.87,0.215816,0.22335,-3.37 +2018-08-16,2,71.97,0.220584,0.214148,3.01 +2018-08-17,2,871.29,0.229461,0.226541,1.29 +2018-08-18,2,938.12,0.226085,0.228271,-0.96 +2018-08-19,1,0.44,0.218516,0.226308,-3.44 +2018-08-20,2,1242.29,0.224176,0.22271,0.66 +2018-08-21,1,0.83,0.207883,0.216589,-4.02 +2018-08-22,2,4833.32,0.22221,0.218997,1.47 +2018-08-23,2,41.45,0.212602,0.211877,0.34 +2018-08-24,2,20000.83,0.219523,0.215223,2.0 +2018-08-25,2,147803.63,0.218418,0.217145,0.59 +2018-08-26,2,1918950.77,0.215656,0.212413,1.53 +2018-08-27,2,107772.6,0.21785,0.218546,-0.32 +2018-08-28,2,142126.55,0.22787,0.230796,-1.27 +2018-08-29,2,378978.48,0.230475,0.228862,0.7 +2018-08-30,2,577607.08,0.220139,0.219376,0.35 +2018-08-31,1,501946.73,0.218983,0.220373,-0.63 +2018-09-01,2,20558.87,0.223495,0.228804,-2.32 +2018-09-02,1,39365.29,0.230767,0.227209,1.57 +2018-09-03,2,42561.32,0.225179,0.222677,1.12 +2018-09-04,2,2042.98,0.224459,0.226074,-0.71 +2018-09-05,2,12987.44,0.228401,0.219043,4.27 +2018-09-06,2,32816.96,0.200822,0.198574,1.13 +2018-09-07,2,4.96,0.209773,0.205843,1.91 +2018-09-08,1,109955.45,0.204559,0.200819,1.86 +2018-09-09,2,183057.2,0.196541,0.194248,1.18 +2018-09-10,1,8.37,0.188897,0.190206,-0.69 +2018-09-11,2,210.83,0.18956,0.196514,-3.54 +2018-09-12,2,12984.65,0.201151,0.19671,2.26 +2018-09-13,2,1269.91,0.204667,0.20563,-0.47 +2018-09-14,2,83.55,0.203046,0.203076,-0.01 +2018-09-15,2,85.16,0.205673,0.20312,1.26 +2018-09-16,2,1234.9,0.206473,0.204616,0.91 +2018-09-17,1,10719.77,0.211988,0.202638,4.61 +2018-09-18,2,11.1,0.200019,0.204979,-2.42 +2018-09-20,1,0.42,0.21055,0.220526,-4.52 +2018-09-21,2,37597.37,0.251561,0.25124,0.13 +2018-09-22,1,0.24,0.236356,0.240144,-1.58 +2018-09-23,2,53.24,0.285404,0.282588,1.0 +2018-09-24,1,502.35,0.260043,0.265521,-2.06 +2018-09-25,2,6395.53,0.243543,0.243118,0.17 +2018-09-26,2,7869.73,0.250531,0.249129,0.56 +2018-09-27,3,3012.08,0.250184,0.255661,-2.14 +2018-09-28,3,3714.32,0.25936,0.255018,1.7 +2018-09-29,3,4452.62,0.252418,0.253825,-0.55 +2018-09-30,3,851.1,0.256767,0.258295,-0.59 +2018-10-01,2,27549.83,0.26217,0.259695,0.95 +2018-10-02,3,2246.88,0.253284,0.254283,-0.39 +2018-10-03,2,4830.92,0.242597,0.242413,0.08 +2018-10-04,3,11936.83,0.245376,0.245723,-0.14 +2018-10-05,3,27882.39,0.241649,0.244866,-1.31 +2018-10-06,1,9764.46,0.24201,0.243995,-0.81 +2018-10-07,2,7655.66,0.241385,0.241722,-0.14 +2018-10-08,3,16283.55,0.243845,0.246558,-1.1 +2018-10-09,3,10359.65,0.245944,0.246729,-0.32 +2018-10-10,3,11304.48,0.241793,0.243852,-0.84 +2018-10-11,3,122224.14,0.216606,0.218775,-0.99 +2018-10-12,3,30439.71,0.214804,0.216323,-0.7 +2018-10-13,2,36305.04,0.213393,0.217214,-1.76 +2018-10-14,3,8375.51,0.210683,0.216253,-2.58 +2018-10-15,3,30721.33,0.213625,0.230929,-7.49 +2018-10-16,2,41474.35,0.221625,0.231667,-4.33 +2018-10-17,3,33121.3,0.228599,0.245653,-6.94 +2018-10-18,3,19617.23,0.23603,0.246735,-4.34 +2018-10-19,2,17092.27,0.236894,0.246808,-4.02 +2018-10-20,3,1054.21,0.235895,0.2525,-6.58 +2018-10-21,3,2759.1,0.240365,0.25098,-4.23 +2018-10-22,2,11991.35,0.237506,0.247797,-4.15 +2018-10-23,3,8866.16,0.236222,0.247071,-4.39 +2018-10-24,3,11247.95,0.236221,0.244379,-3.34 +2018-10-25,3,6908.5,0.234589,0.239254,-1.95 +2018-10-26,3,23823.37,0.23507,0.236924,-0.78 +2018-10-27,2,15996.9,0.231123,0.233873,-1.18 +2018-10-28,2,3381.07,0.230888,0.233422,-1.09 +2018-10-29,3,8362.32,0.230093,0.226838,1.43 +2018-10-30,3,13356.98,0.236351,0.224626,5.22 +2018-10-31,3,12037.39,0.221733,0.223021,-0.58 +2018-11-01,3,31190.88,0.221749,0.225556,-1.69 +2018-11-02,3,16789.82,0.227312,0.233607,-2.69 +2018-11-03,3,9145.22,0.234523,0.239587,-2.11 +2018-11-04,3,3424.89,0.237078,0.244464,-3.02 +2018-11-05,2,7995.15,0.240081,0.244979,-2.0 +2018-11-06,3,81167.16,0.251292,0.25861,-2.83 +2018-11-07,3,20025.25,0.261917,0.260735,0.45 +2018-11-08,3,42545.69,0.256173,0.259355,-1.23 +2018-11-09,3,16287.38,0.2591,0.261084,-0.76 +2018-11-10,3,29256.13,0.259662,0.261832,-0.83 +2018-11-11,3,85415.23,0.265142,0.2733,-2.99 +2018-11-12,3,33448.7,0.267196,0.277167,-3.6 +2018-11-13,3,37853.06,0.264509,0.267056,-0.95 +2018-11-14,3,31796.83,0.228621,0.239471,-4.53 +2018-11-15,3,40063.54,0.230829,0.235271,-1.89 +2018-11-16,2,5020.46,0.240004,0.243947,-1.62 +2018-11-17,3,12930.52,0.242609,0.243576,-0.4 +2018-11-18,3,35193.99,0.247738,0.253197,-2.16 +2018-11-19,3,89484.27,0.231713,0.233274,-0.67 +2018-11-20,3,68022.64,0.195668,0.209009,-6.38 +2018-11-21,3,38070.42,0.200166,0.201872,-0.85 +2018-11-22,2,35873.36,0.192733,0.19788,-2.6 +2018-11-23,3,26276.5,0.181508,0.182226,-0.39 +2018-11-24,3,44245.66,0.172095,0.16377,5.08 +2018-11-25,3,142182.32,0.146905,0.147136,-0.16 +2018-11-26,3,210095.7,0.147489,0.152353,-3.19 +2018-11-27,3,151450.29,0.141305,0.143523,-1.55 +2018-11-28,3,218145.89,0.156383,0.158216,-1.16 +2018-11-29,3,72355.88,0.16246,0.1642,-1.06 +2018-11-30,3,46379.35,0.16119,0.16252,-0.82 +2018-12-01,3,36623.28,0.165805,0.164156,1.0 +2018-12-02,3,35308.9,0.163544,0.16298,0.35 +2018-12-03,3,8685.73,0.153292,0.153949,-0.43 +2018-12-04,3,4847.47,0.148389,0.149427,-0.69 +2018-12-05,3,39142.01,0.141809,0.140858,0.68 +2018-12-06,3,7109.04,0.127616,0.129843,-1.72 +2018-12-07,3,51267.02,0.109517,0.112162,-2.36 +2018-12-08,3,41883.07,0.115669,0.116777,-0.95 +2018-12-09,3,9043.1,0.118969,0.120425,-1.21 +2018-12-10,3,17127.14,0.12202,0.117431,3.91 +2018-12-11,3,22914.48,0.116289,0.113185,2.74 +2018-12-12,3,13662.67,0.115631,0.11272,2.58 +2018-12-13,3,28505.06,0.111296,0.108257,2.81 +2018-12-14,3,15517.14,0.106159,0.099511,6.68 +2018-12-15,3,23628.02,0.101777,0.094848,7.31 +2018-12-16,3,10599.7,0.098316,0.097624,0.71 +2018-12-17,3,32896.51,0.098873,0.104986,-5.82 +2018-12-18,3,53316.61,0.111051,0.108648,2.21 +2018-12-19,3,74017.26,0.117305,0.117528,-0.19 +2018-12-20,3,68072.03,0.119826,0.119423,0.34 +2018-12-21,3,81288.5,0.125455,0.123673,1.44 +2018-12-22,2,10846.42,0.116706,0.117723,-0.86 +2018-12-23,3,43366.89,0.124579,0.122809,1.44 +2018-12-24,3,63712.46,0.126898,0.133316,-4.81 +2018-12-25,3,24286.57,0.120589,0.11939,1.0 +2018-12-26,3,43672.64,0.117862,0.117089,0.66 +2018-12-27,3,34407.11,0.11318,0.112484,0.62 +2018-12-28,3,16175.95,0.114507,0.115356,-0.74 +2018-12-29,3,30817.06,0.118646,0.117373,1.09 +2018-12-30,3,6655.81,0.1155,0.114728,0.67 +2018-12-31,3,6005.23,0.112228,0.112514,-0.25 +2019-01-01,3,9855.8,0.111407,0.111187,0.2 +2019-01-02,3,24535.07,0.116119,0.114798,1.15 +2019-01-03,3,15852.04,0.115741,0.113012,2.42 +2019-01-04,3,8442.6,0.112281,0.111606,0.61 +2019-01-05,3,8901.37,0.113574,0.112669,0.8 +2019-01-06,3,15451.19,0.116567,0.115194,1.19 +2019-01-07,3,28570.31,0.120475,0.121563,-0.9 +2019-01-08,3,12666.83,0.122557,0.121249,1.08 +2019-01-09,3,13505.27,0.122277,0.121586,0.57 +2019-01-10,3,21494.17,0.113173,0.113582,-0.36 +2019-01-11,3,16412.45,0.106727,0.104181,2.44 +2019-01-12,3,4494.24,0.10479,0.103778,0.98 +2019-01-13,3,32997.94,0.103929,0.105487,-1.48 +2019-01-14,3,24317.19,0.106434,0.105025,1.34 +2019-01-15,3,34382.19,0.107077,0.105373,1.62 +2019-01-16,3,16591.07,0.106131,0.105437,0.66 +2019-01-17,3,42640.1,0.106539,0.105711,0.78 +2019-01-18,3,4290.1,0.106071,0.10586,0.2 +2019-01-19,3,6666.91,0.107775,0.107149,0.58 +2019-01-20,3,24838.16,0.10403,0.104228,-0.19 +2019-01-21,2,20826.34,0.103742,0.101847,1.86 +2019-01-22,3,12991.33,0.102431,0.101595,0.82 +2019-01-23,3,15376.18,0.101373,0.101398,-0.02 +2019-01-24,3,4400.7,0.100517,0.100356,0.16 +2019-01-25,3,10593.17,0.099129,0.099426,-0.3 +2019-01-26,3,11698.19,0.101081,0.099933,1.15 +2019-01-27,3,6902.03,0.097915,0.095857,2.15 +2019-01-28,3,26979.21,0.085054,0.087657,-2.97 +2019-01-29,3,24078.32,0.082696,0.082771,-0.09 +2019-01-30,3,18978.95,0.085264,0.084827,0.52 +2019-01-31,3,9091.35,0.083417,0.083943,-0.63 +2019-02-01,3,16588.43,0.080331,0.081016,-0.85 +2019-02-02,3,12212.37,0.081032,0.081758,-0.89 +2019-02-03,3,25143.36,0.079538,0.080971,-1.77 +2019-02-04,3,5016.0,0.078605,0.079632,-1.29 +2019-02-05,3,27535.78,0.076939,0.07737,-0.56 +2019-02-06,3,8769.61,0.074911,0.073999,1.23 +2019-02-07,3,3067.06,0.074319,0.074779,-0.62 +2019-02-08,3,18247.13,0.0739,0.078852,-6.28 +2019-02-09,3,9546.6,0.079772,0.080554,-0.97 +2019-02-10,3,8278.95,0.077932,0.078413,-0.61 +2019-02-11,3,2760.94,0.077403,0.077924,-0.67 +2019-02-12,3,5019.92,0.075608,0.0763,-0.91 +2019-02-13,3,22515.59,0.074263,0.076854,-3.37 +2019-02-14,3,2075.94,0.075043,0.07568,-0.84 +2019-02-15,3,13918.37,0.079598,0.078725,1.11 +2019-02-16,3,5558.32,0.077766,0.078305,-0.69 +2019-02-17,3,3047.92,0.077588,0.078432,-1.08 +2019-02-18,3,6914.73,0.08119,0.081585,-0.48 +2019-02-19,3,18470.97,0.089106,0.088919,0.21 +2019-02-20,3,30172.38,0.089186,0.088927,0.29 +2019-02-21,3,9736.66,0.09073,0.090052,0.75 +2019-02-22,3,11126.05,0.091375,0.089834,1.72 +2019-02-23,3,21865.51,0.095872,0.092327,3.84 +2019-02-24,3,22244.24,0.082124,0.089219,-7.95 +2019-02-25,3,16894.37,0.085696,0.085243,0.53 +2019-02-26,3,4657.38,0.084974,0.084546,0.51 +2019-02-27,3,7599.77,0.08371,0.08374,-0.04 +2019-02-28,3,5512.65,0.083663,0.083933,-0.32 +2019-03-01,3,2551.61,0.085395,0.084958,0.51 +2019-03-02,3,783.61,0.08306,0.08365,-0.71 +2019-03-03,3,8904.41,0.086567,0.087001,-0.5 +2019-03-04,3,6152.24,0.082198,0.082773,-0.7 +2019-03-05,3,1942.38,0.084843,0.084079,0.91 +2019-03-06,3,45131.06,0.085395,0.084599,0.94 +2019-03-07,3,2791.73,0.084552,0.085037,-0.57 +2019-03-08,3,10681.01,0.08738,0.087529,-0.17 +2019-03-09,3,22904.01,0.087314,0.089309,-2.23 +2019-03-10,3,37907.17,0.09893,0.097304,1.67 +2019-03-11,3,11053.81,0.101673,0.102744,-1.04 +2019-03-12,3,54338.52,0.102746,0.102904,-0.15 +2019-03-13,3,73345.36,0.103499,0.105817,-2.19 +2019-03-14,3,72408.22,0.109019,0.108292,0.67 +2019-03-15,3,3871.46,0.104439,0.105798,-1.29 +2019-03-16,3,3435.99,0.10782,0.108259,-0.41 +2019-03-17,3,1351.53,0.107717,0.107789,-0.07 +2019-03-18,3,19442.44,0.110385,0.113208,-2.49 +2019-03-19,3,12926.69,0.112183,0.113845,-1.46 +2019-03-20,3,1185.32,0.110321,0.110383,-0.06 +2019-03-21,3,21839.1,0.104223,0.105985,-1.66 +2019-03-22,3,13293.34,0.104189,0.10698,-2.61 +2019-03-23,3,3078.85,0.108153,0.107312,0.78 +2019-03-24,3,7797.1,0.103867,0.104893,-0.98 +2019-03-25,3,2605.45,0.101997,0.10197,0.03 +2019-03-26,3,13679.28,0.098962,0.100734,-1.76 +2019-03-27,3,19487.26,0.10218,0.105749,-3.38 +2019-03-28,3,6276.66,0.104529,0.105655,-1.07 +2019-03-29,3,9049.44,0.106951,0.107012,-0.06 +2019-03-30,3,2293.05,0.106549,0.106234,0.3 +2019-03-31,3,3308.46,0.106879,0.107288,-0.38 +2019-04-01,3,22539.22,0.109395,0.109927,-0.48 +2019-04-02,3,59577.79,0.115286,0.116313,-0.88 +2019-04-03,3,72032.73,0.128205,0.125298,2.32 +2019-04-04,3,47360.33,0.120319,0.119682,0.53 +2019-04-05,3,32230.78,0.125014,0.125256,-0.19 +2019-04-06,3,5556.86,0.125932,0.125073,0.69 +2019-04-07,3,7495.41,0.12874,0.129281,-0.42 +2019-04-08,3,45181.79,0.131691,0.130179,1.16 +2019-04-09,3,22479.83,0.126225,0.125555,0.53 +2019-04-10,3,14715.27,0.126157,0.12576,0.32 +2019-04-11,3,12594.42,0.119239,0.115962,2.83 +2019-04-12,3,1969.4,0.111633,0.112697,-0.94 +2019-04-13,3,1693.81,0.114447,0.114334,0.1 +2019-04-14,3,14014.2,0.117865,0.115589,1.97 +2019-04-15,3,13342.59,0.115624,0.11498,0.56 +2019-04-16,3,3501.28,0.114571,0.113036,1.36 +2019-04-17,3,27746.32,0.116818,0.115048,1.54 +2019-04-18,3,15316.15,0.117695,0.117361,0.28 +2019-04-19,3,51099.26,0.115983,0.115361,0.54 +2019-04-20,3,804.24,0.115828,0.115406,0.37 +2019-04-21,3,4524.95,0.110842,0.110629,0.19 +2019-04-22,3,7067.36,0.110651,0.111909,-1.12 +2019-04-23,3,13473.26,0.112161,0.111121,0.94 +2019-04-24,3,64061.88,0.103841,0.102366,1.44 +2019-04-25,3,10675.18,0.098695,0.101718,-2.97 +2019-04-26,2,21680.89,0.097067,0.099678,-2.62 +2019-04-27,2,8865.9,0.097033,0.100197,-3.16 +2019-04-28,2,2133.99,0.096271,0.099824,-3.56 +2019-04-29,2,7630.6,0.095292,0.097164,-1.93 +2019-04-30,3,8884.81,0.095578,0.098425,-2.89 +2019-05-01,3,1617.08,0.100447,0.101381,-0.92 +2019-05-02,3,11154.7,0.09882,0.100669,-1.84 +2019-05-03,3,17863.47,0.099912,0.101422,-1.49 +2019-05-04,3,2017.55,0.101474,0.099762,1.72 +2019-05-05,3,5447.14,0.096613,0.097988,-1.4 +2019-05-06,3,5740.01,0.094141,0.096219,-2.16 +2019-05-07,3,17739.65,0.095231,0.096272,-1.08 +2019-05-08,3,6586.56,0.091052,0.092549,-1.62 +2019-05-09,3,27074.73,0.089041,0.090011,-1.08 +2019-05-10,3,22061.7,0.092095,0.091447,0.71 +2019-05-11,3,63997.07,0.098477,0.099088,-0.62 +2019-05-12,3,84006.26,0.098957,0.100318,-1.36 +2019-05-13,3,36809.38,0.101806,0.100661,1.14 +2019-05-14,3,64780.36,0.11014,0.110678,-0.49 +2019-05-15,3,71445.97,0.126892,0.12707,-0.14 +2019-05-16,3,302607.93,0.145724,0.145651,0.05 +2019-05-17,3,163053.69,0.12439,0.124913,-0.42 +2019-05-18,3,44534.72,0.133387,0.133546,-0.12 +2019-05-19,3,25455.39,0.139786,0.139314,0.34 +2019-05-20,3,28770.25,0.134039,0.134428,-0.29 +2019-05-21,3,2438.6,0.134437,0.133686,0.56 +2019-05-22,3,28250.67,0.128189,0.129687,-1.15 +2019-05-23,3,10410.46,0.122313,0.121879,0.36 +2019-05-24,3,5930.48,0.127389,0.126017,1.09 +2019-05-25,3,18037.78,0.127536,0.125611,1.53 +2019-05-26,3,36870.85,0.134534,0.12649,6.36 +2019-05-27,3,19908.77,0.13753,0.135822,1.26 +2019-05-28,3,19625.35,0.139003,0.137233,1.29 +2019-05-29,3,8974.92,0.135821,0.136063,-0.18 +2019-05-30,3,50794.33,0.132941,0.136741,-2.78 +2019-05-31,3,25507.15,0.129388,0.128964,0.33 +2019-06-01,3,15991.11,0.131989,0.132093,-0.08 +2019-06-02,3,12294.72,0.135871,0.135719,0.11 +2019-06-03,3,36287.3,0.13389,0.13364,0.19 +2019-06-04,3,86410.63,0.123812,0.122904,0.74 +2019-06-05,3,22120.11,0.122152,0.122459,-0.25 +2019-06-06,4,20817.37,0.121368,0.121854,-0.4 +2019-06-07,3,21795.26,0.12847,0.126993,1.16 +2019-06-08,3,3541.83,0.125813,0.125138,0.54 +2019-06-09,3,4503.34,0.120449,0.120798,-0.29 +2019-06-10,3,13720.28,0.122812,0.121226,1.31 +2019-06-11,3,13634.33,0.119868,0.120898,-0.85 +2019-06-12,4,2204.55,0.125212,0.124683,0.42 +2019-06-13,3,105.79,0.12531,0.12593,-0.49 +2019-06-14,4,7566.75,0.124098,0.122698,1.14 +2019-06-15,3,3252.49,0.125157,0.125451,-0.23 +2019-06-16,3,14885.78,0.129809,0.129247,0.44 +2019-06-17,4,28151.4,0.131519,0.129078,1.89 +2019-06-18,3,7931.76,0.125836,0.125966,-0.1 +2019-06-19,3,3907.63,0.124717,0.12427,0.36 +2019-06-20,3,2833.24,0.121626,0.121845,-0.18 +2019-06-21,4,4951.67,0.123197,0.123039,0.13 +2019-06-22,3,39949.37,0.128531,0.128388,0.11 +2019-06-23,3,1194.32,0.129369,0.130843,-1.13 +2019-06-24,3,5258.35,0.126537,0.127301,-0.6 +2019-06-25,4,6824.21,0.125369,0.125442,-0.06 +2019-06-26,4,68238.29,0.124377,0.126809,-1.92 +2019-06-27,3,113037.83,0.108025,0.112969,-4.38 +2019-06-28,3,20174.35,0.110144,0.111045,-0.81 +2019-06-29,3,25675.12,0.111026,0.111997,-0.87 +2019-06-30,3,20107.13,0.108023,0.109324,-1.19 +2019-07-01,4,28620.15,0.104299,0.105723,-1.35 +2019-07-02,3,28216.19,0.102191,0.102482,-0.28 +2019-07-03,3,20122.06,0.103717,0.103663,0.05 +2019-07-04,4,46635.47,0.106073,0.102452,3.54 +2019-07-05,2,2909.14,0.100769,0.100157,0.61 +2019-07-06,3,9164.51,0.104673,0.103247,1.38 +2019-07-07,3,2110.07,0.103811,0.104064,-0.24 +2019-07-08,3,3556.63,0.104899,0.104365,0.51 +2019-07-09,4,6929.23,0.102034,0.10196,0.07 +2019-07-10,4,19381.88,0.096224,0.096424,-0.21 +2019-07-11,3,24806.12,0.092795,0.088881,4.4 +2019-07-12,3,37779.56,0.097646,0.096009,1.71 +2019-07-13,3,5210.26,0.09491,0.09588,-1.01 +2019-07-14,3,28961.21,0.088055,0.089768,-1.91 +2019-07-15,3,39037.84,0.085825,0.085413,0.48 +2019-07-16,3,28319.98,0.081896,0.08298,-1.31 +2019-07-17,4,34857.58,0.07909,0.080203,-1.39 +2019-07-18,4,68479.85,0.087022,0.086977,0.05 +2019-07-19,4,25315.42,0.09272,0.090283,2.7 +2019-07-20,3,8043.78,0.095338,0.094032,1.39 +2019-07-21,3,2507.72,0.09211,0.091547,0.62 +2019-07-22,4,4128.49,0.089029,0.08923,-0.22 +2019-07-23,3,3553.16,0.085157,0.085458,-0.35 +2019-07-24,3,6870.5,0.083977,0.083757,0.26 +2019-07-25,3,8020.88,0.086226,0.086242,-0.02 +2019-07-26,3,6921.54,0.085734,0.086232,-0.58 +2019-07-27,3,13337.37,0.085266,0.08564,-0.44 +2019-07-28,3,13745.92,0.083018,0.083642,-0.75 +2019-07-29,3,9783.65,0.083354,0.083882,-0.63 +2019-07-30,3,6518.58,0.083501,0.083331,0.2 +2019-07-31,3,2165.26,0.08407,0.083856,0.25 +2019-08-01,4,1856.44,0.082808,0.082736,0.09 +2019-08-02,3,12480.65,0.082099,0.082517,-0.51 +2019-08-03,3,1766.83,0.082585,0.082225,0.44 +2019-08-04,3,19623.47,0.082804,0.08168,1.38 +2019-08-05,4,3770.97,0.082587,0.082641,-0.07 +2019-08-06,3,3820.19,0.078416,0.07983,-1.77 +2019-08-07,3,8447.06,0.078226,0.078173,0.07 +2019-08-08,3,6064.49,0.076553,0.077089,-0.69 +2019-08-09,3,15994.0,0.074617,0.073914,0.95 +2019-08-10,3,18949.65,0.073622,0.073105,0.71 +2019-08-11,3,6009.5,0.077261,0.076832,0.56 +2019-08-12,3,1437.46,0.075987,0.076523,-0.7 +2019-08-13,3,3214.17,0.073694,0.074534,-1.13 +2019-08-14,3,25966.76,0.071876,0.070928,1.34 +2019-08-15,3,13530.35,0.067852,0.069202,-1.95 +2019-08-16,3,7198.5,0.068362,0.06862,-0.38 +2019-08-17,3,2232.68,0.067792,0.068059,-0.39 +2019-08-18,4,4500.94,0.069814,0.069807,0.01 +2019-08-19,3,2441.62,0.071154,0.071135,0.03 +2019-08-20,3,1684.49,0.069412,0.069662,-0.36 +2019-08-21,3,1247.65,0.066657,0.067053,-0.59 +2019-08-22,4,11224.01,0.06632,0.067727,-2.08 +2019-08-23,4,1835.25,0.068709,0.068524,0.27 +2019-08-24,3,5718.01,0.068666,0.069153,-0.7 +2019-08-25,3,3545.77,0.070083,0.070132,-0.07 +2019-08-26,4,2298.69,0.069959,0.069843,0.17 +2019-08-27,3,5581.11,0.068055,0.06816,-0.15 +2019-08-28,3,18751.86,0.066294,0.066155,0.21 +2019-08-29,3,10835.01,0.061536,0.061942,-0.66 +2019-08-30,3,11783.28,0.06264,0.062293,0.56 +2019-08-31,3,2378.38,0.061242,0.061722,-0.78 +2019-09-01,3,1511.68,0.061672,0.06183,-0.26 +2019-09-02,3,1580.33,0.062695,0.062334,0.58 +2019-09-03,3,3361.35,0.063134,0.063055,0.13 +2019-09-04,3,5619.94,0.062431,0.062034,0.64 +2019-09-05,3,1667.47,0.060831,0.060848,-0.03 +2019-09-06,3,10974.93,0.0591,0.059401,-0.51 +2019-09-07,3,3371.58,0.059111,0.059742,-1.06 +2019-09-08,3,2869.43,0.060891,0.061027,-0.22 +2019-09-09,3,4892.7,0.060288,0.060503,-0.36 +2019-09-10,4,8138.55,0.059904,0.06041,-0.84 +2019-09-11,3,4749.8,0.058501,0.058944,-0.75 +2019-09-12,4,3007.91,0.058302,0.058154,0.25 +2019-09-13,4,7073.15,0.058033,0.057972,0.11 +2019-09-14,4,4658.18,0.058483,0.058067,0.72 +2019-09-15,4,649.09,0.058477,0.058484,-0.01 +2019-09-16,4,1077.0,0.058589,0.058537,0.09 +2019-09-17,4,5946.98,0.063436,0.061906,2.47 +2019-09-18,4,134286.55,0.081251,0.077974,4.2 +2019-09-19,4,197834.2,0.081881,0.081546,0.41 +2019-09-20,4,31087.81,0.075788,0.074806,1.31 +2019-09-21,4,7301.42,0.071496,0.071461,0.05 +2019-09-22,4,13548.15,0.067943,0.068104,-0.24 +2019-09-23,4,21813.03,0.066619,0.066363,0.39 +2019-09-24,4,63372.45,0.055464,0.058473,-5.15 +2019-09-25,4,57718.03,0.055574,0.054888,1.25 +2019-09-26,4,49845.19,0.057285,0.05839,-1.89 +2019-09-27,4,23661.68,0.057922,0.05802,-0.17 +2019-09-28,4,11523.49,0.058309,0.058267,0.07 +2019-09-29,4,6736.92,0.057578,0.057655,-0.13 +2019-09-30,4,11362.4,0.05968,0.059803,-0.21 +2019-10-01,4,12886.56,0.05968,0.060025,-0.57 +2019-10-02,4,3690.88,0.05884,0.058619,0.38 +2019-10-03,4,8798.97,0.058783,0.058678,0.18 +2019-10-04,5,4668.05,0.059141,0.058888,0.43 +2019-10-05,4,5223.83,0.058788,0.058703,0.14 +2019-10-06,4,15352.48,0.058387,0.058298,0.15 +2019-10-07,4,20683.28,0.060975,0.06166,-1.11 +2019-10-08,4,6023.17,0.06235,0.061945,0.65 +2019-10-09,4,5633.61,0.063231,0.062966,0.42 +2019-10-10,3,2266.04,0.061976,0.061839,0.22 +2019-10-11,4,11591.44,0.060714,0.060584,0.21 +2019-10-12,4,6002.07,0.061273,0.061068,0.34 +2019-10-13,4,1773.72,0.06156,0.061771,-0.34 +2019-10-14,4,7107.21,0.063801,0.063582,0.34 +2019-10-15,4,10526.84,0.064192,0.064523,-0.51 +2019-10-16,4,18314.73,0.061846,0.0622,-0.57 +2019-10-17,4,23545.55,0.063937,0.063681,0.4 +2019-10-18,4,15756.5,0.063255,0.063043,0.34 +2019-10-19,4,7426.18,0.062982,0.062903,0.13 +2019-10-20,4,6871.32,0.062946,0.062789,0.25 +2019-10-21,5,3816.26,0.063543,0.063267,0.44 +2019-10-22,5,2750.38,0.064438,0.06413,0.48 +2019-10-23,4,14004.58,0.059316,0.059785,-0.79 +2019-10-24,5,8970.76,0.059776,0.060024,-0.41 +2019-10-25,4,22734.34,0.063764,0.063016,1.19 +2019-10-26,5,36346.38,0.065295,0.064419,1.36 +2019-10-27,5,25326.01,3.3423,0.063837,5135.67 +2019-10-28,5,91869.12,15.846795,0.066135,23861.41 +2019-10-29,5,281621.21,42.207145,0.066933,62958.38 +2019-10-30,4,712350.44,200.049985,0.064434,310372.05 +2019-10-31,5,229847.87,73.840114,0.063617,115970.6 +2019-11-01,5,47311.96,0.070173,0.069143,1.49 +2019-11-02,4,77750.94,268.351635,0.06994,383586.15 +2019-11-03,5,32318.07,0.069226,0.06937,-0.21 +2019-11-04,5,61878.16,130.198254,0.071051,183146.04 +2019-11-05,5,328297.78,0.082942,0.082356,0.71 +2019-11-06,5,117618.07,0.079629,0.079069,0.71 +2019-11-07,6,137243.64,0.074352,0.074568,-0.29 +2019-11-08,6,73273.26,0.071665,0.072795,-1.55 +2019-11-09,6,34995.36,0.072385,0.072382,0.0 +2019-11-10,6,97357.81,0.078173,0.077674,0.64 +2019-11-11,6,70085.66,0.078608,0.078749,-0.18 +2019-11-12,6,155681.23,0.078157,0.077981,0.23 +2019-11-13,5,322442.17,508.700213,0.076686,663256.68 +2019-11-14,6,95065.02,1740.876548,0.073704,2361880.71 +2019-11-15,6,439249.14,902.036746,0.072837,1238339.66 +2019-11-16,6,62397.48,659.570376,0.071615,920900.22 +2019-11-17,6,17056.53,0.071877,0.072136,-0.36 +2019-11-18,6,356526.73,350.630621,0.068361,512812.65 +2019-11-19,6,43992.37,0.064734,0.064839,-0.16 +2019-11-20,6,6731.38,0.064966,0.064852,0.17 +2019-11-21,6,45656.86,0.060064,0.06093,-1.42 +2019-11-22,6,77599.42,0.056861,0.057334,-0.83 +2019-11-23,6,42788.26,0.060365,0.060154,0.35 +2019-11-24,6,178809.47,561.203129,0.058453,959992.79 +2019-11-25,6,104479.68,0.057579,0.056823,1.33 +2019-11-26,5,31067.77,0.057129,0.057222,-0.16 +2019-11-27,6,41105.96,0.057369,0.057621,-0.44 +2019-11-28,5,19664.08,0.05842,0.058362,0.1 +2019-11-29,6,27126.23,0.058771,0.058883,-0.19 +2019-11-30,4,20251.05,0.057525,0.05806,-0.92 +2019-12-01,6,8405.12,0.05678,0.056447,0.59 +2019-12-02,4,28081.39,0.05574,0.056254,-0.91 +2019-12-03,6,11431.38,0.055767,0.055743,0.04 +2019-12-04,4,49393.27,0.055164,0.055329,-0.3 +2019-12-05,5,21358.09,0.054991,0.055038,-0.09 +2019-12-06,5,6699.26,0.055144,0.055162,-0.03 +2019-12-07,5,1445.38,0.05545,0.055571,-0.22 +2019-12-08,5,9757.44,665.5,0.055451,1200055.3 +2019-12-09,4,8040.23,0.055703,0.055179,0.95 +2019-12-10,6,16450.91,3332.21926,0.053564,6220962.53 +2019-12-11,5,3373.02,0.053024,0.053094,-0.13 +2019-12-12,5,8733.06,0.052157,0.052407,-0.48 +2019-12-13,5,2723.92,0.052252,0.052511,-0.49 +2019-12-14,5,7825.49,0.051255,0.051648,-0.76 +2019-12-15,4,10255.37,0.050978,0.050934,0.09 +2019-12-16,5,18527.77,0.048634,0.048694,-0.12 +2019-12-17,5,52059.06,0.044036,0.044705,-1.5 +2019-12-18,5,144087.0,0.044878,0.044792,0.19 +2019-12-19,5,36508.67,0.045481,0.045321,0.35 +2019-12-20,6,11404.15,0.045521,0.045668,-0.32 +2019-12-21,3,6175.14,0.045826,0.045923,-0.21 +2019-12-22,6,26819.81,0.046582,0.046339,0.52 +2019-12-23,3,28323.76,0.045855,0.046203,-0.75 +2019-12-24,5,9965.68,0.045038,0.045065,-0.06 +2019-12-25,4,10356.73,0.044235,0.044382,-0.33 +2019-12-26,3,9209.08,0.044772,0.044887,-0.26 +2019-12-27,4,3595.48,0.04459,0.045278,-1.52 +2019-12-28,4,4154.94,0.045935,0.046105,-0.37 +2019-12-29,4,4287.11,0.046486,0.046117,0.8 +2019-12-30,4,1533.89,0.045849,0.046113,-0.57 +2019-12-31,5,7986.78,0.044798,0.045209,-0.91 +2020-01-01,4,3976.72,0.044912,0.045185,-0.61 +2020-01-02,6,6974.54,2424.241,0.044456,5453049.32 +2020-01-03,4,18242.99,0.044821,0.044963,-0.32 +2020-01-04,5,13452.14,0.045556,0.045487,0.15 +2020-01-05,5,1513.61,0.045584,0.045751,-0.36 +2020-01-06,6,53732.57,0.048725,0.048316,0.85 +2020-01-07,4,23320.55,0.048754,0.049049,-0.6 +2020-01-08,6,26025.2,0.04818,0.048249,-0.14 +2020-01-09,4,9467.08,0.047491,0.047663,-0.36 +2020-01-10,5,35426.54,0.046836,0.04692,-0.18 +2020-01-11,5,3513.93,0.048531,0.048393,0.29 +2020-01-12,2,15312.27,0.048824,0.048772,0.11 +2020-01-13,5,21621.72,0.048006,0.048132,-0.26 +2020-01-14,5,157654.52,0.051478,0.051155,0.63 +2020-01-15,5,145694.52,0.05405,0.054308,-0.47 +2020-01-16,3,1117732.83,0.053691,0.053278,0.78 +2020-01-17,5,470708.59,0.055321,0.059352,-6.79 +2020-01-18,5,270555.86,0.062486,0.062304,0.29 +2020-01-19,3,149165.02,0.061392,0.061597,-0.33 +2020-01-20,3,164791.34,0.06287,0.063134,-0.42 +2020-01-21,5,51339.37,0.063028,0.063267,-0.38 +2020-01-22,4,36521.6,0.062081,0.062437,-0.57 +2020-01-23,6,69917.32,0.058544,0.059183,-1.08 +2020-01-24,4,53235.56,0.057065,0.056854,0.37 +2020-01-25,4,13483.44,0.056135,0.056317,-0.32 +2020-01-26,4,6976.01,0.057448,0.0575,-0.09 +2020-01-27,3,14605.11,0.059222,0.059401,-0.3 +2020-01-28,5,59754.14,0.060556,0.060439,0.19 +2020-01-29,5,59412.2,0.061313,0.061435,-0.2 +2020-01-30,5,46441.84,0.06048,0.06128,-1.31 +2020-01-31,4,12567.88,0.061275,0.061515,-0.39 +2020-02-01,3,13481.45,0.061502,0.06169,-0.31 +2020-02-02,4,36642.18,0.064328,0.06378,0.86 +2020-02-03,4,36263.42,0.064277,0.063948,0.51 +2020-02-04,4,22378.46,0.06499,0.064691,0.46 +2020-02-05,5,73940.9,0.068509,0.067835,0.99 +2020-02-06,5,184535.72,0.070149,0.069903,0.35 +2020-02-07,2,85058.54,0.071873,0.072176,-0.42 +2020-02-08,3,24020.68,0.069724,0.070587,-1.22 +2020-02-09,3,57013.55,0.072307,0.072662,-0.49 +2020-02-10,4,104281.84,0.070369,0.070877,-0.72 +2020-02-11,4,50424.67,0.071965,0.071991,-0.04 +2020-02-12,4,67404.55,0.077583,0.078804,-1.55 +2020-02-13,4,37234.19,0.082342,0.083451,-1.33 +2020-02-14,3,9341.18,0.082426,0.083982,-1.85 +2020-02-15,3,31287.57,0.081201,0.082282,-1.31 +2020-02-16,5,56901.5,0.07098,0.074341,-4.52 +2020-02-17,6,27815.77,0.070118,0.070296,-0.25 +2020-02-18,4,16873.46,0.073256,0.073517,-0.35 +2020-02-19,4,15155.19,0.073789,0.074523,-0.99 +2020-02-20,5,17576.19,0.069468,0.069666,-0.28 +2020-02-21,3,5546.1,0.070255,0.071368,-1.56 +2020-02-22,5,3532.46,2386.065378,0.070368,3390720.01 +2020-02-23,3,156.8,0.072157,0.072264,-0.15 +2020-02-24,3,3835.91,0.068584,0.070039,-2.08 +2020-02-25,3,11558.43,0.065875,0.066061,-0.28 +2020-02-26,5,25060.87,0.058988,0.059959,-1.62 +2020-02-27,3,48991.32,0.06035,0.060051,0.5 +2020-02-28,3,29690.76,0.057665,0.058604,-1.6 +2020-02-29,5,14300.0,0.058142,0.058345,-0.35 +2020-03-01,4,8355.61,0.055431,0.056782,-2.38 +2020-03-02,5,12934.01,0.056943,0.058573,-2.78 +2020-03-03,5,4445.71,0.058265,0.058767,-0.85 +2020-03-04,3,3335.98,0.057579,0.058379,-1.37 +2020-03-05,4,9888.39,0.058738,0.060171,-2.38 +2020-03-06,4,7112.69,0.059585,0.060191,-1.01 +2020-03-07,4,10154.23,0.058874,0.060005,-1.88 +2020-03-08,5,502184.08,6766.837175,0.053396,12672845.89 +2020-03-09,4,23672.79,0.050009,0.050079,-0.14 +2020-03-10,3,9036.87,0.05168,0.051891,-0.41 +2020-03-11,5,23774.53,0.049807,0.050323,-1.03 +2020-03-12,4,43497.46,0.039819,0.038955,2.22 +2020-03-13,4,58095.11,0.036534,0.034811,4.95 +2020-03-14,4,64546.09,0.038082,0.038386,-0.79 +2020-03-15,4,42704.01,0.037972,0.038085,-0.3 +2020-03-16,5,26229.93,0.03512,0.034561,1.62 +2020-03-17,5,29502.84,0.037416,0.036759,1.79 +2020-03-18,3,42395.83,0.036726,0.035883,2.35 +2020-03-19,4,44635.1,0.040315,0.039423,2.26 +2020-03-20,5,48353.94,0.04202,0.041228,1.92 +2020-03-21,3,62309.34,0.040015,0.039474,1.37 +2020-03-22,4,22791.37,0.038968,0.038394,1.5 +2020-03-23,5,13923.59,0.039592,0.03808,3.97 +2020-03-24,3,24642.35,0.040112,0.039754,0.9 +2020-03-25,4,860.01,0.039916,0.039935,-0.05 +2020-03-26,4,1291.67,0.041026,0.041016,0.03 +2020-03-27,3,22139.99,0.041485,0.042057,-1.36 +2020-03-28,4,3085.91,0.039393,0.039538,-0.37 +2020-03-29,3,22292.53,0.038966,0.039013,-0.12 +2020-03-30,5,46718.33,0.040136,0.039745,0.99 +2020-03-31,5,52484.79,0.040554,0.040479,0.18 +2020-04-01,4,35642.39,0.040084,0.039993,0.23 +2020-04-02,5,77619.55,0.041318,0.041284,0.08 +2020-04-03,4,47930.88,0.041479,0.041463,0.04 +2020-04-04,6,42530.6,0.041543,0.041385,0.38 +2020-04-05,5,31827.01,0.042134,0.042226,-0.22 +2020-04-06,5,132978.16,0.047765,0.047639,0.27 +2020-04-07,5,157355.69,0.050061,0.050009,0.11 +2020-04-08,6,68629.79,0.049691,0.049851,-0.32 +2020-04-09,7,12805.75,0.050855,0.050696,0.31 +2020-04-10,5,25225.12,0.047936,0.048327,-0.81 +2020-04-11,4,4239.21,0.048923,0.048976,-0.11 +2020-04-12,4,10479.67,0.04988,0.049864,0.03 +2020-04-13,5,4471.89,0.047674,0.047376,0.63 +2020-04-14,5,54849.34,0.048397,0.04831,0.18 +2020-04-15,4,94954.03,0.04762,0.047387,0.49 +2020-04-16,6,131715.31,0.047641,0.047946,-0.64 +2020-04-17,5,45230.95,0.048921,0.04885,0.15 +2020-04-18,5,39643.61,0.049889,0.049858,0.06 +2020-04-19,4,45597.59,0.049591,0.049439,0.31 +2020-04-20,4,126756.92,0.0508,0.051015,-0.42 +2020-04-21,4,79396.85,0.050305,0.050225,0.16 +2020-04-22,4,55553.71,0.053361,0.053412,-0.1 +2020-04-23,5,189685.43,0.060657,0.061407,-1.22 +2020-04-24,4,141910.31,0.062794,0.062942,-0.24 +2020-04-25,5,62369.87,0.062065,0.062147,-0.13 +2020-04-26,5,61919.38,0.062,0.061928,0.12 +2020-04-27,5,84998.01,0.064205,0.064594,-0.6 +2020-04-28,5,159905.19,0.068908,0.068709,0.29 +2020-04-29,6,157772.76,0.070961,0.070708,0.36 +2020-04-30,5,293181.12,0.070129,0.0706,-0.67 +2020-05-01,4,202281.83,0.071238,0.071568,-0.46 +2020-05-02,5,121197.32,0.074295,0.073846,0.61 +2020-05-03,5,204076.81,0.073819,0.073897,-0.11 +2020-05-04,6,276121.91,0.071129,0.0707,0.61 +2020-05-05,6,110227.12,0.072712,0.072508,0.28 +2020-05-06,5,106095.86,0.072158,0.072088,0.1 +2020-05-07,6,154289.52,0.072072,0.072049,0.03 +2020-05-08,5,98588.2,0.072586,0.072501,0.12 +2020-05-09,5,128552.45,0.072441,0.072472,-0.04 +2020-05-10,6,276346.56,0.063903,0.063738,0.26 +2020-05-11,5,242046.8,0.062944,0.06283,0.18 +2020-05-12,6,188217.82,0.06652,0.067173,-0.97 +2020-05-13,5,138915.99,0.069714,0.069625,0.13 +2020-05-14,6,72632.52,0.069553,0.069534,0.03 +2020-05-15,5,86549.72,0.068742,0.068594,0.22 +2020-05-16,6,45866.64,0.067757,0.067846,-0.13 +2020-05-17,6,937424.15,322580.645161,0.069759,462422156.43 +2020-05-18,5,48556.2,0.069996,0.069989,0.01 +2020-05-19,6,77282.43,0.069441,0.069419,0.03 +2020-05-20,6,148875.96,0.070254,0.070308,-0.08 +2020-05-21,5,239713.85,0.067865,0.06775,0.17 +2020-05-22,4,150351.11,0.067412,0.067642,-0.34 +2020-05-23,4,93995.03,0.067745,0.067715,0.04 +2020-05-24,4,78237.24,0.066623,0.066405,0.33 +2020-05-25,6,93997.42,0.065121,0.064959,0.25 +2020-05-26,6,52605.02,0.065338,0.065002,0.52 +2020-05-27,4,57543.96,0.066101,0.066277,-0.27 +2020-05-28,4,121160.03,0.067378,0.067274,0.15 +2020-05-29,4,120432.04,0.068012,0.068062,-0.07 +2020-05-30,3,178165.51,0.070319,0.070237,0.12 +2020-05-31,5,368857.24,0.073098,0.073374,-0.38 +2020-06-01,4,307412.65,0.07383,0.074247,-0.56 +2020-06-02,5,422910.06,0.079674,0.079452,0.28 +2020-06-03,3,324550.81,0.080216,0.080454,-0.3 +2020-06-04,4,372650.49,0.083214,0.08303,0.22 +2020-06-05,5,285874.18,0.081037,0.081023,0.02 +2020-06-06,4,151722.2,0.079795,0.07987,-0.09 +2020-06-07,4,110219.2,0.078255,0.078351,-0.12 +2020-06-08,6,60941.93,0.079206,0.079322,-0.15 +2020-06-09,6,68070.75,0.078598,0.078559,0.05 +2020-06-10,5,58387.67,0.078462,0.078471,-0.01 +2020-06-11,6,148662.22,0.073653,0.074193,-0.73 +2020-06-12,5,116233.38,0.072879,0.072952,-0.1 +2020-06-13,4,75692.62,0.073143,0.073195,-0.07 +2020-06-14,4,67996.07,0.072311,0.072225,0.12 +2020-06-15,5,79832.2,0.068813,0.068386,0.62 +2020-06-16,5,112614.68,0.071083,0.07098,0.15 +2020-06-17,5,110896.62,0.071823,0.071918,-0.13 +2020-06-18,5,89671.86,0.07052,0.070865,-0.49 +2020-06-19,6,138024.27,0.069658,0.069721,-0.09 +2020-06-20,6,131071.47,0.069341,0.069294,0.07 +2020-06-21,5,45655.46,0.069478,0.069464,0.02 +2020-06-22,5,152818.05,0.070709,0.070788,-0.11 +2020-06-23,5,222230.79,0.07114,0.07121,-0.1 +2020-06-24,5,156590.1,0.070158,0.069909,0.36 +2020-06-25,5,91272.42,0.067721,0.067514,0.31 +2020-06-26,4,112398.2,0.067143,0.067231,-0.13 +2020-06-27,5,66150.0,0.065038,0.063423,2.55 +2020-06-28,4,61640.91,0.063499,0.063615,-0.18 +2020-06-29,4,69754.35,0.064127,0.064109,0.03 +2020-06-30,4,46944.08,0.066323,0.066027,0.45 +2020-07-01,5,105257.94,0.067699,0.067743,-0.06 +2020-07-02,5,179269.03,0.069019,0.06893,0.13 +2020-07-03,4,129370.78,0.067358,0.067341,0.03 +2020-07-04,4,72065.66,0.067323,0.067297,0.04 +2020-07-05,6,74488.44,0.066788,0.066621,0.25 +2020-07-06,5,116642.97,0.070087,0.069587,0.72 +2020-07-07,5,269612.71,0.072048,0.072135,-0.12 +2020-07-08,5,545826.12,0.078771,0.079302,-0.67 +2020-07-09,5,859175.69,0.090558,0.091012,-0.5 +2020-07-10,5,427428.42,0.086796,0.086878,-0.09 +2020-07-11,6,409317.13,0.096176,0.095511,0.7 +2020-07-12,5,317181.38,0.093484,0.093599,-0.12 +2020-07-13,5,384018.43,0.093625,0.093771,-0.16 +2020-07-14,5,251674.11,0.089579,0.089658,-0.09 +2020-07-15,4,105352.22,0.092755,0.092766,-0.01 +2020-07-16,5,94473.06,0.095034,0.095503,-0.49 +2020-07-17,4,548006.28,0.103857,0.103781,0.07 +2020-07-18,4,248774.07,0.101583,0.101392,0.19 +2020-07-19,4,61179.01,0.100534,0.099704,0.83 +2020-07-20,5,83141.09,0.096053,0.09837,-2.35 +2020-07-21,5,262609.88,0.096409,0.096249,0.17 +2020-07-22,5,43364.79,0.095844,0.096225,-0.4 +2020-07-23,6,227346.68,0.099249,0.098924,0.33 +2020-07-24,5,153398.59,0.096623,0.096195,0.44 +2020-07-25,5,77025.42,0.10038,0.098598,1.81 +2020-07-26,5,322876.55,0.100969,0.101412,-0.44 +2020-07-27,5,157002.91,0.098182,0.094729,3.65 +2020-07-28,6,17167.25,0.095035,0.095423,-0.41 +2020-07-29,5,12296.57,0.097346,0.096918,0.44 +2020-07-30,5,17841.57,0.101953,0.095924,6.29 +2020-07-31,5,33190.24,0.096234,0.096213,0.02 +2020-08-01,5,279884.13,0.10542,0.103142,2.21 +2020-08-02,5,721224.23,0.105604,0.10542,0.17 +2020-08-03,5,695012.83,0.106455,0.106083,0.35 +2020-08-04,5,444012.36,0.108115,0.108265,-0.14 +2020-08-05,5,154989.76,0.10807,0.10821,-0.13 +2020-08-06,5,282147.62,0.106784,0.106709,0.07 +2020-08-07,5,185829.04,0.106236,0.104038,2.11 +2020-08-08,4,13813.94,0.103163,0.102813,0.34 +2020-08-09,5,320721.11,0.104701,0.104656,0.04 +2020-08-10,6,374172.18,0.105502,0.10514,0.34 +2020-08-11,5,234786.39,0.105135,0.10223,2.84 +2020-08-12,5,12846.59,0.098864,0.099883,-1.02 +2020-08-13,5,11518.14,0.099945,0.099732,0.21 +2020-08-14,5,344590.11,0.10513,0.104345,0.75 +2020-08-15,5,254746.35,0.105872,0.106,-0.12 +2020-08-16,5,314196.15,0.109124,0.109355,-0.21 +2020-08-17,5,333169.46,0.114208,0.114175,0.03 +2020-08-18,5,239185.58,0.111315,0.111241,0.07 +2020-08-19,5,298555.53,0.106078,0.103517,2.47 +2020-08-20,5,178445.63,0.105215,0.103866,1.3 +2020-08-21,5,353299.04,0.106262,0.104384,1.8 +2020-08-22,4,10949.03,0.100679,0.09997,0.71 +2020-08-23,4,1354.23,0.101247,0.100753,0.49 +2020-08-24,5,280466.82,0.104652,0.104044,0.58 +2020-08-25,4,159720.58,0.101187,0.098624,2.6 +2020-08-26,4,21382.44,0.098517,0.098119,0.41 +2020-08-27,5,127457.53,0.097046,0.094551,2.64 +2020-08-28,5,12542.54,0.095658,0.094798,0.91 +2020-08-29,5,54396.04,0.096359,0.09578,0.6 +2020-08-30,4,123674.94,0.097295,0.097351,-0.06 +2020-08-31,4,239267.1,0.097098,0.096908,0.2 +2020-09-01,5,720123.0,0.09854,0.098803,-0.27 +2020-09-02,5,69727.78,0.094561,0.093575,1.05 +2020-09-03,5,32689.78,0.083284,0.085097,-2.13 +2020-09-04,5,157122.49,0.078136,0.079058,-1.17 +2020-09-05,5,10292.64,0.077718,0.077317,0.52 +2020-09-06,5,12261.13,0.077678,0.075833,2.43 +2020-09-07,5,4011.99,0.076256,0.076612,-0.46 +2020-09-08,6,4976.27,0.076625,0.077208,-0.76 +2020-09-09,4,5638.31,0.078713,0.077978,0.94 +2020-09-10,4,1803.56,0.080513,0.080594,-0.1 +2020-09-11,5,3132.92,0.081259,0.081729,-0.57 +2020-09-12,3,3470.91,0.084463,0.083672,0.95 +2020-09-13,5,3886.31,0.079764,0.081691,-2.36 +2020-09-14,5,5172.83,0.081789,0.080599,1.48 +2020-09-15,4,4318.8,0.07905,0.079716,-0.84 +2020-09-16,5,8474.18,0.079858,0.079083,0.98 +2020-09-17,5,91220.73,0.079675,0.079962,-0.36 +2020-09-18,5,74031.48,0.079663,0.078946,0.91 +2020-09-19,7,19075.21,0.078862,0.078257,0.77 +2020-09-20,4,10402.85,0.078139,0.076977,1.51 +2020-09-21,5,48329.51,0.074969,0.072778,3.01 +2020-09-22,5,4767.04,0.070407,0.070788,-0.54 +2020-09-23,5,40076.45,0.071292,0.069778,2.17 +2020-09-24,5,7302.51,0.070574,0.06926,1.9 +2020-09-25,5,7982.73,0.074142,0.072393,2.42 +2020-09-26,4,37278.39,0.074827,0.07427,0.75 +2020-09-27,4,94639.56,0.073025,0.073058,-0.04 +2020-09-28,5,114810.34,0.074983,0.074587,0.53 +2020-09-29,5,8155.36,0.07309,0.073528,-0.6 +2020-09-30,4,11963.08,0.072971,0.07338,-0.56 +2020-10-01,5,155719.71,0.07528,0.074201,1.45 +2020-10-02,5,6188.24,0.071173,0.070777,0.56 +2020-10-03,4,5118.75,0.071257,0.071288,-0.04 +2020-10-04,3,3933.04,0.07325,0.071827,1.98 +2020-10-05,4,2373.3,0.07316,0.073172,-0.02 +2020-10-06,4,217128.75,0.072469,0.073324,-1.17 +2020-10-07,4,341760.46,0.071828,0.071988,-0.22 +2020-10-08,3,143946.53,0.072899,0.072005,1.24 +2020-10-09,4,329272.06,0.074779,0.074429,0.47 +2020-10-10,4,474925.02,0.077222,0.077088,0.17 +2020-10-11,4,364711.94,0.077576,0.077607,-0.04 +2020-10-12,4,576094.05,0.077558,0.077618,-0.08 +2020-10-13,4,375914.57,0.076609,0.076653,-0.06 +2020-10-14,4,337961.04,0.075215,0.075065,0.2 +2020-10-15,4,257547.79,0.073282,0.073285,-0.0 +2020-10-16,4,449797.41,0.075175,0.075011,0.22 +2020-10-17,6,498928.51,0.080677,0.081182,-0.62 +2020-10-18,4,92332.87,0.080557,0.080655,-0.12 +2020-10-19,5,304377.28,0.084109,0.084425,-0.37 +2020-10-20,4,276463.22,0.08202,0.083138,-1.34 +2020-10-21,5,409867.4,0.083655,0.083767,-0.13 +2020-10-22,4,177128.69,0.085775,0.085675,0.12 +2020-10-23,4,225867.85,0.084502,0.084784,-0.33 +2020-10-24,3,136436.06,0.084416,0.084657,-0.29 +2020-10-25,5,188807.02,0.084129,0.08396,0.2 +2020-10-26,5,157623.29,0.082542,0.082387,0.19 +2020-10-27,6,76253.74,0.081998,0.082044,-0.06 +2020-10-28,4,8817.96,0.080625,0.080142,0.6 +2020-10-29,6,162876.23,0.077361,0.077415,-0.07 +2020-10-30,5,70442.22,0.076973,0.076099,1.15 +2020-10-31,4,25742.73,0.07816,0.078576,-0.53 +2020-11-01,4,56065.31,0.077426,0.077427,-0.0 +2020-11-02,4,107853.01,0.077204,0.077176,0.04 +2020-11-03,5,70577.81,0.074665,0.074592,0.1 +2020-11-04,5,153624.78,0.074567,0.074208,0.48 +2020-11-05,5,533623.12,0.077968,0.077704,0.34 +2020-11-06,5,883464.1,0.083253,0.083122,0.16 +2020-11-07,4,952966.77,0.082564,0.082512,0.06 +2020-11-08,4,126163.98,0.081243,0.081116,0.16 +2020-11-09,4,4857.2,0.080257,0.080102,0.19 +2020-11-10,6,31737.99,0.08128,0.08063,0.81 +2020-11-11,4,48495.36,0.081345,0.081034,0.38 +2020-11-12,5,29138.59,0.08,0.08002,-0.02 +2020-11-13,5,7575.7,0.081142,0.081322,-0.22 +2020-11-14,4,585745.07,0.082194,0.08232,-0.15 +2020-11-15,4,305251.8,0.081285,0.080845,0.54 +2020-11-16,5,223894.43,0.082057,0.081223,1.03 +2020-11-17,4,933957.05,0.085443,0.085525,-0.1 +2020-11-18,4,327950.55,0.083993,0.084204,-0.25 +2020-11-19,4,343255.92,0.084244,0.083881,0.43 +2020-11-20,4,136225.16,0.086214,0.085995,0.25 +2020-11-21,5,2819062.05,0.098695,0.099475,-0.78 +2020-11-22,6,3756515.07,0.104749,0.104917,-0.16 +2020-11-23,4,3465816.52,0.113722,0.113838,-0.1 +2020-11-24,5,3488669.27,0.170352,0.170239,0.07 +2020-11-25,5,2011754.1,0.208895,0.206455,1.18 +2020-11-26,5,1716009.43,0.161948,0.168575,-3.93 +2020-11-27,4,2685684.37,0.184221,0.185881,-0.89 +2020-11-28,3,1410781.67,0.206686,0.207864,-0.57 +2020-11-29,4,1045778.65,0.195011,0.194877,0.07 +2020-11-30,4,1319950.95,0.198954,0.198925,0.01 +2020-12-01,4,638042.94,0.192994,0.191951,0.54 +2020-12-02,4,631918.44,0.183743,0.183582,0.09 +2020-12-03,4,491241.69,0.184164,0.184083,0.04 +2020-12-04,4,1314468.13,0.175984,0.175238,0.43 +2020-12-05,4,594257.5,0.169531,0.16904,0.29 +2020-12-06,4,314251.97,0.173536,0.173778,-0.14 +2020-12-07,4,375748.66,0.170088,0.1704,-0.18 +2020-12-08,5,455178.2,0.16077,0.159239,0.96 +2020-12-09,4,857622.71,0.158017,0.157995,0.01 +2020-12-10,5,436963.96,0.170973,0.169703,0.75 +2020-12-11,3,390136.08,0.158305,0.15821,0.06 +2020-12-12,4,408358.03,0.159232,0.158447,0.5 +2020-12-13,6,854450.12,0.171607,0.170891,0.42 +2020-12-14,6,586774.36,0.171681,0.171625,0.03 +2020-12-15,6,141360.84,0.170661,0.170297,0.21 +2020-12-16,4,808948.25,0.176993,0.17519,1.03 +2020-12-17,5,1540479.49,0.192224,0.191542,0.36 +2020-12-18,5,606977.65,0.183622,0.183613,0.01 +2020-12-19,4,271091.17,0.183856,0.183967,-0.06 +2020-12-20,5,209363.44,0.178792,0.178843,-0.03 +2020-12-21,6,110797.67,0.165743,0.169229,-2.06 +2020-12-22,6,686395.41,0.161586,0.161962,-0.23 +2020-12-23,6,1358208.29,0.138448,0.139618,-0.84 +2020-12-24,6,3309195.29,0.149675,0.144446,3.62 +2020-12-25,6,2680628.58,0.15681,0.156492,0.2 +2020-12-26,6,1502031.6,0.149247,0.148919,0.22 +2020-12-27,7,1599746.11,0.145272,0.145167,0.07 +2020-12-28,3,1658595.28,0.14975,0.149076,0.45 +2020-12-29,6,1307062.55,0.138606,0.138145,0.33 +2020-12-30,5,512763.77,0.133141,0.133093,0.04 +2020-12-31,5,337794.25,0.127025,0.126683,0.27 +2021-01-01,5,278530.29,0.132695,0.133345,-0.49 +2021-01-02,5,75256.46,0.130676,0.130017,0.51 +2021-01-03,6,80333.45,0.132328,0.132209,0.09 +2021-01-04,6,2760994.2,0.147975,0.149607,-1.09 +2021-01-05,6,2369735.15,0.173151,0.179432,-3.5 +2021-01-06,6,6855005.22,0.276063,0.29244,-5.6 +2021-01-07,7,2833847.68,0.352224,0.3499,0.66 +2021-01-08,6,1347696.57,0.292563,0.29932,-2.26 +2021-01-09,5,1813732.32,0.299529,0.301215,-0.56 +2021-01-10,6,1033741.49,0.298377,0.298583,-0.07 +2021-01-11,7,876806.95,0.251253,0.244923,2.58 +2021-01-12,6,1141171.7,0.282687,0.286041,-1.17 +2021-01-13,7,90495.62,0.286246,0.289611,-1.16 +2021-01-14,6,548360.66,0.29909,0.301654,-0.85 +2021-01-15,7,236476.41,0.296941,0.289219,2.67 +2021-01-16,7,191902.59,0.292881,0.295966,-1.04 +2021-01-17,6,88282.33,0.301534,0.302217,-0.23 +2021-01-18,5,684999.03,0.299094,0.300334,-0.41 +2021-01-19,6,1944375.76,0.306766,0.308369,-0.52 +2021-01-20,6,526319.07,0.290399,0.288098,0.8 +2021-01-21,7,2056013.22,0.276087,0.271456,1.71 +2021-01-22,5,315715.61,0.274408,0.257845,6.42 +2021-01-23,7,81952.38,0.275,0.27134,1.35 +2021-01-24,6,643360.45,0.273398,0.273804,-0.15 +2021-01-25,6,707664.27,0.272137,0.271649,0.18 +2021-01-26,5,385259.41,0.257662,0.257414,0.1 +2021-01-27,7,646169.09,0.242451,0.243293,-0.35 +2021-01-28,6,1343563.73,0.268674,0.270505,-0.68 +2021-01-29,7,3859434.25,0.315464,0.320839,-1.68 +2021-01-30,6,1375506.91,0.310444,0.315503,-1.6 +2021-01-31,6,2668396.74,0.321607,0.317921,1.16 diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/summary.json b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/summary.json new file mode 100644 index 00000000..1b8619d1 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/summary.json @@ -0,0 +1,52 @@ +{ + "assets_enumerated": 1194, + "scanned": 1192, + "errors": 0, + "traded_vs_xlm": 11, + "days_with_ref": 1005, + "years": { + "2016": { + "days_with_ref": 1, + "vol_usd": 0.02 + }, + "2017": { + "days_with_ref": 14, + "vol_usd": 31.390000000000004 + }, + "2018": { + "days_with_ref": 228, + "vol_usd": 7594050.72 + }, + "2019": { + "days_with_ref": 365, + "vol_usd": 10688274.440000005 + }, + "2020": { + "days_with_ref": 366, + "vol_usd": 96215632.27999999 + }, + "2021": { + "days_with_ref": 31, + "vol_usd": 39861099.01 + } + }, + "xlm_own_series": { + "2018": { + "days": 21, + "vol_xlm": 4581.33 + }, + "2019": { + "days": 63, + "vol_xlm": 8947.89 + }, + "2020": { + "days": 257, + "vol_xlm": 5238448.71 + }, + "2021": { + "days": 31, + "vol_xlm": 1362866.82 + } + }, + "binance_days": 1000 +} \ No newline at end of file diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/tokens.csv b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/tokens.csv new file mode 100644 index 00000000..2d868bd8 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/tokens.csv @@ -0,0 +1,12 @@ +asset,days,vol_usd_at_par,first,last,median_abs_dev_vs_day_pct,days_dev_gt_10pct,days_dev_gt_25pct,median_abs_dev_vs_binance_pct,binance_days +USD:GB2O5PBQJDAFCNM2U2DIMVAEI7ISOYL4UJDTLN42JYYXAENKBWY6OBKZ,427,127640403.73,2019-11-07,2021-01-31,0.0,40,15,0.52,427 +USD:GDUKMGUGDZQK6YHYA5Z6AY2G4XDSZPSZ3SW5UN3ARVMO6QSRDWP5YLEX,902,18663168.12,2018-08-05,2021-01-31,0.0,19,19,0.63,902 +USDT:GC2PJ6J37QM3DOQ63DVGCXTRC5QOKAJUBQO2XKBXOJCWTCOCP75KWUDX,137,4588203.63,2019-06-06,2021-01-13,931.64,112,107,115809.19,137 +USD:GDSRCV5VTM3U7Y3L6DFRP3PEGBNQMGOWSRTGSBWX6Z3H6C7JHRI4XFJP,544,3360485.07,2018-09-27,2021-01-24,0.73,84,73,1.01,544 +USD:GBNLJIYH34UWO5YZFA3A3HD3N76R6DOI33N4JONUOHEEYZYCAYTEJ5AK,316,72076.94,2019-10-04,2021-01-31,3.54,88,49,3.47,316 +USD:GBUYUAI75XXWDZEKLY66CFYKQPET5JR4EENXZBUZ3YXZ7DS56Z4OKOFU,976,19441.54,2016-10-18,2021-01-31,0.75,22,19,0.7,939 +USD:GAEDZ7BHMDYEMU6IJT3CTTGDUSLZWS5CQWZHGP4XUOIDG5ISH3AFAEK2,271,12275.9,2020-03-07,2021-01-31,1.12,7,1,1.05,271 +USDT:GCCK3X3KRXHZOO5OVQHGTJKMBWTTNRDTYB3F4NM3ZEUK37XFMYFCSCLX,30,2790.93,2020-12-21,2021-01-30,2.44,4,1,4.24,30 +USD:GD3LHBHY6NYJXETU6YO5OYI37F3X6L6C46ZWKBFBT35RBRKACDVMRF5K,17,207.93,2020-10-26,2021-01-31,65.74,9,9,65.64,17 +USDT:GC3YRTJQUXVQMG5STS7MSWVF3OF3Z5MVF6NEWZLKNQAJZX5UUFLMCFMA,15,17.05,2020-11-16,2021-01-21,24.58,15,6,24.27,15 +USD:GA5O7NTUIKYDPZSV3MNMIRFO2LCV5RJBNO7TNN7Y3MDO2IXRBSWDEO3T,2,17.0,2020-06-08,2020-11-12,312462.08,2,2,312385.12,2 diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md new file mode 100644 index 00000000..5a9fd79a --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md @@ -0,0 +1,86 @@ +--- +type: R +status: mature +date: 2026-10-13 +who: stkrolikiewicz +--- + +# R — Pre-run report: dollar tokens traded against XLM before 2021-02 + +Safeguard 3 of the task. Built from the public API and Binance, not from +ClickHouse (no prod read access in the session that made it). Data and the +script that produced it are in [`../data/`](../data/). + +## Method + +1. Every issuer of `USD`, `USDT` and `USDC` on Horizon: **1,194** assets. +2. For each, `GET /v1/assets/{id}/ohlcv?granularity=1M&base_currency=XLM` over + 2015-11 → 2021-01: the candles where the token is the base and XLM the quote, + which is how `canonical.rs` stores every dollar token against XLM except + canonical USDT and USDC. 1,192 scanned (the two canonical ones skipped), 0 + errors. +3. Daily candles of the tokens that traded. Implied XLM/USD at par = `1 / pf_vwap`. +4. Per day: the reference is the USD-weighted median of those markets; each + token's deviation from it; Binance XLM/USDT daily VWAP as an outside check + (Binance data starts 2018-05-31; Horizon and CoinGecko no longer serve this + period publicly). + +**Not covered:** canonical USDT, which is XLM's *quote* and so sits in XLM's own +series. The API has no raw XLM/USDT price for it before 2021 (USD fields are +null), only its volume: 21 days and 4,581 XLM in 2018, 63 days and 8,948 XLM in +2019, 257 days and 5.2 M XLM in 2020. + +## Findings + +**Eleven dollar tokens traded against XLM before 2021-02.** + +| token (issuer prefix) | days | USD volume at par | first → last | median dev. from the day | median dev. from Binance | +|---|---|---|---|---|---| +| USD `GB2O5PBQ` | 427 | 127.6 M | 2019-11 → 2021-01 | 0.0 % | 0.52 % | +| USD `GDUKMGUG` (AnchorUSD) | 902 | 18.7 M | 2018-08 → 2021-01 | 0.0 % | 0.63 % | +| **USDT `GC2PJ6J3`** | 137 | 4.6 M | 2019-06 → 2021-01 | **932 %** | **115,809 %** | +| USD `GDSRCV5V` | 544 | 3.4 M | 2018-09 → 2021-01 | 0.73 % | 1.01 % | +| USD `GBNLJIYH` | 316 | 72 k | 2019-10 → 2021-01 | 3.54 % | 3.47 % | +| USD `GBUYUAI7` | 976 | 19 k | 2016-10 → 2021-01 | 0.75 % | 0.70 % | +| USD `GAEDZ7BH` | 271 | 12 k | 2020-03 → 2021-01 | 1.12 % | 1.05 % | +| USDT `GCCK3X3K` | 30 | 2.8 k | 2020-12 → 2021-01 | 2.44 % | 4.24 % | +| **USD `GD3LHBHY`** | 17 | 208 | 2020-10 → 2021-01 | **65.7 %** | **65.6 %** | +| **USDT `GC3YRTJQ`** | 15 | 17 | 2020-11 → 2021-01 | **24.6 %** | **24.3 %** | +| **USD `GA5O7NTU`** | 2 | 17 | 2020-06 → 2020-11 | **312,462 %** | **312,385 %** | + +**Coverage of the reference** (days with at least one dollar market): + +| year | days | USD volume | median abs. dev. from Binance | p90 | days > 10 % off | +|---|---|---|---|---|---| +| 2016 | 1 | $0.02 | — | — | — | +| 2017 | 14 | $31 | — | — | — | +| 2018 | 228 | $7.6 M | 1.87 % | 5.57 % | 1 | +| 2019 | 365 | $10.7 M | 0.61 % | 2.47 % | 15 | +| 2020 | 366 | $96.2 M | 0.34 % | 1.88 % | 4 | +| 2021-01 | 31 | $39.9 M | 0.85 % | 2.67 % | 0 | + +- **Before 2018 there is no usable reference:** one token, $31 in two years. + Priced history in practice starts in **August 2018** (AnchorUSD), daily from + 2019. +- **Where it exists, par agrees with the market:** median 0.3–1.9 % from + Binance by year. +- **The median filter does not stop USDT `GC2PJ6J3`.** It held par in mid-2019 + (2019-06-06: implied XLM/USD 0.1203 against Binance 0.1219) and then lost it: its + implied XLM/USD is a median **~1,160x** Binance's, ~346 in January 2021. Valued at + $1 its volume is inflated by the same factor, so it holds more than half the + day's USD-at-par volume on **19 days**; there it *is* the weighted median and + sets the reference. (The 932 % in the table is against the day's median, which + it skews itself.) Simulated + with `max_dev` 0.10 and 0.25 alike: **20 days more than 10 % off Binance** in + 2018–2020 with all eleven tokens, **1 day** with the four off-peg tokens + (bold above) left out. + +## So what + +"Every dollar token" as decided on 2026-10-13 gives ~20 days of XLM/USD wrong by up +to three orders of magnitude in 2019–2020, and with it every XLM-quoted price on those days. The +deviation threshold cannot fix it. It needs a per-token exclusion: either a +list from this report (the four bold tokens), or a rule that drops a token whose +median deviation over its history exceeds the threshold. `max_dev` itself barely +matters here (0.10 and 0.25 give the same days); 0.25 keeps the honest small +markets that sit 3–4 % off. diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md new file mode 100644 index 00000000..b3ffdacd --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md @@ -0,0 +1,141 @@ +--- +id: "0332" +title: "USD candles take high/low from USDT-quoted markets, whose USD rate we cannot measure — set the extremes from USDC- and XLM-quoted markets only" +type: BUG +status: active +assignee: akot +related_adr: ["0011", "0287"] +related_tasks: ["0170", "0229", "0236", "0172", "0212", "0286", "0331"] +tags: [layer-backend, priority-high, effort-small, api, ohlcv, methodology, data-correctness, defect] +links: + - "../../../../packages/prices-api/src/assets/queries_ch.rs" + - "../../../../packages/prices-api/src/assets/handlers.rs" + - "../../../../packages/prices-api/tests/ohlcv_it.rs" + - "../../../2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md" + - "../../../../web/portal/public/openapi.json" +history: + - date: "2026-10-14" + status: active + who: akot + note: > + Code in PR #402 (5 commits, not merged). ITs green on CH 26.3.10.60: + ohlcv_it 45, ignored suite 318/0 over 40 targets, workspace 1437/0. + Step 1 measured on prod (AC 6, 7); client notice drafted (AC 10). + Left: merge, compute deploy, sending the notice. + - date: "2026-10-14" + status: active + who: akot + note: > + Decided: fallback buckets get no response flag (measured by AC 7 + instead); ADR 0011 is not amended. Implementation branch opened. + - date: "2026-10-14" + status: active + who: akot + note: > + Opened from Adam's analysis of USD-mode /ohlcv high/low. Methodology change + agreed on Slack the same day: Marek ("wyklucz usdt w liczeniu high/low") + and Oskar (same reason USDT was dropped from the USD reference). Scope is + step 1 only (read path); the per-minute rate (step 2) is a follow-up. +--- + +# USD candles take high/low from USDT-quoted markets + +## Summary + +USD-mode `/ohlcv` merges every market of an asset (pair × source) into one +candle: each market is scaled by one rate per bucket, then `high = max`, +`low = min`. USDT-quoted markets are scaled by a USDT/USD rate we cannot +measure, so their extremes are guesses and max/min picks the worst ones. XLM on +2022-06-18 comes back $0.1767 / $0.0815 against Binance's $0.1135 / $0.1001. +The stored data is correct; the read path's merge is not. Only USDC- and +XLM-quoted markets will set the price fields; every market still counts for +volume. + +- Analysis, case study, prior art: [R-usdt-legs-widen-usd-candles](notes/R-usdt-legs-widen-usd-candles.md) +- Decision: [S-price-setting-markets-are-usdc-and-xlm](notes/S-price-setting-markets-are-usdc-and-xlm.md) +- Measured on prod: [R-step-1-measured-on-production](notes/R-step-1-measured-on-production.md) +- Client notice draft: [G-client-notice-usd-candle-prices](notes/G-client-notice-usd-candle-prices.md) + +## Status: Active + +**Current state:** PR #402 open, not merged; not deployed. Notice drafted, not sent. +Oskar's review (2026-10-14) addressed on the branch: the means follow the same +markets as the prices; a USDC/XLM leg that formed a price but has no USD value +yet leaves the bucket unpriced (0 such buckets in the stored 1d history); an +untracked XLM is no longer cached. The overflow-nulls-the-band trade-off was +kept (0 such buckets on prod). + +## Implementation Plan + +All in `packages/prices-api/src/assets/queries_ch.rs` unless stated. + +1. **XLM id kept apart.** `UsdRefs` (~L703) gains the price-setting quote ids + (USDC + XLM when tracked). `handlers.rs` ~L560 already resolves XLM; today it + sits in `pivots` next to USDT. `pivots` and the `traded` label stay as is. +2. **`band` predicate + per-bucket fallback.** `usd_projection` (~L1003): + `band = valid AND quote_asset_id IN ()`. `usd_aggregates` + (~L979): `if(countIf(band) > 0, If(.., band), If(.., valid))` for + `o`, `h`, `l`, `c`, `meth`. Keep `CLOSE_EXACT` / `HIGH_PUBLISHED` / + `LOW_PUBLISHED` (~L783–799) as single named expressions so `vw`/`pfvw` still + clamp against what the caller sees; keep the `isNull` overflow guards. +3. **Tests, table-driven.** ITs in `tests/ohlcv_it.rs`: USDC+USDT bucket (USDT + extremes excluded, its volume kept); USDT with the larger volume (o/c/method + still from USDC); XLM+USDT (XLM sets the band); USDT-only (unchanged); vwap + clamped into the new band; the 2022-06-18 shape as a regression. + `ohlcv_converts_each_leg_before_merging_across_them` still discriminates. + Plus the ClickHouse-free SQL-string tests. +4. **Contract text.** `/ohlcv` description in `handlers.rs` ~L398 ("against + every quote asset — are merged") rewritten; `openapi.json` regenerated. +5. **Measure and announce.** Read-only before/after on prod; client notice + drafted in `notes/` (sending is Adam's). + +## Acceptance Criteria + +- [x] USD mode: a bucket with a valid USDC- or XLM-quoted row takes + `open`/`high`/`low`/`close`/`method` from those rows only. +- [x] A bucket with no such row returns the same candle as before. +- [x] Volumes and trade counts unchanged for every bucket. +- [x] `vwap` and `pf_vwap` lie within the published `[low, high]`. +- [x] `base_currency=XLM` (`QuoteLeg` arm) unchanged. +- [x] XLM 1d 2022-06-18 returns high ≈ $0.1137, low ≈ $0.1005; XLM p95 high/low + deviation vs Binance recorded before/after and set against the analysis' + step 1+2 figures. Prod: $0.1137 / $0.1005; on the analysis' 557 days step + 1 alone gives its 1.14 % / 2.16 % (see the R- note). +- [x] Affected assets/buckets counted, including non-USDT legs the allowlist + also drops (the analysis' 419 counts USDT only), and fallback buckets + (no USDC/XLM market) counted separately. 37,582 buckets / 419 assets, + all USDT (no other quote is convertible); fallback 1,345 / 160. 1d only. +- [x] ITs pass on CH 26.3.10.60; full ignored suite green. +- [x] OpenAPI description updated in the same PR as the code. +- [x] Client notice drafted in `notes/`. + +## Decisions (2026-10-14, Adam) + +1. **No flag on fallback buckets.** A bucket with no USDC/XLM market is computed + as today and is not marked in the response: a new field changes the response + shape (OpenAPI, portal types, conformance suite, client notice). AC 7 counts + fallback buckets; a flag becomes its own task if there are many. +2. **ADR 0011 is not amended.** The change ships as code, the `/ohlcv` + description and the client notice. The fallback keeps the denomination (USD) + and only narrows which markets set the extremes — the same kind of + data-dependent input choice as today's highest-volume open/close. + +## Follow-ups (not spawned) + +- **Step 2 — per-minute rate for USDC/XLM markets.** Needs `_1m` priced + end-to-end (0286 USD re-pricing) and a choice: read `_1m` at query time, or + store USD extremes in the rollups (schema + re-ingest). AC 6: for XLM, step 1 + alone already gives the analysis' step 1+2 figures; any gain from step 2 + would come from XLM-quoted tokens (ETH, XRP in the analysis). +- **Cheap single fills on the USDC/XLM markets themselves.** After step 1, + most remaining XLM low outliers are Aquarius XLM/USDC fills far below the + market (62 of 121 days >5 % off, 2025–26, drifting from $0.20 to $0.03). This + looks like a stale pool that nobody arbitrages; which pool is not yet + established. The analysis' ±50 % band (rule 3) removed these prints. See the + R- note. + +## Notes + +- Pre-2021 ([[0331]]): USDT-quoted rows carry a par `close_usd` there; the + allowlist still keeps them out of the band when an XLM leg traded — intended. +- Execution: `/gsd-quick --full` with a BRIEF, in a worktree. diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md new file mode 100644 index 00000000..f2a2271c --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md @@ -0,0 +1,72 @@ +--- +title: "Client notice draft: USD candle prices come from USDC- and XLM-quoted markets" +type: generation +status: developing +spawned_from: notes/R-step-1-measured-on-production.md +spawns: [] +tags: [ohlcv, client-notice, high-low] +links: + - "S-price-setting-markets-are-usdc-and-xlm.md" +history: + - date: "2026-10-14" + status: developing + who: akot + note: > + Draft for Adam to send. Effective date left open until the compute + deploy is scheduled. +--- + +# Client notice draft + +Adam sends this. Fill in the date once the deploy is scheduled. + +--- + +**Subject:** Prices API: USD candles now take prices only from USD-measurable markets (effective ``) + +From ``, `GET /assets/{asset}/ohlcv` in USD mode (`base_currency=USD`, +the default) changes which markets set a candle's prices. + +**What changes** + +A candle merges every market of an asset: each trading pair on each venue. +Each market is converted to USD before the merge. A market quoted in Stellar +USDT (the Apay bridge token, `GCQTGZ…`, not Tether) has no reliable USD rate, +so its converted high and low could land far from where the asset traded. On +XLM, 2022-06-18, the API returned a high of $0.1767 and a low of $0.0815; +Binance shows $0.1135 and $0.1001. + +From ``: + +- `open`, `high`, `low`, `close`, `method`, `vwap` and `pf_vwap` come only + from markets quoted in USDC or XLM, when one of them formed a price in the + bucket. That same day now returns $0.1137 / $0.1005. +- If such a market has traded but its USD value is not computed yet, the + bucket's price fields are `null` until it is, rather than taken from the + other markets. This affects only the most recent buckets, and only briefly. +- A bucket in which no USDC- or XLM-quoted market formed a price is computed + exactly as before. +- `volume_base`, `volume_quote_usd`, `trade_count` and `pf_trade_count` are + unchanged and still count every market. +- `method` follows the markets that set the prices. A bucket that traded + mostly against USDT can now report the `assumed-par` of its USDC market. +- `base_currency=XLM` is unchanged. + +The response shape does not change: no field is added, removed or renamed. + +**Who is affected** + +Measured on daily candles since 2021-02: 0.34 % of priced buckets (37,582), +across 419 assets that traded against USDT and against USDC or XLM on the same +day. High or low moves in 16,136 of those buckets, across 340 assets, and +`vwap`/`pf_vwap` can move in any of the 37,582. For XLM, +the 95th-percentile gap to Binance falls from 23 % to 1.2 % for the daily high +and from 44 % to 6.4 % for the daily low. + +**What you need to do** + +Nothing, unless you store candles. Candles you fetched before `` for +the affected assets can differ from what the API returns now; fetch them +again to get the new values. + +The `/ohlcv` description in the API reference states the rule. diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-step-1-measured-on-production.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-step-1-measured-on-production.md new file mode 100644 index 00000000..5d3249b2 --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-step-1-measured-on-production.md @@ -0,0 +1,121 @@ +--- +title: "Step 1 measured on production: XLM vs Binance, affected buckets, and where the remaining cheap lows come from" +type: research +status: mature +spawned_from: notes/R-usdt-legs-widen-usd-candles.md +spawns: [] +tags: [ohlcv, usdt, high-low, measurement, aquarius] +links: + - "R-usdt-legs-widen-usd-candles.md" +history: + - date: "2026-10-14" + status: mature + who: akot + note: > + Read-only measurement on prod (dev_read) of PR #402's SQL against the + code it replaces, 1d buckets 2021-02-01..2026-10-13. AC 6 and AC 7. +--- + +# Step 1 measured on production + +## Method + +- The query is the one PR #402 generates (taken from `system.query_log` of + the `ohlcv_it` run), widened to every asset and grouped by asset × day. +- The old candle is the same query with `band` false everywhere: the fallback + branch is byte-identical to the pre-change SQL, so this is today's API. +- Both run on `prices.price_ohlcv_1d` as `dev_read`, month by month. The + 3.73 GiB per-query limit rejects the whole range at once. +- Binance: `XLMUSDT` 1d klines from `data-api.binance.vision`. +- Only `1d` was measured, not `1h` or `1m`. The scripts stayed in the + session scratchpad. + +## AC 6: XLM vs Binance + +XLM 1d 2022-06-18: $0.1767 / $0.0815 → **$0.1137 / $0.1005** (Binance +$0.1135 / $0.1001). + +p95 of |ours / Binance − 1|: + +| window | days | high: before → after | low: before → after | +|---|---:|---|---| +| the analysis' window | 557 | 8.45 % → **1.14 %** | 12.93 % → **2.16 %** | +| 2022-07 → 2026-08 | 1524 | 27.7 % → 1.33 % | 52.3 % → 8.18 % | +| all | 2081 | 23.2 % → 1.24 % | 43.5 % → 6.44 % | + +Medians after: 0.16 % (high), 0.19 % (low). + +**Step 1 alone reproduces the analysis' "steps 1 + 2" figures.** The analysis +measured 557 days, the ones where `1m` had prices: 2021-02 → 2022-06 and +2026-09 → 2026-10-12. On those days step 1 gives 1.14 % / 2.16 % against the +analysis' 1.14 % / 2.19 %. For XLM, step 2 adds almost nothing, because its +price-setting market is USDC, whose USD rate stays near 1 all day. The analysis' +gains for XLM-quoted tokens (ETH, XRP) are a separate measurement and still +stand. + +Its baseline was lower (8.45 % / 12.93 % against 23.2 % / 43.5 %) only because +the window left out 2022-07 → 2026-08, where most bad days are. + +## AC 7: what changes (11,114,887 priced 1d buckets) + +| | buckets | assets | +|---|---:|---:| +| a leg leaves the band | 37,582 (0.34 %) | 419 | +| … of which USDT | 37,582 | 419 | +| … of which another quote | 0 | 0 | +| high or low changes | 16,136 | 340 | +| open or close changes | 4,519 | — | +| `method` changes | 528 | — | +| band extreme newly `null` (review IN-03) | 0 | — | +| fallback: no valid USDC/XLM leg | 1,345 | 160 | + +- In practice the allowlist drops only USDT. A row on any other quote fails + `convertible`, so it never entered the price before either. +- Where the high moves, the median move is −3.0 %. At p95 the old high was + about 3× the new one. The low moves by a median +3.2 % and a p95 +57 %. +- Fallback buckets are few and falling: 405 in 2021, 17 in 2026. This + supports Decision 1 (no flag). +- Changed buckets per year: + + | 2021 | 2022 | 2023 | 2024 | 2025 | 2026 | + |---:|---:|---:|---:|---:|---:| + | 2,486 | 4,534 | 1,780 | 2,484 | 2,698 | 2,154 | + +## What step 1 leaves: cheap lows on the USDC markets themselves + +After step 1, 121 XLM days have a low more than 5 % off Binance. Market that +sets the low: + +| source | days | low vs Binance (median / worst) | that market's close vs Binance | years | +|---|---:|---|---|---| +| Aquarius | 62 | −56 % / −98 % | −0.03 % | 2025–26 | +| SDEX | 29 | +5.6 % / −34 % | −0.04 % | mostly 2021 | +| Phoenix | 12 | −8 % / −29 % | −0.7 % | 2024 | +| Soroswap | 9 | −18 % / −46 % | −12.7 % | 2024 | +| Sushiswap | 9 | −37 % / −99 % | −26.5 % | 2025–26 | + +- **SDEX**: mostly real cross-venue gaps on the 2021 crash days, which the + analysis also found. +- **Phoenix, Soroswap, Sushiswap**: thin pools trading away from the market. +- **Aquarius**: on a market of thousands of fills a day with a correct close, + single fills print far below the price. The cheap low drifts down steadily: + + | 2025-10 | 2026-03 | 2026-04 | 2026-06 | 2026-08 | 2026-10 | + |---:|---:|---:|---:|---:|---:| + | $0.20 | $0.14 | $0.07 | $0.056 | $0.045 | $0.032 | + + At `1m`, 2026-09-25 08:38 holds 2 fills and 1,001 XLM: 1,000 XLM at + 0.2188 and exactly 1 XLM for 0.0383677 USDC. The arithmetic closes, so this + is a real swap and not a decoding error. It looks like one stale pool that + nobody arbitrages. + + Aquarius lists four XLM/USDC pools: + - a concentrated pool and a constant-product pool at 0.1 %, both deep; + - a stable-curve pool, now empty (`CAQXW3…`); + - a constant-product pool at 0.3 % holding ~$40 (`CARCLMAP…`). + + Which of them printed the fill is not established. Candles carry no pool + id, and stellar.expert's event index for these pools ends at 2025-12-31. +- The analysis' "rule 3" (a ±50 % band around the day's median price) removed + exactly this kind of print: on its window, the worst high went from 395 % to + 12.7 % and the worst low from 85 % to 18.2 %. It was not part of 0332. diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md new file mode 100644 index 00000000..a6efced9 --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md @@ -0,0 +1,83 @@ +--- +title: "Why USDT-quoted markets widen USD candles, and how others pick price-setting trades" +type: research +status: mature +spawns: + - notes/S-price-setting-markets-are-usdc-and-xlm.md + - notes/R-step-1-measured-on-production.md +tags: [ohlcv, usdt, methodology, high-low] +links: [] +history: + - date: "2026-10-14" + status: mature + who: akot + note: "Adam's analysis, moved out of the task README" +--- + +# Why USDT-quoted markets widen USD candles + +A market here is pair × source: XLM/USDC on SDEX, XLM/USDT on SDEX, XLM/USDC on +Aquarius, and so on. + +## How USD mode computes high/low today + +Each market is converted with one rate for the whole bucket, the day's close: +`rate = close_usd / close` of that market. Then `high_usd = high × rate`, +`low_usd = low × rate`, and `high = max`, `low = min` across markets. + +## Case: XLM, 1d, 2022-06-18 + +| market | share of volume | high | low | +|---|---:|---:|---:| +| XLM/USDC, SDEX | 96 % (28.1 M XLM) | $0.1137 | $0.1005 | +| XLM/USDT, SDEX | 4 % (1.2 M XLM) | 0.3242 USDT → **$0.1767** | 0.1496 USDT → **$0.0815** | +| API today (max/min of both) | — | **$0.1767** | **$0.0815** | +| Binance | — | $0.1135 | $0.1001 | + +The USDT market is converted at the end-of-day USDT rate ($0.5449). That day +USDT traded between $0.43 and $0.96, so an evening rate does not fit morning +trades. The API returns the USDT market's extremes. + +## Why max/min is right and its inputs are wrong + +High and low are, by definition, the extremes of real trades; volume plays no +part. A volume-weighted "average high" (weights 0.96 / 0.04) gave tidy numbers +but was rejected: the result is no longer a price anyone traded at. + +A trade on XLM/USDC has a price in USD. A trade on XLM/USDT has a price in USDT; +its USD value is our estimate, price × a USDT rate we compute ourselves. A +maximum of values measured with error is biased up and a minimum down. The more +estimated markets go in, the wider the candle, whatever the truth. + +## Why Stellar USDT has no usable USD rate + +- "USDT" on Stellar (`GCQTGZ…`) is not Tether but an Apay bridge receipt (issuer + domain `dead.apay.io`). It lost its peg in 2022-06; ~$0.15 today. +- Its only USDC market has turned over a median of 0–27 USDC a day since 2023. +- Within one minute it can disagree with the rate implied through XLM. On + 2022-06-18 USDT/USDC printed 0.43 at 00:49 and 0.957 at 01:12 on ~74 USDC an + hour, while the XLM legs implied 0.36–0.65. +- A per-minute rate would not help: the token has no single, reliable USD price. +- Same reason [[0172]]/[[0212]] fenced the USD reference to USDC (Oskar, + 2026-10-14). + +## How others do it + +- **US equities (CTA tape):** every trade has a sale-condition code that decides + whether it may set high/low. Some trades count for volume only. +- **CoinMarketCap:** drops outlier prices and, manually, markets whose price does + not reflect a free market (e.g. an exchange blocking deposits/withdrawals). +- **CoinGecko:** drops tickers with outlier prices (MAD). + +We do not filter by price; we filter by whether the quote asset's USD value is +known. + +## Measured effect (steps 1 and 2 together) + +- 419 assets traded at the same time in USDT and in USDC or XLM since 2021. +- XLM vs Binance, p95 deviation: high 8.45 % → 1.14 %, low 12.93 % → 2.19 %. +- XLM-quoted tokens: median error 2–4× lower (ETH 1.34 % → 0.57 %, XRP + 0.74 % → 0.18 %). + +The analysis does not separate step 1 (drop USDT from the band) from step 2 +(per-minute rate for USDC/XLM markets). diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md new file mode 100644 index 00000000..d5ac1d75 --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md @@ -0,0 +1,58 @@ +--- +title: "Decision: only USDC- and XLM-quoted markets set USD open/high/low/close" +type: synthesis +status: mature +spawned_from: notes/R-usdt-legs-widen-usd-candles.md +spawns: [] +tags: [ohlcv, methodology, high-low, adr-0011] +links: + - "../../../../2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md" +history: + - date: "2026-10-14" + status: mature + who: akot + note: > + Agreed on Slack: Marek ("wyklucz usdt w liczeniu high/low"), Oskar (same + reason USDT was dropped from the USD reference). +--- + +# Decision: only USDC- and XLM-quoted markets set USD prices + +## Conclusion + +Keep the candle definition (high/low = extremes of real trades) but take them +only from trades whose USD value we know. + +1. **Allowlist, not a denylist.** Price-setting quote assets are canonical USDC + and XLM (XLM/USDC is liquid, so XLM's USD rate is well known). Every other + quote leg, USDT included, counts for volume only. A new token (e.g. USDT0) + joins the list once its rate is reliable. +2. **Per bucket.** If a price-setting market has a valid row, open, high, low, + close and `method` come from price-setting markets only. Otherwise the bucket + is computed as today. +3. **Volume, trade counts:** every market. **vwap, pf_vwap:** every convertible + market, clamped into the new `[low, high]`. +4. **Read path only** (`queries_ch.rs`). No stored value changes. +5. **Contract:** the `/ohlcv` description and a client notice. ADR 0011 is not + amended (Adam, 2026-10-14). Fallback buckets carry no response flag. + +## Reasoning + +- The defect is the input to max/min, not max/min itself. +- USDT's USD rate is not measurable (see the R- note), so any USDT extreme in + USD is a guess, and max/min selects the worst guesses. +- Open/close today come from the highest-volume market, which can be USDT; the + same argument applies to them. + +## Alternatives considered + +- **Volume-weighted high/low.** Rejected: not a traded price. +- **Per-minute USDT rate.** Rejected: USDT has no single USD price even within a + minute. +- **Leave as is.** Rejected by Marek and Oskar. + +## Step 2 (not decided here) + +Convert USDC/XLM markets at the rate of the trade's minute, not the bucket close. +Matters for XLM-quoted tokens, since XLM moves within the day. Needs `_1m` +priced end-to-end. Decided after step 1 is measured. diff --git a/lore/1-tasks/active/0340_FEATURE_prices-api-on-prices-api-sorobanscan-com.md b/lore/1-tasks/active/0340_FEATURE_prices-api-on-prices-api-sorobanscan-com.md new file mode 100644 index 00000000..7a6e72b3 --- /dev/null +++ b/lore/1-tasks/active/0340_FEATURE_prices-api-on-prices-api-sorobanscan-com.md @@ -0,0 +1,168 @@ +--- +id: "0340" +title: "Serve the Prices API at prices-api.sorobanscan.com, then move the portal to sorobanscan.com" +type: FEATURE +status: active +related_adr: ["0010"] +related_tasks: ["0339", "0326", "0194", "0186", "0318"] +tags: ["hosting", "portal", "domains", "priority-medium", "effort-medium"] +links: + - https://github.com/rumblefishdev/soroban-block-explorer/blob/develop/lore/1-tasks/backlog/0659_OPS_move-public-domains-to-sorobanscan-com.md + - https://github.com/rumblefishdev/soroban-block-explorer/blob/develop/lore/1-tasks/active/0664_OPS_redirect-old-explorer-hosts.md +history: + - date: 2026-11-05 + status: backlog + who: stkrolikiewicz + note: > + Task created. Step 6 of the explorer's domain move (its task 0659) had no + task here. The explorer moved to sorobanscan.com on 2026-11-05, but the + portal cannot follow while this API stays on *.rumblefish.dev. + - date: 2026-11-05 + status: active + who: stkrolikiewicz + note: > + Started: Step 1 (the second custom domain) on + feat/0340_prices-api-on-prices-api-sorobanscan-com. +--- + +# Serve the Prices API at prices-api.sorobanscan.com, then move the portal to sorobanscan.com + +## Summary + +The Prices API gets a second host, `prices-api.sorobanscan.com`, beside +`prices-api.sorobanscan.rumblefish.dev`. Then the portal moves from +`sorobanscan.rumblefish.dev/api/prices-api/` to `sorobanscan.com/api/prices-api/` +in one window with the backend's own config. The names were agreed by the team +on 2026-11-03 (explorer task 0659, step 6). + +## Status: Active + +**Current state:** Steps 1–4 done (2026-11-06). The portal runs on +`https://sorobanscan.com/api/prices-api/` against `prices-api.sorobanscan.com`, +a Discord sign-in passed there, and the old portal host answers `301`. The old +API host `prices-api.sorobanscan.rumblefish.dev` keeps serving API-key clients. +Step 5's docs part is in progress on `feat/0340_docs-name-new-host`. Still +open: the old host's fate. A key test on the new host passed (2026-11-06). + +- Step 5, docs (2026-11-06): the endpoint reference, the overview, the OAuth + runbook, the skill, the portal's logo link and the comments that state the + current hosts name `sorobanscan.com`. The skill reaches readers from + `master` (its `copyValue`, 0318), so after the next develop→master merge. + - Kept on the old host on purpose: dated SCF evidence and runbooks; the + cache verification and the k6 load test, because they measure API + Gateway and the new host puts Cloudflare in front. They need another + target before the old host retires. + - The Stellar catalog entry (stellar/stellar-dev-skill#141, merged) names + `sorobanscan.rumblefish.dev/prices-api/`; it reaches the portal in one + `301`. + +- Done: Step 3 (2026-11-06, Adam). The Discord application holds the new + redirect URI beside the old one. Checked by an authorize link that showed the + consent screen with `https://prices-api.sorobanscan.com`. +- Done: Step 4 (2026-11-06, about 08:57–09:00 UTC). #412 merged as `8ed247d3`. + - `redirect_uri` in `prices/production/portal-discord-oauth` → the new + callback. The previous version `c34106e0…` is kept as `AWSPREVIOUS`. + - `deploy-production-apigateway` and `-compute`: CORS, `API_BASE_URL`, + `PORTAL_WEB_ORIGIN`. + - `sync-portal-explorer`, with the bundle built against the new host. + - Explorer: `pricesPortalOnLegacyHost: false`, release #719, Delivery from + a laptop. + - Checked: preflight from `sorobanscan.com` answers `204` with that origin and + credentials; sign-in starts with the new `redirect_uri`; the callback lands + on `https://sorobanscan.com/api/prices-api/`; a person signed in. +- Found (2026-11-06): CI on #412 pinned two copies of `apiBaseUrl`: + `PUBLIC_API_BASE_URL` in `web/portal/src/landing/links.ts` and `servers` in + `web/portal/public/openapi.json`. Both moved with it. The logo's `EXPLORER` + link still names the old explorer host (it answers `301`), because nine + `app.spec.tsx` cases pin it. Change it with Step 5. + +- Done: Step 1. #411, deployed with `make deploy-production-apigateway` from + `3dc847d7`. The stack's first deploy since 2026-09-24 also shipped 0309's + `404` for unmapped routes. Regional target + `d-in7vpbow2m.execute-api.eu-central-1.amazonaws.com`. +- Done: Step 2. soroban-infrastructure#9: a proxied CNAME, `prices_api_origin`. +- Decided (2026-11-05): proxied, like the rest of `sorobanscan.com` behind + Cloudflare, with no rules of its own yet. The zone's single Free-plan rate + limit (100 / 10 s per IP) stays the explorer API's, because a paid plan here + allows 25 requests/s; API Gateway's usage plans keep limiting. Managed WAF + comes after watching the traffic. An `X-Edge-Secret` origin lock comes only + with Step 5, while the old host and `execute-api` still answer. The backend + reads no client IP. + +## Context + +- **Why the portal cannot move alone.** Its session cookie is `SameSite=Lax` + and host-only (`portal/auth/cookies.rs`). The browser sends it to the backend + only while page and backend are same-site: today both are under + `rumblefish.dev`. A page on `sorobanscan.com` calling + `prices-api.sorobanscan.rumblefish.dev` is cross-site, so the cookie never + arrives and the portal looks signed out. CORS allows one origin, + `portalWebOrigin`; that is the smaller half. +- **Why a subdomain, not a path on `sorobanscan.com`.** Any host under + `sorobanscan.com` is same-site with the portal. A path would route this API + through the explorer's CloudFront and collide with the `/api/` the explorer + host already uses (hub, portal) and this backend uses (`/api/auth/*`, + `Path=/api/`). Decided 2026-11-05: subdomain, as agreed in 0659. +- **What exists.** The `*.sorobanscan.com` certificate in eu-central-1 + (`c937aeef…`, explorer task 0661) covers `prices-api.sorobanscan.com`, same + AWS account. The `sorobanscan.com` zone is soroban-infrastructure's + `cloudflare/`. The old host is a Route 53 alias to this API Gateway domain. +- **Who.** soroban-infrastructure's README names the migration Karol's. The + Discord application and its redirect URI are Adam's alone (ADR 0010 §6, + `docs/runbooks/portal-oauth-deploy-prep.md`). + +## Implementation Plan + +### Step 1: A second custom domain (CDK) + +`prices-api.sorobanscan.com` on the same API, as the explorer's 0662 did: an +optional domain and its certificate ARN, the regional target as an output. The +old domain keeps serving. + +### Step 2: The record (soroban-infrastructure) + +`prices-api.sorobanscan.com` → that regional target, proxied (see Decided). The +Free plan has one rate-limit rule for the whole zone, already the explorer API's. + +### Step 3: Discord + +Adam adds `https://prices-api.sorobanscan.com/api/auth/callback` as a second +redirect URI, beside the current one. Matching is treated as character-exact +(runbook §1). + +### Step 4: Switch, in one window + +- Here: `apiBaseUrl` and `portalWebOrigin` (`https://sorobanscan.com`) in + `infra/envs/production.json`, and the `redirect_uri` in the Discord OAuth + secret (runbook §2); ApiGateway and Compute. Then the portal bundle, built + against the new API host. +- Explorer: `pricesPortalOnLegacyHost: false` (Delivery), so its old host sends + the portal to `sorobanscan.com` with a `301`, like everything else. +- Signed-in users sign in again once: the cookie is host-only. + +### Step 5: The old API host + +Keep it serving for API-key clients for a while, with docs, the skill +(skills.stellar.org, [[0318]]) and examples pointing at the new host. Then a +`308` or retirement. API Gateway cannot redirect by itself; decide when we get +there. + +## Acceptance Criteria + +- [x] `https://prices-api.sorobanscan.com/v1/…` answers like the old host, + with the same API keys (checked with a key, 2026-11-06). +- [x] A Discord sign-in round-trip completes on + `https://sorobanscan.com/api/prices-api/`, done by a person on production + (2026-11-06). +- [x] `https://sorobanscan.rumblefish.dev/api/prices-api/…` answers `301` to + the same path on `sorobanscan.com` (explorer flag off, 2026-11-06). +- [x] Docs, the skill and the API examples name the new host (2026-11-06; + dated evidence kept as submitted). +- [x] **Docs updated** — README, `docs/runbooks/portal-oauth-deploy-prep.md`, + `infra/README.md` (no host left in it). + +## Notes + +- Explorer side of the same window: soroban-block-explorer task 0664 + (`pricesPortalOnLegacyHost`), umbrella 0659. +- Dated SCF evidence stays as submitted, as in [[0326]] and [[0339]]. diff --git a/lore/1-tasks/active/0343_FEATURE_release-through-deploy-production-master-follows-each-deploy.md b/lore/1-tasks/active/0343_FEATURE_release-through-deploy-production-master-follows-each-deploy.md new file mode 100644 index 00000000..a6c58aca --- /dev/null +++ b/lore/1-tasks/active/0343_FEATURE_release-through-deploy-production-master-follows-each-deploy.md @@ -0,0 +1,135 @@ +--- +id: "0343" +title: "Production is deployed by hand from develop and master trails it — release through deploy/production, and move master only after a successful deploy" +type: FEATURE +status: active +assignee: okarcz +related_adr: [] +related_tasks: ["0141", "0242"] +tags: ["deploy", "release", "ci", "branching", "priority-medium", "effort-medium"] +links: + - "../../../infra/Makefile" + - "../../../infra/src/lib/stacks/cicd-stack.ts" + - "../../../.github/workflows/ci.yml" + - https://github.com/rumblefishdev/stellar-x402/blob/develop/.github/workflows/deploy-production.yml +history: + - date: 2026-11-12 + status: active + who: okarcz + note: > + Task created and activated. Scope so far is the branch flow below; no + code yet. The mechanism (who runs the deploy, how the commit is stamped) + is still open and listed under Open decisions. +--- + +# Production is deployed by hand from develop and master trails it — release through deploy/production, and move master only after a successful deploy + +## Summary + +Anyone reading the repository should be able to trust one thing: the code on +`master` is the code running in production. Today that does not hold. This task +introduces a three-branch release flow, `develop` → `deploy/production` → +`master`, in which production is deployed only from `deploy/production` and +`master` moves only to a commit whose deploy succeeded. + +## Status: Active + +**Current state:** flow described, nothing implemented. Waiting for the +operator's decisions on the open points before any code. + +## Context + +Measured on 2026-11-12: + +- Production is deployed from an operator's machine with + `make deploy-production-*`, from `develop`, one stack at a time. +- `master` is updated later by a release PR. It was 164 commits behind + `develop` (release PR #418, open). +- `master` has no branch protection. The repository has no release tags. +- A deployed Lambda carries no commit id. Which commit the 2026-11-06 Compute + deploy was built from could only be answered from the deployer's reflog. +- One stack was held back on purpose (0242 D8: Observability, three alarms), so + "what `develop` describes" and "what is deployed" differed by design, with + the difference recorded only in a task file. +- `infra/src/lib/stacks/cicd-stack.ts` defines a GitHub OIDC deploy role scoped + to a `production` environment. No workflow uses it. Whether that stack is + deployed was not checked. + +`stellar-x402` already releases through a protected `deploy/production` branch: +a PR from `develop`, a deploy started by the merge, a build stamped with its +commit. This task brings the same order here and adds `master` as the record of +what is live. + +## The flow + +| Branch | Meaning | Who moves it | +| --- | --- | --- | +| `develop` | work in progress | feature PRs | +| `deploy/production` | approved for release | release PR from `develop`, approved, merge commit | +| `master` | confirmed live | the deploy, after it succeeds; fast-forward only | + +1. Feature PRs merge to `develop`, as now. Task documents keep going straight + to `develop`. +2. A release is a PR from `develop` to `deploy/production`. CI runs on it, and + the production `cdk diff` is read on it before approval. +3. The merge starts (or permits) the deploy of **all** stacks from that exact + commit. A stack with no change is a no-op. +4. On success, `master` is fast-forwarded to the deployed commit and the commit + is tagged (for example `prod-2026-11-12`). +5. On failure, `master` does not move. The fix or a revert goes to + `deploy/production` by PR and is deployed like any release. + +Rules that keep it true: + +- `master` takes no PRs and no direct commits; only the deploy moves it. +- Hotfixes go to `deploy/production` by PR and are merged back into `develop`. +- A rollback is a revert PR on `deploy/production`. `master` is never + force-moved. +- No stack is "held back". A resource that must wait is switched off by a + config flag, so every stack can always be deployed from the release commit. +- Changes that are not branch state (ClickHouse schema applies, Ansible runs, + the portal sync, data repairs) are applied from the release commit and noted + on the release tag. + +## Open decisions + +1. **Who runs the deploy.** GitHub Actions on `deploy/production` behind a + `production` environment approval, or the operator's machine with a guard in + `infra/Makefile` that refuses any other commit. Possibly the guard first and + the workflow second. +2. **How the deployed commit is stamped.** Per stack, since a five-stack deploy + can fail part-way. Candidate: an SSM parameter written by each stack. A + Lambda environment variable would restart every Lambda on every release. +3. **Drift check.** A scheduled job that compares the stamps with `master` and + reports a mismatch. +4. **What happens to the current branches.** How `deploy/production` is created + (from `master` after #418, or from `develop`), and how the one commit that + exists only on `master` (`7d52fa9b`) is carried over. +5. **Branch protection.** Rules for `deploy/production` and `master`, and which + identity is allowed to fast-forward `master`. +6. **Non-AWS targets.** How far the Hetzner side (ClickHouse, Ansible) is + brought under the same rule in this task. + +## Implementation Plan + +To be written once the open decisions are answered. + +## Acceptance Criteria + +- [ ] `deploy/production` exists, is protected, and takes only approved PRs +- [ ] A production deploy from any other commit is refused +- [ ] Every stack records the commit it was deployed from, readable without + access to the deployer's machine +- [ ] After a successful deploy `master` points at the deployed commit and + carries a release tag, with no manual step +- [ ] After a failed deploy `master` is unchanged +- [ ] A scheduled check reports when the deployed commit and `master` differ +- [ ] The flow, the hotfix path and the rollback path are documented where + deploys are documented (`infra/README.md`) + +## Notes + +- Out of scope: the catch-up release itself (PR #418) and the 0242 cleanup. + Both are handled separately and do not wait for this task. +- [[0141]] is the history behind `build-lambdas`: a deploy shipped stale Lambda + assets from a laptop. Building in CI removes that class of error. diff --git a/lore/1-tasks/backlog/0131_FEATURE_preroll-usd-coverage-gate.md b/lore/1-tasks/archive/0131_FEATURE_preroll-usd-coverage-gate.md similarity index 85% rename from lore/1-tasks/backlog/0131_FEATURE_preroll-usd-coverage-gate.md rename to lore/1-tasks/archive/0131_FEATURE_preroll-usd-coverage-gate.md index 0c1ac361..daff27fd 100644 --- a/lore/1-tasks/backlog/0131_FEATURE_preroll-usd-coverage-gate.md +++ b/lore/1-tasks/archive/0131_FEATURE_preroll-usd-coverage-gate.md @@ -2,15 +2,26 @@ id: "0131" title: "0088 step-3 pre-roll: gate/warn when 1m USD coverage for the span is below a threshold" type: FEATURE -status: backlog +status: completed related_adr: [] -related_tasks: ["0114", "0088", "0144", "0147", "0145"] +related_tasks: ["0114", "0088", "0144", "0147", "0145", "0286"] tags: [clickhouse, enrichment, backfill, pre-roll, guard, priority-low, effort-small] links: - "../../../packages/prices-clickhouse/schema/preroll-incremental.sql" - "../../../docs/runbooks/continue-soroban-backfill.md" history: - - date: 2026-07-24 + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed as superseded by [[0286]]. The flow this guard was for is gone: + `preroll-incremental.sql` is marked HISTORICAL, DO NOT RUN, and fails on + the post-0286 column count. Phase 3 pre-rolls coarse tiers from + re-ingested 1m that carries close_usd = 0 by design. The coarse re-price + (re-ingest runbook §7b-2) then fixes them, and 0286's AC checks + `reachable_left` ~0 per tier. Nothing was built and there are no + follow-ups. + - date: "2026-07-24" status: backlog who: okarcz note: "Spawned from 0114 future work — the last 0114 AC; belongs in 0088's step-3 pre-roll flow, which hasn't run yet." @@ -81,3 +92,5 @@ exercised once that pre-roll runs — which is why it doesn't fit as a 0114 bloc - [ ] Below-threshold coverage **warns loudly** (hard-refuse opt-in), and the expected-low pre-Soroban tail does not spuriously block the recovery. - [ ] Documented in the 0088 pre-roll runbook. + +*None built: superseded by [[0286]] (see history, 2026-10-06).* diff --git a/lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md b/lore/1-tasks/archive/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md similarity index 94% rename from lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md rename to lore/1-tasks/archive/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md index 4999047e..eb50c3d0 100644 --- a/lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md +++ b/lore/1-tasks/archive/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md @@ -2,7 +2,7 @@ id: "0139" title: "current_price_usd returns duplicate rows — assets is keyed on natural identity, not asset_id" type: BUG -status: active +status: completed assignee: akot related_adr: [] related_tasks: ["0072", "0061", "0067", "0144", "0150", "0129"] @@ -11,6 +11,18 @@ tags: milestone: 2 links: [] history: + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed. The window ran 2026-10-02 (swap 12:46 UTC), PR #382 merged as + 44809656, W14 gap-verify and the W16 next-day checks all green. PR #387 + (phase-3 preflight on live tiers) merged. Checked read-only on prod + 2026-10-06: assets FINAL 211,103 rows = 211,103 distinct asset_ids; + current_price_usd 3,736 rows = current_prices 3,736; 0129's join + cross-check 94,907,504 = 94,907,504. Still owed at close: the second + pass over months ≤ 202202, dropping the __pre0139 tables after it, + W17 period-close after 2026-11-02, and the note to BE. - date: "2026-10-02" status: active who: akot @@ -553,8 +565,11 @@ stopgap. Spawn accordingly. - [x] Determined whether the 3,275 duplicated `asset_id`s are ID collisions or superseded natural-identity rows, with the measurement recorded. **2026-09-30: all 3,315 are collisions, 0 superseded.** -- [ ] `current_price_usd` returns exactly one row per `current_prices` row. -- [ ] Every other view in `views.sql` audited for the same join defect. +- [x] `current_price_usd` returns exactly one row per `current_prices` row. + *Prod 2026-10-06: 3,736 = 3,736.* +- [x] Every other view in `views.sql` audited for the same join defect. + *Fixed at the source instead: `assets` is unique on `asset_id`, so no + join on it can fan out (`asset_id_schema_it.rs`).* - [x] **O2 — the XLM-native and USDC `asset_id`s checked against the 3,279 duplicates** (one query; see the open section above). **2026-09-30: XLM (4), USDC (3) and USDT (111) are clean; no foreign candle is admitted.** @@ -563,15 +578,21 @@ stopgap. Spawn accordingly. the pivot tier's `xlm_usd` — invisibly, because uniform duplication leaves the weighted value unchanged. Low prior, but it must not be assumed. Raised by [[0144]] while answering BE's `volume_base` question. -- [ ] A test fails if the fan-out reappears. -- [ ] BE informed of the resolution. +- [x] A test fails if the fan-out reappears. + *`current_price_usd_returns_one_row_per_current_prices_row` in + `asset_id_schema_it.rs`.* +- [ ] BE informed of the resolution. *Not sent at close; owed.* Carried from [[0129]] when it closed into this task (2026-09-24) — they check the allocator, not the view, so a view-only dedupe does not satisfy them: -- [ ] `SELECT count(), countDistinct(asset_id) FROM prices.assets FINAL` returns - equal values in production. -- [ ] 0129's two-query cross-check (its §Evidence) agrees to the row. -- [ ] An invariant test or probe guards `asset_id` uniqueness going forward. +- [x] `SELECT count(), countDistinct(asset_id) FROM prices.assets FINAL` returns + equal values in production. *Prod 2026-10-06: 211,103 = 211,103.* +- [x] 0129's two-query cross-check (its §Evidence) agrees to the row. + *Prod 2026-10-06, `price_ohlcv_1h` 2024-02…2026-07, `volume_quote > 0`: + 94,907,504 without the join = 94,907,504 with it.* +- [x] An invariant test or probe guards `asset_id` uniqueness going forward. + *`rollup-freshness-probe/src/asset_id_uniqueness.rs` and its alarm.* - [ ] `GET /assets` verified to emit no duplicate asset across a full cursor - walk (extends 0074's pagination test). + walk (extends 0074's pagination test). *No full walk was run. It cannot + emit a duplicate while `asset_id` is unique on prod (above).* diff --git a/lore/1-tasks/backlog/0140_PERF_asset-discovery-reemits-full-registry-hourly.md b/lore/1-tasks/archive/0140_PERF_asset-discovery-reemits-full-registry-hourly.md similarity index 76% rename from lore/1-tasks/backlog/0140_PERF_asset-discovery-reemits-full-registry-hourly.md rename to lore/1-tasks/archive/0140_PERF_asset-discovery-reemits-full-registry-hourly.md index 60a5d9f3..1314ab56 100644 --- a/lore/1-tasks/backlog/0140_PERF_asset-discovery-reemits-full-registry-hourly.md +++ b/lore/1-tasks/archive/0140_PERF_asset-discovery-reemits-full-registry-hourly.md @@ -2,7 +2,8 @@ id: "0140" title: "asset-discovery re-emits the whole asset registry every hour — 0132's defect in a second component" type: PERF -status: backlog +status: completed +assignee: akot related_adr: [] related_tasks: ["0210", "0132", "0133", "0067", "0256", "0226", "0241"] tags: ["priority-medium", "effort-small", "cost", "write-amplification", "clickhouse"] @@ -44,6 +45,20 @@ history: that PR's review: `ensure_seed` still READS the whole registry hourly to check ~20 identities. What remains here is that read and the `oracle-worker` instance. + - date: "2026-10-06" + status: active + who: akot + note: > + Activated together with [[0226]]: one PR on one branch. The oracle-worker `write_assets` and the `ensure_seed` whole-registry read are the same shape as 0226's load, and 0226 removes the registry the oracle write depends on. + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed without code: [[0139]] (PR #382, deployed 2026-10-02 12:30 UTC) + removed both remaining whole-registry paths. Measured on prod: 0 assets + written in 99 asset-discovery runs, largest daily `prices.assets` part + 1–29 rows (was ~210 k), memory 250 → 50 MB. All five ACs met; no + follow-up tasks. --- # `asset-discovery` re-emits the full asset registry every hour @@ -217,10 +232,46 @@ Priority stays at `medium`: what remains is the oracle instance below. load and points here. Same shape as [[0226]] (the oracle, every 5 minutes — twelve times as often, so that one first). +## ✅ Resolved by [[0139]] — measured on prod 2026-10-06 + +No code in this task. [[0139]] (PR #382, a6ecd20d) replaced both remaining +whole-registry paths with `OhlcvWriter::write_absent_assets`, a keyed +`count()` per identity: `ensure_seed` checks only its ~20 seed identities, and +the oracle's `write_assets` behind `count() > known_before` is gone (the oracle +checks its 2 sampled identities). With ids derived by ClickHouse +(`xxh3(identity)`), no watermark is needed. Deployed with the 0139 window: +CloudTrail `UpdateFunctionCode` for both Lambdas at 2026-10-02 12:30:16 UTC. + +| | before (09-28 → 10-02) | after (10-03 → 10-06) | +|---|---|---| +| `prices.assets` largest new part per day (`system.part_log`) | ~210 k rows (whole registry) | **1–2 rows**; 29 on 10-06 = [[0242]]'s classic seed | +| `assets_written` by asset-discovery | — | **0 in 99 runs** since the deploy | +| asset-discovery `Max Memory Used` | ~250 MB | **47–50 MB** | +| asset-discovery average duration | ~4 s | **0.6 s** | +| `loaded asset registry` lines from asset-discovery | 24 a day | **none** after 2026-10-02 11:19 UTC | + +- `system.query_log`, last 24 h: the keyed check ran **1,056 times** = 20 × 24 + (asset-discovery) + 2 × 288 (oracle), 6 ms average, at most 8,546 rows read. +- `prices.assets` holds one copy: 211,103 rows = 211,103 identities. +- The whole-registry `SELECT` still runs ~18 times a day, from the + ledger-processor's cold start (35 in 3 days, one AWS address each, in the + INIT second) and from an operator machine running backfills (18, one + non-AWS address). Neither is this task's: `load_assets()` stays for them by + design ([[0226]] AC 3). +- `OhlcvWriter::write_assets` has no production caller left (one IT, + `asset-discovery/tests/enrichment_survives_it.rs`). Not removed: dead but + harmless, and not worth a PR while the project closes (Adam, 2026-10-06). + ## Acceptance Criteria -- [ ] A zero-discovery hourly run writes **no** `prices.assets` rows. -- [ ] Newly-discovered assets are still persisted (seed path unaffected). -- [ ] The comment sits with the guard it actually describes. -- [ ] Other `write_assets` callers audited for the same defect. -- [ ] `part_log` shows the hourly ~204k rows gone, measured on prod. +- [x] A zero-discovery hourly run writes **no** `prices.assets` rows — 0 in 99 + runs since 2026-10-02 12:30 UTC. +- [x] Newly-discovered assets are still persisted (seed path unaffected) — + `seed_it::seed_writes_only_the_absent_identities` and + `absent_assets_it`, green in #382's CI. +- [x] The comment sits with the guard it actually describes — moot, deleted + with `discover_window` ([[0256]], PR #331). +- [x] Other `write_assets` callers audited for the same defect — AC 4 audit + above; the oracle instance was removed by [[0139]]. +- [x] `part_log` shows the hourly ~204k rows gone, measured on prod — + largest daily part 1–29 rows since 2026-10-03. diff --git a/lore/1-tasks/archive/0149_BUG_sweep-mv-version-ownership-close-usd.md b/lore/1-tasks/archive/0149_BUG_sweep-mv-version-ownership-close-usd.md new file mode 100644 index 00000000..ee84f16e --- /dev/null +++ b/lore/1-tasks/archive/0149_BUG_sweep-mv-version-ownership-close-usd.md @@ -0,0 +1,181 @@ +--- +id: "0149" +title: "Two writers with incompatible version arithmetic own close_usd — the 0114 sweep's repair is overwritten inside the MV window" +type: BUG +status: completed +assignee: akot +related_adr: ["0287", "0317", "0292"] +related_tasks: ["0144", "0146", "0148", "0114", "0095", "0143", "0286", "0203", "0151", "0228"] +tags: + ["priority-medium", "effort-medium", "clickhouse", "data-correctness", "materialized-view", "milestone-M2"] +milestone: 2 +links: + - "../../../packages/enrichment-worker/src/repair.rs" + - "../../../packages/prices-clickhouse/schema/rollups.sql" +history: + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed as superseded by [[0286]] (ADR 0287 §5), like [[0146]]. + Re-measured on prod: 0 coarse rows on any of the six tiers hold a USD + value their children do not support, so TEST E's case cannot recur. The + version arithmetic is unchanged and documented in ADR 0317. Further + paths are hypothetical; no follow-up tasks, by Adam's decision. No code + change, no regression test. + - date: "2026-10-06" + status: active + who: akot + note: "Activated and assigned to akot to work through the options." + - date: "2026-08-05" + status: backlog + who: okarcz + note: > + Spawned from [[0144]] future work (phase 7) — finding 3ii-b, confirmed on + the prod CH pin (TEST E). Deliberately ordered after [[0146]], which + removes most of its consequence. +--- + +# Sweep vs MV: incompatible version arithmetic on `close_usd` + +## Summary + +Two writers update `close_usd` on the same coarse rows and claim the row by +different rules: + +- The [[0114]] coarse sweep wins by **`version + 1`** (`repair.rs:20-22`). +- The rollup MVs write at **`version = sum(version)`** over their sub-rows — + and every enrichment event underneath adds 1 to that sum. + +Two enrichment events and the MV overtakes the sweep. Worse, the MV gets far +more attempts: `mv_ohlcv_15m_to_1h` re-appends the same hour **every 15 minutes +for 8 hours** (`REFRESH EVERY 15 MINUTE`, window `now() - INTERVAL 8 HOUR`). +The sweep gets one. + +Confirmed on CH **26.3.10.60** ([[0144]] `repro/04_sweep_durability.sql`, +TEST E): + +``` +after the sweep repairs it close_usd 0.171 version 401 +after the next MV refresh close_usd 0 version 402 +what the view publishes now (empty) +``` + +So the repair path we already built is defeated inside the re-aggregation +window — the bucket vanishes from `price_usd_series*` again, which is BE's +observation exactly. + +## Why this is ordered late + +**[[0146]] removes most of the consequence.** Once the MVs use `argMaxIf`, the +re-append writes a *correct* value rather than a zero — so being overtaken by +the MV stops being harmful inside the window. And outside the window the MV +does not write at all, so [[0148]]'s historical sweep is unaffected either way. + +That leaves this as a latent trap rather than a live defect: **two writers with +incompatible version arithmetic on one column**, with no mechanism preventing +the next such writer from colliding the same way. Worth closing on its own +terms, not urgent. + +Fixing only this and not [[0146]] would leave the zero-propagation completely +intact — the sweep would win a race to publish a value the MV is about to +recompute as zero anyway. + +## Implementation + +Options to weigh, none obviously correct yet: + +- Give the sweep a version domain the MV cannot reach (e.g. a large offset), at + the cost of making `version` no longer a plain event count. +- Have the sweep write through the same path as the MV so there is one writer. +- Make the sweep skip rows still inside an MV's re-aggregation window entirely, + and rely on [[0146]] for those — smallest change, but encodes the window + boundaries in two places. +- Reconsider whether `close_usd` should be MV-carried at all rather than always + derived by the sweep. + +Related: [[0143]] records that there is no `DEPENDS ON` anywhere in the cascade, +so ordering between tiers is already unsynchronised; any fix here should not +assume refresh ordering it does not control. + +## Re-measured on prod, 2026-10-06 + +Measured as `dev_read` (read-only), 08:23–08:40 UTC. The queries are kept +outside the repo. + +**What changed since this task was written.** + +- The version arithmetic is unchanged. The sweep still writes `version + 1` + (`repair.rs:22`, `ch_enrich.rs:2446/2490/2717`). Every MV still writes + `sum(version)` (`rollups.sql:151-163`), and 0286 kept it. +- [[0146]] was closed as superseded by [[0286]]. Since 2026-09-22 a coarse + `close_usd` is `close × the latest priced child's rate` (ADR 0287 §5, + `rollups.sql:65-75`). The MV writes 0 only when no child is priced, not + whenever the last child is unpriced. TEST E's step 4 now yields the same value + the sweep wrote. +- [[0143]]/[[0203]] (live 2026-10-05) added a third writer: six reconcile MVs + with a 7-day window, hourly. They write only buckets whose `trade_count` or + `volume_base` disagree, so they do not touch a sweep repair by themselves. +- ADR 0317 "Version analysis" already describes the arithmetic (sweep `+1`, + `coarse-repair` `+2`, ties go to the last insert) and accepts that a + re-derived `close_usd` "can briefly go to 0 … until the next sweep pass". +- The one design that would resolve this by construction, a USD rate table with + `close_usd` derived ([[0151]]), was rejected in ADR 0292. +- The fast windows are not all short. 1w re-rolls 60 days and 1M re-rolls + **400 days**, daily. The recurring sweep covers 2 months + (`COARSE_SWEEP_LOOKBACK_MONTHS`). + +**Exposure.** A row is at risk when it holds a USD value that its children do +not support, because the next MV write that wins would replace it. + +| tier (fast window) | rows | priced | last written by sweep | priced, no priced child | +|---|---|---|---|---| +| 15m (2 h) | 10,716 | 2,236 | 0 | 0 | +| 1h (8 h) | 25,444 | 5,948 | 11 | 0 | +| 4h (1 d) | 46,901 | 11,220 | 35 | 0 | +| 1d (7 d) | 117,586 | 29,977 | 241 | 0 | +| 1w (60 d) | 251,015 | 88,003 | 905 | 0 | +| 1M (400 d) | 564,712 | 275,966 | 448 | 0 | + +- "Last written by sweep" is `version > sum(children)`. Each of these rows holds + the USD value its children already give, so the sweep only got there first. + The same holds for `volume_quote_usd`: 0 rows priced over unpriced children, + on every tier. +- Residue at the 1e-12 floor, not this defect: + - 1d: one row priced at 1.0e-12 over children below the floor, and 15 rows + whose `close_usd` is ~1e-13. + - 1M: 1,302 rows at `close_usd = 0` whose 1d children are all below the floor + (max 9.9e-13), all last written by the MV. That is the floor working as + `rollups.sql:70-73` intends. + +**Verdict.** TEST E's case cannot recur. Its zero came from the MV carrying the +last child's `close_usd` when that child was unpriced. Since 0286 the MV +re-prices the bucket from the latest priced child, so it writes the same value +the sweep wrote. The race still happens, because the version arithmetic is +unchanged, but losing it costs nothing. + +## Closing decision (2026-10-06, Adam) + +Closed as superseded by [[0286]] (ADR 0287 §5), like [[0146]]. The measurement +above is the evidence. Other ways the race could matter again are hypothetical, +and we deliberately create no tasks for them: + +- a coarse-only price correction inside an MV window; +- a coarse tier pricing where its children cannot; +- a regression of the rate form; +- a new writer with its own version arithmetic. + +## Acceptance Criteria + +- [x] A single documented owner for `close_usd` on coarse rows, or a version + scheme under which the sweep's repair provably survives an MV refresh. + Met differently: the repair is still overwritten, but with the same + value. The arithmetic and the accepted brief zero are documented in ADR + 0317 "Version analysis"; the rate form in ADR 0287 §5. +- [ ] Regression test on CH 26.3.10.60 reproducing TEST E and showing the + repair now survives. Not written, by decision; the prod measurement of + 2026-10-06 stands in for it. +- [x] The `version` column's semantics restated in the schema header if they + change. They did not change. +- [x] Confirmed not to reintroduce the [[0095]] replace-mode invariants problem. + No schema or MV change was made; every MV is still `APPEND`. diff --git a/lore/1-tasks/backlog/0181_FEATURE_rollup-leading-indicator-alarms.md b/lore/1-tasks/archive/0181_FEATURE_rollup-leading-indicator-alarms.md similarity index 72% rename from lore/1-tasks/backlog/0181_FEATURE_rollup-leading-indicator-alarms.md rename to lore/1-tasks/archive/0181_FEATURE_rollup-leading-indicator-alarms.md index 76c1fc10..891b4d55 100644 --- a/lore/1-tasks/backlog/0181_FEATURE_rollup-leading-indicator-alarms.md +++ b/lore/1-tasks/archive/0181_FEATURE_rollup-leading-indicator-alarms.md @@ -2,7 +2,8 @@ id: "0181" title: "Rollup leading indicators — pending-mutation age, part counts and view_refreshes exceptions" type: FEATURE -status: backlog +status: completed +assignee: akot related_adr: [] related_tasks: ["0137", "0136", "0109", "0134"] tags: @@ -39,10 +40,37 @@ history: since it would be redundant under the permissive model. `storage = users_xml` on both runtime users confirms [[0134]]: we cannot GRANT them. Batch the grant request with the 0136 note already owed to BE. + - date: "2026-10-07" + status: completed + who: akot + note: > + Closed on the operator's decision, two of four parts delivered + elsewhere. view_refreshes exceptions and the empty-1M hole are covered + by [[0203]] (MvRefreshFailingCount, RollupMismatchBuckets for 1M; all + four mv-refresh alarms OK, MvRefreshUnreadable 0 on 96 reads a day). + Part-count and mutation-age indicators NOT built: measured today max 11 + active parts per partition (threshold ~1,000) and 0 pending mutations + in prices. No follow-up tasks. --- # Rollup leading indicators +## ✅ Closed 2026-10-07: two parts delivered by 0203, two not built + +Verified in code and on production, 2026-10-07: + +| indicator | state | +|---|---| +| `view_refreshes` exceptions | **delivered by [[0203]]**: `refresh_waits.rs`, `MvRefreshFailingCount` (non-empty `exception`, or no success for > 2 own periods) plus `MvRefreshWaitingCount`, `MvRefreshDisabledCount`, `MvRefreshUnreadable`. Four `prices-production-mv-refresh-*` alarms, OK since 2026-10-02. `MvRefreshUnreadable` = 0 on 96 reads a day, so `prices_writer` holds `SELECT ON system.view_refreshes` (rollout check in `0142-rollup-mv-reapply.md:329`); the BE ask below is obsolete for this table | +| empty `1M` tier | **covered by [[0203]]**: `RollupMismatchBuckets` with `Table=price_ohlcv_1M` counts closed `1M` buckets missing against `1d`, so an empty `1M` reads > 0 | +| part counts | **not built.** Max active parts per partition in `prices` today: 11 (`price_ohlcv_1m`), against the ~1,000 `parts_to_delay_insert` threshold | +| pending-mutation age | **not built.** 0 pending mutations in `prices` today. Whether `prices_writer` can read `system.mutations` was not verified (`dev_read` cannot `SHOW GRANTS FOR` another user). [[0109]]'s own `system.mutations` preflight criterion is unaffected | + +Closed on the operator's decision without the last two: both read far from +any threshold today, and no follow-up task was spawned. If a 0136-style +mutation backlog recurs, the lagging signals (freshness, mismatch, +`mv-refresh-*`) still fire; only the early warning is missing. + ## Summary [[0137]] alarms when a rollup tier's newest bucket ages past its bound — the @@ -131,8 +159,12 @@ comparing `1M`'s newest bucket against `1w`'s. mTLS user is **measured** and recorded either way. ✅ **Done 2026-08-12** — neither is granted; only `system.parts` is. See §Context. - [ ] Part-count indicator ships (unblocked — build this first). + Not built; closed without it (max 11 parts/partition on 2026-10-07). - [ ] Mutation-age and `view_refreshes` indicators ship, **or** the two-line BE grant request in §Context is raised and its outcome recorded here. + `view_refreshes`: shipped by [[0203]], grant in place. Mutation age: + not built, grant unverified. - [ ] No duplication with [[0109]] — ownership of the `system.mutations` watch is - settled and written down in both tasks. + settled and written down in both tasks. Moot: no watch built here. - [ ] Thresholds recorded with rationale, consistent with [[0137]]'s bounds. + Moot for the unbuilt two; 0203's are in `observability-stack.ts`. diff --git a/lore/1-tasks/archive/0184_FEATURE_portal-hosting-skeleton.md b/lore/1-tasks/archive/0184_FEATURE_portal-hosting-skeleton.md index 8f9ba5fb..9eef1648 100644 --- a/lore/1-tasks/archive/0184_FEATURE_portal-hosting-skeleton.md +++ b/lore/1-tasks/archive/0184_FEATURE_portal-hosting-skeleton.md @@ -203,26 +203,13 @@ directly through execute-api after the failed first attempt (see below): `/health` `200`, `/api-docs-json` `200`, `/v1/assets` `403`, and `Prices-production-{ApiGateway,Compute}` both `UPDATE_COMPLETE`. -> **Production does not match this record, and the deploy is [[0205]].** Every -> acceptance criterion above holds live, but four properties that landed after -> the 2026-08-13 deploy exist only in code: -> -> - the gateway maps `ANY {proxy}` + `{proxy}/{sub}` with **no throttle** — -> neither the original `{proxy+}` nor decision 12's shape, left there by the -> 2026-08-14 deploy attempt. Behaviour is correct at depth 1-2 and `403` at -> depth 3. -> - the access-log bucket, `Cache-Control` on the uploaded objects and the -> trailing-slash redirect are code-only; `/api-tokens` answers `403 -> AccessDenied` rather than `302`. -> - `Prices-production-Compute` **was** deployed (2026-08-14 09:42) as a side -> effect of `cdk deploy` pulling in dependency stacks — the Makefile targets -> do not pass `--exclusively`. So [[0183]]'s handler and `PORTAL_ENABLED=false` -> are live ahead of that task's merge, and `/config` now answers -> `200 {"enabled":false}` with `no-store`. -> -> [[0205]] carries the three deploys, the reason there are three rather than -> one, and the probes to re-run. It also owns deleting this note and its twin in -> `docs/scf/api-endpoints.md`. +> **Resolved 2026-10-06 by [[0205]].** A note here described production as of +> 2026-08-14: the intermediate `{proxy}` + `{proxy}/{sub}` mapping with no +> throttle, and the access logs, upload `Cache-Control` and trailing-slash +> redirect still code-only. None of it is live any more — the prefix moved to +> `/api` ([[0235]]), the gateway serves the greedy `/api/{proxy+}` with the +> per-verb throttle, and the `PortalHosting` stack was deleted on 2026-08-31 +> ([[0194]], [[0195]]). See [[0205]]'s live re-measurement. ## Issues Encountered diff --git a/lore/1-tasks/backlog/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md b/lore/1-tasks/archive/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md similarity index 66% rename from lore/1-tasks/backlog/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md rename to lore/1-tasks/archive/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md index 3f1bb03f..970f93f4 100644 --- a/lore/1-tasks/backlog/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md +++ b/lore/1-tasks/archive/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md @@ -2,7 +2,8 @@ id: "0200" title: "Is the cleanup worker still needed at all? Decide whether prices-production-cleanup is enabled or disabled" type: RESEARCH -status: backlog +status: completed +assignee: akot related_adr: [] related_tasks: ["0088", "0090", "0111", "0167", "0174", "0046", "0063"] tags: @@ -53,10 +54,66 @@ history: stack can NOT silently re-enable cleanup. That acceptance criterion of this task is therefore already DONE, and what remains here is only the enable-or-disable decision itself, now due after M3. + - date: "2026-10-07" + status: completed + who: akot + note: > + DECIDED: prices-production-cleanup stays DISABLED for good; the worker + stays in the code, dark. Measured on prod today: rule DISABLED, last + invocation 2026-07-20 03:00 UTC, none since; 8 EventBridge deploys + since 09-14 left it off. 1m + 15m grow ~0.9 GiB/month (~11 GiB/yr) + against 752 GiB free; oracle_prices is 19.7 MiB. 1m is the source the + 0286 re-ingest is rebuilding, so a 7-day retention would delete it. + No follow-up tasks. --- # Is `prices-production-cleanup` still worth running? +## ✅ Decision — 2026-10-07: DISABLED for good + +**`prices-production-cleanup` stays DISABLED.** No table is expired by a job. +The worker and its rule stay in the code (`enabled: false`), dark; re-enabling +is a deliberate, reviewed edit to `eventbridge-stack.ts`, nothing else. + +Measured on production, 2026-10-07: + +| | measurement | +|---|---| +| rule | `DISABLED`, `cron(0 3 * * ? *)`; target still the cleanup Lambda | +| last invocation | 2026-07-20 03:00 UTC (CloudWatch `Invocations`, last log stream); **0 since** | +| CDK vs live | 8 EventBridge stack deploys 2026-09-14 → 10-05 (CloudTrail `UpdateFunctionCode` by CloudFormation), the rule still `DISABLED` after all of them | +| ClickHouse | no cleanup query in `system.query_log` (kept from 2026-09-08): no `toUInt32(partition) < toYYYYMM(now() - …)` listing, no `DROP PARTITION` on `oracle_prices`. Every 1m/15m `DROP PARTITION` in that window is the 0286 re-ingest (`prices_admin`) or the 0139 rekey (`dev_shared`, `__new` tables) | +| `price_ohlcv_1m` | 28.3 GiB, 903 M rows; live months 0.56–0.82 GiB (2026-07 → 09) | +| `price_ohlcv_15m` | 12.1 GiB; live months 0.30–0.35 GiB | +| `oracle_prices` | **19.7 MiB**, 624 k rows, from 202509 | +| free disk | **752 GiB of 1.72 TiB** (shared; `default` = 614 GiB, `prices` = 193 GiB, of which 132 GiB are re-ingest / 0139 backup tables) | + +Why: + +1. **OHLCV (1m, 15m): growth is not a problem.** ~0.9 GiB/month at the live + rate → ~11 GiB in 12 months, ~22 GiB in 24, against 752 GiB free. The + 0286 re-ingest refilling 1m for 2024-10 → 2026-06 adds a one-off amount + of the same order. The backup tables of the running migrations weigh + more than two years of growth. +2. **1m is now the source, not a cache.** The 0286 phase-3 re-ingest is + rebuilding every 1m month and re-rolling the coarse tiers from it. A + 7-day window would drop that work every night, the same way the + 2026-07-18 run dropped 1m for 2024-10 → 2026-06 (measured today: those + 21 months are absent from `price_ohlcv_1m`, the coarse tables are their + only copy until the re-ingest reaches them). +3. **`oracle_prices`, answered separately:** 19.7 MiB, ~1.5 MiB/month. The + 13-month expiry saves nothing. With no expiry, [[0167]]'s reason for + `usd_rate` (escaping `oracle_prices`' expiry) no longer bites, but + `usd_rate` stays out of `RETENTION` (the test in `cleanup-worker` still + guards it), so re-enabling cleanup later cannot take it. +4. **Replacement signal exists:** `prices-production-ch-disk-free` + (`observability-stack.ts:1145`, task 0204) watches the shared volume. + +Not done here, on purpose: the worker is not deleted (removal would need an +EventBridge deploy and buys nothing), and the "check `describe-rule` before +and after every deploy" lines in the runbooks are left in place. They are now +redundant, not wrong. + The rule has been **DISABLED since 2026-07-20** and prod has been healthy without it for over three weeks. It is time to decide deliberately rather than leave it off by inertia. @@ -148,24 +205,28 @@ should delete the need for it. ## Acceptance Criteria -- [ ] **DECISION RECORDED: `prices-production-cleanup` should be ENABLED or +- [x] **DECISION RECORDED: `prices-production-cleanup` should be ENABLED or DISABLED.** One of the two, stated plainly, with the measurement behind it. A third option (keep it, but change the retention windows or the eligibility predicate) is acceptable — but it must be written as a decision, not left as "needs more thought". -- [ ] Per-table size and growth measured on prod, projected 12/24 months, against - measured free space. -- [ ] `oracle_prices` answered separately from the OHLCV tables, with `usd_rate`'s - dependency on that expiry ([[0167]]) explicitly addressed. -- [ ] **The CDK matches the decision** — `eventbridge-stack.ts` no longer asserts + → DISABLED, 2026-10-07 (Decision section above). +- [x] Per-table size and growth measured on prod, projected 12/24 months, against + measured free space. → ~11 / ~22 GiB against 752 GiB free. +- [x] `oracle_prices` answered separately from the OHLCV tables, with `usd_rate`'s + dependency on that expiry ([[0167]]) explicitly addressed. → point 3. +- [x] **The CDK matches the decision** — `eventbridge-stack.ts` no longer asserts a state that differs from the live rule, so the "check `describe-rule` after every deploy" workaround can be retired from the runbooks and from [[0182]]. -- [ ] If cleanup is retired: a replacement signal (disk-headroom alarm) exists - before the worker is removed, not after. -- [ ] If cleanup is kept: the historical-write hazard is addressed, so a backfill + → `enabled: false` since 0204; held through 8 deploys. The runbook lines + are not removed, only made redundant. +- [x] If cleanup is retired: a replacement signal (disk-headroom alarm) exists + before the worker is removed, not after. → `prices-production-ch-disk-free`; + the worker is not removed. +- [x] (n/a, not kept) If cleanup is kept: the historical-write hazard is addressed, so a backfill or repair run can no longer have its output deleted as it lands. -## ⛔ Until this is decided +## ⛔ Until this is decided (superseded 2026-10-07: decided, DISABLED) **Cleanup stays DISABLED.** That is the deliberate current position, not an oversight — recorded here so the next session does not "fix" it. Do not enable diff --git a/lore/1-tasks/backlog/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md b/lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md similarity index 66% rename from lore/1-tasks/backlog/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md rename to lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md index ff47b8a3..2c41d93e 100644 --- a/lore/1-tasks/backlog/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md +++ b/lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md @@ -2,9 +2,9 @@ id: '0205' title: 'Deploy the portal gateway mapping and verify it live — three deploys 0184 merged but never shipped' type: CHORE -status: backlog +status: completed related_adr: [] -related_tasks: ['0184', '0183', '0185', '0186', '0194', '0141'] +related_tasks: ['0184', '0183', '0185', '0186', '0194', '0141', '0195', '0235', '0309', '0326'] tags: [ layer-infra, @@ -18,7 +18,8 @@ tags: ] milestone: 3 links: - - '../active/0184_FEATURE_portal-hosting-skeleton.md' + - '../../archive/0184_FEATURE_portal-hosting-skeleton.md' + - 'notes/R-live-measurement-2026-10-06.md' - '../../../docs/scf/api-endpoints.md' history: - date: "2026-08-14" @@ -69,10 +70,44 @@ history: both Cache-Control headers, the per-verb throttles, cookie-free access logs, /api-tokens/api/config and [[0185]]'s decision 13 still have no evidence. What remains is the live re-measurement, no deploy expected. + - date: "2026-10-06" + status: active + who: akot + note: > + Activated for the live re-measurement. The portal has since moved twice + ([[0194]]: backend on prices-api.sorobanscan…/api/, [[0326]]: bundle at + sorobanscan…/prices-api/), so each criterion is measured against what + replaced it. + - date: "2026-10-06" + status: active + who: akot + note: > + Measured live, read-only, about 12:10 UTC + ([R-live-measurement-2026-10-06](notes/R-live-measurement-2026-10-06.md)). + The three deploys are moot: PortalHosting was deleted on 2026-08-31 and + [[0235]]'s `/api/{proxy+}` sits under a new parent, so no + variable-child conflict. 10 of 11 criteria hold, or hold for what + replaced their subject. Open: the stale notes about the intermediate + mapping, in the gateway docblock, the OAuth deploy runbook and + [[0184]]'s record. + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed with no deploy: the portal this task was written for no longer + exists. 11 of 11 criteria met — 8 against today's subject, 3 (redirect, + access logs, DependsOn) against what replaced theirs. The last one was + the stale notes: the gateway docblock, the OAuth deploy runbook and + [[0184]]'s record, on this branch. No code change, no follow-up tasks. --- # Deploy the portal gateway mapping and verify it live +> **2026-10-06: no deploy left to do.** Everything below describes production +> as of 2026-08-14. The portal moved off `/api-tokens` and our CloudFront since +> ([[0235]], [[0194]], [[0326]]); the criteria are marked against what replaced +> them — see [R-live-measurement-2026-10-06](notes/R-live-measurement-2026-10-06.md). + ## Summary [[0184]] is merged and its acceptance criteria were measured — but against the @@ -167,28 +202,52 @@ deferred criteria. ## Acceptance Criteria -- [ ] `/api-tokens` returns `302` to `/api-tokens/`, not `403 AccessDenied` -- [ ] `/api-tokens/index.html` carries +- [x] `/api-tokens` returns `302` to `/api-tokens/`, not `403 AccessDenied` + — prefix gone; `sorobanscan…/prices-api` answers `301 /prices-api/` +- [x] `/api-tokens/index.html` carries `Cache-Control: public, max-age=0, must-revalidate` — the entry document must revalidate, or a visitor keeps booting a stale app from a URL that - still resolves -- [ ] Everything under `/api-tokens/assets/` carries + still resolves — `/prices-api/index.html`, measured +- [x] Everything under `/api-tokens/assets/` carries `Cache-Control: public, max-age=31536000, immutable` — safe only because - those names are content-hashed, so a new build is a new URL -- [ ] `/api-tokens/api/a/b/c` returns an empty `404` — greedy matches any depth - again, so the depth-3 `403` is gone -- [ ] The deployed stage carries a throttle entry per verb (`GET`, `POST`, - `DELETE`) at 10 req/s burst 40, with caching off -- [ ] CloudFront access logs are landing in the log bucket, without cookies -- [ ] `/api-tokens/api/config` still answers `200 {"enabled":false}` with + those names are content-hashed, so a new build is a new URL — + `/prices-api/assets/`, all 5 files +- [x] `/api-tokens/api/a/b/c` returns an empty `404` — greedy matches any depth + again, so the depth-3 `403` is gone — `/api/a/b/c` on the API host +- [x] The deployed stage carries a throttle entry per verb (`GET`, `POST`, + `DELETE`) at 10 req/s burst 40, with caching off — and `OPTIONS` +- [x] CloudFront access logs are landing in the log bucket, without cookies + — our distribution is gone; the explorer's land, `IncludeCookies: false` +- [x] `/api-tokens/api/config` still answers `200 {"enabled":false}` with `no-store`, and every other path under the prefix still answers an empty - `404` — the flag is untouched by this task -- [ ] `/health`, `/api-docs-json` and `/v1/assets` (keyless → `403`) unchanged + `404` — the flag is untouched by this task — `/api/config`, now + `{"enabled":true,…}` after [[0194]] opened the portal +- [x] `/health`, `/api-docs-json` and `/v1/assets` (keyless → `403`) unchanged throughout — the data routes must not notice this happening -- [ ] Both "ahead of the deploy" notes deleted -- [ ] The synthesized `PortalHosting` template shows the entry-document +- [x] Both "ahead of the deploy" notes deleted — the `api-endpoints.md` one was + already gone; removed the gateway docblock's, the OAuth runbook's and + [[0184]]'s +- [x] The synthesized `PortalHosting` template shows the entry-document deployment depending on the asset deployment — neither is `DependsOn: null` - any more -- [ ] On a cold cache after the deploy, `/api-tokens/` loads and every asset it + any more — no template any more; `sync-portal-explorer` syncs `assets/*` + first, sequentially +- [x] On a cold cache after the deploy, `/api-tokens/` loads and every asset it references returns `200`; nothing under `/api-tokens/assets/` answers - `403 AccessDenied` + `403 AccessDenied` — per-object misses, no invalidation issued + +## Design Decisions + +### Emerged + +1. **Measured against what replaced each criterion's subject, not closed as + void.** Three criteria name things that are gone (`/api-tokens`, our + CloudFront, the `PortalHosting` template). Each has a successor with the + same job — the explorer's `/prices-api` redirect, its access logs, the + ordered `sync-portal-explorer` — so the property was checked there. +2. **The 0184 note was replaced by a short pointer, not deleted.** An archived + record that loses the paragraph explaining why its numbers differ from + production reads as wrong; three lines saying it was resolved, and by what, + keep it truthful. +3. **No cache invalidation for the cold-cache criterion.** It is a write on + another team's distribution; per-object `Miss from cloudfront` on the + bundle files was taken as enough. diff --git a/lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md b/lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md new file mode 100644 index 00000000..65bdef30 --- /dev/null +++ b/lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md @@ -0,0 +1,80 @@ +--- +title: "Live re-measurement of the portal routes, 2026-10-06 — every criterion that still applies holds" +type: research +status: mature +tags: [portal, measurement, apigateway, cloudfront, production] +links: + - "../README.md" + - "../../../../../docs/scf/api-endpoints.md" +history: + - date: "2026-10-06" + status: mature + who: akot + note: > + Run read-only against production on 2026-10-06 at about 12:10 UTC: + public HTTP requests to both hosts, and AWS reads (API Gateway stage, + CloudFormation stack list, CloudFront distribution config) as + `AWS_PROFILE=stellar`. One point in time. +--- + +# Live re-measurement of the portal routes, 2026-10-06 + +## What changed since the task was written + +The task describes the 2026-08-14 production: the portal on our own +CloudFront (`PortalHosting`, `dojr4epgxo2qp.cloudfront.net`) under +`/api-tokens/`, the gateway mapping the intermediate `{proxy}` + +`{proxy}/{sub}` pair. Neither exists any more: + +- `Prices-production-PortalHosting` is `DELETE_COMPLETE` (2026-08-31 10:09 + UTC), deleted in [[0194]]'s cutover that moved the backend to the API's own + host; [[0195]] removed the stack from the code (`e8ed5c0e`). No CloudFront + distribution of ours serves the portal. +- The backend is `/api/{proxy+}` directly under the root on + `prices-api.sorobanscan.rumblefish.dev` — [[0235]] moved the prefix off + `/api-tokens` (`9d39207e`, 2026-08-28). `/api` is a new parent, so the + one-variable-child conflict that made this task three deploys never arose: + an ordinary gateway deploy created the greedy proxy and deleted the + `/api-tokens` tree. +- The bundle is on the block explorer's distribution `EA2TLS5SS5M87` + (`sorobanscan.rumblefish.dev`), under `/prices-api/` since [[0326]] + (2026-10-02), synced by `make -C infra sync-portal-explorer`. + +So the three deploys are not needed; what follows measures each criterion +against what replaced its subject. + +## Results + +| # | Criterion | Measured | Verdict | +|---|---|---|---| +| 1 | `/api-tokens` → `302` | `sorobanscan…/prices-api` → `301 Location: /prices-api/` (`FunctionGeneratedResponse`); `sorobanscan…/api/` → `301 /prices-api/`. `/api-tokens` on the API host → gateway `404 {"code":"not_found"}` | subject gone; replacement holds | +| 2 | entry document `public, max-age=0, must-revalidate` | `/prices-api/`, `/prices-api/index.html`, `/prices-api/dashboard`: `200 text/html`, 2738 B, `public, max-age=0, must-revalidate` | holds | +| 3 | `assets/*` `public, max-age=31536000, immutable` | the 2 assets `index.html` references and the 3 chunks the entry script loads lazily: `200`, `public, max-age=31536000, immutable` | holds | +| 4 | depth 3 → empty `404` | `/api/a`, `/api/a/b`, `/api/a/b/c`: `404`, 0 bytes (the handler's gate, not the gateway); `/api/auth/me` (depth 3, real route) `200 {"authenticated":false}` | holds | +| 5 | per-verb throttle 10/40, caching off | stage `production`, `api/{proxy+}/{GET,POST,DELETE,OPTIONS}`: rate 10, burst 40, `cachingEnabled: false`. `*/*` default 200/400 | holds | +| 6 | CloudFront access logs, no cookies | no distribution of ours. The explorer's `EA2TLS5SS5M87`: `Logging.Enabled: true`, `IncludeCookies: false`, bucket `production-soroban-explorer-cf-logs`, newest object `EA2TLS5SS5M87.2026-10-06-12.…gz` at 12:18 UTC | subject gone; explorer's holds | +| 7 | `/api/config` `200`, `no-store`; other paths empty `404` | `200 {"enabled":true,"rate_limit_per_second":1}`, `no-store`; with `Origin: https://sorobanscan.rumblefish.dev` adds `Access-Control-Allow-Origin` (that origin) + `Allow-Credentials: true`. `enabled: true` is [[0194]]'s opening, not a regression | holds for today's flag | +| 8 | `/health`, `/api-docs-json`, `/v1/assets` keyless | `200 {"status":"ok","stack":"prices-production"}`; `200` 58 863 B `public, max-age=300` (same at `/api/api-docs-json`); `403 {"message":"Forbidden"}` | holds | +| 9 | "ahead of the deploy" notes deleted | `docs/scf/api-endpoints.md`: already gone. Still present: `api-gateway-stack.ts` docblock of `PORTAL_API_METHODS`, `docs/runbooks/portal-oauth-deploy-prep.md` §preconditions, [[0184]]'s Implementation Notes | open at measurement | +| 10 | entry-document deployment `DependsOn` the asset one | no template. `sync-portal-explorer` runs the `assets/*` sync before the rest, sequentially in one recipe, then one invalidation — the race cannot occur | subject gone; ordering holds | +| 11 | cold cache: every asset `200`, none `403` | all 5 bundle files and the 3 icons `200`, most `Miss from cloudfront`. No invalidation was issued (a write), so "cold" is per-object misses, not a flushed distribution | holds | + +Also seen: `OPTIONS /api/config` with `Origin` + +`Access-Control-Request-Method: DELETE` → `204`, `allow-methods: GET,POST,DELETE`, +`allow-headers: Content-Type,Accept,X-Requested-With`, `max-age: 3600`, +credentials true — a MOCK answer, no invocation. A verb outside the list +(`PATCH /api/config`) gets the gateway's `404 {"code":"not_found"}` ([[0309]]). + +[[0185]]'s decision 13 (`/api-tokens/` in `distributionPaths`) has no subject: +the invalidation is now `'/prices-api/*'` on the explorer's distribution. + +## How it was measured + +`curl -s -D -` (GET; OPTIONS and PATCH where stated) against both hosts, +keeping the status line, `Location`, `Cache-Control`, `Content-Type`, +`Content-Length`, `Access-Control-*`, `X-Cache`, `x-amzn-ErrorType` and the +first bytes of the body. Asset list from the `src`/`href` attributes of +`/prices-api/`, lazy chunks from `assets/…` names inside the entry script. +AWS: `apigateway get-stages` on `02mabge71l`, `apigateway get-resources`, +`cloudformation list-stacks`, `cloudfront get-distribution-config +EA2TLS5SS5M87`, `s3api list-objects-v2` on the log bucket. The scripts stayed outside the repository. diff --git a/lore/1-tasks/backlog/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md b/lore/1-tasks/archive/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md similarity index 78% rename from lore/1-tasks/backlog/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md rename to lore/1-tasks/archive/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md index 8b229945..87ae4ac4 100644 --- a/lore/1-tasks/backlog/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md +++ b/lore/1-tasks/archive/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md @@ -2,7 +2,8 @@ id: "0226" title: "The oracle worker loads all 620,615 assets into memory to write 2 rows, sits at its 256 MB ceiling and OOMs several times a day" type: BUG -status: backlog +status: completed +assignee: akot related_adr: [] related_tasks: ["0223", "0222", "0167", "0112", "0132", "0256", "0241", "0140"] tags: [layer-infra, priority-high, effort-medium, oracle, lambda, memory, observability, ops] @@ -98,6 +99,29 @@ history: task is an efficiency task, not an outage fix — but one more un-merged copy of the registry puts the oracle back at the ceiling. Priority left at high for the operator to lower; not lowered here. + - date: "2026-10-06" + status: active + who: akot + note: > + Activated together with [[0140]]: one PR on one branch, because both rewrite the same oracle-worker pass (the whole-registry load and the `write_assets` at `lib.rs:569`) and both ship through the EventBridge stack, whose deploy carries the cleanup-rule hazard. + - date: "2026-10-06" + status: active + who: akot + note: > + Found already fixed by [[0139]] before any code was written: the oracle + no longer loads the registry (deployed 2026-10-02 12:30 UTC). Measured on + prod: 256 → 52 MB, 7–8 s → 0.75 s, 0 errors and 0 OOM since. ACs 1, 2, + 3 and 5 met; AC 4 (a week without an OOM) ends 2026-10-09 12:44 UTC, + close after it. [[0140]] closed the same day. + - date: "2026-10-07" + status: completed + who: akot + note: > + Closed before AC 4's week ended (2026-10-09 12:44 UTC), on the + operator's decision: 0 OOM and 0 errors from the first new-code run + (2026-10-02 12:44 UTC) to the 2026-10-06 measurement; not re-checked + since. Fixed by [[0139]], no code here. ACs 1, 2, 3 and 5 met; no + follow-up tasks. --- # The oracle worker reads the entire asset registry on every run @@ -312,19 +336,72 @@ no `-errors` alarm at all, `createWorkerLambda` (`lambda-baseline.ts:309`) build does not, that is a separate question. - [[0223]]'s framing decision itself — only the scope gap above is recorded here. +## ✅ Resolved by [[0139]] — measured on prod 2026-10-06 + +No code in this task. [[0139]] (PR #382, adfe4ccc + a6ecd20d) removed the +registry from the oracle's pass: it now calls `OhlcvWriter::write_absent_assets` +for its 2 sampled identities (`oracle-worker/src/lib.rs:562`), one keyed +`count()` each, and the `write_assets` at the old `lib.rs:569` is gone with it +— the collision with [[0140]] described above resolved itself. That is option +1 in its narrowest form; `load_assets()` is untouched for its other callers. + +**Cause, not coincidence.** CloudTrail: `UpdateFunctionCode` at 2026-10-02 +12:30:16 UTC, after #382 merged (11:33 UTC). The last old-code run, 11:52 UTC, +loaded `existing_assets=842792` (four un-merged copies) and used 256 MB in +10.9 s. The schedule was paused for the 0139 window; the first run on the new +code, 12:44 UTC on a cold container, used **51 MB in 1.4 s**. No +`loaded asset registry` line from the oracle since 11:52 UTC. + +| | 2026-09-28 → 10-02 | 2026-10-03 → 10-06 | +|---|---|---| +| `Max Memory Used`, daily max / p50 | **256** / 151–222 MB | **52 / 51 MB**, every day | +| `Runtime.OutOfMemory` lines | 8 (09-24 → 10-02) | **0**; last 2026-10-02 09:17 UTC | +| `AWS/Lambda Errors`, daily | 0–3 | **0** | +| average duration | 6.9–8.2 s | **0.74–0.77 s** | +| `prices-production-oracle-errors` | ALARM ↔ OK several times a day | last fired 2026-10-02 09:18 UTC, OK since | + +- The load was **per invocation**: ~12 cold starts a day against 288 runs, + with p50 at 151–222 MB and one `loaded asset registry` line per run (288 a + day). +- `system.query_log`, last 24 h: the oracle's keyed check is part of 1,056 + runs of `SELECT count() FROM prices.assets WHERE asset_code = ? AND …` + (2 × 288 oracle + 20 × 24 asset-discovery), 6 ms average, ≤ 8,546 rows read. + None of the 53 whole-registry `SELECT`s of the last 3 days is the oracle's: + 35 are ledger-processor cold starts, 18 an operator machine running + backfills — and the oracle could not have paid one, since the read alone + costs ~148 MB. +- The memory is now independent of un-merged copies and of the registry's + size, so the [[0294]] risk "the oracle OOMs while the re-ingest re-emits the + registry" is gone with it. +- `memorySize` stays 256 MB; no EventBridge change was needed. + `prices-production-cleanup` reads `DISABLED` (2026-10-06). + +**Callers of `load_assets()`** (AC 3), all unchanged by this task: +`events-backfill/src/run.rs:185`, `sdex-backfill/src/run.rs:147` (via +`sink.rs:72`), `prices-ledger-processor/src/sink/mod.rs:103` (cold start, +`main.rs:92`; `bin/cli.rs:86`). + +**AC 4 not completed:** the week without an OOM would have ended **2026-10-09 +12:44 UTC**. The task was closed on 2026-10-07, after four clean days. + ## Acceptance Criteria -- [ ] Whether the registry load is per-invocation or per-cold-start is - **measured**, not inferred from the two log lines above. -- [ ] The remedy reduces **peak** memory on a real production invocation, with - `Max Memory Used` recorded before and after. -- [ ] Every other caller of `load_assets()` is enumerated, and the change is +- [x] Whether the registry load is per-invocation or per-cold-start is + **measured**, not inferred from the two log lines above — per + invocation (288 load lines a day against ~12 cold starts). +- [x] The remedy reduces **peak** memory on a real production invocation, with + `Max Memory Used` recorded before and after — 256 → 52 MB ([[0139]]). +- [x] Every other caller of `load_assets()` is enumerated, and the change is shown not to alter their behaviour — or the narrowing is confined to a new - entry point. + entry point — `write_absent_assets` is the new entry point; callers + listed above. - [ ] `prices-production-oracle-errors` shows **zero** `Runtime.OutOfMemory` for a full week after the change. -- [ ] If the deploy touches `eventbridge-stack.ts`, `describe-rule` output for - `prices-production-cleanup` is recorded **before and after**. + Closed early, 2026-10-07: zero through the 2026-10-06 check (four of + the seven days); the rest of the week was not checked. +- [x] If the deploy touches `eventbridge-stack.ts`, `describe-rule` output for + `prices-production-cleanup` is recorded **before and after** — not + touched (no `memorySize` change); the rule reads `DISABLED` on 2026-10-06. ## Notes diff --git a/lore/1-tasks/blocked/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md b/lore/1-tasks/archive/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md similarity index 61% rename from lore/1-tasks/blocked/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md rename to lore/1-tasks/archive/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md index deac377d..f62e2824 100644 --- a/lore/1-tasks/blocked/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md +++ b/lore/1-tasks/archive/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md @@ -2,7 +2,8 @@ id: "0236" title: "Nothing detects an internally inconsistent `price_ohlcv_*` row — and 0229's clamp removed the one surface that used to surface them" type: BUG -status: blocked +status: completed +assignee: akot related_adr: ["0011"] related_tasks: ["0229", "0120", "0182", "0227"] tags: ["priority-medium", "effort-small", "data-correctness", "observability", "ohlcv", "milestone-M2"] @@ -11,7 +12,7 @@ links: - "../../../packages/prices-api/src/assets/queries_ch.rs" - "../../../tools/scripts/conformance-0120.mjs" history: - - date: 2026-08-28 + - date: "2026-08-28" status: backlog who: okarcz note: > @@ -28,6 +29,7 @@ history: - date: "2026-09-29" status: blocked who: akot + by: ["0286"] note: > Detector, alarm, ITs and the ADR-0011 s3 amendment are built on fix/0236_ohlc-band-detector (not yet merged). Blocked on the deploy date: @@ -36,6 +38,22 @@ history: inherit the legacy `low = 0` (measured 22 in `_1w`, 109 in `_1M`), so the alarm would fire on deploy with nothing new broken. No floor in code (Adam's decision); the window moves past those buckets by 2026-10-04. + - date: "2026-10-06" + status: active + who: akot + note: > + Unblocked: the deploy date passed. The detector's own queries read 0 on + all seven tiers on prod, so the alarm will not fire on legacy rows. The + branch is rebased on develop, alongside 0139's and 0203's probe checks, + as block 5c. PR to follow; deploy is Adam's. + - date: "2026-10-06" + status: completed + who: akot + note: > + PR #396 merged (81a48f08) after review; Observability (3 ohlc-band + alarms) and EventBridge (probe Lambda only) deployed 2026-10-06. First + run 13:05 UTC: 7 tiers read, 0 violations, checks_failed 0, alarms OK. + Probe 104 unit tests, 40 ClickHouse ITs. --- # No detector for an internally inconsistent stored candle @@ -126,6 +144,27 @@ bound** — superseded RMT versions are included. - The legacy zero-price rows are pre-0286 residue that [[0286]] phase 3 re-ingests; they are recorded here, not filed as a separate task. +## Pre-deploy check — measured on prod 2026-10-06 + +Read-only, `dev_read`, 10:11 UTC. These are the detector's own queries +(`ohlc_band_queries()`: `FINAL`, 2 days widened by one bucket): + +| tier | band | nonpositive | scanned | oldest bucket in window | +|---|---:|---:|---:|---| +| `_1m` | 0 | 0 | 662,241 | 2026-10-04 10:12 | +| `_15m` | 0 | 0 | 296,212 | 2026-10-04 10:00 | +| `_1h` | 0 | 0 | 178,303 | 2026-10-04 10:00 | +| `_4h` | 0 | 0 | 98,634 | 2026-10-04 08:00 | +| `_1d` | 0 | 0 | 39,812 | 2026-10-04 | +| `_1w` | 0 | 0 | 44,609 | 2026-09-28 | +| `_1M` | 0 | 0 | 26,016 | 2026-10-01 | + +- The legacy `_1w` 2026-09-21 and `_1M` 2026-09-01 buckets have left the + window, so the first run after deploy should read 0. +- Cost: the seven reads together take 0.29 s of server time and read 3.7 M + rows. That is why the check runs as block 5c, before the time-budgeted + mismatch reads (0203), which stay last. + ## Decisions (Adam, 2026-09-28) 1. **Predicate**: band violations **and** priced rows with any price ≤ 0 — the @@ -145,11 +184,64 @@ construction and checked with zero tolerance where it is checked at all. ## Acceptance Criteria -- [ ] The number of internally inconsistent rows per table is **measured on +- [x] The number of internally inconsistent rows per table is **measured on prod** and recorded, before any alarm is designed. -- [ ] If the count is non-zero, the cause is identified and filed as its own - task rather than absorbed here. -- [ ] A recurring check exists wherever the other data-quality probes live, with - its threshold justified by the measured baseline. -- [ ] 0229's clamp is explicitly confirmed as the right behaviour for the read +- [x] If the count is non-zero, the cause is identified and filed as its own + task rather than absorbed here. Band violations are 0. The legacy + zero-price rows are recorded above and left to [[0286]] phase 3. +- [x] A recurring check exists wherever the other data-quality probes live, with + its threshold justified by the measured baseline. Block 5c of + `rollup-freshness-probe`, every 15 min, alarm at >= 1 (baseline 0). +- [x] 0229's clamp is explicitly confirmed as the right behaviour for the read path, or changed — with the decision recorded in [[ADR-0011]] §3. + Confirmed, merged with #396. + +## Implementation Notes + +- `packages/rollup-freshness-probe/src/ohlc_band.rs`: seven per-tier `FINAL` + queries, the `band` and `nonpositive` classes, `ohlc_band_totals` (the one + completeness rule) and the `CandleBandViolations` metric with its two + refusals (`EmptyScan`, `Incomplete`). +- `main.rs` block 5c: the seven reads run in parallel on a `JoinSet`, each + under block 6's `mismatch_read_bound`, server- and client-side. +- Observability: `prices-{env}-ohlc-band-{1,100,10000}` on `zeroLadder`. +- ADR-0011 §3: the clamp stays; the as-stored arm does not clamp. + +## Deployed to production (2026-10-06) + +| step | result | +|---|---| +| merge | #396 → `develop` 81a48f08, 12:43 UTC | +| Observability | 3 alarms created, dashboard 86 alarms, 14:48 CEST | +| EventBridge | only `prices-production-rollup-freshness-probe` changed, 12:57 UTC | +| first run | 13:05:21 UTC: 7 tiers, 0 violations, `checks_failed` 0, 3.1–3.4 s, 51 MB | +| alarms | `ohlc-band-*` INSUFFICIENT_DATA → OK | + +## Design Decisions + +### From Plan + +1. **Predicate, scope, threshold**: Adam's decisions of 2026-09-28 above. + +### Emerged + +2. **Block 5c, not last**: block 6 (0203) is time-budgeted and stays last. + The seven reads cost 0.29 s of server time on prod. +3. **Bounded, parallel reads** (review of #396): a stalled tier is a + recorded failure, not a Lambda timeout that loses block 6 and the log. +4. **One completeness rule** (review): `ohlc_band_totals`, which also + rejects duplicate or unknown tier names. +5. **Review finding 1 declined**: `pf_trade_count > 0` means priced on every + tier, because the ingest floor is the rollup floor (0286 WR-03). Prod had + 0 rows that break this from 2026-09-23 to 2026-10-06. A break would be the + WR-03 defect, so paging on it is right. Documented in the module doc. +6. **The old blocked entry carries `by: ["0286"]`**: the validator requires + `by`, and the block was waiting for pre-0286 buckets to leave the window. + +## Issues Encountered + +- **Rebase onto 299 commits of develop**: 0139 (block 5b) and 0203 (block 6, + time budget, `%reading` log style) had grown the same probe. Four files + conflicted; the band block was fitted in between. +- **`client()` became async on develop**: it now stops the shared reconcile + MVs before a test seeds tiers. The 0236 ITs were adapted in c2eeddaf. diff --git a/lore/1-tasks/archive/0247_FEATURE_backfill-usdc-usd-history-from-an-external-anchor.md b/lore/1-tasks/archive/0247_FEATURE_backfill-usdc-usd-history-from-an-external-anchor.md index b7f1ae38..12cd257f 100644 --- a/lore/1-tasks/archive/0247_FEATURE_backfill-usdc-usd-history-from-an-external-anchor.md +++ b/lore/1-tasks/archive/0247_FEATURE_backfill-usdc-usd-history-from-an-external-anchor.md @@ -98,7 +98,7 @@ Load an external USDC/USD daily (or hourly) series into `prices.usd_rate` for th pre-oracle window, keyed on the canonical Stellar identity. - **Source:** USDC/USD on centralized venues trades against actual dollars and - reaches back to 2018 — CoinGecko, Kraken, CryptoCompare. Pick one, record which + reaches back to 2018 — CoinGecko, third-party, CryptoCompare. Pick one, record which and at what granularity; a daily close is enough for a peg. - **`method` must be its own value**, NOT `'oracle'`. `'oracle'` means a Reflector reading we polled ourselves. Something like `'external'`, added to diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/README.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/README.md index ba562663..befbce20 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/README.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/README.md @@ -160,7 +160,7 @@ Read in this order; every number is reproducible with `analysis/run.sh`. | phase | note | one line | |---|---|---| | 0 | [notes/S-phase0-root-cause.md](notes/S-phase0-root-cause.md) | the 1.0 is ours: `handlers.rs:643` → `queries_ch.rs:933`; 232-asset sweep: USDC is the only fully synthetic series, but `method: peg` sits on 135 assets incl. `native` | -| 1 | [notes/R-source-candidates.md](notes/R-source-candidates.md), [data/sources.csv](data/sources.csv) | 19 candidates; Chainlink primary, Bitstamp fallback, Kraken cross-check; USDT-quoted venues sit 20–30 bps off in stress | +| 1 | [notes/R-source-candidates.md](notes/R-source-candidates.md), [data/sources.csv](data/sources.csv) | 19 candidates; Chainlink primary, Bitstamp fallback, third-party cross-check; USDT-quoted venues sit 20–30 bps off in stress | | 2 | [notes/R-peer-stablecoins.md](notes/R-peer-stablecoins.md) | no peer is a proxy: DAI corr 0.994 (collateral), USDT −0.81 (flight-to), the rest idiosyncratic | | 3 | [notes/S-composition-rule.md](notes/S-composition-rule.md) | primary + flagged fallback/dispute, no averaging, no forward fill; `source`/`quality`/`n_obs` on the wire | | 4 | [notes/S-guardrails.md](notes/S-guardrails.md), `analysis/guardrails.py` | `trade_count = 0` is the only standalone discriminator; thresholds 2–4× outside the noisiest real window; 6 tests green | @@ -175,7 +175,7 @@ measured 6 bps, ES 25 bps, worst day 300 bps. ### From Plan -1. **Chainlink rounds as the primary anchor**, Bitstamp fallback, Kraken +1. **Chainlink rounds as the primary anchor**, Bitstamp fallback, third-party cross-check — chosen on measured dispersion, not on preference. 2. **Backfill via [[0247]]'s design** (`usd_rate`, `method='external'`, view first) rather than a second mechanism. @@ -188,7 +188,7 @@ measured 6 bps, ES 25 bps, worst day 300 bps. has one source and one method; hourly-heartbeat feeds sampled every 2nd–4th round (`stride` column), daily closes unaffected. 5. **Coin Metrics kept as analysis-only** because of CC BY-NC. -6. **Large reproducible CSVs gitignored** (`data/.gitignore`: Kraken trades +6. **Large reproducible CSVs gitignored** (`data/.gitignore`: third-party trades 28 MB, four peer round files ≈ 1 MB each); everything else in `data/` is versioned so the notes' numbers can be checked without refetching. 7. **The `.gitignore` root entry `.claude/`** was added on request; the four diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/analysis/guardrails.py b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/analysis/guardrails.py index ccc86db0..8b69591a 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/analysis/guardrails.py +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/analysis/guardrails.py @@ -3,7 +3,7 @@ thresholds derived from real data and the tests that prove they separate. Thresholds (why each number — measured 2026-09-04 with the snippet at the -bottom of this file over Chainlink USDC/USDT/DAI, Bitstamp, Kraken, OKX daily +bottom of this file over Chainlink USDC/USDT/DAI, Bitstamp, third-party, OKX daily bars; see notes/S-guardrails.md for the table): ZERO_TRADES_SHARE_30D = 0.9 @@ -21,17 +21,17 @@ EXACT_PEG_SHARE_30D = 0.9 share of closes exactly == 1.0 in a 30-day window. Real max over any - window: 0.67 (Kraken, which prints 4 decimals), 0.47 (OKX), 0.40 + window: 0.67 (third-party, which prints 4 decimals), 0.47 (OKX), 0.40 (Bitstamp, Chainlink). Ours: 1.00 for 1 864 consecutive days. 0.9 sits above the noisiest real window with margin and below ours by 0.1. CONSTANT_CLOSE_RUN_DAYS = 30 longest run of identical consecutive closes. Real max: 18 days - (Kraken, 4 dp), 10 (OKX), 4 (Chainlink, Bitstamp). Ours: 1 864. + (third-party, 4 dp), 10 (OKX), 4 (Chainlink, Bitstamp). Ours: 1 864. MIN_RVOL_30D_BPS = 0.1 30-day realized vol of daily log returns. Real minimum over every - window: 0.23 bps (Bitstamp), 0.33 (Chainlink), 0.58 (Kraken), 0.85 + window: 0.23 bps (Bitstamp), 0.33 (Chainlink), 0.58 (third-party), 0.85 (OKX); no real window is ever exactly zero. Ours: 0 on every window before 2026-03-11. 0.1 is less than half the quietest real window. @@ -140,7 +140,7 @@ def test_our_native_control_passes_price_checks() -> None: def test_real_usd_sources_pass() -> None: - for name in ["chainlink_usdc_1d.csv", "bitstamp_usd_1d.csv", "kraken_usd_1d_last720.csv", "okx_usdt_1d.csv", + for name in ["chainlink_usdc_1d.csv", "bitstamp_usd_1d.csv", "third_party_usd_1d_last720.csv", "okx_usdt_1d.csv", "chainlink_usdt_1d.csv", "chainlink_dai_1d.csv", "composed_usdc_usd_1d.csv"]: df = _load(name) if "trades" in df: diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/data/sources.csv b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/data/sources.csv index 31abf50a..972a8917 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/data/sources.csv +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/data/sources.csv @@ -1,6 +1,6 @@ source,pair,quote,coverage_start,granularity,gaps,latency_update_rule,cost,licence_redistribution,api_stability,methodology,integration_effort,keyless_history,march2023_min_close_1h,march2023_close_1d,verdict Bitstamp,USDC/USD,USD,2021-01-01 (probed; API silent before),1m..1d OHLC,none 2021-01..2026-09 (0 missing days),exchange candles; realtime,free public API,exchange ToS; not verified for redistribution,stable v2 API,exchange order book (real USD),low,yes,0.8627 @ 2023-03-11 07:00,0.9685,candidate primary (USD; reaches 2021 not 2018) -Kraken,USDC/USD,USD,OHLC endpoint: last 720 candles only (2024-09-14 for daily); trades endpoint: full history paginated,trades / 1m..1d OHLC (720 cap),none in OHLC window; 580 028 trades fetched for 2023-03,exchange; realtime,free public API (~1 req/s),exchange ToS; not verified for redistribution,stable v0 API,exchange order book (real USD),medium (history must be rebuilt from trades),yes but slow (trades),0.9010 @ 2023-03-11 12:00 (hourly close from trades; low 0.8740),0.9670,candidate fallback / cross-check (USD) +third-party,USDC/USD,USD,OHLC endpoint: last 720 candles only (2024-09-14 for daily); trades endpoint: full history paginated,trades / 1m..1d OHLC (720 cap),none in OHLC window; 580 028 trades fetched for 2023-03,exchange; realtime,free public API (~1 req/s),exchange ToS; not verified for redistribution,stable v0 API,exchange order book (real USD),medium (history must be rebuilt from trades),yes but slow (trades),0.9010 @ 2023-03-11 12:00 (hourly close from trades; low 0.8740),0.9670,candidate fallback / cross-check (USD) Binance,USDC/USDT,USDT,2018-12-15,1m..1d klines,165 missing days: delisted 2022-09-26 03:00 UTC to relist 2023-03-11 14:00 UTC (announcements fetched) — the gap covers the depeg onset and trough,exchange; realtime,free via data-api.binance.vision (already used in this repo),exchange ToS; not verified for redistribution,stable v3 API,exchange order book (USDT-quoted),low,yes,0.9128 @ 2023-03-11 15:00 (first hour after relist),0.9592 (partial day),cross-check only: USDT quote + gap across the event OKX,USDC/USDT,USDT,2020-01-01,1m..1d (1Dutc),none; one idiosyncratic print 2020-03-12 close 0.9395 (610 bps from Binance),exchange; realtime,free public API (100 rows/call),exchange ToS; not verified for redistribution,stable v5 API,exchange order book (USDT-quoted),low,yes,0.8787 @ 2023-03-11 07:00,0.9607,cross-check only (USDT quote) Bybit,USDC/USDT,USDT,2021-10-22,1m..1d,none,exchange; realtime,free public API,exchange ToS; not verified for redistribution,stable v5 API,exchange order book (USDT-quoted),low,yes,0.8840 @ 2023-03-11 07:00,0.9611,cross-check only (USDT quote; starts 2021-10) diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-peer-stablecoins.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-peer-stablecoins.md index 647d7102..b2088643 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-peer-stablecoins.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-peer-stablecoins.md @@ -94,7 +94,7 @@ None of that needs a peer's price to be published as USDC's. ## Consequence for phase 3 -Primary and fallback stay what phase 1 chose (Chainlink, Bitstamp, Kraken): +Primary and fallback stay what phase 1 chose (Chainlink, Bitstamp, third-party): all three *are* USDC/USD. The peer basket enters only as a validation input. The [[0172]] point stands separately: canonical Stellar USDT (`GCQTGZQQ…TG6V`) is an IOU that traded at ~0.13, and its `/ohlcv` series is diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-source-candidates.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-source-candidates.md index 4964a8f2..e5ebfde8 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-source-candidates.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-source-candidates.md @@ -34,7 +34,7 @@ result the brief expected. [data] | source | quote | hourly min close (UTC) | 2023-03-11 daily close | daily low | |---|---|---|---|---| | Bitstamp | USD | **0.8627** @ 03-11 07:00 | 0.9685 | 0.8053 | -| Kraken (from 580 028 trades) | USD | 0.9010 @ 03-11 12:00 | 0.9670 | 0.8740 | +| third-party (from 580 028 trades) | USD | 0.9010 @ 03-11 12:00 | 0.9670 | 0.8740 | | Chainlink on-chain (3 240 rounds read, 2021-02-17 → today) | USD | **0.8800** @ 03-11 07:51:23, 243 rounds that day | 0.9681 | 0.8800 | | OKX | USDT | 0.8787 @ 03-11 07:00 | 0.9607 | 0.8698 | | Bybit | USDT | 0.8840 @ 03-11 07:00 | 0.9611 | 0.8500 | @@ -58,7 +58,7 @@ trough: the Fed's FEDS Note (2025-12-17) says "at its trough, USDC traded at | Bitstamp | 2021-01-01 | 0 | — (API silent before 2021; listing date n/v) | | Chainlink | 2021-02-17 | 1 (of 2 026 days) | 3 phase aggregators, round ids non-contiguous; 3 240 rounds total | | Bybit | 2021-10-22 | 0 | — | -| Kraken OHLC | 2024-09-14 | 0 | endpoint caps at 720 candles; older history only via trades | +| third-party OHLC | 2024-09-14 | 0 | endpoint caps at 720 candles; older history only via trades | Our own series starts 2021-01-25 ([[0247]]). So the **only** key-less USD sources that reach our start are Coin Metrics (daily, non-commercial licence) @@ -72,9 +72,9 @@ From `data/dispersion_daily.csv` (calm = every common day outside 2023-03-08..18 | pair | common days | median calm | p95 calm | max calm | median stress | max stress | |---|---|---|---|---|---|---| | Chainlink – Bitstamp (USD–USD) | 2025 | **0.8** | 6.8 | 36 | **4.0** | 33 | -| Chainlink – Kraken (USD–USD) | 721 | **0.6** | 1.7 | 2.8 | — | — | +| Chainlink – third-party (USD–USD) | 721 | **0.6** | 1.7 | 2.8 | — | — | | Chainlink – Coin Metrics (USD–USD) | 2024 | 0.9 | 5.0 | 83 | 2.1 | 25 | -| Bitstamp – Kraken (USD–USD) | 721 | 0.8 | 2.3 | 6.3 | — | — | +| Bitstamp – third-party (USD–USD) | 721 | 0.8 | 2.3 | 6.3 | — | — | | Bitstamp – Coin Metrics (USD–USD) | 2072 | 0.9 | 6.5 | 105 | 2.4 | 33 | | Binance – Bybit (USDT–USDT) | 1614 | **1.0** | 3.0 | 48 | 1.5 | 19 | | Binance – OKX (USDT–USDT) | 2274 | 1.0 | 5.0 | 610 | 1.0 | 15 | @@ -128,7 +128,7 @@ times, nothing verified. [docs] cost hundreds, history from 2021-02-17. Cost: a few thousand `eth_call`s once, then one call per new round; no key needed today. 2. **Fallback and cross-check: Bitstamp USDC/USD** (from 2021-01-01, real USD - book, free, hourly), with Kraken trades as the second USD venue where the + book, free, hourly), with third-party trades as the second USD venue where the two need arbitration. 3. **Reject as primary:** every USDT-quoted venue (cluster offset above), Binance specifically (gap on the event), Coin Metrics (licence), CoinGecko diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-composition-rule.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-composition-rule.md index 40a48071..b7644a09 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-composition-rule.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-composition-rule.md @@ -26,9 +26,9 @@ history: | step | rule | why (measured) | |---|---|---| -| **primary** | Chainlink USDC/USD rounds, Ethereum mainnet proxy `0x8fFf…18f6` | real USD, verifiable by anyone with an RPC, no licence, 0.25 % deviation + 23 h heartbeat, from 2021-02-17; agrees with Bitstamp to 0.8 bps median calm / 4 bps stress and with Kraken to 0.6 bps | +| **primary** | Chainlink USDC/USD rounds, Ethereum mainnet proxy `0x8fFf…18f6` | real USD, verifiable by anyone with an RPC, no licence, 0.25 % deviation + 23 h heartbeat, from 2021-02-17; agrees with Bitstamp to 0.8 bps median calm / 4 bps stress and with third-party to 0.6 bps | | **fallback** | Bitstamp USDC/USD, used only where the primary has **no observation in the bucket** | the only key-less real-USD book reaching 2021-01-01; same cluster as the primary | -| **cross-check** | Kraken (OHLC last 720 d, trades for 2023-03) and Bitstamp | independent USD venues; USDT-quoted venues are excluded from arbitration because their cluster sits 3–4 bps off in calm and 20–30 bps off in stress (that is USDT/USD, not USDC/USD) | +| **cross-check** | third-party (OHLC last 720 d, trades for 2023-03) and Bitstamp | independent USD venues; USDT-quoted venues are excluded from arbitration because their cluster sits 3–4 bps off in calm and 20–30 bps off in stress (that is USDT/USD, not USDC/USD) | | **disagreement** | if a cross-check differs from the primary by **> 25 bps** on the bucket close, keep the primary value and flag `measured-disputed` | 25 bps is 3.5× the calm p95 between USD sources (≤ 7 bps) and below the stress median between them (4 bps median, 33 bps max). No averaging: a median of venues is a number nobody printed, and the reviewer's question "where did 0.9681 come from" must have a one-word answer | | **gaps** | a bucket with no source publishes **no price** and `quality = missing` | forward-fill is the defect being removed. The only carry allowed is *inside the primary's own heartbeat* on the hourly grain: an hour with no round means "no 0.25 % move since the last round", and the row says `n_obs = 0` | @@ -65,7 +65,7 @@ Against our series on the 2 042 common days: median |diff| 0.8 bps, p99 max/min, close = last; `n_obs` = number of rounds. A day with one round is honestly flat (68 % of Chainlink days), which is why flatness is *not* a synthetic signal in phase 4. -- **Candle from trades (Kraken):** standard OHLC over trade prints, +- **Candle from trades (third-party):** standard OHLC over trade prints, `n_obs` = trades. - **Venues that were closed** (Binance 2022-09-26 → 2023-03-11 14:00) are simply absent for those buckets; they are never interpolated. diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-guardrails.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-guardrails.md index 180df1b1..38b9c4d7 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-guardrails.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-guardrails.md @@ -12,7 +12,7 @@ history: status: seed who: akot note: > - Thresholds measured over Chainlink USDC/USDT/DAI, Bitstamp, Kraken and + Thresholds measured over Chainlink USDC/USDT/DAI, Bitstamp, third-party and OKX daily bars; tests run green on 2026-09-04 (`python3 analysis/guardrails.py`). --- @@ -27,9 +27,9 @@ legitimate twin on real data: | signal | why it false-positives alone [data] | |---|---| | O = H = L = C | 68 % of Chainlink USDC days (one heartbeat round per day), 88 % of USDT days; 20 thin Stellar assets with a single trade a day | -| close exactly 1.0 | 16 % of Kraken days (4-decimal prints), up to 67 % of a 30-day Kraken window; 8–10 % on every other venue | +| close exactly 1.0 | 16 % of third-party days (4-decimal prints), up to 67 % of a 30-day third-party window; 8–10 % on every other venue | | zero realised vol | never on real data, but a 30-day window can be as quiet as 0.23 bps | -| long constant run | Kraken 18 days, OKX 10 days | +| long constant run | third-party 18 days, OKX 10 days | So the invariants are joined with `trade_count`, and the price-shaped ones carry thresholds sized off the *noisiest real window*, not off intuition. @@ -40,9 +40,9 @@ carry thresholds sized off the *noisiest real window*, not off intuition. |---|---|---|---| | `zero_trades_share_30d` | ≥ 0.9 → **reject** | 0.00 on every venue | 1.00 | | `flat_and_no_trades_share_30d` | ≥ 0.9 → reject | 0.00 | 1.00 | -| `exact_peg_share_30d` | ≥ 0.9 → alert | 0.67 (Kraken) | 1.00 | -| `constant_close_run_days` | > 30 → alert | 18 (Kraken) | 1 864 | -| `min_rvol_30d_bps` | < 0.1 bps → alert | 0.23 (Bitstamp), 0.33 (Chainlink), 0.58 (Kraken), 0.85 (OKX); 0 never | 0.00 | +| `exact_peg_share_30d` | ≥ 0.9 → alert | 0.67 (third-party) | 1.00 | +| `constant_close_run_days` | > 30 → alert | 18 (third-party) | 1 864 | +| `min_rvol_30d_bps` | < 0.1 bps → alert | 0.23 (Bitstamp), 0.33 (Chainlink), 0.58 (third-party), 0.85 (OKX); 0 never | 0.00 | | `zero_volume_share_30d` | ≥ 0.9 → alert | 0.00 | 1.00 | "Reject" means the series fails CI (a fixture test over `/ohlcv` for every @@ -56,7 +56,7 @@ production series, one evaluation per day, which fits the [[0125]] dashboard |---|---| | `test_our_usdc_series_fails` | our `/ohlcv` USDC series trips zero-trades, exact-peg, constant-run and rvol | | `test_our_native_control_passes_price_checks` | XLM, a non-stablecoin, trips nothing | -| `test_real_usd_sources_pass` | Chainlink USDC/USDT/DAI, Bitstamp, Kraken, OKX and the composed series trip nothing over their full history | +| `test_real_usd_sources_pass` | Chainlink USDC/USDT/DAI, Bitstamp, third-party, OKX and the composed series trip nothing over their full history | | `test_stress_window_does_not_trigger` | 2023-02 → 2023-04 on Chainlink trips nothing (a depeg is not an anomaly of the *series*) | | `test_a_quiet_real_quarter_does_not_trigger` | the quietest 30 days ever printed on Bitstamp (0.23 bps) pass | | `test_synthetic_injection_is_caught` | a 45-day splice of exactly 1.0 into Bitstamp trips the constant-run check | @@ -80,5 +80,5 @@ production series, one evaluation per day, which fits the [[0125]] dashboard `data/peer_corr.csv`. Thresholds must be re-derived if the surface changes grain or precision: -Kraken's 4-decimal prints are what put `exact_peg_share_30d` at 0.9 rather +third-party's 4-decimal prints are what put `exact_peg_share_30d` at 0.9 rather than 0.5. diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/memo.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/memo.md index b65694f0..17f09412 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/memo.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/memo.md @@ -61,7 +61,7 @@ assumed"; a consumer cannot tell it from "USDC unmeasured". low **0.8800** at 07:51:23 UTC. 2. **Fallback: Bitstamp USDC/USD**, only where the primary has no observation (24 days: the 2021-01-25 → 02-16 tail and a few sparse days). - **Cross-check: Kraken and Bitstamp.** A daily close more than 25 bps from + **Cross-check: third-party and Bitstamp.** A daily close more than 25 bps from a cross-check keeps the primary value and is flagged `measured-disputed` (4 days in five years). No averaging: every published number is one venue's print. @@ -86,7 +86,7 @@ assumed"; a consumer cannot tell it from "USDC unmeasured". ## Rationale - **Dispersion decides the source order.** USD sources (Chainlink, Bitstamp, - Kraken, Coin Metrics) agree to < 1 bps median; USDT-quoted venues agree + third-party, Coin Metrics) agree to < 1 bps median; USDT-quoted venues agree with each other to 1 bps but sit **3–4 bps** off the USD cluster in calm and **20–30 bps** off in stress — that offset is USDT/USD. So USDT-quoted candles are cross-checks after correction, never primaries. Binance diff --git a/lore/1-tasks/archive/0267_FEATURE_usdc-measured-history-from-chainlink-anchor.md b/lore/1-tasks/archive/0267_FEATURE_usdc-measured-history-from-chainlink-anchor.md index 46cac714..47548518 100644 --- a/lore/1-tasks/archive/0267_FEATURE_usdc-measured-history-from-chainlink-anchor.md +++ b/lore/1-tasks/archive/0267_FEATURE_usdc-measured-history-from-chainlink-anchor.md @@ -74,7 +74,7 @@ The acceptance fixture is the falsifying date: **2023-03-11 must close at - The source decision and its evidence: [[0265]] `notes/memo.md`. Primary Chainlink USDC/USD rounds (on-chain, key-less, 2021-02-17 →), fallback - Bitstamp, cross-check Kraken. USDT-quoted venues and peer stablecoins + Bitstamp, cross-check third-party. USDT-quoted venues and peer stablecoins rejected on measured dispersion and correlation. - The composed daily series already exists (`compose_usdc.py` output, 2 049 rows 2021-01-25 → 2026-03-10: 98.6 % measured, 1.2 % fallback, @@ -218,7 +218,7 @@ What shipped, on `feat/0267_usdc-measured-history-from-chainlink-anchor` **The source, its granularity and its fetch date**, because an imported series with no provenance is not evidence ([[0247]]'s fifth criterion). Primary **Chainlink** USDC/USD on-chain rounds, fallback **Bitstamp**, cross-checked -against Kraken; composed by [[0265]]'s `compose_usdc.py` and versioned in this repo at **two +against third-party; composed by [[0265]]'s `compose_usdc.py` and versioned in this repo at **two grains**: - `lore/1-tasks/archive/0265_.../data/composed_usdc_usd_1d.csv` — **2049 daily diff --git a/lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md b/lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md similarity index 94% rename from lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md rename to lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md index 755cfb56..581adbe2 100644 --- a/lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md +++ b/lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md @@ -2,7 +2,7 @@ id: "0274" title: "1-stroop offer-priced fills still publish a USD price — e.g. XAUa at $4,375 on $0.003 of volume" type: FEATURE -status: active +status: completed related_adr: ["0287"] related_tasks: ["0116", "0147", "0252", "0236", "0286", "0278", "0310", "0216"] tags: [layer-backend, layer-api, priority-medium, effort-medium, milestone-M3, data-quality, liquidity, api] @@ -12,7 +12,7 @@ links: - "../../../../packages/prices-clickhouse/schema/current.sql" - "notes/R-offer-priced-dust-population-2026-09-29.md" history: - - date: 2026-09-10 + - date: "2026-09-10" status: backlog who: okarcz note: > @@ -22,7 +22,7 @@ history: asset-level property. Filed rather than absorbed into 0116, because it is a different claim about a different subject — 0116 is about a bucket, this is about an asset. - - date: 2026-09-25 + - date: "2026-09-25" status: backlog who: okarcz note: > @@ -34,7 +34,7 @@ history: 103120 `USD` ($0.10 published vs a 0.5 XLM close) is out of scope here; it goes to its own new task. Retitled, because the old 93% headline describes a population that is now `unpriced`. Original text kept below. - - date: 2026-09-29 + - date: "2026-09-29" status: active who: akot note: > @@ -45,6 +45,14 @@ history: Decided: keep the price and publish what it rests on in a separate `price_basis` field (option C); no `unpriced`, no fourth `price_status` word. Converted to a directory for the measurement note. + - date: "2026-10-05" + status: completed + who: akot + note: > + PR #373 merged to develop as 586713f1: `price_basis` (`trades` / + `offer_dust` / '') on current_prices, the price, list and batch + endpoints, OpenAPI and portal samples. 782 Rust tests, 270 portal tests. + Closed before deploy; the rollout order in Implementation Notes is binding. --- # 1-stroop offer-priced fills still publish a USD price @@ -93,10 +101,11 @@ about $0.20). That is a different defect and has its own task. 0 of 224 thin-but-real assets (at most 5 priced minutes, at least $1 in 24 h) flagged. Known edge: BTC `GBVFOW…`, $86 in 24 h in trades of about 150 stroops each, reads `offer_dust` — see Implementation Notes.* -- [ ] A consumer can tell, without running their own aggregation, that such a +- [x] A consumer can tell, without running their own aggregation, that such a price does not rest on a real market (e.g. it is `unpriced`, or carries a - signal that says so). *Via `price_basis = "offer_dust"`.* -- [ ] `volume_quote_usd` and volume aggregates are explicitly unchanged. + signal that says so). *Via `price_basis = "offer_dust"`. Merged in #373 + (586713f1); reaches consumers only with the rollout, not deployed at close.* +- [x] `volume_quote_usd` and volume aggregates are explicitly unchanged. *By construction: candle tables, rollups and enrichment are not touched.* ## Design (decided 2026-09-29) diff --git a/lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md b/lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md similarity index 100% rename from lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md rename to lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md diff --git a/lore/1-tasks/active/0294_DOCS_scf-milestone-3-verification-package.md b/lore/1-tasks/archive/0294_DOCS_scf-milestone-3-verification-package.md similarity index 68% rename from lore/1-tasks/active/0294_DOCS_scf-milestone-3-verification-package.md rename to lore/1-tasks/archive/0294_DOCS_scf-milestone-3-verification-package.md index 93db0bca..7d896406 100644 --- a/lore/1-tasks/active/0294_DOCS_scf-milestone-3-verification-package.md +++ b/lore/1-tasks/archive/0294_DOCS_scf-milestone-3-verification-package.md @@ -2,17 +2,33 @@ id: "0294" title: "SCF Milestone 3 verification package — evidence doc, form answers, video scenario, deviations" type: DOCS -status: active +status: completed related_adr: [] -related_tasks: ["0102", "0128", "0293", "0260", "0275", "0164", "0249", "0179", "0233", "0239", "0194", "0047", "0295", "0296", "0297", "0311"] +related_tasks: ["0102", "0128", "0293", "0260", "0275", "0164", "0249", "0179", "0233", "0239", "0194", "0047", "0295", "0296", "0297", "0311", "0101", "0139", "0274"] tags: [layer-docs, priority-high, effort-medium, milestone-M3, scf, submission, evidence] milestone: 3 links: + - "../../../docs/scf/milestone-3-evidence.md" + - "../../../docs/scf/milestone-3-rfp-deviations.md" - "../../../docs/scf/milestone-2-evidence.md" - "../../../docs/scf/milestone-2-rfp-deviations.md" - "../../../docs/prices-api-general-overview.md" - "../../../docs/prices-api-load-test-100rps.md" history: + - date: 2026-10-06 + status: completed + who: stkrolikiewicz + note: > + Submitted: Aga sent the SCF Deliverable Verification form on 2026-10-06 + with the shortened answers of #392. The package — + milestone-3-evidence.md (510 lines, 16-page PDF), four declared + deviations, the form answers, the video scenario and a 7:39 film — + merged in #354 (2026-10-05 17:27 UTC) and reached master in #391 three + minutes later; every repo link the PDF and the form make answered 200 + on master. The PDF, the film and their folder are on Google Drive with + link sharing. The form's G- and C-address fields do not apply: the API + owns no Stellar account and no contract. Closing decisions and the gaps + found on the way out are under Design Decisions and Issues Encountered. - date: 2026-09-29 status: active who: claude @@ -206,23 +222,111 @@ issue with its task. Kept here so the package is not written from memory. ## Acceptance Criteria -- [ ] `milestone-3-evidence.md` covers all 9 Tranche 3 criteria, each with the - observable it is graded against and a reproducible source -- [ ] Every §9 Tranche 3 Work bullet is addressed, including the two without a - numbered criterion -- [ ] `milestone-3-rfp-deviations.md` declares every known deviation, including - AC 1, the AC 5 companion numbers and the endpoint-group count -- [ ] Every criterion that is open on 2026-09-18 is closed, declared as a +- [x] `milestone-3-evidence.md` covers all 9 Tranche 3 criteria, each with the + observable it is graded against and a reproducible source — **§5**, each + with a command, a live URL or a CI run, and the date it was read. +- [x] Every §9 Tranche 3 Work bullet is addressed, including the two without a + numbered criterion — **§6**: X-Ray tracing end to end and the CloudWatch + dashboard. +- [x] `milestone-3-rfp-deviations.md` declares every known deviation, including + AC 1, the AC 5 companion numbers and the endpoint-group count — **four**: + AC 1 (the backfill finished early), AC 9 (two metrics flat since), AC 2 + (Redocly and the portal's renderer in place of `openapi-validator` and + Swagger UI), AC 5 (scope and companion numbers). The endpoint-group count + turned out not to be a deviation (Design Decision 7). +- [x] Every criterion that is open on 2026-09-18 is closed, declared as a deviation, or listed under "deliberately not claimed" with an owner — - none is silently absent -- [ ] "What is deliberately not claimed" section present, each row with a - destination -- [ ] Live endpoints + access table current, including how the Stellar team - reaches the dashboard -- [ ] `milestone-3-form-answers.md` and `milestone-3-video-scenario.md` written -- [ ] All cited figures re-run within days of submission, with the date beside - each -- [ ] No claim in the package lacks a task, query or URL behind it + none is silently absent. AC 3, 4, 8 and 9 closed; AC 7 met on the + fresh-account runbook, not run in an empty account ([[0297]] reopens on a + reviewer's request, §8); AC 6's 22 wildcard-resource statements each + named with what limits them. +- [x] "What is deliberately not claimed" section present, each row with a + destination — as **§7 Known issues** and **§8 Limitations**, each row + naming its task (Design Decision 6). +- [x] Live endpoints + access table current, including how the Stellar team + reaches the dashboard — **§9**: read-only access per named reviewer + through IAM Identity Center ([[0295]]'s runbook). +- [x] `milestone-3-form-answers.md` and `milestone-3-video-scenario.md` written + — the form went out with the shortened answers of #392. +- [x] All cited figures re-run within days of submission, with the date beside + each — AC 2, 3, 4, 6 and 8 on 2026-10-05, AC 9's window 09-23 → 09-30. + **Two exceptions, both dated in the package:** AC 5 (2026-09-18, no + re-run by the decision of 09-28) and AC 1 (2026-10-01, Design Decision 3). +- [x] No claim in the package lacks a task, query or URL behind it + +## Implementation Notes + +- `docs/scf/`: `milestone-3-evidence.md` (510 lines, §1–§10), + `milestone-3-rfp-deviations.md`, `milestone-3-form-answers.md`, + `milestone-3-video-scenario.md`, `screenshots/m3-ac2-api-reference.png`, + and `repo-links.lua`, which `build-pdf.sh` now runs. Side edits: the + load-test report, the 0295 dashboard-access runbook, the monitoring report + and the general overview. +- PRs: #354 (the package), #391 (release to master), #392 (shortened answers). + The AC 2 lint and the AC 4 test counts come from the master run of the + earlier release #381 (37298469682). +- Google Drive, link-shared: the evidence PDF, the film (15 clips stitched, + 7:39) and the folder holding both. + +## Issues Encountered + +- **Dead links in the PDF.** Typst renders a relative Markdown link as an + in-document reference, so every link to a repo document went nowhere. + Reported on 2026-10-05; fixed by `repo-links.lua` (Design Decision 4). +- **Two sentences written before the release reached Drive**: AC 7's + "on `master` from the next release" and the access table's "request by + e-mail". Fixed (b943f811, a5816d8e); each PDF went up as a new version under + the same Drive link, checked by its size each time. +- **The film.** QuickTime's variable-frame-rate clips froze their picture, and + one clip's narration ran 12 s late; the stitch padded the frames and cut + 12 s of silence to resync. The ffmpeg scripts were not kept in the repo. +- **Not in the package:** [[0101]]'s AMM history holes (Soroswap + 2026-07-06 → 07-11, Phoenix ~2 % short; blocked) are named in neither the + Milestone 2 nor the Milestone 3 known issues. Found at closing. + +## Design Decisions + +### From Plan + +1. **The M2 layout and the same four documents**: evidence per criterion with + its observable, source and date; deviations in their own document; form + answers and video scenario mirrored from M2. +2. **Launch at 2026-09-23 09:40 CEST** for the AC 9 window and the narrative + ([[0296]]). + +### Emerged + +3. **AC 1 stays at its 2026-10-01 reading** (decided 2026-10-05). A re-read + that day found that the explorer's Galexie export stalled from 2026-10-01 + 19:08 to 10-02 09:07 CEST: no ledger reached the pipeline for 14 h, + `ledger-processor-no-invocations` and `rollup-freshness-1m` fired within + 25 minutes, and `ledger-processor-lag` stayed in ALARM 09:17–13:27 while + the backlog drained. The [[0139]] window then held the writers + 14:08–14:57. Both postdate the reading; neither is in the package. +4. **Repo links in the PDF point at `blob/master`** (`repo-links.lua`): a PDF + has no repository beside it. `build-pdf.sh` stops if the filter no longer + rewrites. +5. **No published address for dashboard access** (decided 2026-10-05): the + evidence says "on request", the submitted Field 4 is "None." and the access + line sits in AC 8. +6. **"What is deliberately not claimed" became §7 Known issues and §8 + Limitations.** M3 is the last tranche, so no row can point at a later + milestone; each names its task instead. +7. **The endpoint-group count is not a deviation**: the evidence counts seven + `/v1` routes as the M2 package did, and the design document's §4 files the + same routes under five headings (evidence §5, AC 4). +8. **Answers shortened before submission** (#392): about 3,700 characters + instead of 8,800, every figure and date unchanged, detail left to the PDF. +9. **`ch-demo-queries.sql` not refreshed**: M3's evidence runs on the API, + CloudWatch and CI, and adds no ClickHouse query. +10. **#391 released #373 ([[0274]], `price_basis`) to master before it was + deployed** — declared in #391's description. The package does not + describe that field. + +## Future Work + +None spawned. Post-delivery work stays with the tasks the evidence names in §7 +and §8. ## Notes diff --git a/lore/1-tasks/active/0326_FEATURE_portal-served-at-prices-api.md b/lore/1-tasks/archive/0326_FEATURE_portal-served-at-prices-api.md similarity index 82% rename from lore/1-tasks/active/0326_FEATURE_portal-served-at-prices-api.md rename to lore/1-tasks/archive/0326_FEATURE_portal-served-at-prices-api.md index d541b113..637e4d9d 100644 --- a/lore/1-tasks/active/0326_FEATURE_portal-served-at-prices-api.md +++ b/lore/1-tasks/archive/0326_FEATURE_portal-served-at-prices-api.md @@ -2,9 +2,9 @@ id: "0326" title: "Serve the portal at /prices-api/ on the explorer's host" type: FEATURE -status: active +status: completed related_adr: [] -related_tasks: ["0194", "0195"] +related_tasks: ["0194", "0195", "0339"] tags: ["portal", "hosting", "effort-small"] links: [] history: @@ -30,6 +30,14 @@ history: /prices-api/ live ~10:55 UTC: sync-portal-explorer uploaded the bundle under prices-api/ (#385), then the explorer's Delivery deploy routed it. Portal and every 301 verified on production. + - date: 2026-11-05 + status: completed + who: stkrolikiewicz + note: > + Superseded by 0339: the portal moved on to /api/prices-api/ before this + task's Compute step shipped. The sign-in round-trip passed there, and + the explorer's 0631 dropped the old api/ and pricing-api/ prefixes on + 2026-11-04. --- # Serve the portal at /prices-api/ on the explorer's host @@ -41,7 +49,12 @@ portal moves to `/prices-api/`; the explorer turns every `/api…` URL into a `301` to the same path under `/prices-api`, query string kept (soroban-block-explorer task 0608), so nothing shared before the move breaks. -## Status: Active +## Status: Completed (superseded by [[0339]]) + +**2026-11-05:** the portal lives at `/api/prices-api/` since 0339; `/prices-api…` +answers `301` there. What this task left open was done under 0339 and the +explorer's 0631. + **Current state:** `/prices-api/` live since 2026-10-02 ~10:55 UTC; `/api…` and `/pricing-api…` answer `301` there. Next: Compute (`PORTAL_HOME`) with the @@ -83,11 +96,13 @@ cookie's `Path=/api/`) do not: that host has no portal bundle on it. - [x] Portal boots at `/prices-api/`, `/prices-api/dashboard`, `/prices-api/docs` (hard refresh included) -- [ ] Discord sign-in round-trip lands on `/prices-api/?…` (after Compute; - until then via the `301`) +- [x] Discord sign-in round-trip lands on `/prices-api/?…` (after Compute; + until then via the `301`): superseded, the landing moved to + `/api/prices-api/` and passed there under [[0339]] (2026-11-05) - [x] `/api/?utm_source=stellar-skill` → `301` `/prices-api/?utm_source=stellar-skill` -- [ ] Old `api/` and `pricing-api/` prefixes dropped from the bucket once - the above holds +- [x] Old `api/` and `pricing-api/` prefixes dropped from the bucket once + the above holds (soroban-block-explorer task 0631, 2026-11-04: 57 and + 18 objects) ## Design Decisions diff --git a/lore/1-tasks/archive/0328_BUG_board-page-ignores-the-computed-assignee.md b/lore/1-tasks/archive/0328_BUG_board-page-ignores-the-computed-assignee.md new file mode 100644 index 00000000..0e476ad4 --- /dev/null +++ b/lore/1-tasks/archive/0328_BUG_board-page-ignores-the-computed-assignee.md @@ -0,0 +1,61 @@ +--- +id: "0328" +title: "The board page recomputes the assignee from the last history entry and ignores the one board.json carries" +type: BUG +status: completed +assignee: akot +related_adr: [] +related_tasks: ["0327", "0149", "0041"] +tags: ["lore-board", "tooling", "effort-small", "priority-medium"] +links: + - "../../board.html" + - "../../../tools/scripts/generate-lore-board.mjs" +history: + - date: "2026-10-06" + status: completed + who: akot + note: > + PR #394 merged (d0399fd2) and deployed to GitHub Pages. getTaskAssignee + returns t.assignee; a test in generate-lore-board.test.mjs pins it + (14/14, red on the old page). The deployed board shows 0149, 0242 and + 0139 as akot. + - date: "2026-10-06" + status: active + who: akot + note: > + Found on 0149: board.json says akot, the board page shows okarcz. + [[0327]] fixed the generator only; the page has its own rule. +--- + +# The board page ignores the computed assignee + +## Summary + +[[0327]] made `getAssignee` in `generate-lore-board.mjs` honour `assignee:`, +then the newest-dated history entry, skipping `who: claude`. `board.json` has +carried the right value since. The page never reads it. +`getTaskAssignee` in `lore/board.html:934` still takes the `who` of the +**last** history entry. On a newest-first history that entry is the author. + +## Evidence (2026-10-06) + +- `board.json` on GitHub Pages: 0149 → `"assignee": "akot"`. +- The board page shows 0149 as `@okarcz`, from its 2026-08-05 spawn entry. +- 0327 checked its fix against the `board.json` diff (25 of 318 tasks moved), + not against the page, so the page kept the old rule. + +## Fix + +- `getTaskAssignee(t)` returns `t.assignee`. The generator already applies the + status rule (active and archive only) and the history fallback, so the page + keeps no rule of its own. +- The same function feeds the filter counts, the card tag, the table and the + modal (`board.html:883, 951, 999, 1052, 1086`), so all of them follow. + +## Acceptance Criteria + +- [x] `board.html` reads `assignee` from `board.json` and has no assignee + logic of its own. +- [x] On the deployed board, 0149, 0242 and 0139 show `@akot`. +- [x] The assignee filter counts match `board.json`'s `assignee` values. By + construction: the filter counts go through the same `getTaskAssignee`. diff --git a/lore/1-tasks/archive/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md b/lore/1-tasks/archive/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md new file mode 100644 index 00000000..ba547277 --- /dev/null +++ b/lore/1-tasks/archive/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md @@ -0,0 +1,162 @@ +--- +id: "0330" +title: "The XDR watch reports BEHIND because stellar-xdr 30 counts as a crate for protocol 29" +type: BUG +status: completed +related_adr: [] +related_tasks: ["0325", "0319", "0098"] +tags: [ci, resilience, ingestion, priority-medium, effort-small] +links: + - "../../../tools/scripts/verify-xdr-protocol-gap.mjs" + - "../../../tools/scripts/verify-xdr-protocol-gap.test.mjs" + - "../../../docs/runbooks/xdr-protocol-watch.md" + - "https://github.com/rumblefishdev/stellar-prices-api/issues/398" + - "https://github.com/stellar/rs-stellar-xdr/releases/tag/v30.0.0" + - "https://github.com/stellar/stellar-core/pull/5478" + - "https://github.com/rumblefishdev/stellar-prices-api/pull/399" + - "https://github.com/rumblefishdev/stellar-prices-api/actions/runs/38240193493" +history: + - date: "2026-10-12" + status: active + who: stkrolikiewicz + note: > + Created from issue #398. The scheduled watch has been red since + 2026-10-09 and reports BEHIND (pin 28, mainnet 29, newest crate 30.0.0). + Nothing is broken: protocol 29 kept the XDR of 28 (0325), and every + production rollup-freshness and ledger-processor alarm is OK. + - date: "2026-10-12" + status: active + who: stkrolikiewicz + note: > + Implemented on fix/0330_xdr-watch-reads-a-later-major-as-the-missing-crate. + A bump target for protocol N is now the newest stable, unyanked version + with pin < major ≤ N, asked for mainnet and core separately. Tier test: + 18/18 pass. The pre-fix script fails 5 of them, exactly the new cases. + Live run against mainnet 29 and crates.io 30.0.0 exits 0 with + "WAITING: no stellar-xdr 29" in both --watch and PR mode. + - date: "2026-10-12" + status: completed + who: stkrolikiewicz + note: > + PR #399 merged to develop as 78765f9e at 11:02:55 UTC. 4 files, + +82 −23. 18 tier cases, 5 of them new. CI 4/4 green. A manual + xdr-protocol-watch run on master (38240193493, 11:03 UTC) was green and + reported "WAITING: no stellar-xdr 29 is published yet". It closed #398 + with the "Closed as not actionable yet" comment. +--- + +# The XDR watch reports BEHIND because stellar-xdr 30 counts as a crate for protocol 29 + +## Summary + +`verify-xdr-protocol-gap.mjs` asks crates.io whether a crate for the needed +protocol exists by comparing `max_stable_version`'s major with it +(`crate.major < needed`). Upstream skipped 29: `stellar-xdr` went from 28.0.1 +to 30.0.0. Since 30 ≥ 29, the script treats 30.0.0 as the crate for protocol +29 and fails BEHIND. A bump target must have a major above our pin and **not +above** the protocol it is for. Read every published version, not only the +newest. + +## Context + +- `stellar-xdr` 30.0.0 was published 2026-10-08. It carries protocol 30 XDR + (CAP-0084 and CAP-0087 ungated). No `stellar-xdr` 29 exists. +- Horizon mainnet and testnet both report current 29 and core-supported 29 + (stellar-core 29.0.0). There is no stellar-core v30 release, and no + testnet or mainnet date for protocol 30. stellar-core merged its "Bump to + protocol 30" PR (#5478) to master on 2026-10-09. +- The watch has been red every day since 2026-10-09 (runs on 10-09, 10-10 and + 10-11). Issue #398 is open at tier BEHIND. +- Production is healthy. `stellar-xdr =28.0.0` decodes P29 (0325), and all + `prices-production-rollup-freshness-*`, `ledger-processor-lag` and + `ledger-processor-no-invocations` alarms are OK. +- BE's `soroban-block-explorer` develop still pins `stellar-xdr = "28"`. + +## Implementation Plan + +### Step 1: Read the published versions + +`publishedCrate()` also returns the stable, unyanked versions from the +`versions` array of the same crates.io document (no second request). A +missing `versions` array is an unreadable document, which is the +`cannot read` path. + +### Step 2: Choose the bump target per protocol + +`bumpTarget(protocol)` = the newest stable version with +`pin < major ≤ protocol`. + +- BEHIND: `pin < current` and `bumpTarget(current)` exists. +- LAGGING: otherwise, `pin < supported` and `bumpTarget(supported)` exists. + This keeps the lead time when mainnet sits on a crate-less protocol (29) + while core already supports the next (30). +- WAITING: neither. The `— WAITING: no stellar-xdr` wording stays, because + master's workflow matches it to close the issue. + +### Step 3: Tests and runbook + +New tier cases: today's state (pin, current pin+1, crate pin+2 only → +WAITING) and the next one (current pin+1, core pin+2, crate pin+2 → +LAGGING). The crates.io mock serves `versions`. Update the runbook's +"published crate" reading and the header comment. + +## Acceptance Criteria + +- [x] Live run against mainnet 29 and crates.io 30.0.0 exits 0 with + `— WAITING: no stellar-xdr 29`, in PR and `--watch` mode. +- [x] Current pin+1, core pin+2, crate pin+2 → LAGGING (exit 1 under `--watch`). +- [x] All existing tier cases still pass, and the pre-fix script fails the new + WAITING case. +- [x] After merge to `develop`, a manual watch run on `master` is green and + closes #398 as "not actionable yet". + +## Implementation Notes + +- `publishedCrate()` returns `releases`: the stable, unyanked versions from + the `versions` array of the crate document. The `major` field is gone. +- `bumpTarget(crate, protocol)` returns the newest release with + `pin < major ≤ protocol`. +- The tier check asks `bumpTarget` for mainnet (BEHIND) and then for core + (LAGGING). WAITING when both are empty. +- The BEHIND report names the target (`bump to …`). The LAGGING report names + the target instead of `max_stable_version`. +- No workflow change on `master` was needed: the workflow checks out + `develop` for the script. + +**Modified tests:** the `published()` mock now also serves `versions`. No +existing case changed. + +## Design Decisions + +### From Plan + +1. **Ask mainnet and core separately.** Mainnet can sit on a crate-less + protocol (29) while core supports the next one, which has a crate (30). + That reading is LAGGING, not WAITING. +2. **Keep the WAITING wording.** `— WAITING: no stellar-xdr` is what + master's workflow matches to close the issue. + +### Emerged + +3. **A range, not an exact major.** The target is `pin < major ≤ N`, not + `major == N`. In chat I first described it as "exactly 29". The range also + covers a pin two protocols behind (pin 27, mainnet 29, only 28 published: + BEHIND, bump to 28). An exact check would call that WAITING. +4. **Same request, `versions` from the crate document.** No call to the + paginated `/versions` endpoint. A document without `versions` is + unreadable, which is the `cannot read` path (red run, no issue). +5. **Yanked versions do not count.** No test covers this. +6. **The WAITING report explains a later major.** When the newest crate is + above the needed protocol, the report says it carries a later protocol's + XDR and is not a bump target. + +## Issues Encountered + +- **`lore_set-task` could not find 0330** (TASK_NOT_FOUND) from the + `issue-398-review` worktree. The MCP server seems to resolve tasks in the + main checkout. The task was never set as the session's current task. +- **The pre-push hook took 3 of 4 nx tasks from cache.** It is unclear + whether the infra `test` target re-ran the changed test file locally. The + test was run directly (18/18), and CI ran it on the PR. +- **Prettier re-padded the whole tier table** in the runbook, which made + the runbook diff larger than the change. diff --git a/lore/1-tasks/archive/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md b/lore/1-tasks/archive/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md new file mode 100644 index 00000000..af1ae343 --- /dev/null +++ b/lore/1-tasks/archive/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md @@ -0,0 +1,130 @@ +--- +id: "0333" +title: "XLM/USD for 2018-01-01 → 2018-02-16" +type: FEATURE +status: active +assignee: akot +related_adr: [] +related_tasks: ["0331"] +tags: [layer-backend, priority-medium, effort-small, enrichment, clickhouse, usd-reference] +links: + - "../active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md" +history: + - date: "2026-10-16" + status: active + who: akot + note: > + Source changed. The exchange had no XLM/USD market from 2017-08-22 to + 2018-02-16 (consecutive trade ids 225906 → 225907), so the OHLCVT files + are empty for the range. The rate is built as the exchange's XLM/BTC × + BTC/USD hourly VWAP from its public trades API. The 2018-01-11 → 01-13 + outage stays without a rate. The 0331 row at 2018-02-16 23:00 is + replaced with the cross rate. + - date: "2026-10-15" + status: active + who: akot + note: > + Activated. First step: fetch the OHLCVT archive and check whether it + carries enough to build the 0331 rate definition (hourly VWAP). + - date: "2026-10-15" + status: backlog + who: akot + note: > + Scope set back to 2018-01-01 → 2018-02-16: priced history starts at the + beginning of 2018. 2017 is out of scope. The source stays the exchange's + downloadable OHLCVT files. + - date: "2026-10-15" + status: backlog + who: akot + note: > + Scope widened from 2018-01-01 → 02-16 to 2017-01-17 → 2018-02-16, the + start of the exchange's XLM/USD market. The downloadable OHLCVT files + are the chosen source. The 68 in-scope 1h rows before 2017-01-17 stay + unpriced. + - date: "2026-10-14" + status: backlog + who: akot + note: > + Spawned from 0331. The hourly XLM/USD series prepared for 0331 starts + 2018-02-16 23:00 UTC; the 47 days before it have no rate and stay + unpriced until this lands. Source not decided yet. +--- + +# XLM/USD for 2018-01-01 → 2018-02-16 + +## Summary + +[[0331]] prices pre-2021 candles from an hourly XLM/USD series loaded into +`prices.usd_rate`. That series starts at 2018-02-16 23:00 UTC. The priced +history should start at the beginning of 2018, so this task fills the 47 days +before it from the same exchange, in the same shape. The exchange had no +XLM/USD market in that period, so the rate is its XLM/BTC × BTC/USD. + +## Context + +- **Why the 0331 series stops at 2018-02-16.** The exchange delisted XLM/USD + and XLM/EUR on 2017-08-22 and relisted them on 2018-02-16. Its trade ids run + without a gap across that date: XLM/USD 225906 is at 2017-08-22 06:05 and + 225907 at 2018-02-16 23:43; XLM/EUR does the same. There was no trade to + fetch, so the OHLCVT files would be empty for the range too. Measured + 2026-10-16. Supersedes the 2026-10-14 reading that the trades API could not + reach older trades. +- **XLM/BTC and BTC/USD traded throughout** on the same exchange and page from + its public trades API: 472,990 XLM/BTC and ~2.3 M BTC/USD trades for + 2017-12-31 → 2018-02-28. XLM/BTC has a trade in 1,074 of the range's 1,127 + hours, median 296 trades an hour. +- **The exchange was down 2018-01-11 07:00 → 01-13 14:00** (53 hours without + an XLM/BTC trade, in four runs). A second exchange's XLM/BTC × BTC/USDT + covers them, but mixes sources and takes USDT at $1, so it is not used. +- **What is left to price** (prod, 2026-10-15): 6,432 1h rows with volume + quoted in XLM or a dollar token, worth 119.2 M XLM of quote volume, all at + `close_usd = 0`. +- **Out of scope:** everything before 2018-01-01. There are 12,819 such 1h + rows (2015-11 → 2017-12), and they stay unpriced. + +## Measurements (2026-10-16, hourly VWAP of each leg) + +| Cross rate compared with | Hourly median / p95 / max | Daily median / max | +|---|---|---| +| The exchange's direct XLM/USD (the 0331 definition), 2018-02-17 12:00 → 02-28 | 0.44 % / 1.87 % / 6.40 % | 0.21 % / 0.64 % | +| Second exchange's XLM/BTC × the exchange's BTC/USD, in range | 0.41 % / 4.19 % / 21.0 % | 0.33 % / 5.87 % | +| Second exchange's XLM/BTC × BTC/USDT (lab cross-check), in range | 0.98 % / 4.67 % / 20.3 % | 0.95 % / 8.06 % | + +The direct pair's first hours after relisting were off the market: at +2018-02-16 23:00 it printed 0.4056 against 0.4505 for the cross rate (11.1 %), +and 3–4 % for the next six hours. The 23:00 hour is already in the 0331 +series as `external`. + +## Decision (amended 2026-10-16, Adam) + +Supersedes the 2026-10-15 source decision. + +- **Range:** 2018-01-01 00:00 → 2018-02-16 22:00 UTC. +- **Source:** the same exchange, as a cross rate: hourly VWAP of XLM/BTC × + hourly VWAP of BTC/USD, both from its public trades API, stamped at the hour + start. An hour gets a rate only when both legs traded in it. +- **The outage hours stay without a rate.** The pass carries a rate for up to + 24 h, so only the candles more than 24 h past the last rate stay unpriced. + XLM's daily candles are priced on every day of the range. +- **The 0331 row at 2018-02-16 23:00 is replaced** with the cross rate. +- **Tag:** `reference_asset = 'third-party'`, as for 0331. + +## Implementation Plan + +1. Build the hourly series for the range plus the 2018-02-16 23:00 hour. +2. Insert as `('native','XLM','','')`, staged `external-candidate`, then + promote; the 23:00 row by a newer version. +3. Re-run 0331's `coarse-repair --pre-usdc-xlm-rate` for 201801–201802, first + 1h, then the wider tiers. +4. Count the candles left unpriced around the outage. + +## Acceptance Criteria + +- [x] Source recorded as third-party. +- [ ] Hourly XLM/USD rows for every hour of 2018-01-01 → 2018-02-16 in which + both legs traded on the exchange. +- [x] Agreement recorded with the direct pair on the overlap and with the + cross-check (median and max deviation). +- [ ] The 2018-02-16 23:00 row replaced with the cross rate. +- [ ] XLM daily USD prices from 2018-01-01 on; candles left unpriced by the + 2018-01-11 → 01-13 outage counted. diff --git a/lore/1-tasks/archive/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md b/lore/1-tasks/archive/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md new file mode 100644 index 00000000..49887958 --- /dev/null +++ b/lore/1-tasks/archive/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md @@ -0,0 +1,102 @@ +--- +id: "0334" +title: "Blend v2.1's Comet BLND/USDC pool is missing from pool_registry — every swap since 2026-10-07 is dropped and both pool alarms fire" +type: BUG +status: completed +assignee: akot +related_adr: [] +related_tasks: ["0300", "0291", "0100", "0323", "0329"] +tags: [layer-backend, priority-high, effort-small, ingest, soroban, comet, observability] +links: + - "../../../packages/prices-ingest-core/src/static_pools.rs" + - "../../../packages/coverage-sweep-probe/allowlist.toml" + - "../../../docs/runbooks/seed-pool-registry.md" + - "https://github.com/blend-capital/docs-v2/blob/main/mainnet-deployments.md" + - "https://github.com/CometDEX/comet-contracts-v1/releases/tag/v1.1.0_contracts_pkg1.0.0_cli25.1.0" +history: + - date: "2026-10-16" + status: active + who: akot + note: "Task created from the unregistered-pool and coverage-sweep alarms; pool verified as Blend v2.1's" + - date: "2026-11-02" + status: completed + who: akot + note: > + Reprice finished under 0338, which has the result and the residual. +--- + +# Blend v2.1's Comet BLND/USDC pool is missing from pool_registry + +## Summary + +Blend v2.1 moved its backstop to a new Comet BLND/USDC pool, +`CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE`. Comet pools have no +factory event, so only `STATIC_POOLS` registers them, and it holds only the old +pool `CAS3FL6T…` ([[0300]]). The live processor drops every swap of the new pool. +It is in `prices-production-ledger-processor-unregistered-pool` (flapping since +2026-10-07, ~12 transitions a day) and makes 393 of the 404 events behind +`prices-production-coverage-sweep-unclassified` (ALARM since 2026-10-12). + +## Verification (2026-10-16) + +- **Official.** `blend-capital/docs-v2` `mainnet-deployments.md` (commit + ef36127c, 2026-10-07) lists it as v2.1's "Comet v1.1 Contract (BLND:USDC + liquidity pool)". The v2.1 backstop `CCS4AZ5O…` is listed there too and in + blend-ui's `.env.production` as `NEXT_PUBLIC_BACKSTOP_V2_1`. It holds ~50% of + the pool's LP supply. +- **Code.** Wasm `d735c339…` is byte-identical (sha256) to CometDEX release + `v1.1.0_contracts_pkg1.0.0_cli25.1.0` (2026-09-11). That release adds + `token_in != token_out` (`ErrSameToken`) to both swap entry points, which closes + the 2026-08-25 self-swap exploit. +- **Payload.** Same `POOL/swap` topic and data map as the old pool (`caller`, + `token_amount_in/out`, `token_in/out`). All 520 lifetime swaps are BLND SAC + `CD25MNVT…` ↔ USDC SAC `CCW67TSZ…`. 730 events in prod ClickHouse match + StellarExpert's count. +- **Price.** Hourly VWAP vs the same hour elsewhere: Soroswap median −0.09% + (p5/p95 −1.8%/+0.8%), SDEX median −0.59% (−3.3%/+1.5%). The only outliers + (≤ −34%) are the launch hours of 2026-10-07, when SDEX and Soroswap disagreed + with each other too. +- **Range.** Pool created 2026-09-30, first swap 2026-10-07 00:16 UTC (ledger + 64,809,271), the same minute as the first WARN. Nothing was traded before. + ~1.55 M BLND / ~9.5 k USDC to 2026-10-16. 80% of swaps come from one arb-like + contract (`CBNVK5PE…`). + +The sweep's two other emitters are not priceable: + +- `CDU47AIU…` (wasm `f3dd3edf…`, 10 events): securities-financing ledger (repo, + stock loan, FX forward; PSG Digital Labs). `trade_opened/closed` carry notional + and `rate_bps`, no token price. Allow-list it. +- `CD35HSTR…` (1 event): an instance of the allow-listed order-book wasm + `4d6abb0d…`, matched only by contract id today. Allow-list this instance. + +## Implementation Plan + +1. Add `CASYAKK3…` (venue `comet`) to `STATIC_POOLS`, with a test on a real swap + of this pool. +2. Allow-list `CDU47AIU…` and `CD35HSTR…` in `coverage-sweep-probe`. +3. Deploy the ledger-processor; its cold start loads the pool. +4. `events-backfill --discover-pools` WRITE (runbook "Factory-less pools"). +5. Reprice from ledger 64,809,271 to the deploy. + +## Acceptance Criteria + +- [x] `STATIC_POOLS` holds `CASYAKK3…`; a test decodes one of its real swaps +- [x] Both sweep emitters allow-listed +- [x] Ledger-processor deployed; no `unregistered_pool_events` WARN after it +- [x] `prices.pool_registry FINAL` has a `comet` row for `CASYAKK3…` +- [x] Minutes from 2026-10-07 00:16 UTC repriced; BLND `comet` 1h candles + exist for those hours +- [x] Next coverage sweep clean; `coverage-sweep-unclassified` back to OK + +## Notes + +- The old pool `CAS3FL6T…`, called frozen after the exploit, still emitted 1,084 + swaps from 2026-09-30 to ~2026-10-11 and is indexed. Its prices were not + checked here. + +## Result (2026-11-02) + +The reprice and pre-roll were done under [[0338]], over ledgers +64,809,264–65,025,000. 2026-10-12–18 matches BE's swap count per day. 93 swaps of +10-07–11 stay undercounted: they share a minute with an enriched row of the old +pool, which outranks the reprice. Comet 1h candles exist for all those hours. diff --git a/lore/1-tasks/archive/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md b/lore/1-tasks/archive/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md new file mode 100644 index 00000000..406de56e --- /dev/null +++ b/lore/1-tasks/archive/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md @@ -0,0 +1,237 @@ +--- +id: "0335" +title: "Bump stellar-xdr 28→30 before the protocol 30 vote — every ledger after it carries a millisecond StellarValue arm that 28 cannot decode" +type: FEATURE +status: completed +related_adr: [] +related_tasks: ["0277", "0330", "0325", "0098", "0091"] +tags: [layer-backend, priority-high, effort-small, phase-live, resilience, ingestion, deployment] +links: + - "../archive/0277_FEATURE_stellar-xdr-28-protocol-28-bump.md" + - "../../../docs/runbooks/xdr-protocol-watch.md" + - "https://github.com/rumblefishdev/soroban-block-explorer/pull/673" + - "https://github.com/stellar/rs-stellar-xdr/releases/tag/v30.0.0" +history: + - date: "2026-10-19" + status: active + who: akot + note: > + Created after BE announced their stellar-xdr 30 bump (BE #673, lore-0649, + merged; BE develop head d1251f30). Our lock still pins xdr-parser at + 31be5f7, so nothing is broken today, but the next `cargo update -p + xdr-parser` fails to build unless stellar-xdr moves to 30 in the same + change. No vote date announced yet. + - date: "2026-10-19" + status: active + who: akot + note: > + Bump done on feat/0335_stellar-xdr-30-protocol-30-bump, not yet + committed. The lock resolves one stellar-xdr (30.0.0) and xdr-parser is + at d1251f30. The only compile break was BE's event API rework + (ADR 0059), not XDR. On 485 real mainnet ledgers the adapted extraction + gives the same 360,847 events as develop. Workspace tests: 1448 passed, + 0 failed. + - date: "2026-10-20" + status: active + who: akot + note: > + PR #406 merged to develop as af504c37 (approved). CI green, including + 328/328 ClickHouse integration tests. Not deployed yet: ledger-processor + and the EventBridge Lambdas are next, then the post-vote crossing + measurement. + - date: "2026-10-20" + status: active + who: akot + note: > + Deployed on the XDR 30 binary: Compute (ledger-processor) at 08:31 UTC + and EventBridge (asset-discovery and the rest) at 09:02 UTC. CodeSha256 + changed on both, with 0 errors after. Stays active until the candle + frontier is measured across the protocol-30 activation ledger. + - date: "2026-10-20" + status: completed + who: akot + note: > + Closed as done before the vote. Pin, lock, event API adaptation, decode + test and both deploys are complete (6 of 7 AC). The post-vote crossing + cannot be measured yet: on 2026-10-20 mainnet, testnet and futurenet + are all on protocol 29. It is not tracked as a task; xdr-protocol-watch + is the backstop. +--- + +# stellar-xdr 28 → 30 for protocol 30 + +## Summary + +Protocol 30 (CAP-0088) makes the millisecond `StellarValueExt` arms +(`SignedMs`, `EmptyTxSetMs`) mandatory from the first ledger after the vote. +Our workspace pins `stellar-xdr = "=28.0.0"` (`Cargo.toml:30`), and the +ledger-processor decodes every Galexie object with it. On 28 the first +protocol-30 ledger fails to decode, the batch errors, and the cursor stops +moving. That is the protocol-27 freeze (0091) all over again. The bump has to +ship and be **deployed** before the vote. + +## Context + +- **BE side is done.** `xdr-parser` on BE `develop` now requires + `stellar-xdr` 30 (BE `Cargo.toml:41`, `version = "30"`). We track it with + `branch = "develop"`, and our lock pins rev `31be5f7` (the proto-28 rev from + 0277). Bumping one without the other puts two XDR majors in the graph and + `decode_object` stops type-checking. The two move together. +- **The compiler will not prove readiness** (0277's correction). CAP-0088 + changes `StellarValueExt`, and we only read `scp_value.close_time`, a plain + field. So a clean `cargo check` says nothing. Proof has to be a decode of a + ledger carrying the new arm through our own path. +- **`closed_at` stays in seconds.** CAP-0088 keeps + `closeTime == closeTimeMs / 1000`, and BE asserts the same in + `crates/xdr-parser/tests/protocol_30_arms.rs`. +- **CAP-0084** adds `ScAddress::MuxedContract` (`W…`). We only construct + `ScAddress::Contract` and never match `ScAddress` exhaustively, so no compile + break is expected. It matters only if a pool swap names a muxed contract. +- **The watch** (`xdr-protocol-watch`, 0098/0330) goes LAGGING once Horizon + reports `core_supported_protocol_version = 30`, because crate 30.0.0 is + already published. That alarm is a backstop, not the plan. +- **Out of our hands:** BE's Galexie on captive core 30 (BE lore-0650). + Protocol 29 stalled on Galexie, not on XDR (0325). + +## Implementation Plan + +1. `Cargo.toml`: `stellar-xdr = "=28.0.0"` → `"=30.0.0"`, keeping the exact pin. +2. `cargo update -p xdr-parser -p stellar-xdr`. The lock must resolve exactly + one `stellar-xdr`, and `xdr-parser` lands on BE `develop`. +3. Fix whatever the compiler reports. +4. Decode test: a ledger with `StellarValueExt::SignedMs` and one with + `EmptyTxSetMs`, packed as Galexie writes them (zstd `LedgerCloseMetaBatch`), + through `decode_object` → `ledger_sequence` / `ledger_close_time` in seconds. +5. Workspace tests, plus the ClickHouse `#[ignore]` suite. +6. Deploy every binary that links `stellar-xdr`. That is the ledger-processor + first (it decodes ledgers), then the EventBridge Lambdas. Confirm + `CodeSha256` changed (0091: a merged bump with a stale binary changes + nothing). +7. After the vote: measure the candle frontier crossing the activation ledger, + as 0277 did. + +## Findings 2026-10-19 + +**The break was BE's event API, not XDR.** Between `31be5f7` and `d1251f30`, BE +replaced `xdr_parser::extract_events` with `LedgerEvents::new(..).extract(tx, +hash)` (ADR 0059: stellar-rpc event ids). `EventSource` is gone, and +`ExtractedEvent` now carries `body: EventBody { event_type, contract_id, +topics, data }`. The new `TxEvents` returns consensus events in `.events` +(tx-level first, then per-op) and diagnostic events separately in +`.diagnostic`. The old function returned the same consensus list with the +diagnostic events appended after it. So an event's position in `.events` is +exactly the old per-tx `event_index`. `prices-ingest-core/src/soroban.rs` is +the only consumer that needed changes. `decompress_zstd`, `deserialize_batch`, +`ParseError` and `scval.rs` are byte-identical between the two revs. + +**Equivalence measured, not assumed.** A scratch dump ran on both `develop` +and the branch over 485 mainnet ledgers from the public data lake (`65,021,700+` +on protocol 29 with V4 meta, `57,000,000+` and `60,000,000+` older). It printed +every consensus event (tx, index, type, contract, topics, data) plus the full +`process_ledger` output. All 360,847 events match, and so do all `amm_ticks` +and `oracle`. The one diff line is the order of two `unresolved` rows in ledger +65,021,722. They come from iterating the `amm_groups` `HashMap`, whose order is +random per process on `develop` too. + +**XDR 30 gates nothing.** There is no `cap_*` feature in `stellar-xdr` +30.0.0, so `SignedMs`/`EmptyTxSetMs` are ungated. 28 rejects both +(`stellar_value_ext.rs:135`, `_ => Err(Invalid)`). `stellar-xdr` 30 also +pulls in `stellar-strkey` 1.0.0 beside our 0.0.13. No types cross between +them. + +**Deploy scope.** Only the ledger-processor decodes ledgers now +(`reconcile.rs:292`). `asset-discovery` no longer calls `decode_object`, which +was 0277's scope correction. The EventBridge Lambdas still link `stellar-xdr`, +so they get rebuilt too. + +**Pre-existing, not ours:** `cargo clippy --workspace --all-targets -D +warnings` fails only on `prices-clickhouse/tests/current_mv_it.rs:1187` +(`too_many_arguments`), a file this branch does not touch. Excluding that +crate, clippy is clean. + +## Deploy 2026-10-20 + +Preflight, read 2026-10-20 at about 08:00 UTC: +`npm run xdr:verify-protocol-gap` reports pinned 30, mainnet current 29, and +**core supports 30**. Validators already run core 30, so the vote can come at +any time. + +| Lambda | `CodeSha256` before | `LastModified` before | +| --- | --- | --- | +| `prices-production-ledger-processor` | `ZFuumECCwyZNKDt1B6lH48YeFxgwH6aHT+1t36FsQzg=` | 2026-10-19 08:26:51 UTC | +| `prices-production-asset-discovery` | `Zaeo0F6RslW+kdc2e5nNxScnikNbjNGkjGIks0ESIos=` | 2026-10-19 08:59:03 UTC | + +Candle frontier before: sdex and aquarius 88 s behind now (07:54 UTC candle). + +**Compute deployed** (`make deploy-production-compute`, stage cache flushed): + +| Lambda | `CodeSha256` after | `LastModified` after | +| --- | --- | --- | +| `prices-production-ledger-processor` | `WCokytgM7mqlxS/+KiQk+QTf665qHwB7a+ssUGTD26g=` | 2026-10-20 08:31:01 UTC | + +Read at 09:00 UTC, about 30 minutes later: Lambda `Errors` = 0 in every +10-minute bucket from 07:30 to 09:10 UTC. sdex and aquarius were 86 s behind, +with 29 and 28 one-minute candles since the deploy. soroswap (5) and sushiswap +(2) also wrote candles since the deploy, so the Soroban event path works on the +new `LedgerEvents` extraction. comet (1.7 h behind) and phoenix (2.7 h) had no +candle since the deploy. That is normal for them: in the previous 24 h they +wrote 12 and 8 candles, and phoenix's longest gap was 16 h. + +**EventBridge deployed** (`make deploy-production-eventbridge`), every Lambda +in the stack at 2026-10-20 09:02:43–44 UTC: + +| Lambda | `CodeSha256` after | +| --- | --- | +| `prices-production-asset-discovery` | `m7eceW0ZGn+gykUhHb3yhn6/haa1q86CFj+LsiH2u18=` (before `Zaeo0F6R…`) | +| `prices-production-oracle` | `+D6AZ+rytOpspELZ37v4iRBiIgj342g6NACnEvrgnY0=` | +| `prices-production-enrichment` | `M6YAUmixrmN6FlTdZkVfr60l2+yX1k97Oe9zYtdcMpc=` | +| `prices-production-supply` | `q8UWEMWZcI0gdsgRSg/VjqaH+91FoUV5w2/h38YTmc0=` | +| `prices-production-coarse-sweep` | `VLMFBYOXdBTXNDEBxS5t7ykJmCnpPCgYnYnm3B2iPX8=` | + +The probes and `cleanup` were rebuilt too. The `prices-production-cleanup` rule +is still `DISABLED` (0200). By 09:13 UTC, oracle (2 runs) and the backfill and +rollup freshness probes (1 run each) had run with 0 errors, and the +ledger-processor had run 119 times with 0 errors. asset-discovery, enrichment +and supply run `rate(1 hour)` and had not run since the deploy yet. + +## Acceptance Criteria + +- [x] Workspace pin `=30.0.0`; `xdr-parser` on BE `develop` at `d1251f30`; one `stellar-xdr` in the lock +- [x] `cargo test --workspace` (1448 passed / 0 failed locally) and the CH ignored suite green (CI on #406: 328/328) +- [x] A test proves a `SignedMs` / `EmptyTxSetMs` ledger decodes through `decode_object` with `close_time` in seconds (`decode.rs`, table-driven) +- [x] The `xdr-parser` event API change preserves extraction: 360,847 events identical on 485 real ledgers +- [x] Ledger-processor deployed on the 30 binary, `CodeSha256` before/after recorded (2026-10-20 08:31 UTC) +- [x] EventBridge Lambdas linking `stellar-xdr` deployed on the 30 binary (2026-10-20 09:02 UTC) +- [ ] Candle frontier measured across the protocol-30 activation ledger (after the vote). Not done: on 2026-10-20 no public network had voted on 30 (Horizon: mainnet, testnet and futurenet all `current_protocol_version` 29). Closed without it. + +## Design Decisions + +### From Plan + +1. **Exact pin `=30.0.0`**, moved together with `xdr-parser` so the lock holds + one XDR major. +2. **Readiness proved by decode, not compile**: the `decode.rs` test packs + `SignedMs` / `EmptyTxSetMs` ledgers as Galexie does and goes through + `decode_object`. + +### Emerged + +3. **Adapted to BE's `LedgerEvents` API instead of pinning the old rev.** The + rev that requires XDR 30 also removed `extract_events`. Staying on the old + API was not possible without forking BE. +4. **`event_index` = position in `TxEvents.events`.** Diagnostic events are + now separate and unread. Equivalence was measured on 485 real ledgers + rather than argued. +5. **Whole EventBridge stack redeployed**, not just asset-discovery, since + every Lambda in it links `stellar-xdr`. +6. **Closed before the vote** at Adam's call. The crossing measurement is + left out, with no follow-up task. + +## Issues Encountered + +- **The compile break was BE's event API (ADR 0059), not XDR.** See Findings. +- **The runbook is stale** (`docs/runbooks/deploy-ledger-processor.md`): it + says asset-discovery decodes ledgers and uses the `soroban-admin` profile. + Not fixed here. +- **The task-opening commit 12a65fd5 also carried an unrelated staged rename + of 0333** (active → archive), done by mistake. diff --git a/lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md b/lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md new file mode 100644 index 00000000..faae8526 --- /dev/null +++ b/lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md @@ -0,0 +1,103 @@ +--- +id: "0336" +title: "coverage-sweep-unclassified stays in ALARM: new instances of the securities-financing ledger wasm page one by one — allow-list the family as not a venue" +type: BUG +status: completed +assignee: akot +related_adr: [] +related_tasks: ["0100", "0323", "0334"] +tags: [layer-infra, observability, coverage, priority-high, effort-small] +links: + - "../../../packages/coverage-sweep-probe/allowlist.toml" + - "../../../packages/coverage-sweep-probe/src/allowlist.rs" + - "../../../docs/runbooks/0100-coverage-sweep-triage.md" +history: + - date: "2026-10-23" + status: active + who: akot + note: "Task created from the daily stuck-alarm digest: coverage-sweep-unclassified in ALARM for 11d 4h" + - date: "2026-10-23" + status: completed + who: akot + note: > + PR #407 merged as 06e788a5. EventBridge deployed at 10:54 UTC (diff: + only the coverage-sweep-probe S3Key). Manual invoke: rows=21, + allowlisted=21, unclassified=0. Alarm OK at 10:55:59 UTC. +--- + +# New instances of the securities-financing ledger wasm keep the sweep in ALARM + +## Summary + +`prices-production-coverage-sweep-unclassified` has been in ALARM since +2026-10-12 07:18 CEST. [[0334]] fixed the residual it was raised for, yet the +manual run after its deploy still found 4 contracts. All four run wasm +`f3dd3edf…`, the securities-financing ledger that [[0334]] allow-listed by +one contract id (`CDU47AIU…`). The family deploys new instances every few +days, so a `[[contract]]` entry per instance would page again every Monday. + +## Context + +Runs from `/aws/lambda/prices-production-coverage-sweep-probe` +(`coverage sweep complete`): + +| Run (UTC) | unclassified | events | Note | +| --- | --- | --- | --- | +| 2026-10-12 05:17 | 3 | 404 | Comet `CASYAKK3…` (393), `CDU47AIU…`, `CD35HSTR…` ([[0334]]) | +| 2026-10-19 05:17 | 7 | 595 | before PR #405 merged | +| 2026-10-19 09:03 | 4 | 15 | manual, after the 0334 deploy; Comet and both 0334 entries gone | + +The four left on 2026-10-19 09:03, all wasm +`f3dd3edfd637a9654054181a9c47391ddafa723697b982698b3c9662a9df974e`, +`trade_opened`/`trade_closed` with shape `sym|u64|bytes -> map`: + +- `CAKQ6DWL…` 8 events, ledgers 64,908,423–64,941,821 +- `CCLN72NM…` 4 events, 64,958,834–64,962,719 +- `CBZYHS2D…` 2 events, 64,944,905–64,959,086 +- `CB4B4ELL…` 1 event, 64,978,560 + +Since [[0323]] the alarm holds the latest run's state, so it stays in ALARM +until a run finds none: the 2026-10-26 run, or a manual invoke after deploy. + +## Design + +[[0100]] D4 forbids a `[[wasm]]` entry without `until`, because a permanent +family-wide ignore would hide the next pool of a real venue. That reason does +not hold for a family whose code is not a market. Add an explicit exception: + +- `[[wasm]]` takes `not_a_venue = true` in place of `until`. Both together, + or neither, fails validation. +- `f3dd3edf…` moves from `CDU47AIU…`'s `[[contract]]` entry to one such + `[[wasm]]` entry. +- Runbook §3 outcome (c) says when to use it: the wasm itself rules out a + market, not the size of the instances seen so far. + +## Acceptance Criteria + +- [x] Validator: `not_a_venue` without `until` accepted; with `until`, or + `until` missing without it, refused (unit tests) +- [x] The five known instances match `wasm:f3dd3edf…` (unit test) +- [x] Runbook §3 describes `not_a_venue` +- [x] EventBridge deployed; one manual invoke logs `coverage sweep complete` + with `unclassified=0` +- [x] `coverage-sweep-unclassified` back to OK + +## Result (2026-10-23) + +- `cdk diff` for `Prices-production-EventBridge` changed only + `CoverageSweepProbeFunction` `.S3Key`. Deployed with + `make -C infra deploy-production-eventbridge`; probe `LastModified` + 10:54:04 UTC, `CodeSha256` `5QXzXnsLtgTvLGb9/CkJIXjAZq7XQBz8gdk3ikopGvI=`. +- Manual invoke at 10:55 UTC, window 64,872,241–65,093,419: `rows=21`, + `allowlisted=21`, `unclassified=0`, `unresolved=0`. +- `Prices/Coverage` `UnclassifiedSwapEvents` = 0 for the 10:55 UTC hour; + `prices-production-coverage-sweep-unclassified` OK at 10:55:59 UTC, after 11 + days in ALARM. + +## Notes + +- The order-book wasm `4d6abb0d…` is allow-listed by two contract ids + (`CBOSXSEZ…`, `CD35HSTR…` from [[0334]]). It is the same pattern and a + candidate for `not_a_venue` if a third instance appears. +- The Comet `[[wasm]]` entry `8abc2891…` (`until = "0300"`) is stale: 0300 + shipped. Not removed here. diff --git a/lore/1-tasks/archive/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md b/lore/1-tasks/archive/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md new file mode 100644 index 00000000..46538b7d --- /dev/null +++ b/lore/1-tasks/archive/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md @@ -0,0 +1,98 @@ +--- +id: "0337" +title: "Three new Comet v1.1 pools (BLNT/USDC, BLNT/XLM, a small BLND/USDC) are missing from pool_registry and the BLNT backfill contract pages the sweep" +type: BUG +status: completed +assignee: okarcz +related_adr: [] +related_tasks: ["0300", "0334", "0336", "0100", "0291"] +tags: [layer-backend, ingest, soroban, comet, observability, coverage, priority-high, effort-small] +links: + - "../../../packages/prices-ingest-core/src/static_pools.rs" + - "../../../packages/coverage-sweep-probe/allowlist.toml" + - "../../../docs/runbooks/0100-coverage-sweep-triage.md" + - "../../../docs/runbooks/seed-pool-registry.md" + - "https://github.com/BLNT-Protocol/blnt-backfill-contract/tree/fe366da68da7f24ef20063c43aaf0b13ea752257" +history: + - date: "2026-10-26" + status: active + who: okarcz + note: "Task created from the coverage-sweep-unclassified alarm of the 2026-10-26 05:17 UTC run" + - date: "2026-11-02" + status: completed + who: akot + note: > + Registry rows and the reprice finished under 0338, which has the result. +--- + +# New Comet pools and the BLNT backfill contract page the sweep + +## Summary + +`prices-production-coverage-sweep-unclassified` went to ALARM at +2026-10-26 05:18 UTC. The Monday run found 4 contracts with 25 swap-shaped +events. The previous run (2026-10-23 10:54, after [[0336]]) found 0. + +Three of the four are Comet pools on wasm `d735c339…`, the CometDEX v1.1 +code [[0334]] registered for Blend's `CASYAKK3…`. Comet has no factory, so +only `STATIC_POOLS` registers a pool. The live processor drops their swaps: +`ledger-processor-unregistered-pool` flapped 00:35–01:56 UTC on 10-26, in +step with the BLNT swaps. The fourth contract is not a market. + +## Findings (StellarExpert public API, 2026-10-26) + +| Contract | What | Created | Swaps in window | +| --- | --- | --- | --- | +| `CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3` | Comet v1.1, BLNT/USDC | 2026-10-25 19:18 by `GDEA2WMO…` | 9 | +| `CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC` | Comet v1.1, BLNT/XLM | 2026-10-25 19:19, same creator | 9 | +| `CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ` | Comet v1.1, BLND/USDC | 2026-09-23 by `GCRKRXU2…` | 4 | +| `CANIE64BDFKLHROLOIY2KTQVBWUZNUKZP75DLXC272QM7TIN5N3M4LVA` | BLNT backfill contract (wasm `dcdc0b18…`) | 2026-10-25 19:30 by `GDEA2WMO…` | 3 (`swap_blnd`) | + +- **Tokens.** BLNT SAC `CCBNWP5L…` (classic `BLNT-GAP2ACGH…`, created + 2026-10-25), USDC SAC `CCW67TSZ…`, XLM SAC `CAS3J7GY…`, BLND SAC + `CD25MNVT…`. Payload is the same `POOL/swap` map as [[0334]]. +- **Prices are coherent.** ~0.0048 USDC/BLNT and ~0.036 XLM/BLNT (implies + XLM ≈ $0.135); the BLND pool ~0.0054 USDC/BLND, near the Blend pool. +- **BLNT swaps come from one contract** `CB2DT56G…` routing USDC → BLNT → + XLM, so each tx swaps in both BLNT pools. The BLND pool's 4 swaps + (~$0.10 in total) come from `CDYYYMHP…`. +- **`CANIE64B…` is not a market.** Verified source + (`BLNT-Protocol/blnt-backfill-contract`): it custodies BLNT v3's initial + allocation; `swap_blnd` is a fixed 1:1 BLND → BLNT conversion (burns + BLND, pays pre-funded BLNT). No admin, no upgrade. Pricing it would pin + BLNT to BLND. + +## Decision + +- Register all three pools as `comet`, the [[0334]] pattern. The BLND pool + barely trades, but the rule stays one rule: a Comet pool is registered. +- Allow-list `CANIE64B…` as a permanent `[[contract]]`. +- No wasm-hash auto-registration for now: three pools in one week come from + one launch. Revisit if Comet pools keep appearing. + +## Implementation Plan + +1. Add the three pools to `STATIC_POOLS` with a decode test on a real swap + of each. +2. Allow-list `CANIE64B…` in `coverage-sweep-probe`. +3. Deploy Compute (ledger-processor cold start loads the pools) and + EventBridge (the allow-list is compiled into the probe). +4. `events-backfill --discover-pools` WRITE. +5. Reprice from ledger 65,099,227 (first swap of `CCNDNFA7…`, 2026-10-23) + to the deploy. +6. Invoke the probe in a later clock hour than 05:17 UTC. + +## Acceptance Criteria + +- [x] `STATIC_POOLS` holds the three pools; a test decodes one real swap of each +- [x] `CANIE64B…` allow-listed with reason and task +- [x] Compute and EventBridge deployed; the running Lambdas carry the change +- [x] No `unregistered_pool_events` WARN for these pools after the deploy +- [x] Registry rows written; range from ledger 65,099,227 repriced +- [x] Manual probe run reads `unclassified = 0`; alarm back to OK + +## Result (2026-11-02) + +The registry write, reprice, pre-roll and manual probe run were done under +[[0338]]: three rows written, ledgers 65,099,220–65,150,000 repriced, 22 dropped +swaps back in the candles, alarm OK at 12:35:41 UTC. diff --git a/lore/1-tasks/archive/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md b/lore/1-tasks/archive/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md new file mode 100644 index 00000000..15e198bf --- /dev/null +++ b/lore/1-tasks/archive/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md @@ -0,0 +1,208 @@ +--- +id: "0338" +title: "coverage-sweep-unclassified stays in ALARM: 0334 and 0337 never finished their registry and reprice steps, and three new emitters are unclassified" +type: BUG +status: completed +assignee: akot +related_adr: [] +related_tasks: ["0334", "0337", "0336", "0242", "0100", "0200"] +tags: [layer-backend, ingest, soroban, comet, observability, coverage, priority-high, effort-small] +links: + - "../../../packages/coverage-sweep-probe/allowlist.toml" + - "../../../packages/prices-ingest-core/src/static_pools.rs" + - "../../../docs/runbooks/0100-coverage-sweep-triage.md" + - "../../../docs/runbooks/seed-pool-registry.md" + - "../../../docs/runbooks/events-sourced-amm-reprice.md" +history: + - date: "2026-11-02" + status: active + who: akot + note: "Task created from the coverage-sweep-unclassified alarm of the 2026-11-02 05:17 UTC run; one task for all of it" + - date: "2026-11-02" + status: completed + who: akot + note: > + PR #409 merged. EventBridge deployed, three registry rows written, 0334 and + 0337 repriced and pre-rolled. Alarm OK at 12:35:41 UTC. Residual recorded. +--- + +# Clear the pool-coverage alarms after 0334 and 0337 + +## Summary + +`prices-production-coverage-sweep-unclassified` has been in ALARM since +2026-10-26 05:18 UTC. The 2026-11-02 run completed with 6 contracts and 594 +events. Three contracts are the Comet pools of [[0337]]: merged and deployed, +but without a `prices.pool_registry` row, which is the only thing the sweep +reads (`sweep.rs:156`). The other three are new emitters. + +The trades are no longer being dropped: the last `unregistered_pool_events` +WARN is 2026-10-26 01:40 UTC, and comet candles exist from the deploys on. What +is missing is the history. Neither [[0334]] nor [[0337]] has been repriced. + +## Findings (2026-11-02) + +### The sweep + +Window 65,040,990–65,262,168. `with_pool` is over all txs of the contract in +the window. + +| Contract | What | Wasm | Events | `with_pool` | Outcome | +| --- | --- | --- | --- | --- | --- | +| `CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3` | Comet v1.1, BLNT/USDC | `d735c339…` | 299 | | Registry row | +| `CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC` | Comet v1.1, BLNT/XLM | `d735c339…` | 281 | | Registry row | +| `CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ` | Comet v1.1, BLND/USDC | `d735c339…` | 6 | | Registry row | +| `CAF5RWQG5GSTEPD5XAKCUH2H77NWA4J6DNFT6TUP2NU6Q6VW3R6HCZYO` | Permissioned order book, grBENJI/USDC | `4d6abb0d…` | 4 | 0/50 | `[[contract]]` | +| `CC5QR5OOQMCPVSL3SJMH7QESCFINWUMX3427CT4NIDBS3FRVDHAM4NAJ` | Tokenised fund aLQD (Alloy Liquid Dollar) | `a85d6d53…` | 1 | 3/8 | `[[contract]]` | +| `CAZE5GVMH5332L6FEGVPPDLEPVYUFGQB7ZOCH2OKKBJCBYZQ5FGKZN7L` | Stableswap "Spreadless USD 3pool" | `ae835560…` | 3 | 0/7 | `[[contract]]` | + +- **Registry.** `prices.pool_registry FINAL` holds 2 `comet` rows, `CAS3FL6T…` + and `CASYAKK3…`. The three pools above are absent. +- **`CC5QR5OO…`.** Unverified source; interface read from the wasm: `issue`, + `redeem`, `nav`, `rebalance`, `trade`, `set_router`. Created 2026-10-26. Its one + `trade` wraps `SoroswapRouter/swap` (allow-listed `CAG5LRYQ…`) and + `SoroswapPair/swap` on registered `CAM7DY53…`. Indexing it would double-count. +- **`CAZE5GVM…`.** Unverified source. An AMM: `get_amp` = 100, `get_reserves`, + `swap_exact_in/out`, `deposit`, `withdraw`, own LP token `SPR-USD3`. USDT0, PYUSD + and USDC, about $29 of reserves, created 2026-10-29 by `GBAXG35E…`. Its three + swaps are 1 unit each. No extractor exists and it has `upgrade()`. +- **`CAF5RWQG…`.** The fourth instance of order-book wasm `4d6abb0d…`. Its + `get_config` returns `base_token` grBENJI (`CBEFW4IK…`) and `quote_token` USDC, + and the book has `whitelist`, `access_control` and admin-set phases and + reference price. Decoded trades are ~75,000 at 0.9999999–1.0000000. +- **The existing reason is wrong.** The three `4d6abb0d…` entries say + "instruments are u64 ids with no token addresses". The u64s are order ids. The + trade event carries no token address, but the book's config does. This also + rules out the `not_a_venue` that [[0336]]'s notes suggest: the code is a market. + +### The history that is missing + +- **[[0334]] was not repriced.** `CASYAKK3…` emitted 174 swaps on 2026-10-12–18 + and `source = 'comet'` has 1 trade on those days. Its 1m candles start at + 2026-10-19 11:02 UTC. BE holds 560 swaps for 2026-10-07–18. Comet candles of + 10-07–11 exist but come from the old pool `CAS3FL6T…`. +- **[[0337]] was not repriced.** The first BLNT candle is 2026-10-26 14:22 UTC; + the 18 BLNT swaps of 10-25 23:35–10-26 01:40 UTC have none. `CCNDNFA7…`'s swaps + from 10-23 18:58 UTC (ledger 65,099,227) are not distinguishable in BLND/USDC + candles and are assumed missing. +- **BLNT has two identities.** The first 10 comet trades (10-26 14:22–16:36 UTC) + sit on `asset_id` 14236029787030528679, `soroban`, contract `CCBNWP5L…`, created + at 14:22:57. From 18:42 UTC, when `sdex` first traded it, the trades sit on + 15676158627979534220, `classic` BLNT-`GAP2ACGH…`, `sac_address` `CCBNWP5L…`. + This is the race in [[0242]]: the SAC traded before anything interned its + classic. PR #390 is still open, so live can do this again for any new SAC. +- **A reprice heals the trades, not the orphan id.** `events-backfill` builds its + registry from `prices.assets` (`run.rs:220`), and the classic row now carries + the SAC, so it writes these minutes under the classic id. The rows under + 14236029787030528679 and its `prices.assets` row stay. +- **The order books.** `CA5YJ5H2…` is grBENJI/USDC like `CAF5RWQG…`; `CBOSXSEZ…` + and `CD35HSTR…` are AMA1/XUSDT. + +### What a reprice can and cannot recover + +`events-backfill` has no venue filter, and an enriched row sits at `version + 1`, +above what the reprice writes for the same ledger. So a reprice fills an empty +minute, and is discarded in a minute that already has an enriched row. That +protects the other venues' USD values, and it limits what comes back here. + +Of `CASYAKK3…`'s 578 swaps up to 2026-10-19, 302 sit in minutes with no comet row +and come back. Up to 286 sit in minutes that already hold an old-pool row: 282 on +10-07–11 and 4 on 10-19. All of 10-12–18 comes back. + +### Timing + +- `prices-production-cleanup` is DISABLED ([[0200]]), so the 1m rows survive. +- `mv_reconcile_*` look back 7 days, and the runbook wants them stopped when a + pre-roll overlaps that window. A range ending 2026-10-27 00:00 UTC stops + overlapping on 2026-11-03 00:00 UTC. +- The pre-roll's 1w and 1M rows for the buckets it cuts short lose on version, so + the reconcile MVs have to rebuild them: week of 10-26 and October are still in + their windows until about 2026-11-08. + +## Decision + +- All of it lives in this task, with no separate venue task. +- **`CAZE5GVM…` is a `[[contract]]`, not a temporary `[[wasm]]`.** A `[[wasm]]` + needs `until` = a task that removes it, and none exists. The contract entry + covers this pool only, so the next pool of the family still pages and reopens + the question. It says plainly that this is a small unindexed AMM. +- `CAF5RWQG…` is a `[[contract]]`, and the reason on the three older order-book + entries is corrected in the same commit. It is no `not_a_venue`. +- `CC5QR5OO…` is a permanent `[[contract]]`. +- Removing the stale `until` entries (`003710b3…`, `95a8e001…`, `8abc2891…`) is + out of scope. +- **The BLNT orphan id goes to [[0242]].** The reprice ends past 16:36 UTC so the + 10 trades reach the classic id. This task deletes nothing on production: after + the reprice the orphan rows are copies, and they join 0242's residual cleanup, + which is where every other SAC-keyed row is removed. +- **The reprice is plain, with no `DELETE` of comet rows.** Up to 286 swaps of + 2026-10-07–11 and 10-19 stay undercounted, which is recorded here as known. + Removing that residual needs comet `DELETE`s on `1m` and on the coarse tiers, + which is not worth the risk for one pool in five days. +- **Two ranges, not one**, minute-aligned, so 10-19–23 is not rewritten for nothing. + +## Implementation Plan + +1. Allow-list the three contracts and correct the order-book reason; fix what + the count-pinning tests report; `cargo test -p coverage-sweep-probe`. +2. Add BLNT's orphan id to [[0242]]'s residual list (done in its Heal log). +3. After merge, in this order: + 1. `make -C infra diff-production`, read in full; then + `make -C infra deploy-production-eventbridge`. + 2. A **new** `events-backfill` from develop. An older binary does not hold the + pools of 0337 and reports a false `to_write=0`. + 3. `--discover-pools --dry-run` (expect `to_write=3`, `change="new"`), WRITE, + dry run again (`to_write=0`). + 4. Invoke the probe in a later clock hour than the last run; expect + `unclassified = 0` and the alarm in OK. + 5. From 2026-11-03 00:15 UTC and before 2026-11-08: reprice ledgers + 64,809,264–65,025,000 and 65,099,220–65,150,000, dry run first. The second + range ends past the orphan BLNT minutes. + 6. `preroll-live-gap.sql` with `start_ts = '2026-10-07 00:00:00'`, + `end_ts = '2026-10-27 00:00:00'`. Never `preroll-amm-reprice.sql`. + 7. Check the repriced 1m rows got `close_usd`; if not, `coarse-repair + --table price_ohlcv_1m --start-month 202610 --end-month 202610 --skip-snapshot`. +4. Close [[0334]], [[0337]] and this task. + +## Acceptance Criteria + +- [x] `CAF5RWQG…`, `CC5QR5OO…` and `CAZE5GVM…` allow-listed with reason and task +- [x] The reason on the three older `4d6abb0d…` entries is corrected +- [x] EventBridge deployed; the running probe carries the change +- [x] `prices.pool_registry FINAL` has `comet` rows for the three 0337 pools +- [x] `source = 'comet'` candles cover 2026-10-12–18 and 10-25 23:35–10-26 01:40 UTC, + on `1m` and on `1h`; the residual of 10-07–11 is recorded with its measured size +- [x] BLNT's 10 comet trades of 10-26 14:22–16:36 UTC exist under the classic id, + and the orphan id is on [[0242]]'s residual list +- [x] Manual probe run reads `unclassified = 0`; alarm back to OK + +## Result (2026-11-02) + +- PR #409 merged as `380aed31`. EventBridge deployed; probe `LastModified` + 12:17:53 UTC, and the deployed `bootstrap` holds the three contracts. `cdk diff` + for that stack changed only `CoverageSweepProbeFunction` `.S3Key`. +- `--discover-pools` WRITE at 12:33:22 UTC: `written=3`, then `to_write=0`. + `comet` holds 5 rows. The binary was `3e0e521a…`, built with `cargo lambda + build --target x86_64-unknown-linux-musl`, since the workstation has no musl-gcc. +- Manual invoke, window 65,046,238–65,267,416: `rows=22`, `allowlisted=22`, + `unclassified=0`, `unresolved=0`. Alarm OK at 12:35:41 UTC. +- Reprice of 65,099,220–65,150,000 (18,717 rows), then 64,809,264–65,025,000 + (86,791 rows). Other venues kept their rows, trades and `close_usd` zero counts, + which confirms that an enriched row outranks the reprice. +- Comet gained 512 trades on 1m: 481 in the first range and 31 in the second. + 10-12–18 matches BE's swap count per day. +- **Residual: 93 swaps of `CASYAKK3…` on 10-07–11**, not the 286 estimated above: + a repriced row with a higher version did replace an old-pool row. The minutes + affected remain undercounted, as decided. +- 334 comet rows at `close_usd = 0` were enriched in about 90 s, before the pre-roll. +- `preroll-live-gap.sql` over 10-07 to 10-27 ran on 2026-11-02, **without `SYSTEM + STOP VIEW` and before the 11-03 start above**, which departs from runbook 0142. + Its reason, a half-refilled `1m`, did not apply: no partition was dropped. No + rollup or `mv-refresh` alarm fired. +- After it, comet agrees on 1m, 15m, 1h, 4h and 1d: 1,726 trades, $23,364.42. The + reconcile MVs finished 1w and 1M: each week equals its days, October is 2,630 + trades, $40,006.36. +- The pre-roll put 72 `sdex` rows at `close_usd = 0` (46, 16, 7, 3 on 15m, 1h, 4h, + 1d). The 13:30 UTC coarse sweep restored every count to its value before. +- BLNT's 10 trades now exist under the classic id. The orphan id's rows stay for + [[0242]]'s cleanup. #390 was merged at 13:28 UTC and is not deployed. diff --git a/lore/1-tasks/archive/0339_FEATURE_portal-served-at-api-prices-api.md b/lore/1-tasks/archive/0339_FEATURE_portal-served-at-api-prices-api.md new file mode 100644 index 00000000..9400a716 --- /dev/null +++ b/lore/1-tasks/archive/0339_FEATURE_portal-served-at-api-prices-api.md @@ -0,0 +1,144 @@ +--- +id: "0339" +title: "Serve the portal at /api/prices-api/ on the explorer's host" +type: FEATURE +status: completed +related_adr: [] +related_tasks: ["0326", "0194"] +tags: ["portal", "hosting", "effort-small"] +links: + - https://github.com/rumblefishdev/soroban-block-explorer/pull/699 + - https://github.com/rumblefishdev/stellar-prices-api/pull/410 +history: + - date: 2026-11-03 + status: active + who: stkrolikiewicz + note: > + Task created. The product portals gather under /api/ on the explorer's + host (team decision): /api/prices-api/ now, /api/x402-facilitator/ and + /api/sorobanscan/ later, /api/ a hub. Explorer side: + soroban-block-explorer task 0660, PR #699. + - date: 2026-11-05 + status: completed + who: stkrolikiewicz + note: > + Compute with the new PORTAL_HOME deployed with 0242's §4e (develop + 43a28399). The OAuth callback lands on /api/prices-api/ directly, a + Discord sign-in round-trip passed on production, and the dead + prices-api/ bucket prefix (18 objects) is dropped after a local copy. +--- + +# Serve the portal at /api/prices-api/ on the explorer's host + +## Summary + +The portal moves from `https://sorobanscan.rumblefish.dev/prices-api/` to +`/api/prices-api/`, under the hub at `/api/` that will also list the other +product portals. The explorer (task 0660, PR #699) serves +`/api/prices-api(/*)` from the bucket prefix `api/prices-api/` and answers +`/prices-api…`, `/pricing-api…` and every other `/api/…` with one `301` to the +same path under `/api/prices-api`, query string kept. Same kind of move as +[[0326]]. + +## Status: Completed + +**Current state:** live on production. The bundle under `api/prices-api/` +(2026-11-04, with the explorer's 0660), Compute with the new `PORTAL_HOME` +(2026-11-05), the old `prices-api/` prefix gone. + +## Context + +The bundle bakes its prefix in (Vite `base`, router `basename`), the sync puts +it under the same key prefix in the explorer's bucket (CloudFront hands S3 the +viewer path unchanged), and the backend's OAuth callback lands the popup on +`/prices-api/`. All three move. On the API's own host nothing +moves: `/api/auth/*`, `/api/key`, `/api/usage`, `/api/config`, the session +cookie's `Path=/api/`, the Discord `redirect_uri`, `portalWebOrigin`. + +## Implementation Plan + +1. `web/portal/src/base-path.ts` + `vite.config.mts`: `BASE_PATH` → + `/api/prices-api/`; specs that pin bundle paths follow. +2. `packages/prices-api/src/portal/auth/mod.rs`: `PORTAL_HOME` → + `/api/prices-api/`; every test asserting a landing follows. +3. `infra/Makefile` `sync-portal-explorer`: upload to `api/prices-api/`, + invalidate `/api/prices-api/*`. +4. Current-state docs (README, `docs/scf/api-endpoints.md`, overview, + runbook, skill) name the new URL. Dated SCF evidence stays as submitted. + +### Deploy order (production, manual) + +1. Here: `make -C infra sync-portal-explorer` — the new bundle under + `api/prices-api/`. Nothing serves it until step 2; the old `prices-api/` + keeps serving. +2. Explorer PR #699: Delivery (CDK) deploy, then the explorer SPA. +3. Here: Compute with the new `PORTAL_HOME`. **Never before step 2:** until + then `/api/prices-api/?signin=…` would `301` to `/prices-api/prices-api/`. + Until step 3 the old home `/prices-api/?signin=…` reaches the portal + through the `301`. + +Then: + +- The old `prices-api/` bucket prefix is dead: drop it, like explorer task + 0631 does for `api/` and `pricing-api/`. Any delete of the old `api/` + prefix must now exclude `api/prices-api/`, or it takes the live portal. +- A person completes a Discord sign-in round-trip on production. + +## Acceptance Criteria + +- [x] Portal boots at `/api/prices-api/`, `/api/prices-api/dashboard`, + `/api/prices-api/docs` (hard refresh included) +- [x] Discord sign-in round-trip lands on `/api/prices-api/?…` (after + Compute; until then via the `301`), done by a person on production + (2026-11-05). An errored callback answers `303` to + `https://sorobanscan.rumblefish.dev/api/prices-api/?signin=failed`. +- [x] `/prices-api/?utm_source=stellar-skill` → one `301` to + `/api/prices-api/?utm_source=stellar-skill` +- [x] Old `prices-api/` prefix dropped from the bucket once the above holds + (`production-soroban-explorer-api-spa`, 18 objects, 2026-11-05; copy in + `~/Backups/explorer-prices-api-prefix-2026-11-05/` on stkrolikiewicz's + laptop, the bucket is unversioned) + +## Design Decisions + +### From Plan + +1. **Backend `/api/` untouched.** Only `PORTAL_HOME`, the landing, moves. + The API host and the explorer host now both have an `/api/`, but they are + different hosts and nothing routes between them; the session cookie is + host-only, so its `Path=/api/` never reaches the explorer's host. ~110 + Location literals in `packages/prices-api/tests/portal_*.rs` follow + `PORTAL_HOME`. + +### Emerged + +2. **`PORTAL_HOME` on the API host is no longer "nothing".** A relative + landing (no `PORTAL_WEB_ORIGIN`) on the API host would now fall into the + backend's own `/api/` namespace instead of the gateway's `404`. Production + always sets the origin; the doc comment says so instead of claiming a 404. +3. **Dev proxy spec pins the real bundle paths.** `dev-proxy.spec.ts` listed + `/api/login`, `/api/@vite/client`… from the 0194 layout; it now lists + `/api/prices-api/…`, the paths Vite actually serves. The enumerated + backend regex is load-bearing again: the bundle shares `/api/`. +4. **Dated SCF evidence left as submitted** (`docs/scf/milestone-*`): the + `301` keeps their links working, as in [[0326]]. +5. **Task created on the branch, not on `develop`.** Asked for explicitly + (nothing pushed straight to `develop`), against the usual rule in + `lore/CLAUDE.md`. +6. **Compute shipped with 0242's §4e**, not alone: `develop` already held + #390, whose D9 holds Compute until after phase-3 stage D. The 301 kept + sign-in working until then. + +## Implementation Notes + +Checked locally on 2026-11-03: `cargo test -p prices-api` (lib 271 + every +`portal_*` suite green), `cargo clippy -p prices-api --all-targets -D +warnings`, `cargo fmt --all --check`, `nx run-many -t lint typecheck test` +on `portal` and the CDK project. `make -C infra build-portal-explorer`: every +asset URL in `dist/index.html` is under `/api/prices-api/`; `vite preview` +boots `/api/prices-api/`, `/api/prices-api/docs` and `/api/prices-api/dashboard`. + +## Notes + +- The usage plan names (`pricing-api--`) and the SSM parameter + `/prices//pricing-api-free-plan-id` are AWS resource names, not URLs. diff --git a/lore/1-tasks/backlog/0149_BUG_sweep-mv-version-ownership-close-usd.md b/lore/1-tasks/backlog/0149_BUG_sweep-mv-version-ownership-close-usd.md deleted file mode 100644 index 3962a3c5..00000000 --- a/lore/1-tasks/backlog/0149_BUG_sweep-mv-version-ownership-close-usd.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -id: "0149" -title: "Two writers with incompatible version arithmetic own close_usd — the 0114 sweep's repair is overwritten inside the MV window" -type: BUG -status: backlog -related_adr: [] -related_tasks: ["0144", "0146", "0148", "0114", "0095", "0143"] -tags: - ["priority-medium", "effort-medium", "clickhouse", "data-correctness", "materialized-view", "milestone-M2"] -milestone: 2 -links: - - "../../../packages/enrichment-worker/src/repair.rs" - - "../../../packages/prices-clickhouse/schema/rollups.sql" -history: - - date: 2026-08-05 - status: backlog - who: okarcz - note: > - Spawned from [[0144]] future work (phase 7) — finding 3ii-b, confirmed on - the prod CH pin (TEST E). Deliberately ordered after [[0146]], which - removes most of its consequence. ---- - -# Sweep vs MV: incompatible version arithmetic on `close_usd` - -## Summary - -Two writers update `close_usd` on the same coarse rows and claim the row by -different rules: - -- The [[0114]] coarse sweep wins by **`version + 1`** (`repair.rs:20-22`). -- The rollup MVs write at **`version = sum(version)`** over their sub-rows — - and every enrichment event underneath adds 1 to that sum. - -Two enrichment events and the MV overtakes the sweep. Worse, the MV gets far -more attempts: `mv_ohlcv_15m_to_1h` re-appends the same hour **every 15 minutes -for 8 hours** (`REFRESH EVERY 15 MINUTE`, window `now() - INTERVAL 8 HOUR`). -The sweep gets one. - -Confirmed on CH **26.3.10.60** ([[0144]] `repro/04_sweep_durability.sql`, -TEST E): - -``` -after the sweep repairs it close_usd 0.171 version 401 -after the next MV refresh close_usd 0 version 402 -what the view publishes now (empty) -``` - -So the repair path we already built is defeated inside the re-aggregation -window — the bucket vanishes from `price_usd_series*` again, which is BE's -observation exactly. - -## Why this is ordered late - -**[[0146]] removes most of the consequence.** Once the MVs use `argMaxIf`, the -re-append writes a *correct* value rather than a zero — so being overtaken by -the MV stops being harmful inside the window. And outside the window the MV -does not write at all, so [[0148]]'s historical sweep is unaffected either way. - -That leaves this as a latent trap rather than a live defect: **two writers with -incompatible version arithmetic on one column**, with no mechanism preventing -the next such writer from colliding the same way. Worth closing on its own -terms, not urgent. - -Fixing only this and not [[0146]] would leave the zero-propagation completely -intact — the sweep would win a race to publish a value the MV is about to -recompute as zero anyway. - -## Implementation - -Options to weigh, none obviously correct yet: - -- Give the sweep a version domain the MV cannot reach (e.g. a large offset), at - the cost of making `version` no longer a plain event count. -- Have the sweep write through the same path as the MV so there is one writer. -- Make the sweep skip rows still inside an MV's re-aggregation window entirely, - and rely on [[0146]] for those — smallest change, but encodes the window - boundaries in two places. -- Reconsider whether `close_usd` should be MV-carried at all rather than always - derived by the sweep. - -Related: [[0143]] records that there is no `DEPENDS ON` anywhere in the cascade, -so ordering between tiers is already unsynchronised; any fix here should not -assume refresh ordering it does not control. - -## Acceptance Criteria - -- [ ] A single documented owner for `close_usd` on coarse rows, or a version - scheme under which the sweep's repair provably survives an MV refresh. -- [ ] Regression test on CH 26.3.10.60 reproducing TEST E and showing the - repair now survives. -- [ ] The `version` column's semantics restated in the schema header if they - change. -- [ ] Confirmed not to reintroduce the [[0095]] replace-mode invariants problem. diff --git a/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md b/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md new file mode 100644 index 00000000..eb874b4f --- /dev/null +++ b/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md @@ -0,0 +1,97 @@ +--- +id: "0341" +title: "Accept a classic asset's SAC C… address as an alias — today it answers 404" +type: FEATURE +status: backlog +related_adr: [] +related_tasks: ["0242", "0139"] +tags: [layer-backend, api, identity, external-request, priority-medium, effort-small] +links: + - "https://github.com/rumblefishdev/stellar-prices-api/issues/416" + - "../../../packages/prices-api/src/identity.rs" + - "../../../packages/prices-api/src/assets/handlers.rs" + - "../../../packages/prices-ingest-core/src/writer.rs" +history: + - date: "2026-11-10" + status: backlog + who: mkowalski + note: > + Returned to backlog, unassigned; handed over to another team member. + No code written. Finding: the alias already exists on develop (0242 D5, + PR #390), but develop is 163 commits ahead of master and it is unclear + what production runs. See "Handover findings". + - date: "2026-11-10" + status: active + who: mkowalski + note: "Activated for implementation." + - date: "2026-11-10" + status: backlog + who: mkowalski + note: > + Created from hackathon feedback by Dominik (Inferera). External request, + tracked in GH #416. Tell Dominik when it ships. +--- + +# Accept a classic asset's SAC C… address as an alias + +> **External request:** suggested by **Dominik (Inferera)** after using the API +> at the hackathon. Tracked in [GH #416](https://github.com/rumblefishdev/stellar-prices-api/issues/416). +> **When this is completed, let Dominik know** and close #416. + +## Handover findings (2026-11-10) + +**The alias is already implemented on `develop`, so most likely no new code is +needed.** Check this before planning work: + +- [[0242]] decision D5, commit `3de11181`, merged to `develop` in PR #390 on + 2026-11-02: `resolve_sac_alias(es)` in + `packages/prices-api/src/assets/queries_ch.rs` maps a SAC `C…` address to its + classic asset (XLM's SAC → `native`). Single-asset, history, oracle and batch + handlers call it. The OpenAPI text says SAC addresses are "accepted as an + alias". `tests/endpoints_it.rs` covers XLM and XCR (incl. a pre-heal duplicate). +- **None of it is on `master`.** `develop` is 163 commits / 20 PRs ahead of + `master` (host move 0340, stellar-xdr 30 bump 0335, pricing fixes 0329/0331/0332, …). +- **It is unclear what production actually runs** (confirmed with the team), so + it's also unclear why the hackathon user got a 404. + +Suggested next steps: find out what production runs, call +`GET /assets/{XLM SAC}` against it, and then either ship via a develop → master +release or close this task as done by 0242. The Implementation Plan below was +written before this was known and is likely obsolete. + +## Summary + +A Soroban caller usually knows a classic asset only by its Stellar Asset +Contract (SAC) address. Asking for e.g. USDC by its `C…` address returns +`404`; only `CODE:ISSUER` works. Resolve a SAC address to the classic asset it +wraps, so both names return the same data. + +## Context + +- Documented behaviour today: `assets/handlers.rs` says classic assets are named + by `CODE:ISSUER` and their SAC addresses "as a rule answer `404`". +- The parser already accepts `C…` (`AssetIdentifier::Contract`, `identity.rs`). +- Ingest already collapses SAC legs into the classic asset and stores the SAC + that wraps each classic asset (`prices-ingest-core/src/writer.rs`, §12.4). +- USDC already resolves by its SAC through a special case (`handlers.rs`, + the id arm of the canonical-USDC check). This task generalises that. +- [[0242]] (SAC minted as a second identity) touches the same mapping. Check + its state first so the alias doesn't resolve to a split identity. + +## Implementation Plan + +1. In the identity resolver, look up a `Contract(c)` as a SAC of a held classic + asset first, and only then as a pure Soroban token. +2. Apply it to every route that takes `{asset_identifier}` (assets, oracles, + history), not only `/assets/{id}`. +3. Update the OpenAPI parameter descriptions that currently promise `404`. +4. Decide whether the response echoes the requested `C…` or the canonical + `CODE:ISSUER`, and document the choice. + +## Acceptance Criteria + +- [ ] `GET /assets/{SAC C…}` returns the same data as `GET /assets/{CODE:ISSUER}` for a held classic asset (incl. USDC, XLM's SAC → `native`) +- [ ] A pure Soroban token and an unknown `C…` behave as before +- [ ] OpenAPI docs no longer say SAC addresses answer `404` +- [ ] Integration test covering at least one non-USDC SAC +- [ ] Dominik (Inferera) informed; GH #416 closed diff --git a/lore/1-tasks/backlog/0342_FEATURE_nav-price-for-tokenised-rwas.md b/lore/1-tasks/backlog/0342_FEATURE_nav-price-for-tokenised-rwas.md new file mode 100644 index 00000000..5eb7fd99 --- /dev/null +++ b/lore/1-tasks/backlog/0342_FEATURE_nav-price-for-tokenised-rwas.md @@ -0,0 +1,63 @@ +--- +id: "0342" +title: "NAV / fundamental price for tokenised RWAs next to the market price — USDY trades below NAV, BENJI and WisdomTree come back unpriced" +type: FEATURE +status: backlog +related_adr: [] +related_tasks: [] +tags: [layer-backend, api, oracles, rwa, external-request, priority-medium, effort-large] +links: + - "https://github.com/rumblefishdev/stellar-prices-api/issues/417" + - "../../../packages/prices-ingest-core/src/writer.rs" + - "../../../packages/prices-api/src/oracles/handlers.rs" +history: + - date: "2026-11-10" + status: backlog + who: mkowalski + note: > + Created from hackathon feedback by Dominik (Inferera). External request, + tracked in GH #417. Tell Dominik when it ships. +--- + +# NAV / fundamental price for tokenised RWAs + +> **External request:** suggested by **Dominik (Inferera)** after using the API +> at the hackathon. Tracked in [GH #417](https://github.com/rumblefishdev/stellar-prices-api/issues/417). +> **When this is completed, let Dominik know** and close #417. + +## Summary + +For a tokenised fund, the trade price isn't the valuation. USDY trades 0.7–2% +below its NAV. Funds that are minted and redeemed through the issuer and rarely +or never trade (BENJI, WisdomTree) get no price at all. Add a NAV / "fundamental" +price, sourced from on-chain feeds such as RedStone, as a separate field next to +the market price. The API could then value fund shares, not only traded tokens. + +## Context + +- Our prices come from trades, so an asset with no trades gets no price. +- RedStone is already ingested, but its samples carry no asset and are stored + under the sentinel id `0` (`prices-ingest-core/src/writer.rs`, `oracle_row`). + Nothing links a feed to a token yet. +- Adding the field changes the public response shape, so it needs an ADR. + +## Implementation Plan + +1. **Research (gate):** list which NAV feeds RedStone (and other providers) + actually publish *on Stellar* for USDY, BENJI and WisdomTree tokens: contract, + feed id, update cadence, and how the value is defined (NAV per share vs. a + price). If no feeds exist on Stellar, decide whether to stop or use issuer data. +2. Map each feed to its Stellar asset identity. +3. ADR: field name (`nav_usd` / `fundamental_price_usd`), source, timestamp and + staleness semantics, and wording that it is issuer-reported, not a market price. +4. Ingest + store per asset; expose it on the asset endpoint(s). Assets with a + NAV but no trades must return a response instead of `404`. + +## Acceptance Criteria + +- [ ] Research note recording which NAV feeds exist on Stellar and for which assets +- [ ] ADR accepted for the new field +- [ ] USDY returns both market price and NAV +- [ ] BENJI / WisdomTree return a NAV even with no trades +- [ ] Docs explain market price vs. NAV +- [ ] Dominik (Inferera) informed; GH #417 closed diff --git a/lore/2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md b/lore/2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md index 17cd84ad..314cd05e 100644 --- a/lore/2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md +++ b/lore/2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md @@ -3,7 +3,7 @@ id: "0011" title: "`base_currency` on the read surfaces is a DENOMINATION, not a quote-leg pair filter" status: accepted deciders: [okarcz, stkrolikiewicz] -related_tasks: ["0170", "0178", "0120", "0127", "0128", "0211", "0165", "0114", "0201", "0116", "0212"] +related_tasks: ["0170", "0178", "0120", "0127", "0128", "0211", "0165", "0114", "0201", "0116", "0212", "0236"] related_adrs: ["0003", "0004", "0008"] tags: [api, read-surface, contract, usd, denomination, provenance, milestone-M2] links: @@ -240,6 +240,68 @@ as the extremes, and scaling by one positive factor is monotonic, so `low <= open <= high` holds within a row and survives `min`/`max` across rows. Only the exact/derived boundary was ever at risk. +#### ✅ AMENDED 2026-09-28 — the clamp is CONFIRMED; stored-row consistency is asserted at the source + +Task [[0236]]. **The 0229 clamp stays: it is the right behaviour for the read +path.** A read API's job is to return well-formed candles, and a consumer cannot +act on `high < close` except by breaking. The alternative the 0229 review floated +— bounding the clamp so only ulp-sized crossings are repaired — would buy a +detection signal by **serving a malformed candle to every consumer** whenever a +source row is genuinely corrupt. That is paying in the wrong currency. + +🔑 **It is safe because stored-row consistency is now asserted at the source, +with ZERO tolerance.** Before 0229 a corrupt stored row leaked into `/ohlcv`, +where [[0120]]'s conformance suite could stumble over it; the clamp removed that +accidental detector, so it was moved to where it belongs. +`rollup-freshness-probe` reads every tier, on a schedule — +[`packages/rollup-freshness-probe/src/ohlc_band.rs`](../../packages/rollup-freshness-probe/src/ohlc_band.rs): + +- **predicate**, per priced row (`pf_trade_count > 0`): `low > least(open, close) + OR high < greatest(open, close) OR low > high OR` any of the four prices `<= 0`. + Exact `Decimal` comparisons, no epsilon — `min`/`max`/`argMin`/`argMax` copy + stored values bit for bit, so any crossing in storage is a real defect; +- **scope**: all seven tiers `price_ohlcv_{1m,15m,1h,4h,1d,1w,1M}`, `FINAL`, a + 48 h window on bucket start widened by one bucket length per tier (a flat 48 h + would never see a `_1w`/`_1M` bucket); +- **signal**: metric `Prices/Rollup` `CandleBandViolations`, alarms + `prices-{env}-ohlc-band-{1,100,10000}`, first rung pinned at 1. + +⚠️ **The as-stored `Denomination::QuoteLeg` arm does NOT clamp O/H/L** — only +`vwap` (`packages/prices-api/src/assets/queries_ch.rs`, the `QuoteLeg` arm, ~L1714). +It applies no rate, so the four prices are the stored decimals and cannot cross +unless the stored row itself is corrupt — which is exactly what the detector +above sees. On that path a corrupt row still reaches the consumer as-is. + +**Baseline, measured on prod 2026-09-28** (read-only, without `FINAL`, every +row of every tier): + +- **Band violations: 0** on every tier, every source, since 2015 — before and + after [[0286]]. +- **Priced rows with a price `<= 0`**: every one an exact zero (none negative), + mostly `low = 0` beside a positive `close`, and **all written before the 0286 + rollout (2026-09-22 12:00 UTC); 0 after it**: + + | tier | before 2026-09-22 12:00 UTC | after | + |---|---:|---:| + | `_1m` | 7,651 | 0 | + | `_15m` | 3,041 | 0 | + | `_1h` | 15,003 | 0 | + | `_4h` | 12,326 | 0 | + | `_1d` | 9,382 | 0 | + | `_1w` | 5,010 | — | + | `_1M` | 4,350 | — | + + These are **upper bounds** — read without `FINAL`, so superseded versions are + included (`_1m` 2026-09 reads 143 without `FINAL`, 52 with it). They are + pre-0286 residue that 0286 phase 3 re-ingests, and they sit outside the + detector's window by construction. The healthy reading is therefore exactly 0. + +⚠️ **No API clamp metric, on purpose.** The only crossing the clamp repairs in +normal operation is the structural ulp between the exact `close_usd` and the +rate-derived extremes — the 1.343e-11 BTC 1h measurement above. Counting it would +page on arithmetic, not on data, and the read path has no per-request metric +path worth adding for it. A corrupt stored row is caught at the source instead. + ### 4. Provenance reuses [[0165]]'s vocabulary `method` is propagated from the underlying `close_usd`, using the values 0165 diff --git a/lore/board.html b/lore/board.html index 3c13b8d4..f1243693 100644 --- a/lore/board.html +++ b/lore/board.html @@ -931,11 +931,10 @@

Stellar Prices API Backlog Board

`; } + // The generator owns the rule (getAssignee, task 0327); the page only + // reads its result (task 0328). function getTaskAssignee(t) { - if (t.status !== 'active' && t.status !== 'archive') return null; - const history = Array.isArray(t.history) ? t.history : []; - const lastEntry = history[history.length - 1]; - return lastEntry?.who || null; + return t.assignee || null; } function filtered() { diff --git a/packages/asset-discovery/Cargo.toml b/packages/asset-discovery/Cargo.toml index 81f6604c..4cbb4e63 100644 --- a/packages/asset-discovery/Cargo.toml +++ b/packages/asset-discovery/Cargo.toml @@ -31,10 +31,10 @@ prices-clickhouse = { path = "../prices-clickhouse" } stellar-xdr = { workspace = true } # Task 0210 symbol stage: Soroban RPC `simulateTransaction` against a token -# contract's `symbol()`. Same set (and the same reqwest feature list) -# oracle-worker uses for its Reflector calls. +# contract's `symbol()`. The envelope and response classification live in +# prices-ingest-core::soroban_rpc since task 0329; reqwest stays for the client +# type in this crate's signatures. stellar-strkey = { workspace = true } -base64 = "0.22" reqwest = { version = "0.12", default-features = false, features = [ "rustls-tls-webpki-roots", "json", diff --git a/packages/asset-discovery/src/symbols.rs b/packages/asset-discovery/src/symbols.rs index 55a7f4eb..90043208 100644 --- a/packages/asset-discovery/src/symbols.rs +++ b/packages/asset-discovery/src/symbols.rs @@ -19,24 +19,16 @@ //! The three-way outcome policy *is* borrowed from `supply-worker::run_supply`, //! because it is what keeps the queue from starving: see [`Outcome`]. -use base64::Engine; use clickhouse::Client; -use serde::Serialize; -use stellar_xdr::{ - ContractId, Hash, HostFunction, InvokeContractArgs, InvokeHostFunctionOp, Limits, Memo, - MuxedAccount, Operation, OperationBody, Preconditions, ReadXdr, ScAddress, ScString, ScSymbol, - ScVal, SequenceNumber, Transaction, TransactionEnvelope, TransactionExt, TransactionV1Envelope, - Uint256, VecM, WriteXdr, +use prices_ingest_core::soroban_rpc::{Simulated, simulate}; +// Re-exported so `main.rs` and the RPC integration test keep their paths; the +// envelope and the Absent/Transient boundary moved to `prices-ingest-core` +// (task 0329), which the ingest's `decimals()` lookup shares. +pub use prices_ingest_core::soroban_rpc::{ + DEFAULT_SOROBAN_RPC, RPC_TIMEOUT_SECS, build_simulate_envelope, http_client, }; - -/// Default public Soroban RPC endpoint (overridable via `SOROBAN_RPC_URL`). -/// Needs no auth — same endpoint and same reasoning as `oracle-worker`. -pub const DEFAULT_SOROBAN_RPC: &str = "https://mainnet.sorobanrpc.com"; - -/// Per-request RPC timeout. Together with [`MAX_CONTRACTS_PER_RUN`] this is the -/// stage's entire time bound: worst case is the product of the two, which must -/// leave the ledger scan its headroom inside the Lambda's 5-minute limit. -pub const RPC_TIMEOUT_SECS: u64 = 5; +use serde::Serialize; +use stellar_xdr::{ScString, ScSymbol, ScVal}; /// How many unresolved contracts one run will attempt. /// @@ -99,92 +91,6 @@ pub struct SymbolStats { pub skipped: usize, } -/// An HTTP client bounded by [`RPC_TIMEOUT_SECS`]. -/// -/// Built here rather than at the call site so the stage's time bound stays next -/// to the constant that states it. -/// -/// # Panics -/// -/// If the client cannot be built — in practice a TLS backend that failed to -/// initialise, which is an environment fault at cold start, identical on every -/// invocation. Deliberately not a silent fallback to `Client::default()`: that -/// client carries **no timeout**, so the stage's whole `MAX_CONTRACTS_PER_RUN × -/// RPC_TIMEOUT_SECS` bound would quietly become unbounded, and 25 hung requests -/// would eat the Lambda's five minutes ahead of the ledger scan that runs after -/// it. A loud failure at startup beats an hourly discovery outage with no -/// obvious cause. -pub fn http_client() -> reqwest::Client { - reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(RPC_TIMEOUT_SECS)) - .build() - .expect("build an HTTP client with a timeout") -} - -/// Base64 `TransactionEnvelope` invoking the zero-argument `func()` on -/// `contract`, for a read-only `simulateTransaction` (source account / fee / seq -/// are not checked by simulation). -/// -/// `None` when `contract` is not a well-formed C-strkey — a data-integrity -/// problem in `prices.assets`, permanent for that row, so the caller sentinels -/// it rather than failing the run. -pub fn build_simulate_envelope(contract: &str, func: &str) -> Option { - let contract_hash = stellar_strkey::Contract::from_string(contract).ok()?.0; - let invoke = InvokeContractArgs { - contract_address: ScAddress::Contract(ContractId(Hash(contract_hash))), - function_name: ScSymbol(func.try_into().ok()?), - args: VecM::default(), - }; - let op = Operation { - source_account: None, - body: OperationBody::InvokeHostFunction(InvokeHostFunctionOp { - host_function: HostFunction::InvokeContract(invoke), - auth: VecM::default(), - }), - }; - let tx = Transaction { - source_account: MuxedAccount::Ed25519(Uint256([0u8; 32])), - fee: 0, - seq_num: SequenceNumber(0), - cond: Preconditions::None, - memo: Memo::None, - operations: vec![op].try_into().ok()?, - ext: TransactionExt::V0, - }; - let envelope = TransactionEnvelope::Tx(TransactionV1Envelope { - tx, - signatures: VecM::default(), - }); - let xdr = envelope.to_xdr(Limits::none()).ok()?; - Some(base64::engine::general_purpose::STANDARD.encode(xdr)) -} - -#[derive(serde::Deserialize)] -struct RpcResponse { - result: Option, - /// JSON-RPC 2.0 reports failure *here*, with HTTP 200 — which is how public - /// endpoints signal quota exhaustion, a disabled method, and a node that is - /// not synced. Without this field such a body deserialises to - /// `result: None`, and every contract in the run sentinels permanently. - #[serde(default)] - error: Option, -} -#[derive(serde::Deserialize)] -struct RpcResult { - #[serde(default)] - results: Vec, - #[serde(default)] - error: Option, - /// Set when the contract's instance or Wasm is archived but restorable — - /// a live token, not one that lacks a symbol. - #[serde(default, rename = "restorePreamble")] - restore_preamble: Option, -} -#[derive(serde::Deserialize)] -struct RpcInvokeResult { - xdr: String, -} - /// Extract a UTF-8 string from a `symbol()` return value. /// /// Both arms are needed: the SEP-41 interface declares `Symbol`, but deployed @@ -227,99 +133,21 @@ pub fn sanitize_symbol(raw: &str) -> Option { /// /// Never returns an error: every failure mode is either Absent (permanent for /// this contract) or Transient (retry next run), which is exactly the decision -/// the caller has to make. +/// the caller has to make. The line between the two is +/// [`prices_ingest_core::soroban_rpc::simulate`]'s. pub async fn resolve_symbol(http: &reqwest::Client, rpc_url: &str, contract: &str) -> Outcome { - let Some(envelope) = build_simulate_envelope(contract, "symbol") else { - tracing::warn!(contract, "not a well-formed contract address; sentinelling"); - return Outcome::Absent; - }; - let body = serde_json::json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "simulateTransaction", - "params": { "transaction": envelope }, - }); - - // A transport-level failure — connection, timeout, non-2xx (a 429 or 5xx - // from the public endpoint lands here), unparseable body — is systemic, not - // a fact about this contract. `error_for_status` is what keeps a - // rate-limited run from sentinelling every contract it touches. - let resp = match http - .post(rpc_url) - .json(&body) - .send() - .await - .and_then(|r| r.error_for_status()) - { - Ok(r) => r, - Err(err) => { - tracing::warn!(contract, error = %err, "symbol() rpc failed; retrying next run"); - return Outcome::Transient; - } - }; - let resp: RpcResponse = match resp.json().await { - Ok(r) => r, - Err(err) => { - tracing::warn!(contract, error = %err, "symbol() rpc body unparseable; retrying next run"); - return Outcome::Transient; - } - }; - - classify(resp, contract) -} - -/// Decide what a parsed simulate response means for this contract. -/// -/// The line between the two failure arms is **whether the simulation actually -/// evaluated the contract**. If it did, its outcome is a fact that will repeat -/// on every future run, so it sentinels. If we never got that far — a JSON-RPC -/// error, archived state, an empty body — we learned nothing about the -/// contract, so it must retry. Getting this backwards is unrecoverable in one -/// direction only: a wrong `Transient` costs one RPC call per run, a wrong -/// `Absent` publishes a permanent empty symbol that nothing re-polls. -fn classify(resp: RpcResponse, contract: &str) -> Outcome { - // Arrives with HTTP 200, so `error_for_status` above cannot see it. - if let Some(err) = resp.error { - tracing::warn!(contract, error = %err, "symbol() rpc returned a json-rpc error; retrying next run"); - return Outcome::Transient; + match simulate(http, rpc_url, contract, "symbol", None).await { + Simulated::Value(v) => symbol_outcome(&v, contract), + Simulated::Absent => Outcome::Absent, + // `as_of` is None, so this arm is unreachable in practice; retrying is + // the safe reading either way. + Simulated::Transient | Simulated::Behind => Outcome::Transient, } - let Some(result) = resp.result else { - tracing::warn!( - contract, - "symbol() rpc returned neither result nor error; retrying next run" - ); - return Outcome::Transient; - }; - if result.restore_preamble.is_some() { - tracing::warn!( - contract, - "symbol() needs a state restore; retrying next run" - ); - return Outcome::Transient; - } - // The simulation ran and the contract itself errored — deterministic, so a - // fact. This is the one arm that stays permanent. - if let Some(err) = result.error { - tracing::debug!(contract, error = %err, "symbol() simulate returned error; sentinelling"); - return Outcome::Absent; - } - if result.results.is_empty() { - tracing::warn!( - contract, - "symbol() simulated without a result or an error; retrying next run" - ); - return Outcome::Transient; - } - - let raw = result - .results - .into_iter() - .next() - .and_then(|r| base64::engine::general_purpose::STANDARD.decode(r.xdr).ok()) - .and_then(|bytes| ScVal::from_xdr(&bytes, Limits::none()).ok()) - .as_ref() - .and_then(scval_to_string); +} +/// A `symbol()` return value as an [`Outcome`]. +fn symbol_outcome(v: &ScVal, contract: &str) -> Outcome { + let raw = scval_to_string(v); match raw.as_deref().and_then(sanitize_symbol) { Some(s) => Outcome::Symbol(s), // It answered, and the answer is unusable. A fact — but a permanent and @@ -529,50 +357,14 @@ mod tests { assert!(sanitize_symbol("v1.2_beta").is_some()); } - fn classify_json(body: &str) -> Outcome { - classify( - serde_json::from_str(body).expect("test body parses"), - "C_TEST", - ) - } - - #[test] - fn a_json_rpc_error_retries_instead_of_sentinelling() { - // JSON-RPC reports this with HTTP 200, so `error_for_status` cannot see - // it. Sentinelling here would name every contract in the run `""` - // permanently, with nothing that ever re-polls them. - let out = classify_json( - r#"{"jsonrpc":"2.0","id":1,"error":{"code":-32601,"message":"method not enabled"}}"#, - ); - assert_eq!(out, Outcome::Transient); - } - #[test] - fn an_empty_or_archived_response_retries() { - // Neither result nor error: we learned nothing about the contract. - assert_eq!( - classify_json(r#"{"jsonrpc":"2.0","id":1}"#), - Outcome::Transient - ); - // Archived-but-restorable state is a live token, not a symbol-less one. + fn an_unusable_answer_is_absent() { + // Answered, but not a string: a fact about the contract, not a retry. + assert_eq!(symbol_outcome(&ScVal::U32(7), "C_TEST"), Outcome::Absent); + let sym = ScVal::Symbol(ScSymbol("XAUM".try_into().unwrap())); assert_eq!( - classify_json(r#"{"result":{"results":[],"restorePreamble":{"minResourceFee":"1"}}}"#), - Outcome::Transient, - ); - // Simulated clean, returned nothing at all — not a fact either. - assert_eq!( - classify_json(r#"{"result":{"results":[]}}"#), - Outcome::Transient - ); - } - - #[test] - fn a_contract_error_is_the_one_permanent_arm() { - // The simulation evaluated the contract and it errored. That repeats on - // every run, so it must leave the queue or it starves the tail. - assert_eq!( - classify_json(r#"{"result":{"results":[],"error":"HostError: missing symbol"}}"#), - Outcome::Absent, + symbol_outcome(&sym, "C_TEST"), + Outcome::Symbol("XAUM".to_string()) ); } @@ -595,37 +387,4 @@ mod tests { assert_eq!(scval_to_string(&ScVal::U32(7)), None); } - - #[test] - fn builds_an_envelope_that_decodes_back_to_symbol_on_the_contract() { - // A locally-derived valid C-strkey: self-contained, no network, and it - // cannot rot the way a pasted mainnet address can. - let contract = stellar_strkey::Contract([7u8; 32]).to_string(); - let b64 = build_simulate_envelope(&contract, "symbol").expect("valid contract address"); - let raw = base64::engine::general_purpose::STANDARD - .decode(&b64) - .expect("valid base64"); - let env = TransactionEnvelope::from_xdr(&raw, Limits::none()).expect("valid xdr"); - let TransactionEnvelope::Tx(v1) = env else { - panic!("expected a v1 envelope") - }; - let OperationBody::InvokeHostFunction(op) = &v1.tx.operations[0].body else { - panic!("expected InvokeHostFunction") - }; - let HostFunction::InvokeContract(args) = &op.host_function else { - panic!("expected InvokeContract") - }; - assert_eq!(args.function_name.0.to_utf8_string().unwrap(), "symbol"); - assert_eq!(args.args.len(), 0, "symbol() takes no arguments"); - assert_eq!( - args.contract_address, - ScAddress::Contract(ContractId(Hash([7u8; 32]))) - ); - } - - #[test] - fn rejects_a_malformed_contract_address() { - assert_eq!(build_simulate_envelope("not-a-contract", "symbol"), None); - assert_eq!(build_simulate_envelope("", "symbol"), None); - } } diff --git a/packages/coarse-sweep-worker/src/main.rs b/packages/coarse-sweep-worker/src/main.rs index 5f09c2df..02181537 100644 --- a/packages/coarse-sweep-worker/src/main.rs +++ b/packages/coarse-sweep-worker/src/main.rs @@ -89,6 +89,8 @@ async fn main() -> Result<(), lambda_runtime::Error> { // Deliberately not env-driven — an operator names the quote leg and // epoch explicitly, via the `coarse-repair` binary. usd_reset: None, + // Opt-in coarse-repair mode only (task 0331). + pre_usdc_xlm_rate: None, }; // Cold start, mirroring the sibling workers: build and probe the mTLS client diff --git a/packages/coverage-sweep-probe/allowlist.toml b/packages/coverage-sweep-probe/allowlist.toml index 9fa70823..3ab6676c 100644 --- a/packages/coverage-sweep-probe/allowlist.toml +++ b/packages/coverage-sweep-probe/allowlist.toml @@ -17,6 +17,9 @@ # the task that registers the family). A permanent family-wide # ignore would hide the next pool of a real venue, so a [[wasm]] # entry without `until` fails validation. +# The one exception is `not_a_venue = true` (task 0336), in +# place of `until`: a family whose code cannot be a market we +# price, read from the wasm, that keeps deploying new instances. # # Every entry needs a non-empty `reason` and `task`. Unknown keys are rejected. # @@ -173,15 +176,33 @@ task = "0100" [[contract]] id = "CA5YJ5H2HH6KUZBGQ7M7R2TX63LQKQVG57Y3HE23NVC5C6Q7BQFMSOI5" wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" -reason = "on-chain order book of tokenised instruments (NewBuyInitiatedTrade); instruments are u64 ids with no token addresses, not priceable here" +reason = "permissioned order book, grBENJI/USDC (whitelist, admin-set phases and reference price); its NewBuyInitiatedTrade carries order ids, no token address, and has no extractor" task = "0100" [[contract]] id = "CBOSXSEZXU7P66O6F3AQV65URR5XUVKMY6CAQ7FT7C5TQLGYNV7IG5I6" wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" -reason = "on-chain order book of tokenised instruments (NewSellInitiatedTrade); instruments are u64 ids with no token addresses, not priceable here" +reason = "permissioned order book, AMA1/XUSDT (whitelist, admin-set phases and reference price); its NewSellInitiatedTrade carries order ids, no token address, and has no extractor" task = "0100" +[[contract]] +id = "CD35HSTRTNL4O3BFXQL75XGAI6AOCWFFAEPR5OCBS7A7HVUDUZOVLCAB" +wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" +reason = "permissioned order book, AMA1/XUSDT, a further instance of the wasm above (whitelist, admin-set phases and reference price); its NewBuyInitiatedTrade carries order ids, no token address, and has no extractor" +task = "0334" + +[[contract]] +id = "CAF5RWQG5GSTEPD5XAKCUH2H77NWA4J6DNFT6TUP2NU6Q6VW3R6HCZYO" +wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" +reason = "permissioned order book, grBENJI/USDC, a further instance of the wasm above (whitelist, admin-set phases and reference price); its trade events carry order ids, no token address, and have no extractor; decoded trades ~75,000 at ~1.00; 0/50 txs hold a registered pool's swap" +task = "0338" + +[[contract]] +id = "CC5QR5OOQMCPVSL3SJMH7QESCFINWUMX3427CT4NIDBS3FRVDHAM4NAJ" +wasm = "a85d6d53dfe894ef6483aad0613359584e5a78629c159cf3dbf53d74395a0404" +reason = "tokenised managed fund aLQD (issue, redeem, nav, rebalance), not a market; its trade event wraps a swap through the Soroswap router, and indexing it double-counts; 3/8 txs hold a registered pool's swap, its one trade among them" +task = "0338" + [[contract]] id = "CBP76I2FRMUKYKIYYBKN3DH7TSWMFAJF2WTDC7Q2OHQFBIVWP7CAAI5Q" wasm = "76e60d025ee6c0322cb19f4846b9587ab172870802251de903e93545371678b6" @@ -194,6 +215,20 @@ wasm = "91d5ab7e075d533a9404373453a548bc21c3d2893d78f9782785985f396b4454" reason = "bonding-curve token launchpad (pad_created, launched, buy, bump_curve); its trade events price a launched token on its own curve, not a pool; first seen 2026-10-04 as a test launch (ANYCHAIN Test, CCRIN5F5…), 3 trades by its creator" task = "0323" +[[contract]] +id = "CANIE64BDFKLHROLOIY2KTQVBWUZNUKZP75DLXC272QM7TIN5N3M4LVA" +wasm = "dcdc0b1860799629c7c88804955dd7373cb6febaf069e2ba863e98ebbaa95c27" +reason = "BLNT v3 backfill contract (verified source BLNT-Protocol/blnt-backfill-contract); swap_blnd is a fixed 1:1 BLND -> BLNT conversion (burns BLND, pays pre-funded BLNT), not a market; no admin, no upgrade" +task = "0337" + +# --- unindexed pools (one contract, not their family) ------------------------- + +[[contract]] +id = "CAZE5GVMH5332L6FEGVPPDLEPVYUFGQB7ZOCH2OKKBJCBYZQ5FGKZN7L" +wasm = "ae835560f930e73b3ac8ce0ab2b26543b815eb6bb61b0d87b894f4dc8dc6e8df" +reason = "Spreadless USD 3pool, a real stableswap AMM (USDT0, PYUSD, USDC; amp 100) with no extractor; ~$29 of reserves and three 1-unit swaps at 2026-11-02; 0/7 txs hold a registered pool's swap; this pool only, so the next pool of the wasm still pages" +task = "0338" + # --- unindexed venue families (temporary, keyed by wasm) ---------------------- [[wasm]] @@ -213,3 +248,11 @@ hash = "8abc28913035c07411ed5d134e6bfeab4723d97ddd4d1a22a0605d35c94d1a36" reason = "Comet weighted pool, Blend backstop BLND/USDC (CAS3FL6T…), a venue not yet indexed; 53k swaps since 2024-05-02" task = "0100" until = "0300" + +# --- non-venue families (permanent, keyed by wasm) ---------------------------- + +[[wasm]] +hash = "f3dd3edfd637a9654054181a9c47391ddafa723697b982698b3c9662a9df974e" +reason = "securities-financing trade ledger (repo, stock loan, FX forward); trade_opened/trade_closed carry notional and rate_bps, no token price; new instances keep appearing (CDU47AIU… in 0334, four more by 2026-10-19), so this is keyed by wasm" +task = "0336" +not_a_venue = true diff --git a/packages/coverage-sweep-probe/src/allowlist.rs b/packages/coverage-sweep-probe/src/allowlist.rs index 08f0d1bc..5bff2e74 100644 --- a/packages/coverage-sweep-probe/src/allowlist.rs +++ b/packages/coverage-sweep-probe/src/allowlist.rs @@ -31,7 +31,8 @@ pub struct ContractEntry { pub task: String, } -/// A whole wasm family, allow-listed until `until` ships. +/// A whole wasm family, allow-listed until `until` ships, or for good when +/// `not_a_venue` (task 0336). #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct WasmEntry { @@ -39,9 +40,14 @@ pub struct WasmEntry { pub hash: String, pub reason: String, pub task: String, - /// The task whose delivery removes this entry. Required (D4). + /// The task whose delivery removes this entry. Required (D4) unless + /// `not_a_venue`. #[serde(default)] pub until: String, + /// The family's code cannot be a market we price (a securities ledger, + /// say), so no task will ever register it. Takes the place of `until`. + #[serde(default)] + pub not_a_venue: bool, } /// Why an allow-list was refused. @@ -87,7 +93,8 @@ impl AllowList { /// Enforce the rules a list must meet before it may hide anything: /// - every entry has a non-empty `reason` and `task`; /// - every `[[wasm]]` entry has a non-empty `until` (D4: a permanent - /// family-wide ignore would hide the next pool of a real venue); + /// family-wide ignore would hide the next pool of a real venue), or + /// `not_a_venue` and no `until` (task 0336); /// - every contract `id` is a valid `C…` strkey (an account key or a typo /// would silently match nothing); /// - every hash, including a contract entry's informational `wasm`, is @@ -127,7 +134,13 @@ impl AllowList { } require(&who, "reason", &e.reason)?; require(&who, "task", &e.task)?; - require(&who, "until", &e.until)?; + if !e.not_a_venue { + require(&who, "until", &e.until)?; + } else if !e.until.trim().is_empty() { + return Err(AllowListError::Invalid(format!( + "{who}: `not_a_venue` and `until` are exclusive" + ))); + } if !hashes.insert(e.hash.as_str()) { return Err(AllowListError::Invalid(format!("{who}: duplicate hash"))); } @@ -174,23 +187,80 @@ mod tests { use super::*; const SUSHI_V3: &str = "003710b383f9da7d650a7f719a7be479110266427817ebbed61d924505fcd7c7"; + const SECURITIES_LEDGER: &str = + "f3dd3edfd637a9654054181a9c47391ddafa723697b982698b3c9662a9df974e"; const AQUARIUS_ROUTER: &str = "CBQDHNBFBZYE4MKPWBSJOPIYLW4SFSXAXUTSXJN76GNKYVYPCKWC6QUK"; #[test] fn embedded_list_parses_with_the_seeded_entries() { let list = AllowList::embedded().expect("embedded allow-list parses"); - assert_eq!(list.contract.len(), 27, "{list:?}"); - assert_eq!(list.wasm.len(), 3, "{list:?}"); - // Every family-wide entry is temporary and names the task that ends it: - // SushiSwap V3 (0290) and the Comet BLND/USDC pool (0300). + assert_eq!(list.contract.len(), 32, "{list:?}"); + assert_eq!(list.wasm.len(), 4, "{list:?}"); + // Every family-wide entry is temporary and names the task that ends it + // (SushiSwap V3, 0290; the Comet BLND/USDC pool, 0300), except the one + // non-venue family (0336). assert!( - list.wasm - .iter() - .all(|w| w.until == "0290" || w.until == "0300"), + list.wasm.iter().all(|w| if w.not_a_venue { + w.hash == SECURITIES_LEDGER + } else { + w.until == "0290" || w.until == "0300" + }), "{list:?}" ); } + #[test] + fn securities_ledger_family_is_allow_listed_by_wasm() { + // The 2026-10-19 sweep (task 0336): four new instances of the wasm + // 0334 allow-listed by one contract id, each paging on its own. + let list = AllowList::embedded().unwrap(); + for id in [ + "CDU47AIU3BKKRTJBTPFHFXLVJG4BGNOXKDYUBDTKRIWAFP2RLWFKL4HV", + "CAKQ6DWLPTKFDHYON346SBAYJJDTVO5DKTMZ7737HBF6NCGASIMOM4U2", + "CCLN72NMUDZVXK6V6W5RQ4GHGHMX437G2FMGT6YKVNB7BAAH7XFLP5SZ", + "CBZYHS2DVYAIOURIAXVUNTTAGTUXH3I4SWFWKN7FAKFKNOKLQ5LIR5YB", + "CB4B4ELLM4YTF7HLITB52OFCYPGHPT763JZBHVUOD4AN4FJR7RWVZXJ6", + ] { + assert_eq!( + list.match_row(id, Some(SECURITIES_LEDGER)), + Some(format!("wasm:{SECURITIES_LEDGER}")), + "{id}" + ); + } + } + + #[test] + fn the_2026_11_02_emitters_are_allow_listed_by_contract() { + // The sweep of 2026-11-02 (task 0338). + let list = AllowList::embedded().unwrap(); + for id in [ + "CAF5RWQG5GSTEPD5XAKCUH2H77NWA4J6DNFT6TUP2NU6Q6VW3R6HCZYO", + "CC5QR5OOQMCPVSL3SJMH7QESCFINWUMX3427CT4NIDBS3FRVDHAM4NAJ", + "CAZE5GVMH5332L6FEGVPPDLEPVYUFGQB7ZOCH2OKKBJCBYZQ5FGKZN7L", + ] { + assert_eq!( + list.match_row(id, None), + Some(format!("contract:{id}")), + "{id}" + ); + } + } + + #[test] + fn the_next_spreadless_pool_still_pages() { + // Task 0338 lists one pool of an AMM we do not index, not its family: a + // new pool has to reach a human. + let list = AllowList::embedded().unwrap(); + let spreadless = "ae835560f930e73b3ac8ce0ab2b26543b815eb6bb61b0d87b894f4dc8dc6e8df"; + assert_eq!( + list.match_row( + "CCCRWH6Q3FNP3I2I57BDLM5AFAT7O6OF6GKQOC6SSJNDAVRZ57SPHGU2", + Some(spreadless) + ), + None + ); + } + #[test] fn comet_pool_family_is_allow_listed_until_0300() { // Phase-1 classification (task 0100): the Comet BLND/USDC pool is a real @@ -276,6 +346,19 @@ mod tests { refused(&wasm_toml(SUSHI_V3, Some(" "))); } + #[test] + fn not_a_venue_takes_the_place_of_until() { + let entry = |extra: &str| { + format!( + "[[wasm]]\nhash = \"{SECURITIES_LEDGER}\"\nreason = \"r\"\ntask = \"0336\"\n{extra}" + ) + }; + AllowList::parse(&entry("not_a_venue = true\n")).unwrap(); + let e = refused(&entry("not_a_venue = true\nuntil = \"0336\"\n")); + assert!(e.contains("exclusive"), "{e}"); + refused(&entry("not_a_venue = false\n")); + } + #[test] fn duplicates_are_refused() { let e = refused(&(contract_toml(GOOD_ID, "a", "1") + &contract_toml(GOOD_ID, "b", "2"))); diff --git a/packages/coverage-sweep-probe/src/lib.rs b/packages/coverage-sweep-probe/src/lib.rs index 9ea5d323..8c451ecc 100644 --- a/packages/coverage-sweep-probe/src/lib.rs +++ b/packages/coverage-sweep-probe/src/lib.rs @@ -39,9 +39,9 @@ pub const UNCLASSIFIED_SWAP_EVENTS: &str = "UnclassifiedSwapEvents"; pub const UNRESOLVED_SWAP_EMITTERS: &str = "UnresolvedSwapEmitters"; /// BE's database (`soroban_events`, `soroban_contracts`). -pub const BE_DATABASE: &str = "default"; +pub const BE_DATABASE: &str = prices_clickhouse::BE_DATABASE; /// Our database (`pool_registry`). -pub const PRICES_DATABASE: &str = "prices"; +pub const PRICES_DATABASE: &str = prices_clickhouse::PROD_DATABASE; /// Client-side `max_execution_time` for every statement, in seconds. The sweep /// runs as `prices_writer`, whose profile carries no execution bound. diff --git a/packages/enrichment-worker/examples/run_ch_pass.rs b/packages/enrichment-worker/examples/run_ch_pass.rs index eb4872f1..0fc68dc2 100644 --- a/packages/enrichment-worker/examples/run_ch_pass.rs +++ b/packages/enrichment-worker/examples/run_ch_pass.rs @@ -30,6 +30,7 @@ async fn main() -> Result<(), Box> { one_shot: env_parse_or("ENRICHMENT_ONE_SHOT", false), time_window: None, usd_reset: None, + pre_usdc_xlm_rate: None, }; let pass = ChEnrichmentPass::new(cfg); diff --git a/packages/enrichment-worker/src/bin/coarse-repair.rs b/packages/enrichment-worker/src/bin/coarse-repair.rs index 57891606..bc02f71e 100644 --- a/packages/enrichment-worker/src/bin/coarse-repair.rs +++ b/packages/enrichment-worker/src/bin/coarse-repair.rs @@ -23,11 +23,19 @@ //! mTLS env (same as sdex-backfill): CH_DOMAIN, MTLS_CERT_PATH, MTLS_KEY_PATH, //! MTLS_CA_PATH. Revert a partition from its FREEZE snapshot with //! `SYSTEM UNFREEZE WITH NAME ''` + `ALTER TABLE … ATTACH`. +//! +//! `price_ohlcv_1m` is accepted too, for a whole-history drain (task 0286 §7b): +//! the hourly Lambda prices at most 200 k rows an hour, which is months for a +//! re-ingested history. Only the fill-the-zeros mode (and `--pre-usdc-xlm-rate`) +//! is allowed there — no `--reset-*` — and only with `--skip-snapshot`, see +//! [`check_table`]. use std::path::PathBuf; use clap::{Parser, ValueEnum}; -use enrichment_worker::ch_enrich::{ChEnrichConfig, UsdResetSpec, external_window_s}; +use enrichment_worker::ch_enrich::{ + ChEnrichConfig, PreUsdcXlmRate, UsdResetSpec, external_window_s, +}; use enrichment_worker::repair::{CoarseRepairConfig, CoarseRepairDriver}; use prices_clickhouse::USDC_ORACLE_EPOCH_S; use tracing::{info, warn}; @@ -69,7 +77,8 @@ struct Args { #[arg(long, env = "MTLS_CA_PATH")] mtls_ca_path: Option, - /// Coarse table to repair (e.g. `price_ohlcv_1h`). One table per run. + /// Table to repair (e.g. `price_ohlcv_1h`). One table per run. + /// `price_ohlcv_1m` takes no `--reset-*` (see `check_table`). #[arg(long)] table: String, @@ -112,12 +121,32 @@ struct Args { #[arg(long, env = "PIVOT_WINDOW_S", default_value_t = 86_400)] pivot_window_s: u32, + /// Before 2021-02-01 only (task 0331): price XLM-quoted candles from the + /// promoted `method = 'external'` XLM/USD rows, and dollar-token-quoted ones + /// through that token's own XLM market within `--pivot-window-s`. Writes only + /// rows whose values differ, so a re-run is idempotent. Refused until the + /// series is promoted. + /// + /// Run `price_ohlcv_1h` (and 1m/15m) first: 4h, 1d, 1w and 1M are rolled up + /// from the priced 1h rows (Σ USD volume, last priced hour's ratio), so + /// `--pivot-window-s` matters only up to 1h, and a wide tier run before its + /// 1h writes nothing. A re-run can set priced rows back to 0: on + /// `price_ohlcv_1m` that needs `--allow-unprice`, elsewhere a snapshot. + #[arg(long, conflicts_with = "reset_quote_asset_id")] + pre_usdc_xlm_rate: bool, + + /// With `--pre-usdc-xlm-rate` on `price_ohlcv_1m`: accept writing priced + /// rows back to 0 (the dry run's `reset` column), with no snapshot. Without + /// it a real run stops at the first such month; earlier months stay written. + #[arg(long, requires = "pre_usdc_xlm_rate")] + allow_unprice: bool, + /// Re-open already-written USD values for this quote `asset_id` so the /// corrected tiers recompute them (task 0182). Requires /// `--reset-not-before`. /// - /// ⚠️ This is the only flag here that DISCARDS a computed value. Everything - /// else in this tool is purely additive — it fills zeros. Use it only to + /// ⚠️ It DISCARDS a computed value, like the 0228 mode and + /// `--pre-usdc-xlm-rate`; the default mode only fills zeros. Use it only to /// correct a *pricing* defect, where the stored number is wrong rather than /// missing, and only against a FREEZE-snapshotted partition. /// @@ -264,6 +293,83 @@ fn validate_month(label: &str, m: u32) -> Result<(), String> { Ok(()) } +/// Which tables this tool may write, and how. +/// +/// The coarse rollups take every mode. `price_ohlcv_1m` is the live base table +/// the hourly Lambda owns, so it takes only the fill-the-zeros mode, which is +/// exactly what the Lambda runs: it writes rows still at zero and discards +/// nothing, so it can run beside the Lambda (task 0286 §7b). A reset there +/// would discard prices every other tier is rolled from, so it is refused. +/// FREEZE is refused too: `prices_writer` cannot take it, and a 1m snapshot +/// protects nothing in a mode that only fills zeros. `--pre-usdc-xlm-rate` is +/// allowed there too: it can discard a price, so the pass refuses a 1m run +/// that would unless `--allow-unprice` is passed. +fn check_table( + table: &str, + resetting: bool, + skip_snapshot: bool, + dry_run: bool, +) -> Result<(), String> { + if !table.starts_with("price_ohlcv_") { + return Err(format!( + "--table {table} is not an OHLCV table (expected price_ohlcv_1m, _15m … _1M)" + )); + } + if table != "price_ohlcv_1m" { + return Ok(()); + } + if resetting { + return Err( + "--reset-* is refused on price_ohlcv_1m: a reset discards prices every \ + coarse tier is rolled from. On 1m only the fill-the-zeros mode runs." + .into(), + ); + } + if !skip_snapshot && !dry_run { + return Err( + "price_ohlcv_1m needs --skip-snapshot: prices_writer cannot FREEZE, and \ + the fill-the-zeros mode discards nothing a snapshot would protect \ + (--pre-usdc-xlm-rate refuses to without --allow-unprice)." + .into(), + ); + } + Ok(()) +} + +/// The snapshot guard of a real run that can discard stored values: reset +/// mode, and `--pre-usdc-xlm-rate` off `price_ohlcv_1m` (on 1m the pass guards +/// that instead, see [`check_table`]). +/// +/// Rollback for a bad reset IS `ATTACH PARTITION` from the shadow copy, so the +/// snapshots must exist. They usually will not have been taken by this tool: +/// `prices_writer` cannot FREEZE and cannot be granted it, so on prod the admin +/// takes them out of band and `--skip-snapshot` is the *correct* flag rather +/// than a shortcut. Refusing that combination outright would make reset mode +/// unusable on the only cluster it exists for. So the operator must say which +/// case they are in. +fn check_snapshots( + table: &str, + resetting: bool, + pre_usdc: bool, + (skip_snapshot, snapshots_verified): (bool, bool), + dry_run: bool, +) -> Result<(), String> { + let discards = resetting || (pre_usdc && table != "price_ohlcv_1m"); + if discards && !dry_run && skip_snapshot && !snapshots_verified { + return Err("--skip-snapshot with --reset-quote-asset-id or \ + --pre-usdc-xlm-rate requires --snapshots-verified: both can discard \ + stored values, and their only \ + rollback is ATTACH PARTITION from the frozen copy.\n\ + If the CH admin already froze this span (runbook Step 3b — the \ + normal prod path, since prices_writer cannot FREEZE), confirm with \ + `ls /var/lib/clickhouse/shadow/ | grep repair_` plus a non-trivial \ + `du`, then pass --snapshots-verified.\n\ + If nobody has, stop: there is no rollback point." + .into()); + } + Ok(()) +} + #[tokio::main] async fn main() -> Result<(), Box> { tracing_subscriber::fmt() @@ -284,39 +390,20 @@ async fn main() -> Result<(), Box> { ) .into()); } - if !args.table.starts_with("price_ohlcv_") || args.table == "price_ohlcv_1m" { - // The driver targets the coarse rollups; 1m is the live base table the - // scheduled Lambda owns and has live writers. Refuse it outright. - return Err(format!( - "--table {} is not a coarse OHLCV table (expected price_ohlcv_15m … _1M; \ - price_ohlcv_1m is the live base table and is off-limits here)", - args.table - ) - .into()); - } - - // Reset mode is the only path here that discards data, so its two footguns - // are refused rather than warned about. - if args.reset_quote_asset_id.is_some() && !args.dry_run { - // 1. Rollback for a bad reset IS `ATTACH PARTITION` from the shadow copy, - // so the snapshots must exist. They usually will not have been taken by - // this tool: `prices_writer` cannot FREEZE and cannot be granted it, so - // on prod the admin takes them out of band and `--skip-snapshot` is the - // *correct* flag rather than a shortcut. Refusing that combination - // outright would make reset mode unusable on the only cluster it exists - // for. So require the operator to say which case they are in. - if args.skip_snapshot && !args.snapshots_verified { - return Err("--skip-snapshot with --reset-quote-asset-id requires \ - --snapshots-verified: a reset discards stored values, and its only \ - rollback is ATTACH PARTITION from the frozen copy.\n\ - If the CH admin already froze this span (runbook Step 3b — the \ - normal prod path, since prices_writer cannot FREEZE), confirm with \ - `ls /var/lib/clickhouse/shadow/ | grep repair_` plus a non-trivial \ - `du`, then pass --snapshots-verified.\n\ - If nobody has, stop: there is no rollback point." - .into()); - } - } + check_table( + &args.table, + args.reset_quote_asset_id.is_some(), + args.skip_snapshot, + args.dry_run, + )?; + + check_snapshots( + &args.table, + args.reset_quote_asset_id.is_some(), + args.pre_usdc_xlm_rate, + (args.skip_snapshot, args.snapshots_verified), + args.dry_run, + )?; // The window guards below run in dry-run mode TOO (task 0228 review WR-01): // the runbooks say "dry run first", and a dry run that accepts a window the // real run then refuses is a rehearsal of nothing. Only the snapshot guard @@ -434,6 +521,9 @@ async fn main() -> Result<(), Box> { // clap's `requires` makes the mixed cases unreachable. _ => None, }, + pre_usdc_xlm_rate: args.pre_usdc_xlm_rate.then_some(PreUsdcXlmRate { + allow_unprice: args.allow_unprice, + }), }; let repair_cfg = CoarseRepairConfig { @@ -487,7 +577,12 @@ async fn main() -> Result<(), Box> { driver.preflight().await?; - if args.skip_snapshot && !args.dry_run { + if args.skip_snapshot && !args.dry_run && args.table == "price_ohlcv_1m" { + info!( + allow_unprice = args.allow_unprice, + "snapshot disabled on price_ohlcv_1m: nothing is discarded unless --allow-unprice" + ); + } else if args.skip_snapshot && !args.dry_run { warn!( "snapshot DISABLED (--skip-snapshot): partitions will be repaired with no FREEZE \ backup — only safe when the 1m source can rebuild this coarse table" @@ -546,11 +641,22 @@ async fn main() -> Result<(), Box> { m.snapshot_name.as_deref().unwrap_or("-") ); } + if args.pre_usdc_xlm_rate { + println!( + "--pre-usdc-xlm-rate: zeros_before = rows to write, enriched = rows written, \ + zeros_after = rows left unpriced, reset = priced rows set back to 0" + ); + } println!( - "{} month(s): {} enriched, {} left at the no_reference floor{}", + "{} month(s): {} enriched, {} {}{}", summary.months.len(), summary.total_enriched(), summary.total_remaining(), + if args.pre_usdc_xlm_rate { + "left unpriced" + } else { + "left at the no_reference floor" + }, if args.dry_run { " [DRY RUN — nothing written]" } else { @@ -558,11 +664,34 @@ async fn main() -> Result<(), Box> { } ); + // A real 1m run refuses the first month that would set priced rows to 0; + // the dry run reports the count, so say what the real run would do with it. + let unprice = summary.total_reset(); + let first = summary.months.iter().find(|m| m.rows_reset > 0); + if let (true, true, Some(m)) = (args.pre_usdc_xlm_rate, args.dry_run, first) { + if args.table == "price_ohlcv_1m" { + if !args.allow_unprice { + println!( + "A real run would stop at month {} ({unprice} priced row(s) in all \ + would be set back to 0); months before it would be written. \ + Pass --allow-unprice to accept.", + m.month + ); + } + } else { + println!( + "A real run would set {unprice} priced row(s) back to 0, from month {} \ + on; only the snapshot can restore them.", + m.month + ); + } + } + // The reset's own safety check, computed rather than left to the operator's // arithmetic. `rows_reset` > `rows_enriched` means values were discarded and // not recomputed — the one outcome worse than the defect being repaired. let (reset, enriched) = (summary.total_reset(), summary.total_enriched()); - if reset > 0 { + if reset > 0 && !args.pre_usdc_xlm_rate { println!("{reset} row(s) re-opened by the USD reset, {enriched} recomputed"); println!( "Run the damage check (runbook Appendix A, 'Extra verification') on \ @@ -606,3 +735,61 @@ async fn main() -> Result<(), Box> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::{check_snapshots, check_table}; + + /// (table, resetting, pre_usdc, skip_snapshot, snapshots_verified, dry_run, refused) + #[rustfmt::skip] + const SNAPSHOT_CASES: &[(&str, bool, bool, bool, bool, bool, bool)] = &[ + ("price_ohlcv_1h", true, false, true, false, false, true), + ("price_ohlcv_1h", false, true, true, false, false, true), + ("price_ohlcv_1M", false, true, true, false, false, true), + ("price_ohlcv_1h", false, true, true, true, false, false), + ("price_ohlcv_1h", false, true, false, false, false, false), + ("price_ohlcv_1h", false, true, true, false, true, false), + ("price_ohlcv_1m", false, true, true, false, false, false), + ("price_ohlcv_1h", false, false, true, false, false, false), + ]; + + #[test] + fn a_real_run_that_can_discard_needs_a_snapshot_or_its_confirmation() { + for &(t, reset, pre, skip, verified, dry, refused) in SNAPSHOT_CASES { + let got = check_snapshots(t, reset, pre, (skip, verified), dry); + assert_eq!( + got.is_err(), + refused, + "{t} reset={reset} pre={pre} skip={skip} verified={verified} dry={dry}" + ); + } + } + + #[test] + fn coarse_tables_take_every_mode() { + assert!(check_table("price_ohlcv_1h", true, false, false).is_ok()); + assert!(check_table("price_ohlcv_1M", false, true, false).is_ok()); + } + + #[test] + fn a_non_ohlcv_table_is_refused() { + assert!(check_table("assets", false, true, false).is_err()); + } + + #[test] + fn one_minute_takes_the_fill_the_zeros_mode_without_a_snapshot() { + assert!(check_table("price_ohlcv_1m", false, true, false).is_ok()); + assert!(check_table("price_ohlcv_1m", false, false, true).is_ok()); + } + + #[test] + fn one_minute_refuses_a_reset_even_in_a_dry_run() { + assert!(check_table("price_ohlcv_1m", true, true, false).is_err()); + assert!(check_table("price_ohlcv_1m", true, true, true).is_err()); + } + + #[test] + fn one_minute_refuses_a_real_run_with_a_snapshot() { + assert!(check_table("price_ohlcv_1m", false, false, false).is_err()); + } +} diff --git a/packages/enrichment-worker/src/ch_enrich.rs b/packages/enrichment-worker/src/ch_enrich.rs index dd21fca1..147985c5 100644 --- a/packages/enrichment-worker/src/ch_enrich.rs +++ b/packages/enrichment-worker/src/ch_enrich.rs @@ -110,7 +110,7 @@ use tracing::{info, warn}; /// Re-exported from `prices-clickhouse` (the single source of truth, also used by /// `sdex-backfill`) so the `asset_id` the backfill interns under these issuers /// matches the `quote_asset_id` enriched here — no hand-synced literal to drift. -use prices_clickhouse::{USDC_ISSUER, USDT_ISSUER}; +use prices_clickhouse::{DOLLAR_TOKEN_PRED, USDC_ISSUER, USDT_ISSUER}; #[derive(Debug, thiserror::Error)] pub enum ChEnrichError { @@ -447,6 +447,15 @@ pub enum ChEnrichError { until a later run." )] ResetRequiresOneShot { quote_asset_id: u64 }, + + #[error("pre-2021 XLM/USD rate mode refused: {0}")] + PreUsdcXlmRateRefused(String), + + #[error( + "pre-2021 XLM/USD step stalled: {remaining} row(s) still differ after the last \ + batch, and {reset} priced row(s) were set back to 0 before it stopped" + )] + PreUsdcXlmRateStalled { remaining: u64, reset: u64 }, } /// Opt-in reset of **already-written** USD columns, so a corrected pricing tier @@ -999,6 +1008,25 @@ pub struct ChEnrichConfig { /// computed value if an operator explicitly names the quote leg and epoch. /// See [`UsdResetSpec`]. pub usd_reset: Option, + /// Opt-in coarse-repair mode (task 0331), `None` on every scheduled path. + /// When set the pass runs only the pre-2021 XLM/USD step. + pub pre_usdc_xlm_rate: Option, +} + +/// Options of the pre-2021 XLM/USD step (task 0331). +#[derive(Debug, Clone, Copy, Default)] +pub struct PreUsdcXlmRate { + /// Let a `price_ohlcv_1m` run set priced rows back to 0 (1m has no snapshot). + pub allow_unprice: bool, +} + +/// The pre-2021 step's counts: rows a run writes, priced rows it sets back to +/// 0, and rows left unpriced after it. +#[derive(Debug, Clone, Copy, Default, clickhouse::Row, Deserialize)] +pub struct PreUsdcCounts { + pub pending: u64, + pub unprice: u64, + pub unpriced: u64, } impl Default for ChEnrichConfig { @@ -1016,6 +1044,7 @@ impl Default for ChEnrichConfig { one_shot: false, time_window: None, usd_reset: None, + pre_usdc_xlm_rate: None, } } } @@ -1508,7 +1537,13 @@ impl ChEnrichmentPass { /// The last step (task 0208) is `assert_reset_epoch_is_covered`: a /// pivot leg's `not_before` below the table's first priced reference /// candle is refused, because nothing could refill the rows in between. + /// + /// With [`ChEnrichConfig::pre_usdc_xlm_rate`] set it runs only that mode's + /// refusals ([`Self::assert_pre_usdc_admissible`]). pub async fn assert_reset_is_admissible(&self) -> Result<(), ChEnrichError> { + if self.cfg.pre_usdc_xlm_rate.is_some() { + return self.assert_pre_usdc_admissible().await.map(|_| ()); + } let Some(spec) = self.cfg.usd_reset.as_ref() else { return Ok(()); }; @@ -2028,6 +2063,103 @@ impl ChEnrichmentPass { Ok(()) } + /// The pre-2021 step's refusals; returns native XLM's id. The driver calls + /// this once up front, so a dry run refuses too. + pub async fn assert_pre_usdc_admissible(&self) -> Result { + let refuse = |m: &str| Err(ChEnrichError::PreUsdcXlmRateRefused(m.to_string())); + if self.cfg.usd_reset.is_some() { + return refuse("it cannot be combined with a --reset-* spec"); + } + let Some(xlm) = self.resolve_reference_ids().await?.xlm else { + return refuse("native XLM does not resolve in assets"); + }; + let sql = format!( + "SELECT count() FROM ({}) WHERE rts < toDateTime({})", + usd_rate_series_sql(&self.cfg.database, XLM_RATE_IDENTITY), + prices_clickhouse::PRE_USDC_END_S, + ); + if self.client.query(&sql).fetch_one::().await? == 0 { + return refuse( + "usd_rate holds no method = 'external' third-party XLM row before \ + 2021-02-01 (promote the staged series first)", + ); + } + Ok(xlm) + } + + fn pre_usdc_statements(&self, xlm: u64) -> (String, String) { + let (db, tbl) = (&self.cfg.database, &self.cfg.table); + pre_usdc_sql(db, tbl, xlm, self.cfg.pivot_window_s, self.cfg.time_window) + } + + /// [`PreUsdcCounts`] now; `pending` is the dry-run figure. + pub async fn pre_usdc_counts(&self, xlm: u64) -> Result { + let (_, count) = self.pre_usdc_statements(xlm); + Ok(self.client.query(&count).fetch_one().await?) + } + + /// The pre-2021 XLM/USD step (task 0331), drained to a fixed point. In its + /// stats `candidates_after` is the rows left unpriced and `rows_reset` the + /// priced rows set back to 0. + pub async fn run_pre_usdc(&self, xlm: u64) -> Result { + let start = std::time::Instant::now(); + let before = self.pre_usdc_counts(xlm).await?; + let allow = self.cfg.pre_usdc_xlm_rate.is_some_and(|m| m.allow_unprice); + if before.unprice > 0 && self.cfg.table == "price_ohlcv_1m" && !allow { + return Err(ChEnrichError::PreUsdcXlmRateRefused(format!( + "it would set {} priced price_ohlcv_1m row(s) back to 0, and 1m has no \ + snapshot to roll back to (pass --allow-unprice to accept)", + before.unprice + ))); + } + let (insert, _) = self.pre_usdc_statements(xlm); + let insert = self.client.query(&insert).bind(self.cfg.batch_size); + let last = std::cell::Cell::new(before); + let (remaining, batches) = self + .drain( + before.pending, + async || Ok(insert.clone().execute().await?), + async || { + last.set(self.pre_usdc_counts(xlm).await?); + Ok(last.get().pending) + }, + "pre-2021 XLM/USD step stalled — rows still differ", + ) + .await?; + let stats = pre_usdc_stats(before, last.get(), remaining, batches, self.cfg.one_shot)?; + Ok(ChPassStats { + duration_ms: start.elapsed().as_millis() as u64, + ..stats + }) + } + + /// Runs `insert` until `pending` reaches 0 or stops falling (logged as + /// `stalled`), at most `effective_max_batches` times. Returns + /// `(remaining, batches)`. + async fn drain( + &self, + mut remaining: u64, + insert: impl AsyncFn() -> Result<(), ChEnrichError>, + pending: impl AsyncFn() -> Result, + stalled: &str, + ) -> Result<(u64, u32), ChEnrichError> { + let mut batches = 0; + for _ in 0..self.effective_max_batches() { + if remaining == 0 { + break; + } + insert().await?; + let after = pending().await?; + batches += 1; + if after >= remaining { + info!(remaining = after, table = %self.cfg.table, "{stalled}"); + return Ok((after, batches)); + } + remaining = after; + } + Ok((remaining, batches)) + } + /// Tier 2 — the peg-pivot deep-history backbone, over the fixed `watermark` /// snapshot. Each per-batch pivot computes its volume-weighted reference/USDC /// rate **inline** ([`pivot_sql`]) rather than from a pre-materialized table, @@ -2066,8 +2198,8 @@ impl ChEnrichmentPass { &self, usdc_id: u64, watermark: u32, - mut remaining: u64, - mut batches: u32, + remaining: u64, + batches: u32, ) -> Result<(u64, u32), ChEnrichError> { let sql = external_sql( &self.cfg.database, @@ -2076,31 +2208,17 @@ impl ChEnrichmentPass { &self.window_pred("timestamp"), ); let stale = external_window_s(&self.cfg.table); - for _ in 0..self.effective_max_batches() { - if remaining == 0 { - break; - } - self.client - .query(&sql) - .bind(watermark) - .bind(stale) - .bind(self.cfg.batch_size) - .execute() - .await?; - let after = self.count_candidates(watermark).await?; - batches += 1; - if after >= remaining { - info!( - remaining = after, - table = %self.cfg.table, - "external tier drained — remaining candles have no imported USD rate for their bucket" - ); - remaining = after; - break; - } - remaining = after; - } - Ok((remaining, batches)) + let insert = self.client.query(&sql).bind(watermark).bind(stale); + let insert = insert.bind(self.cfg.batch_size); + let (remaining, n) = self + .drain( + remaining, + async || Ok(insert.clone().execute().await?), + async || self.count_candidates(watermark).await, + "external tier drained — remaining candles have no imported USD rate for their bucket", + ) + .await?; + Ok((remaining, batches + n)) } /// Returns the updated `(remaining, batches)`. @@ -2174,6 +2292,11 @@ impl ChEnrichmentPass { /// pin the boundary and assert that candles newer than the snapshot are /// deferred, not enriched, in the current pass. pub async fn run_through(&self, watermark: u32) -> Result { + if self.cfg.pre_usdc_xlm_rate.is_some() { + return self + .run_pre_usdc(self.assert_pre_usdc_admissible().await?) + .await; + } let start = std::time::Instant::now(); // Task 0182 — re-open already-written USD values so the tiers below can @@ -2597,6 +2720,55 @@ pub fn external_window_s(table: &str) -> u32 { bucket_width_s(table).max(86_400) } +/// The pre-2021 step's stats from its counts before and after the drain: +/// `rows_reset` is the priced rows actually set back to 0. A one-shot run that +/// leaves rows still differing is an error, because the mode can discard +/// values and a partial run must not read as a finished one. +fn pre_usdc_stats( + before: PreUsdcCounts, + after: PreUsdcCounts, + remaining: u64, + batches: u32, + one_shot: bool, +) -> Result { + let reset = before.unprice.saturating_sub(after.unprice); + if one_shot && remaining > 0 { + return Err(ChEnrichError::PreUsdcXlmRateStalled { remaining, reset }); + } + Ok(ChPassStats { + batches, + candidates_before: before.pending, + candidates_after: after.unpriced, + rows_enriched: before.pending.saturating_sub(remaining), + rows_reset: reset, + ..Default::default() + }) +} + +/// `usd_rate` identity `(asset_kind, asset_code, issuer_address, contract_address)` +/// and the `reference_asset` to require, `""` for any. +const USDC_RATE_IDENTITY: [&str; 5] = ["credit", "USDC", USDC_ISSUER, "", ""]; +const XLM_RATE_IDENTITY: [&str; 5] = ["native", "XLM", "", "", prices_clickhouse::XLM_RATE_SOURCE]; + +/// The imported (`method = 'external'`) rate series of one `usd_rate` identity, +/// as an ASOF right side keyed `k = 1`. Equality on `method`: the staged +/// `external-candidate` rows share its prefix (`external_rate.rs`). +fn usd_rate_series_sql(db: &str, id: [&str; 5]) -> String { + let [kind, code, issuer, contract, source] = id; + let source = if source.is_empty() { + String::new() + } else { + format!(" AND reference_asset = '{source}'") + }; + format!( + "SELECT 1 AS k, timestamp AS rts, usd_rate AS usd \ + FROM {db}.usd_rate FINAL \ + WHERE asset_kind = '{kind}' AND asset_code = '{code}' \ + AND issuer_address = '{issuer}' AND contract_address = '{contract}' \ + AND method = 'external'{source} AND usd_rate > 0" + ) +} + /// External statement (task 0268): a USDC-quoted candle the oracle tier left at /// `close_usd = 0` is priced from the MEASURED USDC/USD rate task 0267 imports /// into `prices.usd_rate`, instead of falling through to the peg tier's `× $1`. @@ -2702,6 +2874,7 @@ pub fn external_window_s(table: &str) -> u32 { /// `external_tier_never_overwrites_a_candle_the_oracle_tier_priced` proves the /// `close_usd` outcome end to end. fn external_sql(db: &str, tbl: &str, usdc_id: u64, window: &str) -> String { + let series = usd_rate_series_sql(db, USDC_RATE_IDENTITY); let bend = bucket_end_expr(tbl, "timestamp"); let epoch = prices_clickhouse::USDC_ORACLE_EPOCH_S; let columns = insert_columns(); @@ -2732,13 +2905,7 @@ fn external_sql(db: &str, tbl: &str, usdc_id: u64, window: &str) -> String { AND timestamp < toDateTime({epoch}) \ AND timestamp <= toDateTime(?){window} \ ) AS p \ - ASOF LEFT JOIN ( \ - SELECT 1 AS k, timestamp AS rts, usd_rate AS usd \ - FROM {db}.usd_rate FINAL \ - WHERE asset_kind = 'credit' AND asset_code = 'USDC' \ - AND issuer_address = '{USDC_ISSUER}' AND contract_address = '' \ - AND method = 'external' AND usd_rate > 0 \ - ) AS r \ + ASOF LEFT JOIN ( {series} ) AS r \ ON r.k = p.k AND r.rts < p.bend \ WHERE r.usd > 0 \ AND (toUInt32(p.bend) - toUInt32(r.rts)) <= ? \ @@ -2747,6 +2914,174 @@ fn external_sql(db: &str, tbl: &str, usdc_id: u64, window: &str) -> String { ) } +/// `prices.assets` rows of a dollar token other than canonical USDC (task 0331). +/// Keyed on code/issuer/contract, never `asset_type` (fixtures disagree on it). +fn dollar_token_asset_pred() -> String { + format!( + "{DOLLAR_TOKEN_PRED} AND NOT (asset_code = 'USDC' AND issuer_address = '{USDC_ISSUER}')" + ) +} + +/// Rows the pre-2021 XLM/USD step may write: before [`PRE_USDC_END_S`], quoted +/// in native XLM or a dollar token. Uncorrelated, so the driver's month +/// enumeration, the statement and its pending count share this one string. +/// +/// [`PRE_USDC_END_S`]: prices_clickhouse::PRE_USDC_END_S +pub fn pre_usdc_scope_pred(db: &str) -> String { + format!( + "timestamp < toDateTime({end}) AND volume_quote > 0 \ + AND quote_asset_id IN (SELECT asset_id FROM {db}.assets FINAL \ + WHERE (asset_code = 'XLM' AND issuer_address = '' AND contract_address = '') \ + OR ({dollar}))", + end = prices_clickhouse::PRE_USDC_END_S, + dollar = dollar_token_asset_pred(), + ) +} + +/// Pre-2021 step (task 0331). Up to `_1h`, `close_usd = close × f × rate`: +/// `rate` is the newest `external` XLM/USD row before the bucket end (within +/// [`external_window_s`]); `f` is 1 on an XLM quote, else the dollar token's +/// XLM price from its own XLM markets (both orientations, pf columns) at or +/// before the bucket start, within `pivot_window_s`. Wider tiers roll up their +/// `price_ohlcv_1h` rows as `preroll.sql` does (Σ USD volume, `close` × the +/// last priced hour's `close_usd / close`), so `_1h` must be priced first. +/// +/// Writes only rows whose computed pair differs from the stored one, so the +/// batch loop terminates, a re-run writes nothing, and a newer rate or a +/// narrower window re-prices (an unreachable row computes 0). +/// +/// Returns `(insert, counts)` over one body; the insert binds the `LIMIT`. +fn pre_usdc_sql( + db: &str, + tbl: &str, + xlm: u64, + pivot_window_s: u32, + window: Option<(u32, u32)>, +) -> (String, String) { + let end = prices_clickhouse::PRE_USDC_END_S; + let bend = bucket_end_expr(tbl, "timestamp"); + // `[s - back, upper(e))` of the driver's month window `[s, e)`, if any. + let span = |back: u32, upper: &dyn Fn(String) -> String| { + window.map_or(String::new(), |(s, e)| { + let (s, e) = (s.saturating_sub(back), upper(format!("toDateTime({e})"))); + format!(" AND timestamp >= toDateTime({s}) AND timestamp < {e}") + }) + }; + let scope = pre_usdc_scope_pred(db); + let parents = format!("FROM {db}.{tbl} FINAL WHERE {scope}{}", span(0, &|e| e)); + let cols = prices_clickhouse::CANDLE_COLUMNS; + let list = |f: &dyn Fn(&str) -> String| cols.iter().map(|c| f(c)).collect::>(); + let all = cols.join(", "); + let from = |a: &str| list(&|c| format!("{a}.{c} AS {c}")).join(", "); + // Float64, then a NULL-on-overflow parse: a Decimal product wraps silently + // and an out-of-range CAST aborts the batch. + let dec = + |f: String| format!("ifNull(toDecimal128OrNull(toString({f}), 14), toDecimal128(0, 14))"); + let inner = if bucket_width_s(tbl) > 3_600 { + let tiny = "toDecimal128('0.000000000001', 14)"; + format!( + "SELECT {p}, {cu} AS new_cu, {vqu} AS new_vqu \ + FROM (SELECT {all} {parents}) AS p \ + LEFT JOIN ( \ + SELECT b.asset_id AS a, b.quote_asset_id AS q, b.source AS s, b.timestamp AS ts, \ + sum(toDecimal256(h.volume_quote_usd, 14)) AS vqu, \ + argMaxIf(toFloat64(h.close_usd) / toFloat64(h.close), h.timestamp, \ + h.close_usd >= {tiny} AND h.close >= {tiny}) AS ratio \ + FROM ( \ + SELECT asset_id, quote_asset_id, source, timestamp, close, close_usd, \ + volume_quote_usd \ + FROM {db}.price_ohlcv_1h FINAL \ + WHERE timestamp < toDateTime({end}){hours} \ + ) AS h \ + ASOF JOIN ( \ + SELECT asset_id, quote_asset_id, source, timestamp, {bend} AS bend {parents} \ + ) AS b \ + ON h.asset_id = b.asset_id AND h.quote_asset_id = b.quote_asset_id \ + AND h.source = b.source AND h.timestamp >= b.timestamp \ + WHERE h.timestamp < b.bend \ + GROUP BY a, q, s, ts \ + ) AS c \ + ON c.a = p.asset_id AND c.q = p.quote_asset_id AND c.s = p.source \ + AND c.ts = p.timestamp", + p = from("p"), + cu = dec("toFloat64(p.close) * c.ratio".into()), + // Σ in Decimal256, parsed back: a Decimal(38, 14) sum wraps silently. + vqu = dec("c.vqu".into()), + hours = span(0, &|e| bucket_end_expr(tbl, &e)), + ) + } else { + let dollar = format!( + "SELECT asset_id FROM {db}.assets FINAL WHERE {}", + dollar_token_asset_pred() + ); + // Decimal256 sums: a Decimal(38, 14) sum wraps silently, a Float64 one + // depends on merge order and would make the differs filter flap. + let amt = |a: &str, b: &str| { + format!("toFloat64(sum(toDecimal256(if(asset_id = {xlm}, {a}, {b}), 14)))") + }; + let txlm = format!( + "{} / {}", + amt("pf_volume", "pf_price_volume"), + amt("pf_price_volume", "pf_volume") + ); + let xr = format!( + "if(r.usd > 0 AND (toUInt32(pt.bend) - toUInt32(r.rts)) <= {}, toFloat64(r.usd), 0.)", + external_window_s(tbl) + ); + let usd = |col: &str| dec(format!("toFloat64(pt.{col}) * {xr} * pt.xf")); + format!( + "SELECT {pt}, {cu} AS new_cu, {vqu} AS new_vqu \ + FROM ( \ + SELECT {p}, p.k AS k, p.bend AS bend, \ + if(p.quote_asset_id = {xlm}, 1., if(t.txlm > 0 \ + AND (toUInt32(p.timestamp) - toUInt32(t.tts)) <= {pivot_window_s}, \ + t.txlm, 0.)) AS xf \ + FROM (SELECT {all}, 1 AS k, {bend} AS bend {parents}) AS p \ + ASOF LEFT JOIN ( \ + SELECT if(asset_id = {xlm}, quote_asset_id, asset_id) AS tok, \ + timestamp AS tts, {txlm} AS txlm \ + FROM {db}.{tbl} FINAL \ + WHERE timestamp < toDateTime({end}){refs} \ + AND pf_trade_count > 0 AND pf_volume > 0 AND pf_price_volume > 0 \ + AND ((quote_asset_id = {xlm} AND asset_id IN ({dollar})) \ + OR (asset_id = {xlm} AND quote_asset_id IN ({dollar}))) \ + GROUP BY tok, tts \ + ) AS t \ + ON t.tok = p.quote_asset_id AND t.tts <= p.timestamp \ + ) AS pt \ + ASOF LEFT JOIN ( {series} ) AS r \ + ON r.k = pt.k AND r.rts < pt.bend", + pt = from("pt"), + p = from("p"), + cu = usd("close"), + vqu = usd("volume_quote"), + // The token-market scan: the month plus its look-back. + refs = span(pivot_window_s, &|e| e), + series = usd_rate_series_sql(db, XLM_RATE_IDENTITY), + ) + }; + let differs = "x.close_usd != x.new_cu OR x.volume_quote_usd != x.new_vqu"; + let out = list(&|c| match c { + "close_usd" => "x.new_cu AS close_usd".into(), + "volume_quote_usd" => "x.new_vqu AS volume_quote_usd".into(), + "version" => "x.version + 1 AS version".into(), + c => format!("x.{c}"), + }) + .join(", "); + ( + format!( + "INSERT INTO {db}.{tbl} ({}) SELECT {out} FROM ({inner}) AS x \ + WHERE {differs} ORDER BY x.timestamp LIMIT ?", + insert_columns() + ), + format!( + "SELECT countIf({differs}) AS pending, \ + countIf(x.close_usd > 0 AND x.new_cu = 0) AS unprice, \ + countIf(x.new_cu = 0 AND x.close > 0) AS unpriced FROM ({inner}) AS x" + ), + ) +} + /// The count the 0268 reset mode refuses on: canonical USDC readings in /// `oracle_prices` — the table the oracle tier READS, not `usd_rate` — stamped /// below `USDC_ORACLE_EPOCH_S`. Pure, so the string is unit-testable; bound @@ -3072,6 +3407,17 @@ fn pivot_sql(db: &str, tbl: &str, ref_id: u64, usdc_id: u64, window: &str) -> St mod tests { use super::*; + /// lore-0331 moved the code list into `DOLLAR_TOKEN_PRED`; the text is unchanged. + #[test] + fn dollar_token_asset_pred_renders_the_pre_0331_text() { + let want = format!( + "contract_address = '' AND issuer_address != '' \ + AND asset_code IN ('USD', 'USDT', 'USDC') \ + AND NOT (asset_code = 'USDC' AND issuer_address = '{USDC_ISSUER}')" + ); + assert_eq!(dollar_token_asset_pred(), want); + } + #[test] fn peg_sql_is_none_without_stablecoins() { assert!(peg_sql("prices", "price_ohlcv_1m", &[], "").is_none()); @@ -4191,16 +4537,20 @@ mod tests { #[test] fn every_timezone_sensitive_expression_pins_utc() { for t in ["price_ohlcv_1d", "price_ohlcv_1w", "price_ohlcv_1M"] { - let sql = external_sql("prices", t, 3, ""); - for f in ["addDays(", "addWeeks(", "addMonths(", "toDate("] { - for (i, _) in sql.match_indices(f) { - let tail = &sql[i..]; - let close = tail.find(')').unwrap(); - assert!( - tail[..close].ends_with("'UTC'"), - "{t}: {f} without an explicit 'UTC': {}", - &tail[..close + 1] - ); + for sql in [ + external_sql("prices", t, 3, ""), + pre_usdc_sql("prices", t, 9, 86_400, None).0, + ] { + for f in ["addDays(", "addWeeks(", "addMonths(", "toDate("] { + for (i, _) in sql.match_indices(f) { + let tail = &sql[i..]; + let close = tail.find(')').unwrap(); + assert!( + tail[..close].ends_with("'UTC'"), + "{t}: {f} without an explicit 'UTC': {}", + &tail[..close + 1] + ); + } } } } @@ -4710,6 +5060,14 @@ mod tests { reset_sql("prices", "price_ohlcv_1m", &spec, ""), ), ("pivot_sql", pivot_sql("prices", "price_ohlcv_1m", 5, 3, "")), + ( + "pre_usdc_sql", + pre_usdc_sql("prices", "price_ohlcv_1m", 9, 86_400, None).0, + ), + ( + "pre_usdc_sql rollup", + pre_usdc_sql("prices", "price_ohlcv_1d", 9, 86_400, None).0, + ), ] } @@ -5383,4 +5741,83 @@ mod tests { "an unresolved USDC must be refused by the epoch guard itself, got {res:?}" ); } + + #[test] + fn external_sql_reads_the_usdc_external_series() { + let want = format!( + "ASOF LEFT JOIN ( SELECT 1 AS k, timestamp AS rts, usd_rate AS usd \ + FROM prices.usd_rate FINAL WHERE asset_kind = 'credit' AND asset_code = 'USDC' \ + AND issuer_address = '{USDC_ISSUER}' AND contract_address = '' \ + AND method = 'external' AND usd_rate > 0 ) AS r" + ); + assert!(external_sql("prices", "price_ohlcv_1h", 3, "").contains(&want)); + } + + /// (remaining, one_shot, before unprice, after unprice, Ok(rows_reset) or Err(reset)). + type StatsCase = (u64, bool, u64, u64, Result); + + #[test] + fn pre_usdc_stats_reports_resets_written_and_refuses_a_partial_run() { + let counts = |unprice| PreUsdcCounts { + pending: 10, + unprice, + unpriced: 4, + }; + #[rustfmt::skip] + let cases: [StatsCase; 4] = [ + (0, true, 3, 0, Ok(3)), + (0, true, 3, 3, Ok(0)), + (2, false, 3, 1, Ok(2)), + (2, true, 5, 1, Err(4)), + ]; + for (remaining, one_shot, b, a, want) in cases { + let got = pre_usdc_stats(counts(b), counts(a), remaining, 1, one_shot); + match (got, want) { + (Ok(s), Ok(reset)) => { + assert_eq!(s.rows_reset, reset); + assert_eq!(s.rows_enriched, 10 - remaining); + assert_eq!((s.candidates_before, s.candidates_after), (10, 4)); + } + ( + Err(ChEnrichError::PreUsdcXlmRateStalled { + remaining: r, + reset, + }), + Err(want), + ) => { + assert_eq!((r, reset), (remaining, want)); + } + (got, want) => { + panic!("remaining={remaining} one_shot={one_shot}: {got:?} vs {want:?}") + } + } + } + } + + #[test] + fn pre_usdc_sql_renders_its_contract() { + let end = format!( + "timestamp < toDateTime({})", + prices_clickhouse::PRE_USDC_END_S + ); + let month = " AND timestamp >= toDateTime(5) AND timestamp < toDateTime(9)"; + for t in GRAINS { + for w in [None, Some((5, 9))] { + let (ins, cnt) = pre_usdc_sql("prices", t, 9, 86_400, w); + let rollup = bucket_width_s(t) > 3_600; + for needle in ["method = ", "reference_asset = 'third-party'"] { + assert_eq!(ins.matches(needle).count(), usize::from(!rollup), "{ins}"); + } + assert_eq!(ins.contains("argMaxIf("), rollup, "{t}"); + for needle in ["LIKE", "startsWith", "candidate"] { + assert!(!ins.contains(needle), "{t}: {needle}"); + } + assert!(ins.matches(&end).count() >= 2, "{ins}"); + assert_eq!(ins.contains(month), w.is_some(), "{t}"); + assert_eq!((ins.matches('?').count(), cnt.matches('?').count()), (1, 0)); + let inner = &cnt[cnt.find("AS unpriced FROM (").unwrap() + 18..cnt.len() - 6]; + assert!(ins.contains(&format!("FROM ({inner}) AS x WHERE")), "{t}"); + } + } + } } diff --git a/packages/enrichment-worker/src/main.rs b/packages/enrichment-worker/src/main.rs index c9392813..8fc46ac8 100644 --- a/packages/enrichment-worker/src/main.rs +++ b/packages/enrichment-worker/src/main.rs @@ -69,6 +69,8 @@ async fn main() -> Result<(), lambda_runtime::Error> { // would re-zero and recompute the same rows forever. Deliberately not // env-driven — there is no ENRICHMENT_USD_RESET. usd_reset: None, + // Opt-in coarse-repair mode only (task 0331). + pre_usdc_xlm_rate: None, }; tracing::info!( diff --git a/packages/enrichment-worker/src/repair.rs b/packages/enrichment-worker/src/repair.rs index 7f03e481..c9461c7d 100644 --- a/packages/enrichment-worker/src/repair.rs +++ b/packages/enrichment-worker/src/repair.rs @@ -32,7 +32,9 @@ use clickhouse::{Client, Row}; use serde::{Deserialize, Serialize}; use tracing::{debug, info, warn}; -use crate::ch_enrich::{ChEnrichConfig, ChEnrichError, ChEnrichmentPass, repair_target_pred}; +use crate::ch_enrich::{ + ChEnrichConfig, ChEnrichError, ChEnrichmentPass, pre_usdc_scope_pred, repair_target_pred, +}; /// One coarse-repair run over `[start_month, end_month]` (inclusive `YYYYMM`). #[derive(Debug, Clone)] @@ -206,6 +208,12 @@ impl CoarseRepairDriver { /// the month's first instant; `addMonths(..., 1)` is the exclusive upper /// bound. Both cast to `UInt32` to match [`MonthWindow`]. async fn months_with_zeros(&self) -> Result, ChEnrichError> { + let e = &self.cfg.enrich; + let pred = if e.pre_usdc_xlm_rate.is_some() { + pre_usdc_scope_pred(&e.database) + } else { + repair_target_pred(&e.database, &e.table, e.usd_reset.as_ref()) + }; let sql = format!( "SELECT toYYYYMM(timestamp) AS month, \ toUInt32(toUnixTimestamp(toDateTime(min(toStartOfMonth(timestamp))))) AS start_ts, \ @@ -217,11 +225,6 @@ impl CoarseRepairDriver { GROUP BY month ORDER BY month", db = self.cfg.enrich.database, tbl = self.cfg.enrich.table, - pred = repair_target_pred( - &self.cfg.enrich.database, - &self.cfg.enrich.table, - self.cfg.enrich.usd_reset.as_ref(), - ), start = self.cfg.start_month, end = self.cfg.end_month, ); @@ -273,9 +276,12 @@ impl CoarseRepairDriver { // green having touched nothing. Found by the 0228 prove run and its two // review rounds. `reset_step` calls the SAME method — one list, not two // that must be kept in step by hand. - ChEnrichmentPass::with_client(self.client.clone(), self.cfg.enrich.clone()) - .assert_reset_is_admissible() - .await?; + let pass = ChEnrichmentPass::with_client(self.client.clone(), self.cfg.enrich.clone()); + // Task 0331: its refusals, and native XLM resolved once for every month. + let xlm = match self.cfg.enrich.pre_usdc_xlm_rate { + Some(_) => Some(pass.assert_pre_usdc_admissible().await?), + None => pass.assert_reset_is_admissible().await.map(|()| None)?, + }; let months = self.months_with_zeros().await?; info!( count = months.len(), @@ -302,18 +308,37 @@ impl CoarseRepairDriver { break; } - if self.cfg.dry_run { - info!( - month = mw.month, - zeros = mw.zeros, - "coarse repair: DRY RUN — would repair" - ); + // Partition-bounded pass over this one month. `one_shot` decides + // whether it drains the month fully (manual repair) or a bounded + // `max_batches` chunk with overflow deferring to the next run (the + // recurring hourly sweep) — see [`CoarseRepairConfig::one_shot`]. + let mut ecfg = self.cfg.enrich.clone(); + ecfg.time_window = Some((mw.start_ts, mw.end_ts)); + ecfg.one_shot = self.cfg.one_shot; + let pass = ChEnrichmentPass::with_client(self.client.clone(), ecfg); + // Task 0331: rows to write, rows left unpriced, priced rows set to 0. + let counts = match xlm { + Some(x) => Some(pass.pre_usdc_counts(x).await?), + None => None, + }; + let (zeros, after, reset) = counts.map_or((mw.zeros, mw.zeros, 0), |c| { + (c.pending, c.unpriced, c.unprice) + }); + + // A 0331 month with nothing to write gets no FREEZE either. + if self.cfg.dry_run || zeros == 0 { + let what = if zeros == 0 { + "nothing to write" + } else { + "DRY RUN — would repair" + }; + info!(month = mw.month, zeros, "coarse repair: {what}"); summary.months.push(MonthRepair { month: mw.month, - zeros_before: mw.zeros, - zeros_after: mw.zeros, + zeros_before: zeros, + zeros_after: after, rows_enriched: 0, - rows_reset: 0, + rows_reset: reset, snapshot_name: None, }); continue; @@ -334,16 +359,10 @@ impl CoarseRepairDriver { None }; - // Partition-bounded pass over this one month. `one_shot` decides - // whether it drains the month fully (manual repair) or a bounded - // `max_batches` chunk with overflow deferring to the next run (the - // recurring hourly sweep) — see [`CoarseRepairConfig::one_shot`]. - let mut ecfg = self.cfg.enrich.clone(); - ecfg.time_window = Some((mw.start_ts, mw.end_ts)); - ecfg.one_shot = self.cfg.one_shot; - let stats = ChEnrichmentPass::with_client(self.client.clone(), ecfg) - .run() - .await?; + let stats = match xlm { + Some(x) => pass.run_pre_usdc(x).await?, + None => pass.run().await?, + }; info!( month = mw.month, @@ -363,7 +382,8 @@ impl CoarseRepairDriver { }); // Defensive cross-check: the enumeration count and the pass's own // pre-count should agree (both are the month's zero population). - if mw.zeros != stats.candidates_before { + // In the 0331 mode the pass counts rows to write, not scope rows. + if xlm.is_none() && mw.zeros != stats.candidates_before { warn!( month = mw.month, enumerated = mw.zeros, diff --git a/packages/enrichment-worker/tests/ch_enrich_it.rs b/packages/enrichment-worker/tests/ch_enrich_it.rs index ac994a4a..94d24c97 100644 --- a/packages/enrichment-worker/tests/ch_enrich_it.rs +++ b/packages/enrichment-worker/tests/ch_enrich_it.rs @@ -9,12 +9,16 @@ //! at the end, so they never touch `prices` and can run in parallel. use clickhouse::Client; -use enrichment_worker::ch_enrich::{ChEnrichConfig, ChEnrichError, ChEnrichmentPass, UsdResetSpec}; +use enrichment_worker::ch_enrich::{ + ChEnrichConfig, ChEnrichError, ChEnrichmentPass, PreUsdcXlmRate, UsdResetSpec, +}; use enrichment_worker::repair::{ - CoarseRepairConfig, CoarseRepairDriver, CoarseSweepConfig, run_coarse_sweep, + CoarseRepairConfig, CoarseRepairDriver, CoarseSweepConfig, MonthRepair, run_coarse_sweep, }; use prices_clickhouse::asset_id::fixture::{AssetFixture, assets_insert, fetch_ids}; -use prices_clickhouse::{USDC_ISSUER, USDC_ORACLE_EPOCH_S, USDT_ISSUER}; +use prices_clickhouse::{ + PRE_USDC_END_S, USDC_ISSUER, USDC_ORACLE_EPOCH_S, USDT_ISSUER, XLM_RATE_SOURCE, +}; use std::fmt::Display; // The fixture assets. Each displays as its id, derived from its identity @@ -159,6 +163,7 @@ fn cfg(db: &str) -> ChEnrichConfig { one_shot: false, time_window: None, usd_reset: None, + pre_usdc_xlm_rate: None, } } @@ -2210,17 +2215,24 @@ fn utc_day_start(ts: u32) -> u32 { ts - ts % 86_400 } -/// Seed one `usd_rate` row for canonical USDC under an explicit `method`, so a -/// test can stage BOTH series for the same bucket and prove which one wins. -/// [`seed_external_rate`] is the `'external'` case and carries the day-start +/// Seed one `usd_rate` row of identity `(asset_kind, asset_code, issuer_address)` +/// and `reference_asset` under an explicit `method`, so a test can stage BOTH +/// series for the same bucket and prove which one wins. [`seed_external_rate`] +/// is the canonical USDC `'external'` case and carries the day-start /// convention note. -async fn seed_usd_rate(client: &Client, db: &str, ts: u32, rate: f64, method: &str) { +async fn seed_usd_rate( + client: &Client, + db: &str, + (kind, code, issuer, source): (&str, &str, &str, &str), + (ts, rate, method): (u32, f64, &str), + version: u64, +) { client .query(&format!( "INSERT INTO {db}.usd_rate \ (asset_kind, asset_code, issuer_address, contract_address, \ timestamp, usd_rate, method, reference_asset, hops, version) VALUES \ - ('credit', 'USDC', '{USDC_ISSUER}', '', {ts}, {rate}, '{method}', '', 0, 1)" + ('{kind}', '{code}', '{issuer}', '', {ts}, {rate}, '{method}', '{source}', 0, {version})" )) .execute() .await @@ -3185,7 +3197,8 @@ async fn a_pivot_leg_prefers_the_oracle_rate_over_the_external_rate() { let db = "it_enrich_0228_oracle_wins"; let client = setup_0228_pivot(db, "price_ohlcv_1d", DEPEG_DAY).await; // The oracle says 0.95 at day start; the import says 0.9681 an hour LATER. - seed_usd_rate(&client, db, DEPEG_DAY, 0.95, "oracle").await; + let usdc = ("credit", "USDC", USDC_ISSUER, ""); + seed_usd_rate(&client, db, usdc, (DEPEG_DAY, 0.95, "oracle"), 1).await; seed_external_rate(&client, db, DEPEG_DAY + 3_600, DEPEG_RATE).await; let mut c = cfg(db); @@ -5708,3 +5721,439 @@ async fn the_epoch_is_measured_on_the_pass_s_own_table() { .await .unwrap(); } + +// --- Task 0331: pre-2021 candles priced from the hourly XLM/USD rate --- + +const USDX: AssetFixture = AssetFixture::new("USD", "classic", "GUSDX", ""); +/// 2020-06-01 00:00 UTC, a Monday. +const D0331: u32 = 1_590_969_600; +const H1: &str = "price_ohlcv_1h"; +const WIDE: [&str; 4] = [ + "price_ohlcv_4h", + "price_ohlcv_1d", + "price_ohlcv_1w", + "price_ohlcv_1M", +]; +const END: u32 = PRE_USDC_END_S; + +const fn h(n: u32) -> u32 { + D0331 + n * 3600 +} + +/// (ts, rate, method) of `('native','XLM','')` rows in `usd_rate`. +type Rate = (u32, f64, &'static str); +/// (label, table, ts, asset, quote, close, volume_base, volume_quote, close_usd, volume_quote_usd) +#[rustfmt::skip] +type Leg = (&'static str, &'static str, u32, AssetFixture<'static>, AssetFixture<'static>, f64, f64, f64, f64, f64); + +const XLM_RATE: (&str, &str, &str, &str) = ("native", "XLM", "", XLM_RATE_SOURCE); +/// An `external` XLM row from another source, which no 0331 reader may take: +/// at h(30) it would price the h(48) and h(53) legs. +const OTHER_XLM_RATE: (&str, &str, &str, &str) = ("native", "XLM", "", "other-source"); +const RATES_0331: &[Rate] = &[ + (h(10), 0.25, "external"), + (h(23), 0.20, "external"), + (h(24), 0.50, "external"), + (h(29), 0.40, "external"), + (h(11), 9.0, "external-candidate"), + (h(81), 0.25, "external"), + (END - 3600, 0.30, "external"), +]; + +// No dollar-quoted expectation equals par: a token is 5 XLM via USDT/XLM, 3 via +// XLM/USDX and 4.5 where both USDT orientations pool at h(13). Wide tiers are +// Σ of their hours' USD volume and close × the last priced hour's ratio. +#[rustfmt::skip] +const LEGS_0331: &[Leg] = &[ + ("FOO/XLM 1h", H1, h(10), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), + ("FOO/XLM ignores the candidate's 9.0", H1, h(11), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), + ("FOO/XLM 23:00 takes 23:00, not midnight", H1, h(23), FOO, XLM, 10.0, 3.0, 30.0, 2.0, 6.0), + ("FOO/XLM, rate 20 h old", H1, h(48), FOO, XLM, 10.0, 3.0, 30.0, 4.0, 12.0), + ("FOO/XLM, newest rate 25 h old", H1, h(53), FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), + ("FOO/XLM, rate 2 h old", H1, h(82), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), + ("USDT/XLM", H1, h(10), USDT, XLM, 5.0, 10.0, 50.0, 1.25, 12.5), + ("XLM/USDX", H1, h(10), XLM, USDX, 0.4, 30.0, 10.0, 0.3, 7.5), + ("FOO/USDT via USDT/XLM", H1, h(10), FOO, USDT, 2.0, 3.0, 6.0, 2.5, 7.5), + ("FOO/USDX via XLM/USDX", H1, h(10), FOO, USDX, 2.0, 3.0, 6.0, 1.5, 4.5), + ("FOO/USDT, market 2 h old", H1, h(12), FOO, USDT, 2.0, 3.0, 6.0, 2.5, 7.5), + ("USDT/XLM pooled", H1, h(13), USDT, XLM, 5.0, 10.0, 50.0, 1.25, 12.5), + ("XLM/USDT pooled: XLM's own USD candle", H1, h(13), XLM, USDT, 0.24, 40.0, 10.0, 0.27, 11.25), + ("FOO/USDT via the pooled market", H1, h(13), FOO, USDT, 2.0, 3.0, 6.0, 2.25, 6.75), + ("FOO/USDT, market 69 h old", H1, h(82), FOO, USDT, 2.0, 3.0, 6.0, 0.0, 0.0), + ("FOO/XLM 2021-01-31 23:00", H1, END - 3600, FOO, XLM, 10.0, 3.0, 30.0, 3.0, 9.0), + ("FOO/XLM at 2021-02-01", H1, END, FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), + ("XLM/USDC (canonical)", H1, h(10), XLM, USDC, 0.25, 40.0, 10.0, 0.0, 0.0), + ("FOO/USDC (canonical)", H1, h(10), FOO, USDC, 4.0, 1.0, 4.0, 0.0, 0.0), + ("4h FOO/XLM", WIDE[0], h(8), FOO, XLM, 10.0, 6.0, 60.0, 2.5, 15.0), + ("1d FOO/XLM, hours at 0.25 and 0.20", WIDE[1], D0331, FOO, XLM, 10.0, 9.0, 90.0, 2.0, 21.0), + ("1d FOO/XLM, last hour unpriced", WIDE[1], h(48), FOO, XLM, 10.0, 6.0, 60.0, 4.0, 12.0), + ("1d FOO/USDT", WIDE[1], D0331, FOO, USDT, 2.0, 9.0, 18.0, 2.25, 21.75), + ("1w FOO/XLM, hours at three rates", WIDE[2], D0331, FOO, XLM, 10.0, 18.0, 180.0, 2.5, 40.5), + ("1w FOO/XLM ending at 2021-02-01", WIDE[2], END - 7 * 86_400, FOO, XLM, 10.0, 3.0, 30.0, 3.0, 9.0), + ("1M FOO/XLM 2021-01", WIDE[3], 1_609_459_200, FOO, XLM, 10.0, 3.0, 30.0, 3.0, 9.0), + ("1w FOO/XLM at 2021-02-01", WIDE[2], END, FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), +]; + +async fn seed_leg(client: &Client, db: &str, leg: &Leg) { + let (_, table, ts, asset, quote, close, vb, vq, _, _) = *leg; + client + .query(&format!( + "INSERT INTO {db}.{table} \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + VALUES ({ts}, {asset}, {quote}, 'sdex', {close}, {close}, {close}, {close}, \ + {vb}, {vq}, 0, 0, {close}, 1, 1)" + )) + .execute() + .await + .unwrap(); +} + +/// (close_usd, volume_quote_usd, version) of a leg, read FINAL. +async fn usd_of(client: &Client, db: &str, leg: &Leg) -> (f64, f64, u64) { + let (_, table, ts, asset, quote, ..) = *leg; + client + .query(&format!( + "SELECT toFloat64(close_usd), toFloat64(volume_quote_usd), version \ + FROM {db}.{table} FINAL \ + WHERE asset_id = {asset} AND quote_asset_id = {quote} AND timestamp = {ts}" + )) + .fetch_one::<(f64, f64, u64)>() + .await + .unwrap() +} + +async fn setup_0331(db: &str, rates: &[Rate], legs: &[Leg]) -> Client { + let client = setup_scratch(db).await; + client + .query(&assets_insert(db, &[XLM, USDC, USDT, FOO, USDX])) + .execute() + .await + .unwrap(); + for r in rates { + seed_usd_rate(&client, db, XLM_RATE, *r, 1).await; + } + seed_usd_rate(&client, db, OTHER_XLM_RATE, (h(30), 9.0, "external"), 1).await; + for l in legs { + seed_leg(&client, db, l).await; + } + client +} + +fn cfg_0331(db: &str, table: &str) -> ChEnrichConfig { + ChEnrichConfig { + table: table.to_string(), + one_shot: true, + pre_usdc_xlm_rate: Some(PreUsdcXlmRate::default()), + ..cfg(db) + } +} + +fn written_in(table: &str) -> u64 { + LEGS_0331 + .iter() + .filter(|l| l.1 == table && l.8 != 0.0) + .count() as u64 +} + +async fn assert_legs<'a>( + client: &Client, + db: &str, + legs: impl IntoIterator, + scale: impl Fn(&str) -> f64, +) -> Vec { + let mut versions = Vec::new(); + for leg in legs { + let (cu, vqu, v) = usd_of(client, db, leg).await; + let (want_cu, want_vqu) = (leg.8 * scale(leg.0), leg.9 * scale(leg.0)); + assert!( + (cu - want_cu).abs() < 1e-9 && (vqu - want_vqu).abs() < 1e-9, + "{}: got ({cu}, {vqu}), want ({want_cu}, {want_vqu})", + leg.0 + ); + versions.push(v); + } + versions +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_prices_xlm_and_dollar_quoted_legs() { + let db = "test_ch_enrich_pre_usdc_prices"; + let client = setup_0331(db, RATES_0331, LEGS_0331).await; + let run = async |table: &str| { + let pass = ChEnrichmentPass::new(cfg_0331(db, table)); + pass.run().await.unwrap().rows_enriched + }; + assert_eq!( + run(WIDE[1]).await, + 0, + "a wide tier before its 1h writes nothing" + ); + for table in std::iter::once(H1).chain(WIDE) { + assert_eq!(run(table).await, written_in(table), "{table}"); + } + let versions = assert_legs(&client, db, LEGS_0331, |_| 1.0).await; + // Only written rows bump their version; unreachable and out-of-scope ones stay at 1. + let want: Vec = LEGS_0331 + .iter() + .map(|l| 1 + u64::from(l.8 != 0.0)) + .collect(); + assert_eq!(versions, want); + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_rewrites_only_values_that_differ() { + let db = "test_ch_enrich_pre_usdc_rewrite"; + let client = setup_0331(db, RATES_0331, LEGS_0331).await; + let run = async |pivot_window_s| { + let pass = ChEnrichmentPass::new(ChEnrichConfig { + pivot_window_s, + ..cfg_0331(db, H1) + }); + let stats = pass.run().await.unwrap(); + (stats.rows_enriched, stats.rows_reset) + }; + let h1 = || LEGS_0331.iter().filter(|l| l.1 == H1); + assert_eq!(run(86_400).await, (written_in(H1), 0)); + let first = assert_legs(&client, db, h1(), |_| 1.0).await; + + assert_eq!( + run(86_400).await, + (0, 0), + "a re-run with the same rate writes nothing" + ); + assert_eq!( + assert_legs(&client, db, h1(), |_| 1.0).await, + first, + "and bumps no version" + ); + + // A corrected h(10) rate re-prices exactly the priced 1h legs of h(10)..h(13). + seed_usd_rate(&client, db, XLM_RATE, (h(10), 0.5, "external"), 2).await; + let on_h10: Vec<&str> = h1() + .filter(|l| (h(10)..=h(13)).contains(&l.2) && l.8 != 0.0) + .map(|l| l.0) + .collect(); + assert_eq!(run(86_400).await, (on_h10.len() as u64, 0)); + let doubled = |l: &str| if on_h10.contains(&l) { 2.0 } else { 1.0 }; + assert_legs(&client, db, h1(), doubled).await; + + // A 1 h window can no longer reach the 2 h old USDT market: written back to 0. + assert_eq!(run(3600).await, (1, 1)); + let narrowed = |l: &str| { + if l == "FOO/USDT, market 2 h old" { + 0.0 + } else { + doubled(l) + } + }; + assert_legs(&client, db, h1(), narrowed).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_repair_refuses_unpromoted_rates_and_previews_exactly() { + let driver = |client: &Client, db: &str, dry_run, snapshot| { + CoarseRepairDriver::with_client( + client.clone(), + CoarseRepairConfig { + enrich: cfg_0331(db, H1), + start_month: 202_001, + end_month: 202_103, + snapshot, + dry_run, + one_shot: true, + deadline: None, + }, + ) + }; + let refused = |res: &Result<_, ChEnrichError>, why: &str| matches!(res, Err(ChEnrichError::PreUsdcXlmRateRefused(m)) if m.contains(why)); + // (i) Only a staged row: refused before any write, in a dry run too. + let db = "test_ch_enrich_pre_usdc_refuse"; + let client = setup_0331(db, &[(h(10), 0.25, "external-candidate")], &LEGS_0331[..1]).await; + for dry_run in [true, false] { + let res = driver(&client, db, dry_run, false).run().await.map(|_| ()); + assert!(refused(&res, "promote"), "{res:?}"); + } + assert_eq!(usd_of(&client, db, &LEGS_0331[0]).await.2, 1); + // (ii) Never combined with a reset. + let spec = UsdResetSpec { + quote_asset_id: 1, + not_before: 1, + not_after: None, + require_external_rate: false, + require_pivot_usdc_rate: false, + }; + let res = ChEnrichmentPass::new(ChEnrichConfig { + usd_reset: Some(spec), + ..cfg_0331(db, H1) + }) + .assert_reset_is_admissible() + .await; + assert!(refused(&res, "--reset-*"), "{res:?}"); + // (iii) Native XLM does not resolve. + client + .query(&format!("TRUNCATE TABLE {db}.assets")) + .execute() + .await + .unwrap(); + let res = driver(&client, db, true, false).run().await.map(|_| ()); + assert!(refused(&res, "native XLM"), "{res:?}"); + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); + + // (iv) The dry run reports exactly what the real run then writes; a month + // with nothing to write is skipped, FREEZE included. + let db = "test_ch_enrich_pre_usdc_preview"; + let client = setup_0331(db, RATES_0331, LEGS_0331).await; + // (month, to write / written, left unpriced, set back to 0, frozen) + let months = async |dry_run, snapshot| { + let summary = driver(&client, db, dry_run, snapshot).run().await.unwrap(); + let n = |m: &MonthRepair| { + if dry_run { + m.zeros_before + } else { + m.rows_enriched + } + }; + let row = |m: &MonthRepair| { + let frozen = m.snapshot_name.is_some(); + (m.month, n(m), m.zeros_after, m.rows_reset, frozen) + }; + summary.months.iter().map(row).collect::>() + }; + let want = [(202_006, 13, 2, 0, false), (202_101, 1, 0, 0, false)]; + assert_eq!(months(true, false).await, want); + assert_eq!( + usd_of(&client, db, &LEGS_0331[0]).await.2, + 1, + "the dry run wrote nothing" + ); + assert_eq!(months(false, false).await, want); + let rerun = [(202_006, 0, 2, 0, false), (202_101, 0, 0, 0, false)]; + assert_eq!(months(false, true).await, rerun); + + // (v) On 1m a run that would set a priced row back to 0 needs allow_unprice. + #[rustfmt::skip] + let legs: &[Leg] = &[ + ("USDT/XLM", "price_ohlcv_1m", h(10), USDT, XLM, 5.0, 10.0, 50.0, 1.25, 12.5), + ("FOO/USDT, market 2 h old", "price_ohlcv_1m", h(12), FOO, USDT, 2.0, 3.0, 6.0, 2.5, 7.5), + ]; + for l in legs { + seed_leg(&client, db, l).await; + } + let run = async |pivot_window_s, allow_unprice| { + ChEnrichmentPass::new(ChEnrichConfig { + pivot_window_s, + pre_usdc_xlm_rate: Some(PreUsdcXlmRate { allow_unprice }), + ..cfg_0331(db, "price_ohlcv_1m") + }) + .run() + .await + .map(|s| (s.rows_enriched, s.rows_reset)) + }; + assert_eq!(run(86_400, false).await.unwrap(), (2, 0)); + let res = run(3600, false).await.map(|_| ()); + assert!(refused(&res, "--allow-unprice"), "{res:?}"); + assert_legs(&client, db, legs, |_| 1.0).await; + assert_eq!(run(3600, true).await.unwrap(), (1, 1)); + assert_legs(&client, db, &legs[..1], |_| 1.0).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_repair_reaches_across_a_month_edge() { + let db = "test_ch_enrich_pre_usdc_month_edge"; + let day = |n: u32| D0331 + n * 86_400; + // 2020-06-29 is a Monday: its week runs into July. + let (week, jun30, jul1, jul2) = (day(28), day(29), day(30), day(31)); + let rates: &[Rate] = &[ + (jun30 + 10 * 3600, 0.25, "external"), + (jul1, 0.30, "external"), + (jul2 + 10 * 3600, 0.20, "external"), + ]; + #[rustfmt::skip] + let legs: &[Leg] = &[ + ("USDT/XLM 06-30 20:00", H1, jun30 + 20 * 3600, USDT, XLM, 5.0, 10.0, 50.0, 1.25, 12.5), + ("FOO/USDT 07-01 via June's market", H1, jul1, FOO, USDT, 2.0, 3.0, 6.0, 3.0, 9.0), + ("FOO/XLM 06-30 10:00", H1, jun30 + 10 * 3600, FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), + ("FOO/XLM 07-02 10:00", H1, jul2 + 10 * 3600, FOO, XLM, 10.0, 3.0, 30.0, 2.0, 6.0), + ("1w FOO/XLM from 06-29 rolls up July's hour", WIDE[2], week, FOO, XLM, 10.0, 6.0, 60.0, 2.0, 13.5), + ]; + let client = setup_0331(db, rates, legs).await; + for table in [H1, WIDE[2]] { + CoarseRepairDriver::with_client( + client.clone(), + CoarseRepairConfig { + enrich: cfg_0331(db, table), + start_month: 202_006, + end_month: 202_007, + snapshot: false, + dry_run: false, + one_shot: true, + deadline: None, + }, + ) + .run() + .await + .unwrap(); + } + assert_legs(&client, db, legs, |_| 1.0).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); +} + +/// A wide tier's USD volume is a Σ of its hours; past Decimal(38, 14) it is +/// written as 0, never as the sum wrapped negative (three hours of 6e23 do). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_rollup_volume_never_wraps() { + let db = "test_ch_enrich_pre_usdc_rollup_overflow"; + #[rustfmt::skip] + let day: Leg = ("1d FOO/XLM over three huge hours", WIDE[1], D0331, FOO, XLM, 10.0, 9.0, 90.0, 2.5, 0.0); + let client = setup_0331(db, RATES_0331, &[day]).await; + for ts in [h(10), h(11), h(12)] { + client + .query(&format!( + "INSERT INTO {db}.{H1} \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + VALUES ({ts}, {FOO}, {XLM}, 'sdex', 10, 10, 10, 10, 3, 30, \ + '600000000000000000000000', 2.5, 10, 1, 1)" + )) + .execute() + .await + .unwrap(); + } + let pass = ChEnrichmentPass::new(cfg_0331(db, WIDE[1])); + assert_eq!(pass.run().await.unwrap().rows_enriched, 1); + assert_legs(&client, db, &[day], |_| 1.0).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); +} diff --git a/packages/events-backfill/src/discover.rs b/packages/events-backfill/src/discover.rs index 7b5f5939..e113e6fd 100644 --- a/packages/events-backfill/src/discover.rs +++ b/packages/events-backfill/src/discover.rs @@ -291,10 +291,14 @@ mod tests { } const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; + const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; + const BLNT_USDC: &str = "CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3"; + const BLNT_XLM: &str = "CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC"; + const BLND_USDC_SMALL: &str = "CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ"; - fn comet_row() -> PoolRegistryRow { + fn comet_row(id: &str) -> PoolRegistryRow { PoolRegistryRow { - contract_id: COMET.to_string(), + contract_id: id.to_string(), venue: "comet".to_string(), token0: String::new(), token1: String::new(), @@ -303,19 +307,47 @@ mod tests { } } - /// Task 0300 D3b: over an EMPTY table the factory-less Comet pool is the - /// one row to write — whatever the range, since it has no factory event. + /// Task 0300 D3b: over an EMPTY table the factory-less Comet pools are the + /// rows to write — whatever the range, since they have no factory event. #[test] fn a_static_pool_missing_from_the_table_is_written_once() { let (reg, persisted) = preload(Registries::new()); - assert_eq!(reg.pool_rows_unpersisted(&persisted), vec![comet_row()]); + assert_eq!( + reg.pool_rows_unpersisted(&persisted), + vec![ + comet_row(BLNT_XLM), + comet_row(COMET), + comet_row(COMET_V1_1), + comet_row(BLND_USDC_SMALL), + comet_row(BLNT_USDC), + ] + ); } - /// ...and once the table holds it, a re-run writes nothing. + /// ...a table holding the two Blend pools writes only the three of task + /// 0337 (production's state), and a table holding all five writes nothing. #[test] fn a_static_pool_already_in_the_table_is_not_rewritten() { let mut loaded = Registries::new(); - loaded.load_pool_rows(&[comet_row()]); + loaded.load_pool_rows(&[comet_row(COMET), comet_row(COMET_V1_1)]); + let (reg, persisted) = preload(loaded); + assert_eq!( + reg.pool_rows_unpersisted(&persisted), + vec![ + comet_row(BLNT_XLM), + comet_row(BLND_USDC_SMALL), + comet_row(BLNT_USDC), + ] + ); + + let mut loaded = Registries::new(); + loaded.load_pool_rows(&[ + comet_row(COMET), + comet_row(COMET_V1_1), + comet_row(BLNT_USDC), + comet_row(BLNT_XLM), + comet_row(BLND_USDC_SMALL), + ]); let (reg, persisted) = preload(loaded); assert!(reg.pool_rows_unpersisted(&persisted).is_empty()); } @@ -336,11 +368,15 @@ mod tests { assert_eq!( written, vec![ + (BLNT_XLM, "comet"), (COMET, "comet"), + (COMET_V1_1, "comet"), ( "CAZ4Z273BBAAFL5NYNQJKEMZDQBRCPKAS4GOXDUFXPSE56M4ONBJUOVD", "soroswap" ), + (BLND_USDC_SMALL, "comet"), + (BLNT_USDC, "comet"), ] ); } diff --git a/packages/events-backfill/src/run.rs b/packages/events-backfill/src/run.rs index 4791e915..0ccb428c 100644 --- a/packages/events-backfill/src/run.rs +++ b/packages/events-backfill/src/run.rs @@ -21,9 +21,10 @@ use serde_json::Value; use tracing::{info, warn}; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, DEFAULT_BACKOFF_MS, LedgerSoroban, OhlcvCandle, OhlcvWriter, - RawSorobanEvent, Registries, UnresolvedPool, UnresolvedPoolSwap, process_soroban_event_rows, - retry_with_backoff, + AssetRegistry, BE_DATABASE, CandleAccumulator, DEFAULT_BACKOFF_MS, DecimalsResolver, + DecimalsRow, LedgerSoroban, OhlcvCandle, OhlcvWriter, PRICES_DATABASE, RawSorobanEvent, + Registries, UnresolvedPool, UnresolvedPoolSwap, decode_resolving, preload_sac_resolver, + process_soroban_event_rows, retry_with_backoff, }; use crate::cli::Cli; @@ -47,11 +48,48 @@ where .map_err(EventsBackfillError::from) } -/// Run the shared seam over one ledger's events, merge the resulting ticks into -/// the **run-level** per-source accumulators, then move every candle for a minute -/// strictly older than this ledger's minute out of the accumulators and into the -/// per-source write buffers. Synchronous (no I/O): buffered candles are written -/// later, in batches, at chunk end. +/// Run the shared seam over one ledger's events. +/// +/// A trade naming a Soroban token whose decimals are not known yet is dropped by +/// the decode, never guessed (task 0329). Those tokens are resolved over RPC, +/// persisted (not in a dry-run), and the ledger is decoded again — decoding is +/// deterministic and the first result is discarded whole. Whatever is still in +/// `missing_decimals` afterwards is volume this run did not reprice. +#[allow(clippy::too_many_arguments)] +async fn decode_ledger( + ledger: u32, + closed_at: i64, + events: &[RawSorobanEvent], + reg: &mut Registries, + assets: &mut AssetRegistry, + decimals: &mut DecimalsResolver, + writer: &OhlcvWriter, + dry_run: bool, +) -> Result { + decode_resolving( + decimals, + assets, + ledger, + |assets| { + let mut out = LedgerSoroban::default(); + process_soroban_event_rows(ledger, closed_at, events, reg, assets, &mut out); + out + }, + async |rows: &[DecimalsRow]| { + if dry_run { + return Ok(()); + } + retry_write(|| async { writer.write_decimals(rows).await }).await + }, + ) + .await +} + +/// Merge one decoded ledger's ticks into the **run-level** per-source +/// accumulators, then move every candle for a minute strictly older than this +/// ledger's minute out of the accumulators and into the per-source write +/// buffers. Synchronous (no I/O): buffered candles are written later, in +/// batches, at chunk end. /// /// Because events arrive in ledger (hence minute) order and the accumulators /// persist across chunks, only the current minute stays open — so a minute split @@ -59,19 +97,17 @@ where /// RMT-colliding partials. #[allow(clippy::too_many_arguments)] fn accumulate_ledger( - ledger: u32, + out: LedgerSoroban, closed_at: i64, - events: &[RawSorobanEvent], - reg: &mut Registries, - assets: &mut AssetRegistry, accumulators: &mut HashMap<&'static str, CandleAccumulator>, buffers: &mut HashMap<&'static str, Vec>, ticks_by_source: &mut HashMap<&'static str, u64>, failed_by_source: &mut HashMap<&'static str, u64>, raw_unresolved: &mut Vec, + sac_unproven: &mut Vec, + missing_decimals: &mut u64, ) { - let mut out = LedgerSoroban::default(); - process_soroban_event_rows(ledger, closed_at, events, reg, assets, &mut out); + *missing_decimals += out.trades_missing_decimals as u64; for (source, tick) in out.amm_ticks { accumulators.entry(source).or_default().merge(tick); *ticks_by_source.entry(source).or_default() += 1; @@ -80,6 +116,7 @@ fn accumulate_ledger( *failed_by_source.entry(source).or_default() += *swaps as u64; } raw_unresolved.extend(out.unresolved); + sac_unproven.extend(out.sac_unproven); // Minute-START timestamp of this ledger — the same key the accumulator buckets // on (`(closed_at/60)*60`), NOT the minute index; flush_older_than compares @@ -184,6 +221,27 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { // write only newly seen assets and resolve every seeded pool. let existing_assets = writer.load_assets().await?; let mut assets = AssetRegistry::from_existing(existing_assets); + // Task 0242 D1: BE's is_sac set, then one proof per SAC `assets` cannot + // resolve, both before the first chunk (this path reads no SAC events). + // Dry-run too, so its `unproven sac swaps` matches the write's. + // `execute_preloads_the_sac_resolver_before_the_first_chunk` pins this call. + let sac = + preload_sac_resolver(writer.client(), BE_DATABASE, PRICES_DATABASE, &mut assets).await?; + info!( + sac_contracts = sac.candidates, + sac_proofs = sac.proofs, + sacs_verified = sac.verified, + sacs_rejected = sac.rejected, + "preloaded SAC proofs (task 0242)" + ); + if sac.rejected > 0 { + warn!( + sacs_rejected = sac.rejected, + "SAC proofs that did not verify: their swaps are skipped, not minted (task 0242 D2)" + ); + } + writer.load_decimals(&mut assets).await?; + let mut decimals = DecimalsResolver::backfill_from_env(); let mut reg = reprice_registry(writer.load_pool_registry().await?)?; // Every AMM pool has a `venue` entry (the registry superset); its strkeys are @@ -214,6 +272,8 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { let mut total_events: u64 = 0; let mut total_candles: u64 = 0; let mut raw_unresolved: Vec = Vec::new(); + // One entry per swap skipped on an is_sac leg with no proof (task 0242 D2). + let mut sac_unproven: Vec = Vec::new(); // Ledgers present in soroban_events but absent from default.ledgers (no close // time): their events are skipped but COUNTED, so the gap is visible instead // of a silent LEFT-JOIN drop. @@ -225,6 +285,9 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { // not cover this ledger" — it can only be a corrupt or unexpected row, and // any non-zero count means those fills ARE in the wrong order. let mut apply_order_fallbacks: u64 = 0; + // AMM trades the run could not reprice because a token's `decimals()` did + // not resolve (task 0329) — trades, not legs. + let mut missing_decimals: u64 = 0; // Run-level state (persists across chunks): one accumulator per source and // one write buffer per source. `flush_older_than` keeps the current minute @@ -255,17 +318,27 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { if let Some(l) = cur_ledger && !cur_missing { - accumulate_ledger( + let out = decode_ledger( l, cur_closed_at, &cur_events, &mut reg, &mut assets, + &mut decimals, + &writer, + cli.dry_run, + ) + .await?; + accumulate_ledger( + out, + cur_closed_at, &mut accumulators, &mut buffers, &mut ticks_by_source, &mut failed_by_source, &mut raw_unresolved, + &mut sac_unproven, + &mut missing_decimals, ); } cur_ledger = Some(r.ledger_sequence); @@ -348,17 +421,27 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { if let Some(l) = cur_ledger && !cur_missing { - accumulate_ledger( + let out = decode_ledger( l, cur_closed_at, &cur_events, &mut reg, &mut assets, + &mut decimals, + &writer, + cli.dry_run, + ) + .await?; + accumulate_ledger( + out, + cur_closed_at, &mut accumulators, &mut buffers, &mut ticks_by_source, &mut failed_by_source, &mut raw_unresolved, + &mut sac_unproven, + &mut missing_decimals, ); } @@ -428,6 +511,18 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { ); } + if !sac_unproven.is_empty() { + let contracts: std::collections::BTreeSet<&str> = + sac_unproven.iter().map(String::as_str).collect(); + let sample: Vec<&str> = contracts.iter().copied().take(20).collect(); + warn!( + swaps = sac_unproven.len(), + contracts = contracts.len(), + ?sample, + "swaps skipped on SACs with no proof (task 0242 D2); the month is NOT complete" + ); + } + if ledgers_missing_close > 0 { warn!( ledgers_missing_close, @@ -456,6 +551,8 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { unresolved_genuine.len(), dropped_swaps, apply_order_fallbacks, + sac_unproven.len() as u64, + missing_decimals, ); info!( elapsed_s = run_start.elapsed().as_secs(), @@ -558,6 +655,9 @@ pub(crate) fn resolve_transaction_index(application_order: i16) -> (u16, bool) { (application_order as u16, false) } +/// Summary label the 0286 orchestrator reads (`tools/scripts/reingest_0286.py`). +pub(crate) const UNPROVEN_SAC_LABEL: &str = "unproven sac swaps:"; + #[allow(clippy::too_many_arguments)] fn print_summary( ticks_by_source: &HashMap<&'static str, u64>, @@ -568,6 +668,8 @@ fn print_summary( unresolved_contracts: usize, dropped_swaps: u64, apply_order_fallbacks: u64, + unproven_sac_swaps: u64, + missing_decimals: u64, ) { println!(); println!("=== events-backfill complete ==="); @@ -596,6 +698,12 @@ fn print_summary( // event row, not a missing join row (task 0304). Non-zero means the range // is not repaired. println!("negative apply order: {apply_order_fallbacks}"); + // Always printed, 0 included. Non-zero means swaps were skipped on SACs with + // no proof, and the 0286 orchestrator STOPs on it (task 0242 D2). + println!("{UNPROVEN_SAC_LABEL:<27}{unproven_sac_swaps}"); + // Always printed, 0 included: non-zero means a token's `decimals()` did not + // resolve and its trades are NOT in this range's candles (task 0329). + println!("trades dropped (decimals): {missing_decimals}"); } /// The registry the reprice reads events for: the LOADED `pool_registry` plus @@ -664,6 +772,7 @@ mod tests { reg.soroswap .register(POOL.to_string(), T0.to_string(), T1.to_string()); let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(T1.to_string(), 7); let mut accs: HashMap<&'static str, CandleAccumulator> = HashMap::new(); let mut buffers: HashMap<&'static str, Vec> = HashMap::new(); let mut ticks: HashMap<&'static str, u64> = HashMap::new(); @@ -682,17 +791,25 @@ mod tests { ticks: &mut HashMap<&'static str, u64>, unresolved: &mut Vec, ) { - accumulate_ledger( + let mut out = LedgerSoroban::default(); + process_soroban_event_rows( ledger, closed_at, &[soroswap_swap(ledger, closed_at)], reg, assets, + &mut out, + ); + accumulate_ledger( + out, + closed_at, accs, buffers, ticks, &mut HashMap::new(), unresolved, + &mut Vec::new(), + &mut 0, ); } @@ -782,7 +899,98 @@ mod tests { ); } + // ---- task 0242: a SAC leg is proven or skipped, never minted ---------- + + const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; + const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + + fn xcr() -> prices_ingest_core::AssetIdentity { + prices_ingest_core::AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + } + } + + /// One XLM-SAC / XCR-SAC Soroswap swap through `accumulate_ledger`: the + /// candles of its minute and the run's unproven list. + fn xcr_swap(assets: &mut AssetRegistry) -> (Vec, Vec) { + let mut reg = Registries::new(); + reg.venue.insert(POOL.to_string(), Venue::Soroswap); + reg.soroswap + .register(POOL.to_string(), T0.to_string(), XCR_SAC.to_string()); + let mut accs = HashMap::new(); + let mut unproven = Vec::new(); + let mut out = LedgerSoroban::default(); + process_soroban_event_rows( + 100, + 1_700_000_000, + &[soroswap_swap(100, 1_700_000_000)], + &mut reg, + assets, + &mut out, + ); + accumulate_ledger( + out, + 1_700_000_000, + &mut accs, + &mut HashMap::new(), + &mut HashMap::new(), + &mut HashMap::new(), + &mut Vec::new(), + &mut unproven, + &mut 0, + ); + let candles = accs.values_mut().flat_map(|a| a.flush_all()).collect(); + (candles, unproven) + } + + #[test] + fn a_swap_on_an_unproven_sac_is_skipped_and_counted_and_a_proven_one_prices() { + let candidates = || std::collections::HashSet::from([XCR_SAC.to_string()]); + + let mut assets = AssetRegistry::from_existing(vec![]); + assert_eq!(assets.sac_address_of(&xcr()).as_deref(), Some(XCR_SAC)); + assets.set_sac_candidates(candidates()); + let (candles, unproven) = xcr_swap(&mut assets); + assert!(candles.is_empty(), "no tick"); + assert_eq!(unproven, [XCR_SAC]); + assert_eq!(assets.pending_new().count(), 0, "nothing interned"); + + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_sac_candidates(candidates()); + let proof = prices_ingest_core::SacProofRow { + contract_id: XCR_SAC.to_string(), + sep11: format!("XCR:{XCR_ISSUER}"), + }; + assert_eq!( + prices_ingest_core::apply_sac_proofs(&mut assets, &[proof]), + (1, 0) + ); + let (candles, unproven) = xcr_swap(&mut assets); + assert!(unproven.is_empty()); + assert_eq!(candles.len(), 1, "one tick, one candle"); + assert_eq!( + (&candles[0].base, &candles[0].quote), + (&xcr(), &prices_ingest_core::AssetIdentity::Native) + ); + } + + /// `execute` needs BE's and our tables, so no test runs it. It must arm the + /// registry before its first chunk: without the preload every swap on a SAC + /// whose classic is not in `assets` is skipped, and with the candidates gone + /// too it would be minted again (task 0242 D1/D2). `sac_proof_it` runs the + /// preload itself against real tables. + #[test] + fn execute_preloads_the_sac_resolver_before_the_first_chunk() { + let src = include_str!("run.rs"); + let execute = &src[src.find("pub async fn execute(").unwrap()..]; + let preload = execute.find("preload_sac_resolver(").unwrap(); + let first_chunk = execute.find("stream_chunk(").unwrap(); + assert!(preload < first_chunk, "the preload runs before any chunk"); + } + const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; + const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; /// Task 0300 D3a: the static pools never mask an unseeded `pool_registry` /// — the guard is about the table, and still fires on an empty one. @@ -803,9 +1011,10 @@ mod tests { loaded.venue.insert(AQUA.to_string(), Venue::Aquarius); let reg = reprice_registry(loaded).expect("a seeded table reprices"); - assert_eq!(reg.venue.len(), 2); + assert_eq!(reg.venue.len(), 1 + prices_ingest_core::STATIC_POOLS.len()); assert_eq!(reg.venue.get(AQUA), Some(&Venue::Aquarius)); assert_eq!(reg.venue.get(COMET), Some(&Venue::Comet)); + assert_eq!(reg.venue.get(COMET_V1_1), Some(&Venue::Comet)); } // ---- task 0139: only new identities, and before their candles ---------- diff --git a/packages/prices-api/src/assets/dto.rs b/packages/prices-api/src/assets/dto.rs index a0608e8f..2247f22e 100644 --- a/packages/prices-api/src/assets/dto.rs +++ b/packages/prices-api/src/assets/dto.rs @@ -582,7 +582,8 @@ pub struct Candle { /// `chainlink` or `bitstamp` (task 0267, task 0265's composed history). /// /// ⚠️ **Populated only on canonical USDC's OWN synthesized series** (§6, the - /// `ohlcv_peg_series` path). `None` on every other asset — not because those + /// `ohlcv_peg_series` path), and on native XLM's own series before 2021-02-01, as + /// `third-party` (`ohlcv_native_usd`). `None` on every other asset — not because those /// candles have no provenance, but because the candle TABLES carry no /// provenance column to report it from. That gap is task 0268's Issue 9 and /// is deliberately out of scope here; a value invented for those rows would @@ -613,8 +614,8 @@ pub struct Candle { /// better than the $1 assumption it replaces, but a different instrument /// on a different venue. /// - /// Same scope as [`Candle::source`]: non-null only on USDC's own synthesized - /// series, `None` everywhere else and on the peg fallback. + /// Same scope as [`Candle::source`]: non-null only on the synthesized USDC and + /// pre-2021 XLM series, `None` everywhere else and on the peg fallback. pub quality: Option, } diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index 336fc9c5..f2249199 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -81,8 +81,8 @@ fn min_volume_error(v: Option) -> Option { ("asset_identifier" = String, Path, description = "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or \ the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic \ - asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract \ - addresses, which as a rule answer `404`"), + asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the \ + answer names the asset by `CODE:ISSUER` (or `native`)"), ("min_volume_usd" = Option, Query, minimum = 0, description = "Drop every venue whose trailing 24-hour USD volume is at or below this value, then \ recompute `vwap_24h` over the venues that remain. Applied exactly as given: it can \ @@ -114,6 +114,10 @@ pub async fn get_price( if let Some(resp) = min_volume_error(q.min_volume_usd) { return resp; } + let id = match queries_ch::resolve_sac_alias(state.ch(), id).await { + Ok(id) => id, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; match queries_ch::current_price(state.ch(), &id).await { Ok(Some(row)) => { @@ -143,8 +147,8 @@ pub async fn get_price( ("asset_identifier" = String, Path, description = "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or \ the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic \ - asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract \ - addresses, which as a rule answer `404`") + asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the \ + answer names the asset by `CODE:ISSUER` (or `native`)") ), responses( (status = 200, description = "Asset detail", body = AssetDetail), @@ -164,6 +168,10 @@ pub async fn get_asset( Ok(id) => id, Err(e) => return errors::bad_request(errors::INVALID_ID, e.to_string()), }; + let id = match queries_ch::resolve_sac_alias(state.ch(), id).await { + Ok(id) => id, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; match queries_ch::asset_detail(state.ch(), &id).await { Ok(Some(row)) => { @@ -371,6 +379,42 @@ pub struct OhlcvParams { pub base_currency: Option, } +/// The USD references for `/ohlcv`. A lookup ERROR fails: that is the +/// database misbehaving, not an absent asset. +/// +/// USDC must be tracked (see `UsdRefs::usdc`). XLM and USDT are optional: an +/// untracked reference cannot be any candle's quote leg. A missing USDT only +/// loses the `traded` label. A missing XLM also drops XLM from the markets that +/// set USD prices, so the caller does not cache that result. +async fn resolve_usd_refs( + state: &AppState, +) -> Result { + let (usdc_id, xlm_id, usdt_id) = ( + usdc_identifier(), + AssetIdentifier::Native, + usdt_identifier(), + ); + let (usdc, xlm, usdt) = tokio::join!( + queries_ch::resolve_asset_id(state.ch(), &usdc_id), + queries_ch::resolve_asset_id(state.ch(), &xlm_id), + queries_ch::resolve_asset_id(state.ch(), &usdt_id), + ); + let xlm = xlm?; + if xlm.is_none() { + tracing::error!("ohlcv: XLM is untracked; USD prices come from USDC markets only"); + } + let usdt = usdt?; + if usdt.is_none() { + tracing::warn!("ohlcv: USDT is untracked; its rows go unlabelled"); + } + match usdc? { + Some(usdc) => Ok(queries_ch::UsdRefs { usdc, xlm, usdt }), + None => Err(clickhouse::error::Error::Custom( + "canonical USDC is not tracked".to_string(), + )), + } +} + /// `GET /assets/{asset_identifier}/ohlcv` — candlestick history. /// /// `base_currency` **denominates** the series; it does not select a quote leg @@ -381,8 +425,10 @@ pub struct OhlcvParams { /// /// Price fields are **absent, not dropped**, on a bucket with no USD value. /// Candles merge across sources — and, in USD mode, across quote legs — per -/// bucket. `backfill_note` appears only for `timeframe=all` while the backfill -/// is still running. +/// bucket; in USD mode the USDC/XLM legs set the prices and both means when one +/// formed a price. +/// `backfill_note` appears only for `timeframe=all` while the backfill is still +/// running. #[utoipa::path( get, path = "/assets/{asset_identifier}/ohlcv", @@ -396,15 +442,32 @@ pub struct OhlcvParams { only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but \ has no USD value yet is returned with its price fields `null`\nrather than omitted, so a \ chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote \ - asset — are merged per bucket, in ascending time order.\n\nThe window is `timeframe`, ending \ + asset — are merged per bucket, in ascending time order,\nand all of them count toward the \ + volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` \ + and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a \ + price in the bucket, and from every market otherwise. If\nsuch a market has no USD value \ + yet, those fields are `null`. Before 2021-02-01, candles\nquoted in XLM or in a \ + dollar token are converted with an imported, measured\nXLM/USD rate (`method` is \ + `external`); a dollar token is converted through its own XLM\nmarket and stays unpriced \ + without one.\n\nFor native XLM itself (`GET /assets/native/ohlcv`) before 2021-02-01, a bucket \ + covered by\nthe imported hourly XLM/USD rate is built from that rate instead: `method` is \ + `external`,\n`derived` is `true` and `source` is `third-party`. `open` and `close` are the \ + bucket's\nfirst and last hourly rate, and `high` and `low` are the extremes of the hourly \ + averages,\nnot of individual trades. The volumes and trade count are those of XLM traded \ + against\ndollar tokens (USD, USDT and USDC of any issuer, except four that lost their \ + peg) in\nevery hour of the bucket; an hour without a rate of its own is valued in USD at \ + the\nnewest rate of the previous 24 hours. From 2021-02-01 the volumes are those of \ + XLM's own\nmarkets against canonical USDC and USDT only. At\n`15m` and `1m` a bucket appears only where XLM traded, and takes the rate of its \ + hour. A\nbucket the rate does not cover is built as above.\n\n\ + The window is `timeframe`, ending \ now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is \ rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the \ historical backfill is still running.", params( ("asset_identifier" = String, Path, description = "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or \ the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic \ - asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract \ - addresses, which as a rule answer `404`"), + asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the \ + answer names the asset by `CODE:ISSUER` (or `native`)"), ("timeframe" = Option, Query, description = "Window ending now: `1h`, `24h` (default), `7d`, `30d`, `1y` or `all` (from Stellar \ genesis). `start` overrides its start"), @@ -533,6 +596,10 @@ pub async fn get_ohlcv( } // Resolve the base asset. + let id = match queries_ch::resolve_sac_alias(state.ch(), id).await { + Ok(id) => id, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; let asset_id = match queries_ch::resolve_asset_id(state.ch(), &id).await { Ok(Some(a)) => a, Ok(None) => return errors::not_found("unknown asset"), @@ -546,70 +613,42 @@ pub async fn get_ohlcv( // // The three references are resolved by natural identity, mirroring the // enrichment worker's own resolve_reference_ids so read and write cannot - // disagree about which USDC is canonical. All three must be tracked: their - // absence is a server-side data gap, not "no candles", so it stays a 503 - // rather than being masked as an empty 200 (which looks like a healthy asset - // with no history). - let denomination = - match base_currency { - BaseCurrency::Usd => { - // Resolved once per AppState and shared thereafter — see - // AppState::usd_refs. The three identities are constants and the - // database derives their ids from them, so re-reading `assets - // FINAL` on every request is pure waste on a p95-bounded path. - let refs = state - .usd_refs() - .get_or_try_init(|| async { - // Concurrent: they do not depend on each other. - let (usdc_id, xlm_id, usdt_id) = - (usdc_identifier(), AssetIdentifier::Native, usdt_identifier()); - let (usdc, xlm, usdt) = tokio::join!( - queries_ch::resolve_asset_id(state.ch(), &usdc_id), - queries_ch::resolve_asset_id(state.ch(), &xlm_id), - queries_ch::resolve_asset_id(state.ch(), &usdt_id), - ); - // The pivots are best-effort: an untracked reference cannot - // be any candle's quote leg, so it only costs the `traded` - // label. A lookup ERROR still fails — that is the database - // misbehaving, not an absent asset. - let mut pivots = Vec::new(); - for found in [xlm, usdt] { - match found { - Ok(Some(id)) => pivots.push(id), - Ok(None) => tracing::debug!( - "ohlcv pivot reference not tracked; rows on this leg go unlabelled" - ), - Err(e) => return Err(e), - } - } - // USDC must resolve: see UsdRefs::usdc. - match usdc { - Ok(Some(usdc)) => Ok(queries_ch::UsdRefs { usdc, pivots }), - Ok(None) => Err(clickhouse::error::Error::Custom( - "canonical USDC is not tracked".to_string(), - )), - Err(e) => Err(e), - } - }) - .await; - match refs { - Ok(r) => queries_ch::Denomination::Usd(r.clone()), - Err(e) => { - tracing::error!(error = %e, "ohlcv reference resolution failed"); - return errors::service_unavailable( - errors::QUOTE_UNAVAILABLE, - "pricing in the requested base_currency is unavailable", - ); + // disagree about which USDC is canonical. Only USDC must be tracked; see + // `resolve_usd_refs` for XLM and USDT. + let denomination = match base_currency { + BaseCurrency::Usd => { + // Resolved once per AppState and shared thereafter — see + // AppState::usd_refs. The three identities are constants and the + // database derives their ids from them, so re-reading `assets + // FINAL` on every request is pure waste on a p95-bounded path. + let refs = match state.usd_refs().get() { + Some(r) => Ok(r.clone()), + // Cached only once XLM resolves: a missing XLM changes + // prices, so it must not outlive the gap it came from. + None => resolve_usd_refs(&state).await.inspect(|r| { + if r.xlm.is_some() { + let _ = state.usd_refs().set(r.clone()); } + }), + }; + match refs { + Ok(r) => queries_ch::Denomination::Usd(r), + Err(e) => { + tracing::error!(error = %e, "ohlcv reference resolution failed"); + return errors::service_unavailable( + errors::QUOTE_UNAVAILABLE, + "pricing in the requested base_currency is unavailable", + ); } } - BaseCurrency::Xlm => { - match resolve_reference(state.ch(), AssetIdentifier::Native, base_currency).await { - Ok(id) => queries_ch::Denomination::QuoteLeg(id), - Err(resp) => return resp, - } + } + BaseCurrency::Xlm => { + match resolve_reference(state.ch(), AssetIdentifier::Native, base_currency).await { + Ok(id) => queries_ch::Denomination::QuoteLeg(id), + Err(resp) => return resp, } - }; + } + }; // The validated window is exactly what binds into SQL — the same eff_start // the point-count check measured, so there is one interpretation of the @@ -627,23 +666,19 @@ pub async fn get_ohlcv( limit: OHLCV_MAX_POINTS, }; // ADR 0011 §6: canonical USDC is only ever a quote leg, so a normal query - // for it matches zero rows and no amount of filter-dropping helps — its - // series is synthesized instead. Keyed on the requested identity, not on a - // resolved asset_id, because 0139 has ids serving more than one identity. - // ADR 0011 §6: canonical USDC is only ever a quote leg, so a normal query // for it matches zero rows in EITHER denomination and no amount of // filter-dropping helps — its series is synthesized instead. // // Matched on the natural identity (case-insensitively: `AssetIdentifier` - // preserves the code's case verbatim) OR on the resolved id. The id arm is - // what catches USDC addressed by its SAC contract address, which is a - // different identity for the same asset. + // preserves the code's case verbatim) OR on the resolved id. USDC's SAC + // address already arrives here as the classic identity (task 0242 alias); + // the id arm stays as a backstop. // ⚠️ [[0139]] means an `asset_id` can serve more than one identity, so the // id arm could in principle route a colliding asset here. Accepted: the // alternative is that a legitimate USDC identity silently falls back to the // known-empty path, which is the defect this task exists to remove. let (peg_usdc, peg_xlm) = match &args.denomination { - queries_ch::Denomination::Usd(refs) => (Some(refs.usdc), refs.pivots.first().copied()), + queries_ch::Denomination::Usd(refs) => (Some(refs.usdc), refs.xlm), queries_ch::Denomination::QuoteLeg(x) => { match queries_ch::resolve_asset_id(state.ch(), &usdc_identifier()).await { Ok(Some(u)) => (Some(u), Some(*x)), @@ -659,6 +694,8 @@ pub async fn get_ohlcv( .eq_ignore_ascii_case(&usdc_identifier().to_canonical()) }); + // Native XLM in USD: its pre-2021 buckets come from the imported rate. + let native_xlm = queries_ch::native_xlm_usd(&args.denomination, asset_id); let mut data = if is_peg_asset { // Both references anchor the series: USDC is the rate's identity, XLM is // the market the buckets come from (and, in XLM mode, the denominator). @@ -683,6 +720,11 @@ pub async fn get_ohlcv( Ok(d) => d, Err(e) => return errors::db_error(&e, "ohlcv peg series"), } + } else if let Some(xlm) = native_xlm { + match queries_ch::ohlcv_native_usd(state.ch(), args, xlm).await { + Ok(d) => d, + Err(e) => return errors::db_error(&e, "ohlcv native usd"), + } } else { match queries_ch::ohlcv(state.ch(), args).await { Ok(d) => d, diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index 9ef713d0..e04cba8d 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -6,6 +6,9 @@ //! field order MUST match the `SELECT` column order. use clickhouse::Client; +use prices_clickhouse::{ + DOLLAR_TOKEN_PRED, OFF_PEG_DOLLAR_TOKENS, PRE_USDC_END_S, XLM_RATE_SOURCE, +}; use crate::assets::dto::Candle; use crate::common::cursor::Cursor; @@ -506,6 +509,117 @@ fn resolve_asset_id_sql(where_sql: &str) -> String { ) } +// ---------------------------------------------------------------------------- +// SAC alias (task 0242, D5) +// ---------------------------------------------------------------------------- + +/// A classic identity named by its SAC address. Positional: field order is the +/// [`sac_alias_sql`] SELECT order. +#[derive(Debug, clickhouse::Row, serde::Deserialize)] +struct SacAliasRow { + sac_address: String, + asset_code: String, + issuer_address: String, +} + +/// The classic identities (`native` / `credit`) of `n` bound SAC addresses, +/// read through `identity_by_contract`, the one SAC→classic resolver of the +/// read side (views.sql), so a fix there reaches the API too. The view reads +/// the stored `sac_address`; the ingest writes it on every classic row (0 of +/// 211,044 classic rows without it on 2026-10-06), and an address cannot be +/// turned back into its classic, so a read cannot derive it. +fn sac_alias_sql(n: usize) -> String { + let binds = vec!["?"; n].join(", "); + format!( + "SELECT contract AS sac_address, asset_code, issuer_address \ + FROM identity_by_contract \ + WHERE asset_kind != 'contract' AND contract IN ({binds})" + ) +} + +/// A classic row's identifier: XLM with no issuer is `native`. +fn alias_identifier(code: String, issuer: String) -> AssetIdentifier { + if code == "XLM" && issuer.is_empty() { + AssetIdentifier::Native + } else { + AssetIdentifier::Classic { code, issuer } + } +} + +/// The distinct contract addresses in `ids`, in first-seen order — the only +/// identifiers an alias can replace. Empty means no lookup is needed. +fn sac_alias_candidates(ids: &[AssetIdentifier]) -> Vec { + let mut out: Vec = Vec::new(); + for id in ids { + if let AssetIdentifier::Contract(c) = id + && !out.contains(c) + { + out.push(c.clone()); + } + } + out +} + +/// Replace each `Contract(c)` that `aliases` names with its classic; every +/// other identifier passes through. +fn apply_sac_aliases( + ids: Vec, + aliases: &std::collections::HashMap, +) -> Vec { + ids.into_iter() + .map(|id| match id { + AssetIdentifier::Contract(c) => aliases + .get(&c) + .cloned() + .unwrap_or(AssetIdentifier::Contract(c)), + other => other, + }) + .collect() +} + +/// Answer a classic asset's SAC address as that classic asset (task 0242, D5). +/// +/// Run once at the handler boundary, before any lookup. Together with +/// [`identity_where`]'s contract arm (`a.contract_address = ?`) a `C…` +/// identifier matches `contract_address = ? OR sac_address = ?`, and the +/// classic wins when both rows exist (a pre-heal SAC `Contract` row). The +/// response then echoes `CODE:ISSUER` / `native` through `to_canonical`. +/// No query when `ids` holds no contract. +pub async fn resolve_sac_aliases( + ch: &Client, + ids: Vec, +) -> Result, clickhouse::error::Error> { + let candidates = sac_alias_candidates(&ids); + if candidates.is_empty() { + return Ok(ids); + } + let sql = sac_alias_sql(candidates.len()); + let mut q = ch.query(&sql); + for c in candidates { + q = q.bind(c); + } + let aliases = q + .fetch_all::() + .await? + .into_iter() + .map(|r| { + ( + r.sac_address, + alias_identifier(r.asset_code, r.issuer_address), + ) + }) + .collect(); + Ok(apply_sac_aliases(ids, &aliases)) +} + +/// [`resolve_sac_aliases`] for one identifier. +pub async fn resolve_sac_alias( + ch: &Client, + id: AssetIdentifier, +) -> Result { + Ok(resolve_sac_aliases(ch, vec![id]).await?.remove(0)) +} + // ---------------------------------------------------------------------------- // OHLCV (overview §4.2) // ---------------------------------------------------------------------------- @@ -706,12 +820,31 @@ pub struct UsdRefs { /// indistinguishable from the anomalous same-signature rows on other legs /// and would be dropped. Its absence is a real server-side data gap. pub usdc: u64, - /// XLM and canonical USDT, the two pivot references. **Optional**: they only - /// select the `traded` label. An untracked reference cannot be any candle's - /// quote leg, so the branch simply never matches — refusing to serve the - /// endpoint over a missing label would turn a cosmetic dependency into an - /// outage. - pub pivots: Vec, + /// XLM and canonical USDT, the two pivot references. **Optional**: an + /// untracked reference cannot be any candle's quote leg, so its branch simply + /// never matches — refusing to serve the endpoint over it would turn a data + /// gap into an outage. XLM is also a price-setting quote and the peg series' + /// anchor. + pub xlm: Option, + pub usdt: Option, +} + +impl UsdRefs { + /// The pivots that select the `traded` label. + pub fn pivots(&self) -> Vec { + self.xlm.into_iter().chain(self.usdt).collect() + } + + /// The quote legs whose rows set a USD candle's prices. Never empty. + /// + /// ⚠️ XLM's USD value comes from the XLM/USDC pivot, which task 0286 + /// documents as quantised on dust days (e.g. 2023-03-11) until its + /// re-ingest and stage-D re-enrichment finish. Until then such a bucket + /// takes its prices from that XLM leg alone. + pub fn price_setting(&self) -> Vec { + // A token joins here once its USD rate is reliable (lore-0332). + std::iter::once(self.usdc).chain(self.xlm).collect() + } } /// What `base_currency` asks for. Per [ADR 0011] it **denominates**; it does not @@ -780,7 +913,9 @@ pub struct OhlcvArgs { /// no clamp: it is derived through the same `rate` as `h_x`/`l_x`, and scaling by /// one positive factor is monotonic, so `l_x <= o_x <= h_x` holds within a row and /// survives `min`/`max` across rows. Only the exact/derived boundary breaks. -const CLOSE_EXACT: &str = "argMaxIf(c_x, (volume_base, quote_asset_id), valid)"; +fn close_exact(p: &str) -> String { + format!("argMaxIf(c_x, (volume_base, quote_asset_id), {p})") +} /// The published `high` / `low`, before rendering. Named because `vw` clamps /// against these rather than against the raw aggregates — the response has to be @@ -795,8 +930,50 @@ const CLOSE_EXACT: &str = "argMaxIf(c_x, (volume_base, quote_asset_id), valid)"; /// NULL the whole expression is NULL, so `greatest(vwap, NULL)` leaves vwap /// unclamped on that side — which is correct: a bound we could not compute must /// not be used to move a value we could. -const HIGH_PUBLISHED: &str = "if(isNull(maxIf(h_x, valid)), NULL, greatest(maxIf(h_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))"; -const LOW_PUBLISHED: &str = "if(isNull(minIf(l_x, valid)), NULL, least(minIf(l_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))"; +fn high_published(p: &str) -> String { + let c = close_exact(p); + format!("if(isNull(maxIf(h_x, {p})), NULL, greatest(maxIf(h_x, {p}), {c}))") +} +fn low_published(p: &str) -> String { + let c = close_exact(p); + format!("if(isNull(minIf(l_x, {p})), NULL, least(minIf(l_x, {p}), {c}))") +} + +/// The USD candle's price inputs, computed once per bucket and named, so every +/// output column and both clamps read the same values (lore-0332). +/// +/// `_ub`: a USDC/XLM market formed a price in the bucket (`ps`). Then the +/// prices and both means come from those markets only (`band`, `bandc`); +/// otherwise from every market (`valid`, `convertible`), as before lore-0332. +/// Such a market with no USD value yet (`ps` but not `band`) leaves the bucket +/// unpriced (`_np`) rather than handing it to the other markets, and a band +/// extreme that overflows stays NULL for the same reason. +fn usd_with() -> String { + let pick = |f: &dyn Fn(&str) -> String, band: &str, all: &str| { + format!("if(_ub, {}, {})", f(band), f(all)) + }; + let open = pick( + &|p| format!("argMaxIf(o_x, (volume_base, quote_asset_id), {p})"), + "band", + "valid", + ); + let meth = pick( + &|p| format!("argMaxIf(meth, (volume_base, quote_asset_id), {p})"), + "band", + "valid", + ); + let high = pick(&high_published, "band", "valid"); + let low = pick(&low_published, "band", "valid"); + let close = pick(&close_exact, "band", "valid"); + let vwap = pick(&vwap_raw, "bandc", "convertible"); + let pf_vwap = pick(&pf_vwap_usd_raw, "band", "valid"); + format!( + "WITH countIf(ps) > 0 AS _ub, \ + (if(_ub, countIf(band), countIf(valid)) = 0) AS _np, \ + {open} AS _op, {high} AS _hi, {low} AS _lo, {close} AS _cl, \ + {meth} AS _me, {vwap} AS _vw, {pf_vwap} AS _pfvw " + ) +} /// The volume-weighted mean before clamping — task 0229's review, finding 1. /// @@ -829,9 +1006,13 @@ const LOW_PUBLISHED: &str = "if(isNull(minIf(l_x, valid)), NULL, least(minIf(l_x /// bucket whose dust leg carries most of the volume the clamp will move the /// published value to `low` or `high` — a mean of all trades cannot always sit /// inside the range of some of them. -const VWAP_RAW: &str = "toDecimal128OrNull(toString( \ - sumIf(toFloat64(w_x) * toFloat64(volume_base), convertible) \ - / nullIf(sumIf(toFloat64(volume_base), convertible), 0)), 14)"; +fn vwap_raw(p: &str) -> String { + format!( + "toDecimal128OrNull(toString( \ + sumIf(toFloat64(w_x) * toFloat64(volume_base), {p}) \ + / nullIf(sumIf(toFloat64(volume_base), {p}), 0)), 14)" + ) +} /// Smallest `close` / `close_usd` a USD rate may be derived from — a /// **precision precondition**, not a plausibility band. @@ -876,9 +1057,13 @@ const PRECISION_FLOOR: &str = prices_clickhouse::PRICE_FLOOR_SQL; /// reports: before task 0286's migration the column does not exist, and its /// DEFAULT is `volume_quote`, which a fixture or a legacy writer may never have /// filled. -const PF_VWAP_USD_RAW: &str = "nullIf(toDecimal128OrNull(toString( \ - sumIf(toFloat64(pfpv_x), valid) \ - / nullIf(sumIf(toFloat64(pf_volume), valid), 0)), 14), 0)"; +fn pf_vwap_usd_raw(p: &str) -> String { + format!( + "nullIf(toDecimal128OrNull(toString( \ + sumIf(toFloat64(pfpv_x), {p}) \ + / nullIf(sumIf(toFloat64(pf_volume), {p}), 0)), 14), 0)" + ) +} /// The quote-leg arm's price gate, spelled once. Unlike the USD arm there is no /// `valid` to hang it on: that expression exists to say "this row can be @@ -980,21 +1165,23 @@ const PF_ROWS: &str = "pf_trade_count > 0 AND close >= toDecimal128('0.000000000 /// can read it — see [`OUTER_ALIASES`] for why the column tail is not something /// to maintain by hand in three places. fn usd_aggregates() -> String { + // The price inputs are the `usd_with` names, so `h`, `l`, `vw` and `pfvw` + // clamp against the same bounds the caller sees. format!( - "if(countIf(valid) = 0, NULL, toString(argMaxIf(o_x, (volume_base, quote_asset_id), valid))) AS o, \ - if(countIf(valid) = 0, NULL, toString({HIGH_PUBLISHED})) AS h, \ - if(countIf(valid) = 0, NULL, toString({LOW_PUBLISHED})) AS l, \ - if(countIf(valid) = 0, NULL, toString({CLOSE_EXACT})) AS c, \ + "if(_np, NULL, toString(_op)) AS o, \ + if(_np, NULL, toString(_hi)) AS h, \ + if(_np, NULL, toString(_lo)) AS l, \ + if(_np, NULL, toString(_cl)) AS c, \ toString(sum(volume_base)) AS vb, \ toString(sum(volume_quote_usd)) AS vqu, \ - if(countIf(valid) = 0 OR isNull({VWAP_RAW}), NULL, \ - toString(least(greatest({VWAP_RAW}, {LOW_PUBLISHED}), {HIGH_PUBLISHED}))) AS vw, \ + if(_np OR isNull(_vw), NULL, \ + toString(least(greatest(_vw, _lo), _hi))) AS vw, \ toUInt64(sum(trade_count)) AS tc, \ - nullIf(if(countIf(valid) = 0, NULL, argMaxIf(meth, (volume_base, quote_asset_id), valid)), '') AS meth, \ - if(countIf(valid) = 0, NULL, toUInt8(1)) AS drv, \ + nullIf(if(_np, NULL, _me), '') AS meth, \ + if(_np, NULL, toUInt8(1)) AS drv, \ toNullable(toUInt64(sum(pf_trade_count))) AS pftc, \ - if(isNull({PF_VWAP_USD_RAW}), NULL, \ - toString(least(greatest({PF_VWAP_USD_RAW}, {LOW_PUBLISHED}), {HIGH_PUBLISHED}))) AS pfvw, \ + if(_np OR isNull(_pfvw), NULL, \ + toString(least(greatest(_pfvw, _lo), _hi))) AS pfvw, \ {CLOSE_DIVERGENT_PLACEHOLDER}, \ {PROVENANCE_NULL_TAIL}" ) @@ -1009,7 +1196,9 @@ fn usd_projection(refs: &UsdRefs, granularity: Granularity) -> String { let floor = PRECISION_FLOOR; // The whole classification, extracted so its arm ORDER is testable // without a ClickHouse — see [`usd_method_expr`]. - let meth_arm = usd_method_expr(usdc, &refs.pivots, granularity); + let meth_arm = usd_method_expr(usdc, &refs.pivots(), granularity); + let ids: Vec = refs.price_setting().iter().map(|i| i.to_string()).collect(); + let price_setting = ids.join(", "); // ⚠️ `pf_trade_count > 0` is a conjunct of `valid` rather than a filter on // the subquery: `valid` gates the PRICE expressions only, while // `sum(volume_base)`, `sum(volume_quote_usd)` and `sum(trade_count)` @@ -1022,12 +1211,19 @@ fn usd_projection(refs: &UsdRefs, granularity: Granularity) -> String { // price". The prices take `valid`; `vwap` takes `convertible`, because ADR // 0287 §7 weighs every fill (review C2). Spelling `valid` in terms of // `convertible` is what keeps them one decision rather than two. + // + // lore-0332: `band` and `bandc` are the same two predicates on the USDC/XLM + // legs; `ps` asks only whether such a leg formed a price, so an unconverted + // one still claims the bucket (see `usd_with`). format!( "timestamp, volume_base, volume_quote_usd, trade_count, quote_asset_id, \ pf_trade_count, pf_volume, \ (close >= {floor} AND close_usd >= {floor} \ AND (quote_asset_id = {usdc} OR close_usd != close)) AS convertible, \ (convertible AND pf_trade_count > 0) AS valid, \ + (pf_trade_count > 0 AND quote_asset_id IN ({price_setting})) AS ps, \ + (valid AND ps) AS band, \ + (convertible AND quote_asset_id IN ({price_setting})) AS bandc, \ toFloat64(close_usd) / nullIf(toFloat64(close), 0) AS rate, \ toDecimal128OrNull(toString(toFloat64(open) * rate), 14) AS o_x, \ toDecimal128OrNull(toString(toFloat64(high) * rate), 14) AS h_x, \ @@ -1063,7 +1259,7 @@ fn quote_leg_aggregates() -> String { let ql_high = format!("maxIf(high, {PF_ROWS})"); let ql_low = format!("minIf(low, {PF_ROWS})"); // The price-forming mean, in the stored denomination. Same zero rule as - // `PF_VWAP_USD_RAW`. + // `pf_vwap_usd_raw`. let pf_vwap_raw = format!( "nullIf(toDecimal128OrNull(toString( \ sumIf(toFloat64(pf_price_volume), {PF_ROWS}) \ @@ -1541,12 +1737,15 @@ pub async fn ohlcv_peg_series( /// USDC leg at or after the epoch was priced from a measured reading. /// 4. `quote_asset_id = usdc` -> **`''`**. Scaled, pre-epoch, and no imported /// rate covers the bucket — no tier produces this. Null is the honest answer -/// to "which input priced this"; it also keeps USDC off arm 5 if it is ever -/// tracked as a pivot. -/// 5. `quote_asset_id IN (pivots)` -> **`traded`** (ADR 0011 §4; see [`ohlcv`]). +/// to "which input priced this"; it also keeps USDC off the arms below, e.g. if +/// it is ever tracked as a pivot. +/// 5. `timestamp < PRE_USDC_END_S` -> **`external`** (task 0331): before +/// 2021-02-01 only `coarse-repair --pre-usdc-xlm-rate` writes a non-USDC +/// `close_usd`, from the imported XLM/USD rate. +/// 6. `quote_asset_id IN (pivots)` -> **`traded`** (ADR 0011 §4; see [`ohlcv`]). /// Omitted ENTIRELY when no pivot reference is tracked — an empty `IN ()` is /// a ClickHouse syntax error, and the label is optional. -/// 6. `''` -> the fallback, which [`ohlcv`]'s `nullIf` turns into a JSON `null`. +/// 7. `''` -> the fallback, which [`ohlcv`]'s `nullIf` turns into a JSON `null`. /// A `multiIf` with no else is an error, so this arm always closes the list. /// /// ⚠️ The bare column names below (`close_usd`, `close`, `timestamp`) resolve to @@ -1580,12 +1779,7 @@ pub(crate) fn usd_method_expr(usdc: u64, pivots: &[u64], granularity: Granularit // the plain `+ INTERVAL` form slides the coverage window's upper day by the // server's offset — and by a whole hour across a DST boundary. The worker // guards this at length; the read side must not reintroduce it. - let bucket_end = match granularity { - Granularity::D1 => "addDays(timestamp, 1, 'UTC')".to_string(), - Granularity::W1 => "addWeeks(timestamp, 1, 'UTC')".to_string(), - Granularity::Mo1 => "addMonths(timestamp, 1, 'UTC')".to_string(), - g => format!("timestamp + {}", g.seconds()), - }; + let bucket_end = bucket_end_sql(granularity, "timestamp"); // The imported days, as ONE uncorrelated set built per query (ClickHouse // renders this as a single `CreatingSet` node, not per row). let imported_days = format!( @@ -1612,18 +1806,34 @@ pub(crate) fn usd_method_expr(usdc: u64, pivots: &[u64], granularity: Granularit quote_asset_id = {usdc} AND close_usd = close, 'assumed-par', \ quote_asset_id = {usdc} AND timestamp >= toDateTime({epoch}), 'oracle', \ quote_asset_id = {usdc}, '', \ + timestamp < toDateTime({pre}), 'external', \ {traded_arm}\ - '') AS meth" + '') AS meth", + // Task 0331: keys on the timestamp alone because only `coarse-repair + // --pre-usdc-xlm-rate` writes a non-USDC close_usd below this boundary. + // Another pre-2021 pricer needs its own arm above this one. + pre = prices_clickhouse::PRE_USDC_END_S, ) } +/// `col` plus one bucket of `g`, pinned to UTC (see [`usd_method_expr`]). +fn bucket_end_sql(g: Granularity, col: &str) -> String { + match g { + Granularity::D1 => format!("addDays({col}, 1, 'UTC')"), + Granularity::W1 => format!("addWeeks({col}, 1, 'UTC')"), + Granularity::Mo1 => format!("addMonths({col}, 1, 'UTC')"), + g => format!("{col} + {}", g.seconds()), + } +} + /// Read merged candles for one asset at the chosen grain, denominated per /// [ADR 0011]. /// /// Per-source rows are collapsed (`FINAL`) then merged per bucket: `high=max`, /// `low=min`, volumes + `trade_count` summed, `vwap` volume-weighted, and /// `open`/`close` from the highest-volume source (`argMax(.., volume_base)`). -/// Ascending by timestamp. +/// In USD mode open/high/low/close/method come from the USDC/XLM-quoted rows when +/// the bucket has a valid one (lore-0332). Ascending by timestamp. /// /// ## 🔑 In USD mode the conversion happens BEFORE the merge, and the order is /// load-bearing @@ -1717,20 +1927,21 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou // ascending for output. An ASC+LIMIT would instead return the OLDEST N and // silently drop the recent candles a chart actually wants. `ts` is ISO-8601 // (`%Y-%m-%dT%H:%i:%SZ`), so lexicographic `ts ASC` == chronological order. - let (projection, aggregates) = match args.denomination { + let (projection, with, aggregates) = match args.denomination { Denomination::Usd(ref refs) => ( // Per-row scaling — see the ordering note above. `valid` gates // both the arithmetic and the classification, so a row that // cannot be priced contributes to volume and trade_count but // never to a price or a method. usd_projection(refs, args.granularity), - // `countIf(valid) = 0` is what produces §5's price-less bucket: + usd_with(), + // `_np` is what produces §5's price-less bucket: // NULL across every price field, while the volume columns below // still aggregate over all rows. // `c` is EXACT (`close_usd` as stored) while `h`/`l` are derived // through `toFloat64`, so the two are on different scales and can // cross — task 0229. `least`/`greatest` pull the derived extremes - // back over the exact close; see the CLOSE_EXACT note above. + // back over the exact close; see the `close_exact` note above. usd_aggregates(), ), // As stored: no conversion, so nothing is derived and there is no USD @@ -1744,6 +1955,7 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou volume_quote_usd, trade_count, pf_trade_count, pf_volume, \ pf_price_volume" .to_string(), + String::new(), // ⚠️ `vw` is clamped into `[min(low), max(high)]` here too — task 0229's // review, finding 1. This arm applies no rate, so `o`/`h`/`l`/`c` are // the stored decimals and cannot cross; the merged vwap still can, @@ -1796,7 +2008,7 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou // `ohlcv_and_peg_series_project_the_same_alias_tail` is the guard. let sql = format!( "SELECT {OUTER_ALIASES} FROM ( \ - SELECT \ + {with}SELECT \ formatDateTime(timestamp, '%Y-%m-%dT%H:%i:%SZ') AS ts, \ {aggregates} \ FROM ( SELECT {projection} FROM {table} FINAL WHERE {conds} ) \ @@ -1821,6 +2033,188 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou q.fetch_all::().await } +/// The aggregates of [`native_usd_series_sql`] over its rows +/// `(bkt, hk, hts, hr, hs, hq, hx, htc, hxr, hxp)`, in `OUTER_ALIASES` order +/// minus `ts`. Prices and provenance come from the rate rows (`hk = 1`), volume +/// from every row; `hxr`/`hxp` are the priced XLM amount × rate, and amount. +fn native_usd_aggregates() -> String { + // Float64, parsed back: a Decimal × Decimal product wraps silently. + let dec = |f: &str| format!("toDecimal128OrNull(toString({f}), 14)"); + // The worst quality of the bucket wins; an unknown label ranks worst. + let rank = "multiIf(hq = '', 0, hq = 'measured', 1, hq = 'measured-disputed', 2, \ + hq = 'fallback', 3, 4)"; + format!( + "toNullable(toString(argMinIf(hr, hts, hk))) AS o, \ + toNullable(toString(maxIf(hr, hk))) AS h, toNullable(toString(minIf(hr, hk))) AS l, \ + toNullable(toString(argMaxIf(hr, hts, hk))) AS c, \ + toString(sum(hx)) AS vb, toString(ifNull({vqu}, toDecimal128(0, 14))) AS vqu, \ + toNullable(toString(least(greatest(ifNull({vw}, argMaxIf(hr, hts, hk)), \ + minIf(hr, hk)), maxIf(hr, hk)))) AS vw, \ + toUInt64(sum(htc)) AS tc, toNullable('external') AS meth, \ + toNullable(toUInt8(1)) AS drv, {PF_NULL_TAIL}, \ + if(uniqExactIf(hs, hk) = 1, nullIf(anyIf(hs, hk), ''), NULL) AS src, \ + nullIf(argMaxIf(hq, {rank}, hk), '') AS qual", + // NULL (→ 0) only on overflow, ≥ 1e24 USD: unreachable before 2021. + vqu = dec("sum(hxr)"), + vw = dec("if(sum(hxp) > 0, sum(hxr) / sum(hxp), avgIf(toFloat64(hr), hk))"), + ) +} + +/// XLM's markets against a dollar token other than the off-peg ones, in either +/// orientation; each leg starts on a key prefix (see [`ohlcv_peg_series`]). +fn dollar_leg_pred(xlm: u64) -> String { + let off_peg = OFF_PEG_DOLLAR_TOKENS.map(|(code, issuer)| format!("('{code}', '{issuer}')")); + let d = format!( + "SELECT asset_id FROM assets FINAL WHERE {DOLLAR_TOKEN_PRED} \ + AND (asset_code, issuer_address) NOT IN ({})", + off_peg.join(", ") + ); + format!( + "((asset_id = {xlm} AND quote_asset_id IN ({d})) \ + OR (asset_id IN ({d}) AND quote_asset_id = {xlm}))" + ) +} + +/// Native XLM's own USD candles before [`PRE_USDC_END_S`], built from the +/// promoted hourly XLM/USD rate (task 0331). Only typed integers are +/// interpolated, so nothing binds. +fn native_usd_series_sql(args: &OhlcvArgs, xlm: u64) -> String { + let (pre, iv) = (PRE_USDC_END_S, args.granularity.interval_sql()); + // ` AND {lhs}toDateTime(t){rhs}` when the window bound `t` is set. + let w = |t: Option, lhs: &str, rhs: &str| { + t.map_or(String::new(), |t| format!(" AND {lhs}toDateTime({t}){rhs}")) + }; + let (amt, legs) = ( + format!("if(asset_id = {xlm}, volume_base, volume_quote)"), + dollar_leg_pred(xlm), + ); + let since = w(args.start, "timestamp >= ", ""); + // `reference_asset` by equality, as `coarse-repair` reads it, so `source` is + // `third-party` by construction. + let rates = |cols: &str, window: String| { + format!( + "SELECT 1 AS k, timestamp AS rts, usd_rate AS rate, \ + CAST(reference_asset AS String) AS rsrc, CAST(quality AS String) AS rq{cols} \ + FROM usd_rate FINAL \ + WHERE asset_kind = 'native' AND asset_code = 'XLM' AND issuer_address = '' \ + AND contract_address = '' AND method = 'external' \ + AND reference_asset = '{XLM_RATE_SOURCE}' AND usd_rate > 0 \ + AND timestamp < toDateTime({pre}){window}" + ) + }; + let hours = if args.granularity.seconds() < 3600 { + // 1m/15m: a bucket only where XLM traded, at its hour's rate. `quote_asset_id` + // is not a key prefix; read-only on prod (2026-10-15) a pre-2021 month holds + // ≤ 302,183 rows on 1m and ≤ 94,724 on 15m, so the 5000-bucket worst case + // (3 months) reads under 1 M rows. + let until = w(args.end, "timestamp <= ", ""); + format!( + "SELECT t.bkt AS bkt, 1 AS hk, r.rts AS hts, r.rate AS hr, r.rsrc AS hs, \ + r.rq AS hq, t.tx AS hx, t.ttc AS htc, \ + toFloat64(t.tx) * toFloat64(r.rate) AS hxr, toFloat64(t.tx) AS hxp \ + FROM ( SELECT timestamp AS bkt, toStartOfHour(timestamp, 'UTC') AS th, \ + sumIf({amt}, {legs}) AS tx, sumIf(trade_count, {legs}) AS ttc \ + FROM price_ohlcv_{g} FINAL \ + WHERE (asset_id = {xlm} OR quote_asset_id = {xlm}) \ + AND timestamp < toDateTime({pre}){since}{until} \ + GROUP BY timestamp ) AS t \ + INNER JOIN ( {r} ) AS r ON r.rts = t.th", + g = args.granularity.as_str(), + r = rates( + "", + w(args.start, "timestamp >= toStartOfHour(", ", 'UTC')") + &until + ), + ) + } else { + // ≥1h: the bucket set and prices from the rate rows (`hk = 1`); volume from + // every dollar-leg hour, in USD at the newest rate at or before it within 24 h + // (`ch_enrich::external_sql` at 1h), else 0. + let bkt = + |c: &str| format!("toDateTime(toStartOfInterval({c}, INTERVAL {iv}, 'UTC'), 'UTC')"); + let ok = "ifNull(r.rate > 0 AND toUInt32(v.vend) - toUInt32(r.rts) <= 86400, 0)"; + format!( + "SELECT bkt, 1 AS hk, rts AS hts, rate AS hr, rsrc AS hs, rq AS hq, \ + toDecimal128(0, 14) AS hx, toUInt64(0) AS htc, 0. AS hxr, 0. AS hxp \ + FROM ( {r} ) \ + UNION ALL \ + SELECT {vb} AS bkt, 0 AS hk, v.vh AS hts, toDecimal128(0, 14) AS hr, '' AS hs, \ + '' AS hq, v.vx AS hx, v.vtc AS htc, \ + if({ok}, toFloat64(v.vx) * ifNull(toFloat64(r.rate), 0.), 0.) AS hxr, \ + if({ok}, toFloat64(v.vx), 0.) AS hxp \ + FROM ( SELECT 1 AS k, timestamp AS vh, timestamp + INTERVAL 1 HOUR AS vend, \ + sum({amt}) AS vx, toUInt64(sum(trade_count)) AS vtc \ + FROM price_ohlcv_1h FINAL \ + WHERE {legs} AND timestamp < toDateTime({pre}){since}{until} \ + GROUP BY timestamp ) AS v \ + ASOF LEFT JOIN ( {back} ) AS r ON r.k = v.k AND r.rts < v.vend", + r = rates( + &format!(", {} AS bkt", bkt("timestamp")), + w(args.start, "bkt >= ", "") + &w(args.end, "bkt <= ", ""), + ), + vb = bkt("v.vh"), + back = rates("", w(args.start, "timestamp >= ", " - 86400")), + until = args.end.map_or(String::new(), |t| { + let end = bucket_end_sql(args.granularity, &format!("toDateTime({t}, 'UTC')")); + format!(" AND timestamp < {end}") + }), + ) + }; + format!( + "SELECT {OUTER_ALIASES} FROM ( \ + SELECT formatDateTime(bkt, '%Y-%m-%dT%H:%i:%SZ', 'UTC') AS ts, {agg}, bkt \ + FROM ( {hours} ) \ + GROUP BY bkt HAVING sum(hk) > 0 ORDER BY bkt DESC LIMIT {limit} \ + ) ORDER BY ts ASC", + agg = native_usd_aggregates(), + limit = args.limit, + ) +} + +/// `own` minus every bucket `synth` covers, plus `synth`: ascending, the last +/// `limit` kept, as [`ohlcv`] keeps the most recent. The dedup compares strings: +/// `synth` prints UTC, [`ohlcv`] the server's timezone, which must be UTC (as its +/// `Z` suffix already assumes). +fn merge_native_usd(mut synth: Vec, own: Vec, limit: usize) -> Vec { + let have: std::collections::HashSet = + synth.iter().map(|c| c.timestamp.clone()).collect(); + synth.extend(own.into_iter().filter(|c| !have.contains(&c.timestamp))); + synth.sort_by(|a, b| a.timestamp.cmp(&b.timestamp)); + synth.split_off(synth.len().saturating_sub(limit)) +} + +/// XLM's id when `asset_id` is native XLM in USD, the one case +/// [`ohlcv_native_usd`] serves; `None` in XLM mode and for every other asset. +pub fn native_xlm_usd(denomination: &Denomination, asset_id: u64) -> Option { + match denomination { + Denomination::Usd(refs) if refs.xlm == Some(asset_id) => refs.xlm, + _ => None, + } +} + +/// Whether a window starting at `start` can hold a pre-2021 bucket. +fn reaches_pre_usdc(start: Option) -> bool { + start.is_none_or(|s| s < i64::from(PRE_USDC_END_S)) +} + +/// Native XLM in USD: the synthesized pre-2021 series over [`ohlcv`] unchanged, +/// which still serves every bucket no promoted rate covers. +pub async fn ohlcv_native_usd( + ch: &Client, + args: OhlcvArgs, + xlm: u64, +) -> Result, clickhouse::error::Error> { + let limit = usize::try_from(args.limit).unwrap_or(usize::MAX); + let synth_sql = reaches_pre_usdc(args.start).then(|| native_usd_series_sql(&args, xlm)); + let synth = async { + let Some(sql) = &synth_sql else { + return Ok(Vec::new()); + }; + ch.query(sql).fetch_all::().await + }; + let (synth, own) = tokio::try_join!(synth, ohlcv(ch, args))?; + Ok(merge_native_usd(synth, own, limit)) +} + /// How far a bucket's `close` may sit from its price-forming mean before the /// response says so: one percent. /// @@ -2082,6 +2476,24 @@ mod tests { ); } + /// Task 0331: a priced pre-2021 row outside the USDC arms is `external`, + /// and the arm sits after every USDC arm and before `traded`. + #[test] + fn usd_method_expr_labels_priced_pre_usdc_rows_external() { + let arm = format!( + "timestamp < toDateTime({}), 'external', ", + prices_clickhouse::PRE_USDC_END_S + ); + for pivots in [&[7u64][..], &[]] { + let sql = usd_method_expr(2, pivots, Granularity::H1); + let at = sql.find(&arm).unwrap_or_else(|| panic!("{sql}")); + assert!(at > sql.find("quote_asset_id = 2, '', ").unwrap(), "{sql}"); + if let Some(traded) = sql.find("'traded'") { + assert!(at < traded, "{sql}"); + } + } + } + /// Task 0139: every SELECT whose id lands in a Rust u64 field casts with /// `toUInt64`, so the readers work on the UInt32 schema and the UInt64 one. /// A bare `a.asset_id` read into a u64 misparses a UInt32 column. @@ -2335,6 +2747,108 @@ mod tests { ); } + const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; + const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + const XLM_SAC: &str = "CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA"; + + fn xcr() -> AssetIdentifier { + AssetIdentifier::Classic { + code: "XCR".into(), + issuer: XCR_ISSUER.into(), + } + } + + /// Task 0242: classic rows only, one bind per address, positional order + /// matching [`SacAliasRow`]. + #[test] + fn sac_alias_sql_reads_classic_rows_through_identity_by_contract() { + use clickhouse::Row; + + let sql = sac_alias_sql(2); + for needle in [ + "FROM identity_by_contract", + "asset_kind != 'contract'", + "contract IN (?, ?)", + ] { + assert!(sql.contains(needle), "missing {needle}: {sql}"); + } + let expected: Vec = SacAliasRow::COLUMN_NAMES + .iter() + .map(|s| (*s).to_string()) + .collect(); + assert_eq!(projected_columns(&sql), expected, "SQL: {sql}"); + } + + #[test] + fn alias_identifier_maps_xlm_to_native_and_the_rest_to_classic() { + for (code, issuer, expected) in [ + ("XLM", "", AssetIdentifier::Native), + ("XCR", XCR_ISSUER, xcr()), + ( + "XLM", + XCR_ISSUER, + AssetIdentifier::Classic { + code: "XLM".into(), + issuer: XCR_ISSUER.into(), + }, + ), + ] { + assert_eq!( + alias_identifier(code.into(), issuer.into()), + expected, + "{code}:{issuer}" + ); + } + } + + /// Only contracts need a lookup; none means no query at all. + #[test] + fn sac_alias_candidates_are_the_distinct_contracts() { + let c = |s: &str| AssetIdentifier::Contract(s.into()); + for (ids, expected) in [ + (vec![], vec![]), + (vec![AssetIdentifier::Native, xcr()], vec![]), + ( + vec![c(XCR_SAC), AssetIdentifier::Native, c(XLM_SAC), c(XCR_SAC)], + vec![XCR_SAC, XLM_SAC], + ), + ] { + assert_eq!(sac_alias_candidates(&ids), expected, "{ids:?}"); + } + } + + /// A matched SAC becomes its classic; an unmatched contract and every + /// non-contract pass through in order. + #[test] + fn apply_sac_aliases_replaces_only_matched_contracts() { + let c = |s: &str| AssetIdentifier::Contract(s.into()); + let other = stellar_strkey::Contract([9u8; 32]).to_string(); + let aliases = std::collections::HashMap::from([ + (XCR_SAC.to_string(), xcr()), + (XLM_SAC.to_string(), AssetIdentifier::Native), + ]); + let got = apply_sac_aliases( + vec![ + c(XCR_SAC), + c(&other), + xcr(), + c(XLM_SAC), + AssetIdentifier::Native, + ], + &aliases, + ); + assert_eq!( + got, + vec![ + xcr(), + c(&other), + xcr(), + AssetIdentifier::Native, + AssetIdentifier::Native + ] + ); + } + #[test] fn identity_where_native_is_literal_no_binds() { let (sql, binds) = identity_where(&AssetIdentifier::Native); @@ -2889,8 +3403,58 @@ mod tests { fn usd_refs() -> UsdRefs { UsdRefs { usdc: 2, - pivots: vec![1], + xlm: Some(1), + usdt: None, + } + } + + /// lore-0332: a bucket where a USDC/XLM market formed a price takes its + /// prices and both means from those markets; any other bucket renders the + /// pre-0332 expressions, byte for byte. + #[test] + fn usd_prices_come_from_the_price_setting_legs_when_the_bucket_has_one() { + let with = usd_with(); + assert!(with.starts_with("WITH countIf(ps) > 0 AS _ub, "), "{with}"); + assert!( + with.contains("(if(_ub, countIf(band), countIf(valid)) = 0) AS _np"), + "an unconverted USDC/XLM leg leaves the bucket unpriced: {with}" + ); + // o, h, l, c, meth, vw, pfvw. + assert_eq!(with.matches("if(_ub, ").count(), 8, "{with}"); + for (band, all) in [("band", "valid"), ("bandc", "convertible")] { + let (b, a) = match band { + "band" => (high_published(band), high_published(all)), + _ => (vwap_raw(band), vwap_raw(all)), + }; + assert!(with.contains(&format!("if(_ub, {b}, {a})")), "{with}"); } + for f in [low_published, close_exact, pf_vwap_usd_raw] { + assert!( + with.contains(&format!("if(_ub, {}, {})", f("band"), f("valid"))), + "{with}" + ); + } + for before in [ + "argMaxIf(c_x, (volume_base, quote_asset_id), valid)", + "if(isNull(maxIf(h_x, valid)), NULL, greatest(maxIf(h_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))", + "if(isNull(minIf(l_x, valid)), NULL, least(minIf(l_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))", + "sumIf(toFloat64(w_x) * toFloat64(volume_base), convertible)", + "sumIf(toFloat64(pfpv_x), valid)", + ] { + assert!( + with.contains(before), + "fallback drifted from `{before}`: {with}" + ); + } + // The output columns read only the named inputs. + let agg = usd_aggregates(); + for gone in ["countIf(", "maxIf(", "minIf(", "argMaxIf(", "sumIf("] { + assert!(!agg.contains(gone), "`{gone}` belongs in usd_with: {agg}"); + } + let refs = usd_refs(); + assert_eq!(refs.price_setting(), vec![2, 1]); + assert_eq!(refs.pivots(), vec![1]); + assert_eq!(UsdRefs { xlm: None, ..refs }.price_setting(), vec![2]); } /// A candle's prices come only from the rows that held a price-forming @@ -2907,6 +3471,15 @@ mod tests { valid.contains("pf_trade_count > 0"), "`valid` must exclude a row whose bucket formed no price: {valid}" ); + assert!( + projection.contains( + ") AS valid, (pf_trade_count > 0 AND quote_asset_id IN (2, 1)) AS ps, \ + (valid AND ps) AS band, \ + (convertible AND quote_asset_id IN (2, 1)) AS bandc, " + ), + "`band`/`bandc` narrow `valid`/`convertible` to the USDC/XLM legs \ + (lore-0332): {projection}" + ); // The pf columns the aggregates weigh have to reach the outer SELECT. for col in ["pf_trade_count", "pf_volume", "pfpv_x"] { assert!( @@ -3097,19 +3670,18 @@ mod tests { #[test] fn the_merged_vwap_weights_every_row_including_the_ones_that_formed_no_price() { let usd = usd_aggregates(); + let with = usd_with(); assert!( - VWAP_RAW.contains("sumIf(toFloat64(w_x) * toFloat64(volume_base), convertible)"), + with.contains(&format!( + "if(_ub, {}, {}) AS _vw", + vwap_raw("bandc"), + vwap_raw("convertible") + )), "the USD merge weights every row it can convert, not only the \ - price-forming ones: {VWAP_RAW}" - ); - assert!( - !VWAP_RAW.contains("valid"), - "`valid` carries `pf_trade_count > 0`, which is what this undoes: {VWAP_RAW}" + price-forming ones: {with}" ); assert!( - usd.contains(&format!( - "if(countIf(valid) = 0 OR isNull({VWAP_RAW}), NULL," - )), + usd.contains("if(_np OR isNull(_vw), NULL,"), "a bucket with no price-forming fill still publishes no vwap: {usd}" ); @@ -3160,9 +3732,16 @@ mod tests { /// price-forming volume to weigh: a zero here would be read as a price. #[test] fn the_price_forming_vwap_is_clamped_into_the_published_band_and_never_zero() { - for (name, agg) in [ - ("Denomination::Usd", usd_aggregates()), - ("Denomination::QuoteLeg", quote_leg_aggregates()), + // The USD arm names its mean in `usd_with` and clamps it in the list. + let usd_mean = usd_with() + .split(" AS _pfvw") + .next() + .and_then(|head| head.rsplit(" AS _vw, ").next()) + .unwrap_or_default() + .to_string(); + for (name, agg, mean) in [ + ("Denomination::Usd", usd_aggregates(), Some(usd_mean)), + ("Denomination::QuoteLeg", quote_leg_aggregates(), None), ] { let pfvw = agg .split(" AS pfvw") @@ -3174,6 +3753,7 @@ mod tests { pfvw.contains("least(greatest("), "{name}'s pf_vwap must be clamped into [low, high]: {pfvw}" ); + let pfvw = mean.unwrap_or(pfvw); assert!( pfvw.contains("nullIf("), "{name}'s pf_vwap must be NULL, not 0, with no price-forming \ @@ -3263,4 +3843,128 @@ mod tests { quality: None, } } + + #[test] + fn native_usd_series_sql_renders_every_grain() { + let want: Vec<&str> = OUTER_ALIASES.split(", ").skip(1).collect(); + assert_eq!(aliases_of(&native_usd_aggregates()), want); + use Granularity::*; + for g in [M1, M15, H1, H4, D1, W1, Mo1] { + let args = OhlcvArgs { + asset_id: 7, + denomination: Denomination::QuoteLeg(7), + granularity: g, + start: Some(1_600_000_000), + end: Some(1_600_086_400), + limit: 5000, + }; + let sql = native_usd_series_sql(&args, 7); + let fine = g.seconds() < 3600; + assert!( + sql.starts_with(&format!("SELECT {OUTER_ALIASES} FROM (")), + "{sql}" + ); + // One rate read on 1m/15m; ≥1h adds the 24 h look-back for volume. + let reads = if fine { 1 } else { 2 }; + let rate = "method = 'external' AND reference_asset = 'third-party' AND"; + assert_eq!(sql.matches("method = ").count(), reads, "{g:?}"); + assert_eq!(sql.matches(rate).count(), reads, "{g:?}"); + for bad in ["LIKE", "startsWith", "candidate", "SETTINGS", "?"] { + assert!(!sql.contains(bad), "{g:?} {bad}: {sql}"); + } + let pre = format!("toDateTime({PRE_USDC_END_S})"); + assert_eq!(sql.matches(&pre).count(), reads + 1, "{g:?}: {sql}"); + let issuers = OFF_PEG_DOLLAR_TOKENS.map(|(_, i)| i); + for needle in [pre.as_str(), "'UTC'", DOLLAR_TOKEN_PRED] + .iter() + .chain(&issuers) + { + assert!(sql.contains(needle), "{g:?} {needle}: {sql}"); + } + // ≥1h reads volume from 1h on key prefixes only; 1m/15m scan their + // own grain with XLM on either side. + let table = if fine { g.as_str() } else { "1h" }; + assert!( + sql.contains(&format!("price_ohlcv_{table} FINAL")), + "{g:?}: {sql}" + ); + assert_eq!(sql.matches("price_ohlcv_").count(), 1, "{g:?}: {sql}"); + let either = sql.contains("(asset_id = 7 OR quote_asset_id = 7)"); + assert_eq!(either, fine, "{g:?}: {sql}"); + let interval = format!("INTERVAL {}", g.interval_sql()); + assert!(fine || sql.contains(&interval), "{g:?}: {sql}"); + // Calendar arithmetic is pinned to UTC, as in `usd_method_expr`. + let call = |at: usize| { + let mut depth = 0; + for (j, ch) in sql[at..].char_indices() { + match ch { + '(' => depth += 1, + ')' if depth == 1 => return &sql[at..at + j], + ')' => depth -= 1, + _ => {} + } + } + panic!("unbalanced at {at}: {sql}") + }; + for f in [ + "addDays(", + "addWeeks(", + "addMonths(", + "toStartOf", + "formatDateTime(", + ] { + for (i, _) in sql.match_indices(f) { + assert!(call(i).ends_with("'UTC'"), "{g:?}: {}", call(i)); + } + } + for iv in [ + "- INTERVAL", + "+ INTERVAL 1 DAY", + "+ INTERVAL 1 WEEK", + "+ INTERVAL 1 MONTH", + ] { + assert!(!sql.contains(iv), "{g:?} {iv}: {sql}"); + } + } + } + + #[test] + fn native_usd_routing_and_skip() { + let usd = Denomination::Usd(usd_refs()); + assert_eq!(native_xlm_usd(&usd, 1), Some(1)); + assert_eq!(native_xlm_usd(&usd, 2), None); + assert_eq!(native_xlm_usd(&Denomination::QuoteLeg(1), 1), None); + let pre = i64::from(PRE_USDC_END_S); + let got = [None, Some(pre - 1), Some(pre), Some(pre + 1)].map(reaches_pre_usdc); + assert_eq!(got, [true, true, false, false]); + } + + #[test] + fn merge_native_usd_replaces_orders_and_caps() { + let at = |d: &&str, synth: bool| Candle { + timestamp: format!("2021-{d}"), + source: synth.then(|| "third-party".to_string()), + ..candle_with(None, None) + }; + let tag = + |c: &Candle| c.timestamp[5..].to_string() + ["", "*"][c.source.is_some() as usize]; + // (synthetic days, limit, merged days, `*` = synthetic) + let cases: [(&[&str], usize, &str); 3] = [ + (&["01-30", "01-31"], 5000, "01-29 01-30* 01-31* 02-01"), + (&["01-30", "01-31"], 2, "01-31* 02-01"), + (&[], 5000, "01-29 01-31 02-01"), + ]; + for (synth, limit, want) in cases { + let synth = synth.iter().map(|d| at(d, true)).collect(); + let own = ["01-29", "01-31", "02-01"] + .iter() + .map(|d| at(d, false)) + .collect(); + let got: Vec = merge_native_usd(synth, own, limit) + .iter() + .map(tag) + .collect(); + assert_eq!(got.join(" "), want, "limit {limit}"); + } + } } diff --git a/packages/prices-api/src/batch/handlers.rs b/packages/prices-api/src/batch/handlers.rs index 46297edc..5cef874e 100644 --- a/packages/prices-api/src/batch/handlers.rs +++ b/packages/prices-api/src/batch/handlers.rs @@ -19,7 +19,9 @@ use crate::state::AppState; /// /// Validates the whole list first (400 if any identifier is malformed or the /// batch is empty / over `MAX_BATCH`). Assets with no current-price row are -/// returned in `not_found` rather than failing the request. Uncached. +/// returned in `not_found` rather than failing the request. A found entry is +/// named by the asset's canonical identifier, so a SAC address answers under +/// its classic name (task 0242 D5); `not_found` echoes what was asked. Uncached. #[utoipa::path( post, path = "/prices/batch", @@ -28,8 +30,12 @@ use crate::state::AppState; description = "The same snapshot `GET /assets/{asset_identifier}/price` returns, for up to 100 assets\nin \ one request. The whole list is validated first: a malformed identifier, an empty list\nor \ more than 100 entries fails the entire request with a 400. Assets without a current\nprice \ - are listed in `not_found` instead of failing the call. Results follow the order of\nthe \ - request; a duplicated identifier is answered each time. Not cached.", + are listed in `not_found` instead of failing the call. `prices` follows the order of the\nrequest \ + with the not-found identifiers left out, and a duplicated identifier is answered each\ntime. \ + Each entry's `asset` is the canonical identifier of the asset found: a classic asset's\nStellar \ + Asset Contract address is answered as `CODE:ISSUER` (or `native`), so a request naming\nboth \ + forms gets two identical entries. `not_found` echoes each identifier as it was asked,\nin \ + canonical form. Not cached.", request_body = BatchRequest, responses( (status = 200, description = "Current prices + not-found list", body = BatchResponse), @@ -67,6 +73,14 @@ pub async fn post_batch( } } + // A SAC address answers as its classic (task 0242); `not_found` still + // echoes what was asked. + let asked: Vec = ids.iter().map(AssetIdentifier::to_canonical).collect(); + let ids = match queries_ch::resolve_sac_aliases(state.ch(), ids).await { + Ok(ids) => ids, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; + // One query for the whole batch (vs. a per-asset N+1 loop), then map each // requested identifier back to its row by natural-identity key. let rows = match queries_ch::current_prices_batch(state.ch(), &ids).await { @@ -78,7 +92,7 @@ pub async fn post_batch( let mut prices = Vec::new(); let mut not_found = Vec::new(); - for id in &ids { + for (id, asked) in ids.iter().zip(asked) { match by_key.get(&queries_ch::IdentKey::of(id)) { Some(row) => prices.push(PriceResponse::from_row( id.to_canonical(), @@ -96,7 +110,7 @@ pub async fn post_batch( price_basis: row.price_basis.clone(), }, )), - None => not_found.push(id.to_canonical()), + None => not_found.push(asked), } } diff --git a/packages/prices-api/src/config.rs b/packages/prices-api/src/config.rs index cc1770ed..4f4cf7b4 100644 --- a/packages/prices-api/src/config.rs +++ b/packages/prices-api/src/config.rs @@ -94,7 +94,7 @@ pub struct AppConfig { /// [`Self::load_portal_eligibility`], or directly by a test. pub portal_eligibility: Option, /// The origin the portal's bundle is served from, when that is not this - /// backend's own host (task 0194): `https://sorobanscan.rumblefish.dev`. + /// backend's own host (task 0194): `https://sorobanscan.com`. /// /// Read from `PORTAL_WEB_ORIGIN`, which `compute-stack.ts` sets from /// `portalWebOrigin` — the same value `api-gateway-stack.ts` names in the @@ -581,7 +581,7 @@ mod web_origin_tests { Some(" "), Some("/"), Some("sorobanscan.rumblefish.dev"), - Some("https://sorobanscan.rumblefish.dev/prices-api/"), + Some("https://sorobanscan.rumblefish.dev/api/prices-api/"), Some("https://sorobanscan.rumblefish.dev?x=1"), Some("https://"), ] { diff --git a/packages/prices-api/src/identity.rs b/packages/prices-api/src/identity.rs index ee01ae0c..01a29ec3 100644 --- a/packages/prices-api/src/identity.rs +++ b/packages/prices-api/src/identity.rs @@ -4,7 +4,8 @@ //! of three textual forms: //! - `native` → the native XLM asset //! - `{code}:{issuer}` → a classic credit asset (issuer is a G-strkey) -//! - `{contract_address}` → a Soroban token / SAC (a C-strkey) +//! - `{contract_address}` → a Soroban token / SAC (a C-strkey); a classic +//! asset's SAC is aliased to that classic (task 0242) //! //! This module only *parses + validates the shape* (real strkey CRC validation //! via `stellar-strkey`). Resolving a parsed identity to the internal diff --git a/packages/prices-api/src/lib.rs b/packages/prices-api/src/lib.rs index 151e8872..d916f2e0 100644 --- a/packages/prices-api/src/lib.rs +++ b/packages/prices-api/src/lib.rs @@ -136,8 +136,8 @@ fn app_inner( // on a host whose root belongs to another application (task 0194). // // `Access-Control-Allow-Origin: *`, on both copies (task 0195). The - // portal's Swagger UI is a page on `sorobanscan.rumblefish.dev` fetching - // this document from `prices-api.sorobanscan.rumblefish.dev` — cross-origin, + // portal's Swagger UI is a page on `sorobanscan.com` fetching this + // document from `prices-api.sorobanscan.com` — cross-origin, // and read by `fetch`, not by navigation, so without an allow header the // browser refuses to hand the bytes to the page. `*` rather than the // portal's origin, deliberately: the document is anonymous, public and diff --git a/packages/prices-api/src/openapi/descriptions.rs b/packages/prices-api/src/openapi/descriptions.rs index 6a6a3f8a..316d19b2 100644 --- a/packages/prices-api/src/openapi/descriptions.rs +++ b/packages/prices-api/src/openapi/descriptions.rs @@ -59,7 +59,10 @@ pub(super) const SCHEMAS: &[(&str, &str)] = &[ that has no USD value: either the conversion has not caught up with the newest \ buckets yet, or the bucket traded only against a quote asset with no USD reference. \ `volume_base`, `volume_quote_usd` and `trade_count` are always present, so such a \ - bucket still shows its activity.", + bucket still shows its activity.\n\nThe exception is native XLM before 2021-02-01 \ + (`GET /assets/native/ohlcv`): a bucket the imported hourly XLM/USD rate covers takes \ + its prices from that rate and its volumes from XLM's trades against dollar tokens \ + (`method` `external`, `derived` `true`).", ), ( "ErrorEnvelope", @@ -346,7 +349,10 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Closing price of the bucket, in `base_currency`: the LAST price-forming trade of \ this bucket, never one carried over from an earlier one. `null` when the bucket has \ no price — see `pf_trade_count`. Exact in USD mode — the other price fields are \ - scaled, see `derived`.", + scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when \ + one of them formed a price in the bucket; otherwise every market does. On native \ + XLM's synthesized buckets before 2021-02-01 it is the bucket's last hourly XLM/USD \ + rate.", ), ( "Candle", @@ -366,27 +372,39 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ measured. Normally `close` is exact and the other price fields are scaled from the \ quote-asset values with one rate per bucket, so the true USD high may have occurred \ at a different moment than the quote-asset high. For USDC, which has no trades of \ - its own, every price field comes from the rate and `derived` is `true`.\n\n`null` \ + its own, every price field comes from the rate and `derived` is `true`, as it does for \ + native XLM before 2021-02-01 where the hourly XLM/USD rate covers the bucket.\n\n`null` \ when the price fields are `null`, and always `null` for `base_currency=XLM`.", ), ( "Candle", "high", "Highest price among this bucket's price-forming trades, in `base_currency`; `null` \ - when the bucket has no price — see `pf_trade_count`.", + when the bucket has no price — see `pf_trade_count`. In USD mode only markets \ + quoted in USDC or XLM set it when one of them formed a price in the bucket; \ + otherwise every market does. On native XLM's synthesized buckets before 2021-02-01 \ + it is the highest hourly XLM/USD rate of the bucket, an hourly average rather than \ + a trade.", ), ( "Candle", "low", "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` \ - when the bucket has no price — see `pf_trade_count`.", + when the bucket has no price — see `pf_trade_count`. In USD mode only markets \ + quoted in USDC or XLM set it when one of them formed a price in the bucket; \ + otherwise every market does. On native XLM's synthesized buckets before 2021-02-01 \ + it is the lowest hourly XLM/USD rate of the bucket, an hourly average rather than \ + a trade.", ), ( "Candle", "method", "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was \ measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* \ - `external` — an imported, measured USDC/USD series supplied the rate.\n* `oracle` — \ + `external` — an imported, measured USD series supplied the rate: USDC/USD on a \ + USDC-quoted candle; before 2021-02-01, XLM/USD on a \ + candle quoted in XLM or in a dollar token (USD, USDT or a USDC other than \ + Circle's), the latter converted through that token's own XLM market.\n* `oracle` — \ a measured Reflector reading supplied the rate.\n* `traded` — priced through a \ reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series \ (`GET /assets/USDC:/ohlcv`): no measured USDC/USD observation covered the \ @@ -415,13 +433,19 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ cases: when the price fields are `null`; always for \ `base_currency=XLM`, where nothing is converted; and for a USDC-quoted bucket \ below the oracle epoch that carries a converted rate no imported series covers \ - — nothing can attribute it, and a label would be a claim.", + — nothing can attribute it, and a label would be a claim.\n\nIn USD mode the label \ + follows the markets that set the prices: those quoted in USDC or XLM when one of \ + them formed a price in the bucket, otherwise every market. A bucket that traded \ + mostly against USDT can therefore report the `assumed-par` of its USDC market.", ), ( "Candle", "open", "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of \ - this bucket. `null` when the bucket has no price — see `pf_trade_count`.", + this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD \ + mode only markets quoted in USDC or XLM set it when one of them formed a price in \ + the bucket; otherwise every market does. On native XLM's synthesized buckets \ + before 2021-02-01 it is the bucket's first hourly XLM/USD rate.", ), ( "Candle", @@ -438,8 +462,8 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ traded, its `volume_base` and `trade_count` are reported, and it has NO price — \ every price field is `null` rather than carrying a dust print. Always at most \ `trade_count`; `null` only on the synthesized USDC self-series \ - (`GET /assets/USDC:/ohlcv`), which is built from rate observations and has \ - no trades to count.", + (`GET /assets/USDC:/ohlcv`) and on native XLM's synthesized buckets before \ + 2021-02-01, which are built from rate observations.", ), ( "Candle", @@ -447,16 +471,19 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Volume-weighted average price of the bucket's price-forming trades, in \ `base_currency` — `vwap` with the dust left out, and the value `close_divergent` \ compares `close` against.\n\nDenominated exactly like `close` and bounded by the \ - published `low` and `high`, so the response stays self-consistent. `null` when the \ - bucket has no price-forming volume, and on the synthesized USDC self-series. Never \ - `0`.", + published `low` and `high`, so the response stays self-consistent. In USD mode it \ + weighs the same markets that set those (see `/ohlcv`). `null` when the \ + bucket has no price-forming volume, on the synthesized USDC self-series and on \ + native XLM's synthesized buckets before 2021-02-01. Never `0`.", ), ( "Candle", "quality", - "How confident the imported series is in that day's observation. Present only \ + "How confident the imported series is in its observation; on a bucket built from \ + several, the least confident of them. Present only \ alongside `source`, i.e. only on the synthesized USDC self-series \ - (`GET /assets/USDC:/ohlcv`) for buckets whose `method` is `external`; \ + (`GET /assets/USDC:/ohlcv`) and on native XLM before 2021-02-01 \ + (`GET /assets/native/ohlcv`), for buckets whose `method` is `external`; \ `null` on every other asset, on the `peg` fallback, and whenever the price fields \ are `null`.\n\n* `measured` — a real observation from the primary feed.\n* \ `measured-disputed` — observed, but a cross-check against a second independent \ @@ -470,9 +497,10 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ ( "Candle", "source", - "Which outside USD series the imported rate came from: `chainlink` or `bitstamp`.\n\n\ - Present only on the synthesized USDC self-series \ - (`GET /assets/USDC:/ohlcv`), and there only for buckets whose `method` is \ + "Which outside USD series the imported rate came from: `chainlink` or `bitstamp` for \ + USDC, `third-party` for XLM.\n\nPresent only on the synthesized USDC self-series \ + (`GET /assets/USDC:/ohlcv`) and on native XLM before 2021-02-01 \ + (`GET /assets/native/ohlcv`), and there only for buckets whose `method` is \ `external`. It is `null` on every other asset — not because those candles lack \ provenance, but because the stored candles carry no column to report it from — \ `null` on an `oracle` bucket (a poll has no outside series, even where an outranked \ @@ -488,14 +516,18 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ ( "Candle", "trade_count", - "Number of trades in the bucket. The documented maximum, 2^53 − 1, is the largest \ - integer JSON carries exactly — a transport limit, not a domain one.", + "Number of trades in the bucket; on native XLM's synthesized buckets before \ + 2021-02-01, only XLM's trades against dollar tokens, so possibly `0`. The documented \ + maximum, 2^53 − 1, is the largest integer JSON carries exactly — a transport limit, \ + not a domain one.", ), ( "Candle", "volume_base", "Volume in units of the asset — the base side of each trade, over EVERY trade in \ - the bucket including the ones too small to form a price. It is therefore a \ + the bucket including the ones too small to form a price (on native XLM's \ + synthesized buckets before 2021-02-01, its trades against dollar tokens only, so \ + possibly `0`). It is therefore a \ complete activity figure and not a price-quality signal: `pf_trade_count` is the \ one to read for that, and a bucket that traded only in such amounts reports its \ volume here with no price at all.\n\nVolume is undistorted by those trades — \ @@ -507,20 +539,26 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "volume_quote_usd", "USD volume of the bucket, summed over the trades that have a USD value. A trade not \ yet priced counts in `volume_base` and `trade_count` but not here, so this is the \ - USD volume that can be accounted for, not the bucket's total restated in USD.", + USD volume that can be accounted for, not the bucket's total restated in USD. On \ + native XLM's synthesized buckets before 2021-02-01 it covers XLM's trades against \ + dollar tokens only, each hour valued at the newest XLM/USD rate of the previous \ + 24 hours.", ), ( "Candle", "vwap", "Volume-weighted average price of the bucket, in `base_currency`, over the trades it \ holds — including the ones too small to form a price. `pf_vwap` is the same mean \ - with those left out. `null` when the bucket has no price.\n\nWHICH trades it \ + with those left out. `null` when the bucket has no price. On native XLM's synthesized \ + buckets before 2021-02-01 it is the hourly rates weighted by its dollar-token volume, \ + or their mean without any.\n\nWHICH trades it \ weighs depends on the denomination, so the two can differ on a bucket that holds a \ source with no usable price. With `base_currency=XLM` it is every row of the \ bucket, as stored. With `USD` it is every row that has a usable rate to convert it \ - with — a row whose own price is below the precision floor, or that has no USD value \ - yet, cannot be restated in USD and is left out of the mean rather than counted at \ - zero.\n\nBounded by the published `low` and `high` so the response stays \ + with, on the markets that set the prices (see `/ohlcv`) — a row whose own price is \ + below the precision floor, or that has no USD value yet, cannot be restated in USD \ + and is left out of the mean rather than counted at zero.\n\nBounded by the \ + published `low` and `high` so the response stays \ self-consistent. Those come from the price-forming trades only, so on a bucket \ where the trades that formed no price carry most of the volume the raw mean falls \ outside that range and the value reported is the nearer bound. Compare it with \ diff --git a/packages/prices-api/src/oracles/handlers.rs b/packages/prices-api/src/oracles/handlers.rs index babd4e0c..a6a9b8db 100644 --- a/packages/prices-api/src/oracles/handlers.rs +++ b/packages/prices-api/src/oracles/handlers.rs @@ -29,8 +29,8 @@ use crate::state::AppState; ("asset_identifier" = String, Path, description = "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or \ the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic \ - asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract \ - addresses, which as a rule answer `404`") + asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the \ + answer names the asset by `CODE:ISSUER` (or `native`)") ), responses( (status = 200, description = "Oracle readings", body = OraclesResponse), @@ -50,6 +50,10 @@ pub async fn get_oracles( Ok(id) => id, Err(e) => return errors::bad_request(errors::INVALID_ID, e.to_string()), }; + let id = match assets_q::resolve_sac_alias(state.ch(), id).await { + Ok(id) => id, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; let asset_id = match assets_q::resolve_asset_id(state.ch(), &id).await { Ok(Some(aid)) => aid, diff --git a/packages/prices-api/src/portal/auth/cookies.rs b/packages/prices-api/src/portal/auth/cookies.rs index 8f13bd5b..98cfbf06 100644 --- a/packages/prices-api/src/portal/auth/cookies.rs +++ b/packages/prices-api/src/portal/auth/cookies.rs @@ -28,8 +28,8 @@ //! the state check. `Lax` sends cookies on top-level GET navigation and //! withholds them from cross-site subrequests, which is precisely the //! distinction wanted. It survives the split across two hosts (task 0194) -//! because those hosts are **same-site**: `sorobanscan.rumblefish.dev` and -//! `prices-api.sorobanscan.rumblefish.dev` share a registrable domain, and +//! because those hosts are **same-site**: `sorobanscan.com` and +//! `prices-api.sorobanscan.com` share a registrable domain (task 0340), and //! `SameSite` is judged on that, not on origin. Had the backend moved to an //! unrelated domain this would have had to become `SameSite=None`, i.e. no //! CSRF protection from the cookie layer at all. diff --git a/packages/prices-api/src/portal/auth/issue.rs b/packages/prices-api/src/portal/auth/issue.rs index dfd3ebf1..9f30e890 100644 --- a/packages/prices-api/src/portal/auth/issue.rs +++ b/packages/prices-api/src/portal/auth/issue.rs @@ -696,7 +696,7 @@ mod tests { ]; for (i, a) in fixed.iter().enumerate() { assert!(a.starts_with("?issue=")); - assert!(format!("{PORTAL_HOME}{a}").starts_with("/prices-api/?")); + assert!(format!("{PORTAL_HOME}{a}").starts_with("/api/prices-api/?")); for b in &fixed[i + 1..] { assert_ne!(a, b); } diff --git a/packages/prices-api/src/portal/auth/mod.rs b/packages/prices-api/src/portal/auth/mod.rs index 09484f68..6f094c84 100644 --- a/packages/prices-api/src/portal/auth/mod.rs +++ b/packages/prices-api/src/portal/auth/mod.rs @@ -95,7 +95,7 @@ pub const LOGOUT_PATH: &str = "/api/auth/logout"; /// origin is the one that handles OAuth. The portal has several pages now and /// this did not change: the page it lands on decides where to go next; this /// handler still will not. -const PORTAL_HOME: &str = "/prices-api/"; +const PORTAL_HOME: &str = "/api/prices-api/"; /// Appended to [`PORTAL_HOME`] when the visitor declined at Discord's consent /// screen, so [0185]'s page can say "sign-in cancelled" instead of silently @@ -215,10 +215,13 @@ impl AuthState { /// The callback runs on this backend's host, where the session cookie is /// set; the page the visitor is sent back to lives on /// `AppConfig::portal_web_origin`. A relative `Location` would keep them on - /// the API host, where `/prices-api/` is nothing — the gateway's own - /// `404`. `None` keeps the bare path, which is the same-origin deployment - /// and what every existing test sees. A builder, like [`Self::with_issue`], - /// so every constructor and test stays valid. + /// the API host, where `/api/prices-api/` is not the page: it falls under + /// this backend's own [`PORTAL_API_PREFIX`](super::PORTAL_API_PREFIX) and + /// gets the router's empty `404` (task 0339; the explorer host's `/api/` + /// and this one share a name, not a route). `None` keeps the bare path, + /// which is the same-origin deployment and what every existing test sees. + /// A builder, like [`Self::with_issue`], so every constructor and test + /// stays valid. pub fn with_web_origin(mut self, origin: Option<&str>) -> Self { if let Some(origin) = origin { self.home = format!("{origin}{PORTAL_HOME}").into(); @@ -1232,7 +1235,10 @@ mod tests { vec!["a=1; Path=/".into(), "b=2; Path=/".into()], ); assert_eq!(response.status(), StatusCode::SEE_OTHER); - assert_eq!(response.headers().get(LOCATION).unwrap(), "/prices-api/"); + assert_eq!( + response.headers().get(LOCATION).unwrap(), + "/api/prices-api/" + ); assert_eq!(response.headers().get_all(SET_COOKIE).iter().count(), 2); assert_eq!(response.headers().get("cache-control").unwrap(), "no-store"); } @@ -1323,7 +1329,7 @@ mod tests { assert_ne!(CANCELLED_QUERY, FAILED_QUERY); assert_eq!(ERROR_ACCESS_DENIED, "access_denied"); for query in [CANCELLED_QUERY, FAILED_QUERY] { - assert!(format!("{PORTAL_HOME}{query}").starts_with("/prices-api/?")); + assert!(format!("{PORTAL_HOME}{query}").starts_with("/api/prices-api/?")); } } @@ -1333,7 +1339,7 @@ mod tests { /// delete this to do it. #[test] fn the_only_redirect_targets_are_the_portal_itself() { - assert_eq!(PORTAL_HOME, "/prices-api/"); + assert_eq!(PORTAL_HOME, "/api/prices-api/"); assert!(PORTAL_HOME.starts_with('/')); assert!(!PORTAL_HOME.starts_with("//")); for query in [ @@ -1353,7 +1359,7 @@ mod tests { &issue::too_young_query(173), &issue::capped_query("2026-09-01"), ] { - assert!(format!("{PORTAL_HOME}{query}").starts_with("/prices-api/?")); + assert!(format!("{PORTAL_HOME}{query}").starts_with("/api/prices-api/?")); } } @@ -1363,19 +1369,19 @@ mod tests { #[test] fn the_landing_is_prefixed_with_the_configured_origin_and_nothing_else() { let bare = AuthState::new(None, discord::Endpoints::default()); - assert_eq!(&*bare.home, "/prices-api/"); + assert_eq!(&*bare.home, "/api/prices-api/"); let kept = AuthState::new(None, discord::Endpoints::default()).with_web_origin(None); - assert_eq!(&*kept.home, "/prices-api/"); + assert_eq!(&*kept.home, "/api/prices-api/"); let hosted = AuthState::new(None, discord::Endpoints::default()) .with_web_origin(Some("https://sorobanscan.example")); - assert_eq!(&*hosted.home, "https://sorobanscan.example/prices-api/"); + assert_eq!(&*hosted.home, "https://sorobanscan.example/api/prices-api/"); let response = redirect(&format!("{}{CANCELLED_QUERY}", hosted.home), vec![]); assert_eq!( response.headers()[LOCATION], - "https://sorobanscan.example/prices-api/?signin=cancelled" + "https://sorobanscan.example/api/prices-api/?signin=cancelled" ); } diff --git a/packages/prices-api/src/portal/auth/state_token.rs b/packages/prices-api/src/portal/auth/state_token.rs index b232cb5c..fa144683 100644 --- a/packages/prices-api/src/portal/auth/state_token.rs +++ b/packages/prices-api/src/portal/auth/state_token.rs @@ -277,7 +277,7 @@ pub fn accept( /// /// ⚠️ **This authorises nothing and must never be used to.** It chooses /// between two fixed literals, both failures, both reachable by anyone who -/// types `/prices-api/?issue=failed` into a link — so a forged claim buys an +/// types `/api/prices-api/?issue=failed` into a link — so a forged claim buys an /// attacker nothing they did not already have. Everything a claim could /// matter for goes through [`accept`], which verifies both halves before /// reading a byte of either. `None` on anything that is not a token of ours diff --git a/packages/prices-api/src/portal/mod.rs b/packages/prices-api/src/portal/mod.rs index ba173c6b..b31f3235 100644 --- a/packages/prices-api/src/portal/mod.rs +++ b/packages/prices-api/src/portal/mod.rs @@ -69,12 +69,15 @@ use crate::config::AppConfig; /// /// `/api/` is the portal's backend on the API's own hostname. The bundle shared /// the prefix from task 0194 (2026-08-31) until task 0326 moved it to -/// `sorobanscan.rumblefish.dev/prices-api/` (the explorer now answers `301` on -/// `/api…` there). Nothing routes between them either way, because they are on -/// different HOSTS — the bundle is served from the block explorer's -/// distribution, and the page calls this backend cross-origin and same-site. So -/// from in here the prefix is simply ours end to end; a bundle path that -/// arrives (it should not) is a plain `404`, the same as any unrouted path. +/// `sorobanscan.rumblefish.dev/prices-api/`, and shares it again since task +/// 0339 at `/api/prices-api/` on the explorer's host, `sorobanscan.com` since +/// task 0340 (the explorer answers +/// `301` there on every other `/api/…`, `/prices-api…` and `/pricing-api…`). +/// Nothing routes between them either way, because they are on different +/// HOSTS — the bundle is served from the block explorer's distribution, and +/// the page calls this backend cross-origin and same-site. So from in here the +/// prefix is simply ours end to end; a bundle path that arrives (it should +/// not) is a plain `404`, the same as any unrouted path. /// /// Replaces [0161]'s `/*` + `/api/*` convention, which produced /// `/api/api/…` for an app that is itself called "api". The OAuth redirect URI diff --git a/packages/prices-api/src/state.rs b/packages/prices-api/src/state.rs index d602b83a..343af530 100644 --- a/packages/prices-api/src/state.rs +++ b/packages/prices-api/src/state.rs @@ -47,7 +47,8 @@ impl AppState { } } - /// Memoized reference ids; resolved once per state, then shared. + /// Memoized reference ids; resolved once per state, then shared. Not set + /// while XLM is untracked (see `handlers::resolve_usd_refs`). pub fn usd_refs(&self) -> &OnceCell { &self.usd_refs } diff --git a/packages/prices-api/tests/endpoints_it.rs b/packages/prices-api/tests/endpoints_it.rs index b8acc0a4..bbeb9ecb 100644 --- a/packages/prices-api/tests/endpoints_it.rs +++ b/packages/prices-api/tests/endpoints_it.rs @@ -27,6 +27,11 @@ fn issuer() -> &'static str { prices_clickhouse::USDC_ISSUER } +/// XLM's SAC, stored on the XLM row as prod holds it (task 0242). +const XLM_SAC: &str = "CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA"; +const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; +const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + /// Create + seed a scratch db with assets, current prices, oracle prices, and /// backfill progress; return a client scoped to it. async fn setup(db: &str) -> Client { @@ -44,9 +49,17 @@ async fn setup(db: &str) -> Client { prices_clickhouse::apply_sql(&admin, &rewrite(prices_clickhouse::INIT_SQL, db)) .await .unwrap(); + // The SAC alias reads this view (task 0242); the rest of views.sql needs + // tables these tests do not seed. + let view = prices_clickhouse::VIEWS_SQL + .split(';') + .find(|s| s.contains("CREATE OR REPLACE VIEW prices.identity_by_contract")) + .expect("views.sql defines identity_by_contract"); + let view = &view[view.find("CREATE OR REPLACE VIEW").unwrap()..]; + admin.query(&rewrite(view, db)).execute().await.unwrap(); let assets = [ - AssetFixture::new("XLM", "native", "", ""), + AssetFixture::new("XLM", "native", "", "").with_sac(XLM_SAC), AssetFixture::new("USDC", "credit", issuer(), ""), ]; admin @@ -305,6 +318,186 @@ async fn asset_detail_unknown_is_404() { teardown(db).await; } +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn asset_detail_unknown_contract_is_404() { + let db = "it_ep_detail_unknown_c_0242"; + let client = setup(db).await; + let (status, _) = get(client, &format!("/v1/assets/{}", contract())).await; + assert_eq!(status, StatusCode::NOT_FOUND); + teardown(db).await; +} + +/// Seed XCR as a classic row carrying its SAC, priced at 0.25. `pre_heal` adds +/// the second identity task 0242 heals: a Contract row for the SAC, priced at +/// 9.0, so a lookup that reached it would show. +async fn seed_xcr(db: &str, pre_heal: bool) { + let admin = Client::default().with_url(ch_url()); + let classic = AssetFixture::new("XCR", "credit", XCR_ISSUER, "").with_sac(XCR_SAC); + let sac = AssetFixture::new("", "contract", "", XCR_SAC); + let usdc = AssetFixture::new("USDC", "credit", issuer(), "").id(); + let mut rows = vec![classic]; + let mut prices = vec![format!( + "({}, 0.25, 0.25, 10, '2026-02-10 12:00:30')", + classic.id() + )]; + // One XCR/USDC hour per identity, `close_usd = close` (par quote). + let candle = |id: String, px: f64, trades: u32| { + format!( + "('2026-02-10 12:00:00', {id}, {usdc}, 'soroswap', {px}, {px}, {px}, {px}, \ + 10, 10, {px}, {px}, {trades}, 1)" + ) + }; + let mut candles = vec![candle(classic.id(), 0.25, 7)]; + if pre_heal { + rows.push(sac); + prices.push(format!( + "({}, 9.0, 9.0, 1, '2026-02-10 12:00:30')", + sac.id() + )); + candles.push(candle(sac.id(), 9.0, 1)); + } + admin + .query(&format!( + "INSERT INTO {db}.price_ohlcv_1h \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote_usd, close_usd, vwap, trade_count, version) VALUES {}", + candles.join(", ") + )) + .execute() + .await + .unwrap(); + admin + .query(&assets_insert(db, &rows)) + .execute() + .await + .unwrap(); + admin + .query(&format!( + "INSERT INTO {db}.current_prices \ + (asset_id, price_usd, vwap_24h, volume_24h_usd, updated_at) VALUES {}", + prices.join(", ") + )) + .execute() + .await + .unwrap(); +} + +/// Task 0242 (D5): a classic asset's SAC address answers as the classic, and +/// the classic wins over a pre-heal Contract row for the same address. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn sac_address_answers_as_its_classic_asset() { + let xcr = format!("XCR:{XCR_ISSUER}"); + for (db, pre_heal) in [ + ("it_ep_sac_alias_0242", false), + ("it_ep_sac_alias_pre_heal_0242", true), + ] { + let client = setup(db).await; + seed_xcr(db, pre_heal).await; + + let (status, json) = get(client.clone(), &format!("/v1/assets/{XCR_SAC}")).await; + assert_eq!(status, StatusCode::OK, "{db}: body={json}"); + assert_eq!(json["asset"], xcr, "{db}"); + assert_eq!(json["asset_kind"], "credit", "{db}"); + assert_eq!(json["code"], "XCR", "{db}"); + assert_eq!(json["issuer"], XCR_ISSUER, "{db}"); + assert_eq!(json["contract"], "", "{db}"); + + let (status, json) = get(client.clone(), &format!("/v1/assets/{XCR_SAC}/price")).await; + assert_eq!(status, StatusCode::OK, "{db}: body={json}"); + assert_eq!(json["asset"], xcr, "{db}"); + approx(&json["price_usd"], 0.25); + + let (status, json) = get(client.clone(), &format!("/v1/oracles/{XCR_SAC}")).await; + assert_eq!(status, StatusCode::OK, "{db}: body={json}"); + assert_eq!(json["asset"], xcr, "{db}"); + + // The classic's series only: without the alias the clean case is 404 + // and the pre-heal case answers the SAC row's 9.0 candle. + let uri = format!( + "/v1/assets/{XCR_SAC}/ohlcv?granularity=1h\ + &start=2026-02-10T00:00:00Z&end=2026-02-11T00:00:00Z&base_currency=USD" + ); + let (status, json) = get(client, &uri).await; + assert_eq!(status, StatusCode::OK, "{db}: body={json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), 1, "{db}: body={json}"); + approx(&data[0]["close"], 0.25); + assert_eq!(data[0]["trade_count"], 7, "{db}"); + + teardown(db).await; + } +} + +/// XLM's SAC answers as `native`, off the stored `sac_address` alone. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn xlm_sac_address_answers_as_native() { + let db = "it_ep_xlm_sac_0242"; + let client = setup(db).await; + + let (status, json) = get(client.clone(), &format!("/v1/assets/{XLM_SAC}")).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + assert_eq!(json["asset"], "native"); + assert_eq!(json["asset_kind"], "native"); + assert_eq!(json["code"], "XLM"); + assert_eq!(json["is_active"], true); + + let (status, json) = get(client, &format!("/v1/assets/{XLM_SAC}/price")).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + assert_eq!(json["asset"], "native"); + approx(&json["price_usd"], 0.5); + + teardown(db).await; +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn batch_answers_sac_addresses_as_their_classic_assets() { + let db = "it_ep_batch_sac_0242"; + let client = setup(db).await; + seed_xcr(db, false).await; + + let body = json!({ "assets": [XCR_SAC, XLM_SAC] }); + let (status, json) = post(client, "/v1/prices/batch", body).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + let assets: Vec<&str> = json["prices"] + .as_array() + .unwrap() + .iter() + .map(|p| p["asset"].as_str().unwrap()) + .collect(); + assert_eq!(assets, [format!("XCR:{XCR_ISSUER}").as_str(), "native"]); + assert_eq!(json["not_found"], json!([])); + + teardown(db).await; +} + +/// Task 0242: an unpriced SAC is listed in `not_found` as the address the +/// client sent, not as its classic. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn batch_not_found_echoes_the_sac_address_asked() { + let db = "it_ep_batch_sac_nf_0242"; + let client = setup(db).await; + let classic = AssetFixture::new("XCR", "credit", XCR_ISSUER, "").with_sac(XCR_SAC); + Client::default() + .with_url(ch_url()) + .query(&assets_insert(db, &[classic])) + .execute() + .await + .unwrap(); + + let body = json!({ "assets": [XCR_SAC] }); + let (status, json) = post(client, "/v1/prices/batch", body).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + assert_eq!(json["prices"], json!([])); + assert_eq!(json["not_found"], json!([XCR_SAC])); + + teardown(db).await; +} + #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn batch_returns_found_and_not_found() { diff --git a/packages/prices-api/tests/ohlcv_it.rs b/packages/prices-api/tests/ohlcv_it.rs index 25cbbb59..9612b196 100644 --- a/packages/prices-api/tests/ohlcv_it.rs +++ b/packages/prices-api/tests/ohlcv_it.rs @@ -2919,3 +2919,609 @@ async fn ohlcv_the_usdc_peg_series_reports_no_price_forming_fields() { teardown(db).await; } + +/// A USDT leg's USD value rests on a rate nobody measures, so in a bucket that +/// also traded against USDC its extremes must not become the candle's (lore-0332). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_a_usdt_leg_no_longer_sets_the_usd_high_and_low() { + let db = "it_ohlcv_usdt_band_0332"; + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + exec(&admin, &assets_insert(db, &[BAZ, USDT])).await; + // Shaped like XLM 1d 2022-06-18: USDC 96 % at the peg, USDT 4 % at 0.5449. + exec( + &admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1d \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) VALUES \ + ('2022-06-18 00:00:00', {BAZ}, {USDC}, 'sdex', 0.105, 0.1137, 0.1005, 0.11, 96, 10.56, 10.56, 0.11, 0.11, 40, 1), \ + ('2022-06-18 00:00:00', {BAZ}, {USDT}, 'sdex', 0.2, 0.3242, 0.1496, 0.2, 4, 0.8, 0.43592, 0.10898, 0.2, 3, 1)" + ), + ) + .await; + + let uri = format!( + "/v1/assets/BAZ:{}/ohlcv?granularity=1d&start=2022-06-18T00:00:00Z\ + &end=2022-06-18T00:00:00Z&base_currency=USD", + iss() + ); + let (status, json) = get(client, &uri).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), 1, "body={json}"); + let c = &data[0]; + let num = |k: &str| c[k].as_str().unwrap().parse::().unwrap(); + assert!( + num("high") < 0.17, + "the USDT leg's high (0.1767) leaked in: {c}" + ); + assert!( + num("low") > 0.09, + "the USDT leg's low (0.0815) leaked in: {c}" + ); + approx(&c["high"], 0.1137); + approx(&c["low"], 0.1005); + approx(&c["open"], 0.105); + approx(&c["close"], 0.11); + approx(&c["volume_base"], 100.0); + assert_eq!(c["method"], "assumed-par", "{c}"); + + teardown(db).await; +} + +/// BAZ on 2026-04-01, one 1h bucket per case. USDC legs sit at the peg (rate +/// 1), USDT legs at rate 0.5, the XLM leg at rate 0.25. +async fn seed_price_setting_0332(db: &str, admin: &Client) { + exec(admin, &assets_insert(db, &[BAZ, USDT])).await; + // lore-0332. 10:00 USDC + USDT; 11:00 USDT has the larger volume; + // 12:00 XLM + USDT; 13:00 USDT only; 14:00 USDT pulls vwap above the band; + // 15:00 USDC formed no price + USDT; 16:00 XLM + a larger USDT; + // 17:00 USDT only, overflowing; 18:00 USDC overflowing + USDT; + // 19:00 USDC formed a price but has no USD value yet + USDT. + exec( + admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1h \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) VALUES \ + ('2026-04-01 10:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.2, 0.9, 1.1, 100, 105, 105, 1.1, 1.05, 10, 1), \ + ('2026-04-01 10:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ + ('2026-04-01 11:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.2, 0.9, 1.1, 10, 11, 11, 1.1, 1.1, 2, 1), \ + ('2026-04-01 11:00:00', {BAZ}, {USDT}, 'sdex', 2.1, 2.2, 1.9, 2.1, 1000, 2100, 1050, 1.05, 2.1, 20, 1), \ + ('2026-04-01 12:00:00', {BAZ}, {XLM}, 'sdex', 10, 12, 9, 10, 100, 1000, 250, 2.5, 10, 7, 1), \ + ('2026-04-01 12:00:00', {BAZ}, {USDT}, 'sdex', 6, 8, 2, 5, 20, 100, 50, 2.5, 5, 3, 1), \ + ('2026-04-01 13:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ + ('2026-04-01 14:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.1, 0.9, 1.0, 10, 10, 10, 1.0, 1.0, 2, 1), \ + ('2026-04-01 14:00:00', {BAZ}, {USDT}, 'sdex', 4, 6, 2, 4, 990, 3960, 1980, 2.0, 4.0, 50, 1), \ + ('2026-04-01 15:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ + ('2026-04-01 16:00:00', {BAZ}, {XLM}, 'sdex', 10, 12, 9, 10, 10, 100, 25, 2.5, 10, 2, 1), \ + ('2026-04-01 16:00:00', {BAZ}, {USDT}, 'sdex', 6, 8, 2, 5, 200, 1000, 500, 2.5, 5, 9, 1), \ + ('2026-04-01 17:00:00', {BAZ}, {USDT}, 'sdex', 1000, 1000, 1000, 0.000000000001, 10, 0.00000000001, 10, 10000000000, 1000, 3, 1), \ + ('2026-04-01 18:00:00', {BAZ}, {USDC}, 'sdex', 1000, 1000, 1000, 0.000000000001, 10, 0.00000000001, 10, 10000000000, 1000, 3, 1), \ + ('2026-04-01 18:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ + ('2026-04-01 19:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.2, 0.9, 1.1, 10, 11, 0, 0, 1.1, 2, 1), \ + ('2026-04-01 19:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1)" + ), + ) + .await; + // 15:00: a USDC leg that traded only dust is not `valid`, so not `band`. + exec( + admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1h \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, \ + pf_trade_count, pf_volume, pf_price_volume, version) VALUES \ + ('2026-04-01 15:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.2, 0.9, 1.1, 10, 11, 11, 1.1, 1.1, 2, \ + 0, 0, 0, 1)" + ), + ) + .await; +} + +/// An untracked XLM is not cached: once it is tracked, the same `AppState` +/// lets XLM set the prices again without a restart (lore-0332, review). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_does_not_cache_an_untracked_xlm() { + let db = "it_ohlcv_untracked_xlm_0332"; + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + exec(&admin, &assets_insert(db, &[BAZ, USDT])).await; + // Case 3's shape: XLM sets 2.5–3.0, USDT would widen it to 1.0–4.0. + exec( + &admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1h \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) VALUES \ + ('2026-04-01 12:00:00', {BAZ}, {XLM}, 'sdex', 10, 12, 9, 10, 100, 1000, 250, 2.5, 10, 7, 1), \ + ('2026-04-01 12:00:00', {BAZ}, {USDT}, 'sdex', 6, 8, 2, 5, 20, 100, 50, 2.5, 5, 3, 1)" + ), + ) + .await; + exec( + &admin, + "ALTER TABLE assets DELETE WHERE asset_type = 'native' SETTINGS mutations_sync = 2", + ) + .await; + + let state = AppState::new(client); + let uri = format!( + "/v1/assets/BAZ:{}/ohlcv?granularity=1h&start=2026-04-01T12:00:00Z\ + &end=2026-04-01T12:00:00Z&base_currency=USD", + iss() + ); + let high = |state: AppState| { + let uri = uri.clone(); + async move { + let resp = app(&config(), state) + .oneshot(Request::builder().uri(&uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let json: Value = serde_json::from_slice(&bytes).unwrap(); + json["data"][0]["high"] + .as_str() + .unwrap() + .parse::() + .unwrap() + } + }; + // Untracked: XLM does not set the prices, so USDT widens the high. + assert!((high(state.clone()).await - 4.0).abs() < 1e-6); + exec(&admin, &assets_insert(db, &[XLM])).await; + // Tracked now, on the same state: XLM sets the high again. + assert!((high(state).await - 3.0).abs() < 1e-6); + + teardown(db).await; +} + +/// In USD mode USDC- and XLM-quoted rows set the prices when the bucket has +/// one; a USDT-only bucket is priced as before (lore-0332). Before-fix values +/// in the comments show what each case discriminates. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one() { + let db = "it_ohlcv_price_setting_0332"; + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + seed_price_setting_0332(db, &admin).await; + + let uri = format!( + "/v1/assets/BAZ:{}/ohlcv?granularity=1h&start=2026-04-01T10:00:00Z\ + &end=2026-04-01T19:00:00Z&base_currency=USD", + iss() + ); + let (status, json) = get(client, &uri).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), 10, "body={json}"); + let at = |hour: &str| { + let ts = format!("2026-04-01T{hour}:00:00Z"); + data.iter() + .find(|c| c["timestamp"] == ts.as_str()) + .unwrap_or_else(|| panic!("no bucket at {ts}: {json}")) + }; + + type Case = ( + &'static str, + &'static str, + [f64; 4], + &'static str, + f64, + Option, + ); + let cases: [Case; 7] = [ + // label, hour, [open, high, low, close], method, volume_base, vwap (= pf_vwap) + // was high 1.5, low 0.5, vwap 155/150 + ( + "1 usdc+usdt", + "10", + [1.0, 1.2, 0.9, 1.1], + "assumed-par", + 150.0, + Some(1.05), + ), + // was open 1.05, close 1.05, `traded` + ( + "2 usdt larger", + "11", + [1.0, 1.2, 0.9, 1.1], + "assumed-par", + 1010.0, + None, + ), + // was high 4.0, low 1.0 + ( + "3 xlm+usdt", + "12", + [2.5, 3.0, 2.25, 2.5], + "traded", + 120.0, + None, + ), + ( + "4 usdt only", + "13", + [1.0, 1.5, 0.5, 1.0], + "traded", + 50.0, + Some(1.0), + ), + // was high 3.0, close 2.0, vwap 1.99; then 1.1, the USDT mean clamped + ( + "5 usdt mean", + "14", + [1.0, 1.1, 0.9, 1.0], + "assumed-par", + 1000.0, + Some(1.0), + ), + // was 1.0, 1.2, 0.9, 1.1 with `band` not requiring `valid` + ( + "6 usdc dust+usdt", + "15", + [1.0, 1.5, 0.5, 1.0], + "traded", + 60.0, + None, + ), + // was open 3.0, high 4.0, low 1.0 + ( + "7 xlm smaller", + "16", + [2.5, 3.0, 2.25, 2.5], + "traded", + 210.0, + None, + ), + ]; + for (label, hour, ohlc, method, vb, vwap) in cases { + let c = at(hour); + let num = |k: &str| c[k].as_str().map(|s| s.parse::().unwrap()); + for (field, want) in ["open", "high", "low", "close"].into_iter().zip(ohlc) { + let got = num(field).unwrap_or_else(|| panic!("case {label}: no {field}: {c}")); + assert!( + (got - want).abs() < 1e-6, + "case {label}: {field} {got} != {want}: {c}" + ); + } + assert_eq!(c["method"], method, "case {label}: {c}"); + assert!( + (num("volume_base").unwrap() - vb).abs() < 1e-6, + "case {label}: {c}" + ); + if let Some(want) = vwap { + for field in ["vwap", "pf_vwap"] { + let got = num(field).unwrap_or_else(|| panic!("case {label}: no {field}: {c}")); + assert!( + (got - want).abs() < 1e-6, + "case {label}: {field} {got} != {want}: {c}" + ); + } + } + } + // An extreme that overflows stays null in the fallback (8) and in the band + // (9, was high 3.0): the USDT leg does not stand in for the USDC one. + for (label, hour) in [("8 usdt overflow", "17"), ("9 usdc overflow+usdt", "18")] { + let c = at(hour); + assert!(c["high"].is_null(), "case {label}: {c}"); + assert_eq!(c["close"], "10000000000", "case {label}: {c}"); + } + // 10: a USDC leg with no USD value yet leaves the bucket unpriced rather + // than priced by USDT (was 1.0, 1.5, 0.5, 1.0); its volume still counts. + let c10 = at("19"); + for field in ["open", "high", "low", "close", "vwap", "pf_vwap", "method"] { + assert!(c10[field].is_null(), "case 10: {field}: {c10}"); + } + approx(&c10["volume_base"], 60.0); + // Case 1: the USDT leg still counts toward volume and trades. + let c1 = at("10"); + approx(&c1["volume_quote_usd"], 155.0); + assert_eq!(c1["trade_count"], 15, "case 1: {c1}"); + assert_eq!(c1["pf_trade_count"], 15, "case 1: {c1}"); + + teardown(db).await; +} + +/// Before 2021-02-01 a priced candle quoted in XLM or a dollar token took the +/// imported XLM/USD rate, so it is labelled `external` (lore-0331). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_labels_pre_usdc_candles_external() { + let db = "it_ohlcv_pre_usdc_0331"; + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + exec(&admin, &assets_insert(db, &[USDT])).await; + // (day, quote, close, close_usd, method). close_usd != close: an exact-par + // dollar row is dropped as a peg signature. + let cases = [ + ("2021-01-30", USDT, 2.0, 1.98, "external"), + ("2021-01-31", XLM, 10.0, 2.5, "external"), + ("2021-02-01", XLM, 10.0, 3.0, "traded"), + ]; + for (day, quote, close, cu, _) in cases { + exec( + &admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1d \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + VALUES ('{day} 00:00:00', {FOO}, {quote}, 'sdex', {close}, {close}, {close}, {close}, \ + 1, {close}, {cu}, {cu}, {close}, 1, 1)" + ), + ) + .await; + } + let uri = format!( + "/v1/assets/FOO:{}/ohlcv?granularity=1d&start=2021-01-30T00:00:00Z\ + &end=2021-02-01T00:00:00Z&base_currency=USD", + iss() + ); + let (status, json) = get(client, &uri).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), cases.len(), "body={json}"); + for (c, (day, _, _, cu, method)) in data.iter().zip(cases) { + assert_eq!(c["method"], method, "{day}: {c}"); + approx(&c["close"], cu); + } + teardown(db).await; +} + +// A dollar token that is not off-peg, and one that is (lore-0331). +const USD_X: AssetFixture = AssetFixture::new("USD", "credit", USDT_ISSUER, ""); +const OFF: (&str, &str) = prices_clickhouse::OFF_PEG_DOLLAR_TOKENS[0]; +const OFF_PEG: AssetFixture = AssetFixture::new(OFF.0, "credit", OFF.1, ""); + +/// XLM rates as `rate_method`: 24 hourly on 2021-01-31 (0.33 at 00:00, 0.31 at +/// 05:00, 0.34 at 22:00, 0.35 at 23:00; max 0.36, min 0.30), three on 01-27/28, +/// three after the seam; 9.0 decoys staged or from another source. Plus XLM's +/// dollar, off-peg and non-dollar markets. +async fn seed_native_usd(db: &str, admin: &Client, m: &str) { + exec(admin, &assets_insert(db, &[USDT, USD_X, OFF_PEG])).await; + exec( + admin, + &format!( + "INSERT INTO {db}.usd_rate (asset_kind, asset_code, issuer_address, contract_address, \ + timestamp, usd_rate, method, reference_asset, quality, hops, version) \ + SELECT 'native', 'XLM', '', '', toDateTime('2021-01-31 00:00:00') + number * 3600, \ + toDecimal128(30 + (number + 3) % 7, 14) / 100, '{m}', 'third-party', \ + 'measured', 0, 1 FROM numbers(24) \ + UNION ALL SELECT 'native', 'XLM', '', '', toDateTime(t), toDecimal128(r, 14), \ + m, s, q, 0, 1 FROM values('t String, r String, m String, s String, q String', \ + ('2021-01-27 06:00:00', '0.20', '{m}', 'third-party', 'measured'), \ + ('2021-01-28 12:00:00', '0.25', '{m}', 'third-party', 'fallback'), \ + ('2021-01-28 13:00:00', '0.27', '{m}', 'third-party', 'measured-disputed'), \ + ('2021-02-01 00:00:00', '0.5', '{m}', 'third-party', 'measured'), \ + ('2021-02-01 01:00:00', '0.5', '{m}', 'third-party', 'measured'), \ + ('2021-02-01 02:00:00', '0.5', '{m}', 'third-party', 'measured'), \ + ('2021-01-31 05:00:00', '9', 'external-candidate', 'third-party', 'measured'), \ + ('2021-01-30 12:00:00', '9', 'external-candidate', 'third-party', 'measured'), \ + ('2021-01-30 06:00:00', '9', 'external', 'other-source', 'measured'))" + ), + ) + .await; + // (grain, timestamp, base, quote, close, close_usd, volume_base, volume_quote, trades). + // close_usd != close: an exact-par dollar row is dropped as a peg signature. + #[rustfmt::skip] + let candles = [ + ("1h", "2021-01-31 05:00", XLM, USDT, 0.5, 0.49, 100.0, 50.0, 2), + ("1h", "2021-01-31 05:00", XLM, USDC, 0.48, 0.4801, 10.0, 4.8, 1), + ("1h", "2021-01-31 05:00", USD_X, XLM, 2.0, 0.0, 25.0, 50.0, 3), + ("1h", "2021-01-31 05:00", OFF_PEG, XLM, 2.0, 0.0, 500.0, 1000.0, 9), + ("1h", "2021-01-31 05:00", FOO, XLM, 1.0, 0.0, 777.0, 777.0, 11), + ("1h", "2021-01-31 22:00", USD_X, XLM, 2.0, 0.0, 20.0, 40.0, 1), + // 01-28: 01:00 carries 01-27 06:00's rate (20 h), 06:00 has none within + // 24 h, 20:00 carries 13:00's. + ("1h", "2021-01-28 01:00", XLM, USDT, 0.5, 0.49, 10.0, 5.0, 1), + ("1h", "2021-01-28 06:00", XLM, USDT, 0.5, 0.49, 20.0, 10.0, 1), + ("1h", "2021-01-28 20:00", USD_X, XLM, 2.0, 0.0, 15.0, 30.0, 1), + ("1h", "2021-02-01 00:00", XLM, USDC, 0.3215, 0.3216, 50.0, 16.075, 1), + ("4h", "2021-01-31 04:00", XLM, USDT, 0.5, 0.49, 100.0, 50.0, 2), + ("1d", "2021-01-30 00:00", XLM, USDT, 0.2905, 0.29, 5.0, 1.4525, 1), + ("1d", "2021-01-31 00:00", XLM, USDT, 0.5, 0.49, 999.0, 499.5, 1), + ("1d", "2021-02-01 00:00", XLM, USDC, 0.3221, 0.3222, 7.0, 2.2547, 1), + ("1w", "2021-01-25 00:00", XLM, USDT, 0.5, 0.49, 999.0, 499.5, 1), + ("1w", "2021-02-01 00:00", XLM, USDC, 0.3221, 0.3222, 7.0, 2.2547, 1), + ("1M", "2021-01-01 00:00", XLM, USDT, 0.5, 0.49, 999.0, 499.5, 1), + ("1M", "2021-02-01 00:00", XLM, USDC, 0.3221, 0.3222, 7.0, 2.2547, 1), + ("15m", "2021-01-31 05:15", FOO, XLM, 1.0, 0.0, 5.0, 5.0, 1), + ("15m", "2021-01-31 05:30", XLM, USDT, 0.5, 0.49, 100.0, 50.0, 2), + ("15m", "2021-01-31 05:30", OFF_PEG, XLM, 2.0, 0.0, 500.0, 1000.0, 9), + ]; + for (g, ts, base, quote, p, cu, vb, vq, tc) in candles { + exec( + admin, + &format!( + "INSERT INTO {db}.price_ohlcv_{g} \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + VALUES ('{ts}:00', {base}, {quote}, 'sdex', {p}, {p}, {p}, {p}, \ + {vb}, {vq}, 0, {cu}, {p}, {tc}, 1)" + ), + ) + .await; + } +} + +/// One expected candle: (timestamp, close, and for a synthesized one +/// [open, high, low, volume_base, trade_count, volume_quote_usd, vwap] and quality). +type NativeRow = (&'static str, f64, Option<([f64; 7], &'static str)>); + +/// Seeds `db` with `rate_method` rates, then checks per case (query, candles, +/// synthesized candles, expected rows). Ascending timestamps: no duplicate bucket. +async fn check_native_usd( + db: &str, + rate_method: &str, + cases: &[(&str, usize, usize, &[NativeRow])], +) { + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + seed_native_usd(db, &admin, rate_method).await; + for (query, n, synth_n, rows) in cases { + let uri = format!("/v1/assets/native/ohlcv?{query}&base_currency=USD"); + let (status, json) = get(client.clone(), &uri).await; + assert_eq!(status, StatusCode::OK, "{query}: {json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), *n, "{query}: {json}"); + let ts: Vec<&Value> = data.iter().map(|c| &c["timestamp"]).collect(); + assert!( + ts.windows(2).all(|w| w[0].as_str() < w[1].as_str()), + "{query}: {ts:?}" + ); + let synth = data.iter().filter(|c| c["source"] == "third-party").count(); + assert_eq!(synth, *synth_n, "{query}: {json}"); + for (at, close, s) in rows.iter() { + let c = data.iter().find(|c| c["timestamp"] == *at).expect(at); + approx(&c["close"], *close); + let Some(([o, h, l, vb, tc, vqu, vw], quality)) = s else { + assert!(c["source"].is_null(), "{query} {at}: {c}"); + continue; + }; + let fields = [ + "open", + "high", + "low", + "volume_base", + "volume_quote_usd", + "vwap", + ]; + for (f, v) in fields.iter().zip([o, h, l, vb, vqu, vw]) { + approx(&c[f], *v); + } + assert_eq!(c["trade_count"].as_f64(), Some(*tc), "{query} {at}: {c}"); + let tail = [ + &c["method"], + &c["derived"], + &c["quality"], + &c["pf_trade_count"], + ]; + let want = [ + "external".into(), + true.into(), + (*quality).into(), + Value::Null, + ]; + assert!(tail.into_iter().eq(&want), "{query} {at}: {c}"); + assert!( + c["pf_vwap"].is_null() && c["close_divergent"].is_null(), + "{c}" + ); + } + } + teardown(db).await; +} + +const NATIVE_DAY: &str = "granularity=1d&start=2021-01-30T00:00:00Z&end=2021-02-01T00:00:00Z"; +const NATIVE_HOUR: &str = "granularity=1h&start=2021-01-31T00:00:00Z&end=2021-01-31T23:00:00Z"; +const NATIVE_15M: &str = "granularity=15m&start=2021-01-31T05:00:00Z&end=2021-01-31T06:45:00Z"; +const NATIVE_WEEK: &str = "granularity=1w&start=2021-01-18T00:00:00Z&end=2021-02-01T00:00:00Z"; + +/// Native XLM's pre-2021 USD buckets come from the promoted hourly rate; the +/// rest is today's series (lore-0331). XLM mode never routes here +/// (`ohlcv_native_xlm_mode_ignores_the_promoted_rate`). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_native_usd_synthesizes_pre_2021_from_the_rate() { + // The 01-27 → 01-31 rates and 01-28 + 01-31 volume, worst quality `fallback`. + let jan = Some(( + [0.20, 0.36, 0.20, 260.0, 10.0, 73.3, 73.3 / 240.0], + "fallback", + )); + #[rustfmt::skip] + let cases: [(&str, usize, usize, &[NativeRow]); 8] = [ + (NATIVE_DAY, 3, 1, &[ + // Only a staged or other-source rate covers 01-30: today's candle. + ("2021-01-30T00:00:00Z", 0.29, None), + // 05:00 legs 100 + 10 + 50 and 22:00's 40; off-peg, FOO and the + // stored 1d volume left out. + ("2021-01-31T00:00:00Z", 0.35, Some(([0.33, 0.36, 0.30, 200.0, 7.0, 63.2, 0.316], "measured"))), + // Rates after the seam are ignored. + ("2021-02-01T00:00:00Z", 0.3222, None), + ]), + // Volume from every hour; USD 10 × 0.20 + 30 × 0.27, the 06:00 hour unpriced. + ("granularity=1d&start=2021-01-28T00:00:00Z&end=2021-01-29T00:00:00Z", 1, 1, &[ + ("2021-01-28T00:00:00Z", 0.27, Some(([0.25, 0.27, 0.25, 60.0, 3.0, 10.1, 0.2525], "fallback"))), + ]), + // Every hour from the rate; the own 05:00 candle replaced. + (NATIVE_HOUR, 24, 24, &[ + ("2021-01-31T05:00:00Z", 0.31, Some(([0.31, 0.31, 0.31, 160.0, 6.0, 49.6, 0.31], "measured"))), + ("2021-01-31T22:00:00Z", 0.34, Some(([0.34, 0.34, 0.34, 40.0, 1.0, 13.6, 0.34], "measured"))), + ("2021-01-31T00:00:00Z", 0.33, Some(([0.33, 0.33, 0.33, 0.0, 0.0, 0.0, 0.33], "measured"))), + ]), + ("granularity=1h&start=2021-01-31T22:00:00Z&end=2021-02-01T02:00:00Z", 3, 2, &[ + ("2021-02-01T00:00:00Z", 0.3216, None), + ]), + // Only where XLM traded: FOO/XLM alone at 05:15, off-peg left out at 05:30. + (NATIVE_15M, 2, 2, &[ + ("2021-01-31T05:15:00Z", 0.31, Some(([0.31, 0.31, 0.31, 0.0, 0.0, 0.0, 0.31], "measured"))), + ("2021-01-31T05:30:00Z", 0.31, Some(([0.31, 0.31, 0.31, 100.0, 2.0, 31.0, 0.31], "measured"))), + ]), + // Each bucket must land on the stored one's timestamp, or both show. + ("granularity=4h&start=2021-01-31T00:00:00Z&end=2021-01-31T20:00:00Z", 6, 6, &[ + ("2021-01-31T04:00:00Z", 0.33, Some(([0.30, 0.33, 0.30, 160.0, 6.0, 49.6, 0.31], "measured"))), + ("2021-01-31T20:00:00Z", 0.35, Some(([0.32, 0.35, 0.32, 40.0, 1.0, 13.6, 0.34], "measured"))), + ]), + (NATIVE_WEEK, 2, 1, &[ + ("2021-01-25T00:00:00Z", 0.35, jan), + ("2021-02-01T00:00:00Z", 0.3222, None), + ]), + ("granularity=1M&start=2021-01-01T00:00:00Z&end=2021-02-01T00:00:00Z", 2, 1, &[ + ("2021-01-01T00:00:00Z", 0.35, jan), + ("2021-02-01T00:00:00Z", 0.3222, None), + ]), + ]; + check_native_usd("it_ohlcv_native_usd_0331", "external", &cases).await; +} + +/// Before promotion (only `external-candidate` rates) the series is today's. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_native_usd_before_promotion_equals_today() { + #[rustfmt::skip] + let cases: [(&str, usize, usize, &[NativeRow]); 4] = [ + (NATIVE_DAY, 3, 0, &[ + ("2021-01-30T00:00:00Z", 0.29, None), + ("2021-01-31T00:00:00Z", 0.49, None), + ("2021-02-01T00:00:00Z", 0.3222, None), + ]), + (NATIVE_HOUR, 1, 0, &[("2021-01-31T05:00:00Z", 0.4801, None)]), + (NATIVE_15M, 1, 0, &[("2021-01-31T05:30:00Z", 0.49, None)]), + (NATIVE_WEEK, 2, 0, &[("2021-01-25T00:00:00Z", 0.49, None)]), + ]; + check_native_usd("it_ohlcv_native_usd_pre_0331", "external-candidate", &cases).await; +} + +/// `base_currency=XLM` never takes the synthesized series: promoting the rate +/// changes nothing in XLM mode (lore-0331). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_native_xlm_mode_ignores_the_promoted_rate() { + let mut got = Vec::new(); + for (db, m) in [ + ("it_ohlcv_native_xlm_0331", "external"), + ("it_ohlcv_native_xlm_pre_0331", "external-candidate"), + ] { + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + seed_native_usd(db, &admin, m).await; + let mut data = Vec::new(); + for q in [NATIVE_DAY, NATIVE_HOUR, NATIVE_15M, NATIVE_WEEK] { + let uri = format!("/v1/assets/native/ohlcv?{q}&base_currency=XLM"); + let (status, json) = get(client.clone(), &uri).await; + assert_eq!(status, StatusCode::OK, "{q}: {json}"); + let candles = json["data"].as_array().unwrap(); + assert!(candles.iter().all(|c| c["source"].is_null()), "{q}: {json}"); + data.push(json["data"].clone()); + } + got.push(data); + teardown(db).await; + } + assert_eq!(got[0], got[1]); +} diff --git a/packages/prices-api/tests/portal_auth.rs b/packages/prices-api/tests/portal_auth.rs index 7b685660..295fcc09 100644 --- a/packages/prices-api/tests/portal_auth.rs +++ b/packages/prices-api/tests/portal_auth.rs @@ -462,7 +462,7 @@ async fn login_lands_an_unwired_issue_round_trip_on_failed() { let open = signed_in_app(true); let refused = fetch(&open, &format!("{LOGIN_PATH}?action=issue"), &[]).await; assert_eq!(refused.status, StatusCode::SEE_OTHER); - assert_eq!(refused.location(), "/prices-api/?issue=failed"); + assert_eq!(refused.location(), "/api/prices-api/?issue=failed"); assert!(refused.set_cookies().is_empty()); // And not a body the browser would render as text. assert!(refused.body.is_empty(), "{:?}", refused.body); @@ -493,11 +493,11 @@ async fn neither_arm_503s_on_a_deployment_with_no_credentials() { let signin = fetch(&router, LOGIN_PATH, &[]).await; assert_eq!(signin.status, StatusCode::SEE_OTHER); - assert_eq!(signin.location(), "/prices-api/?signin=not_open"); + assert_eq!(signin.location(), "/api/prices-api/?signin=not_open"); let issue = fetch(&router, &format!("{LOGIN_PATH}?action=issue"), &[]).await; assert_eq!(issue.status, StatusCode::SEE_OTHER); - assert_eq!(issue.location(), "/prices-api/?issue=failed"); + assert_eq!(issue.location(), "/api/prices-api/?issue=failed"); } // --------------------------------------------------------------------------- @@ -524,7 +524,7 @@ async fn a_complete_round_trip_signs_the_visitor_in() { // real deployment has — is covered by the `app_with_keys` tests below. assert_eq!( reply.location(), - "/prices-api/", + "/api/prices-api/", "the callback must land on the portal, and on a literal" ); @@ -655,7 +655,7 @@ async fn a_first_sign_in_issues_a_key_and_lands_on_the_welcome() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!(gateway.with(|s| s.create_calls), 1); assert_eq!(gateway.with(|s| s.named(&key_name()).len()), 1); @@ -673,7 +673,7 @@ async fn a_returning_sign_in_adopts_the_key_and_lands_plain() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/"); + assert_eq!(reply.location(), "/api/prices-api/"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!( gateway.with(|s| s.create_calls), @@ -704,7 +704,7 @@ async fn a_refused_sign_in_leaves_an_existing_session_alone() { let reply = sign_in_against(&app_against(&mock)).await; - assert_eq!(reply.location(), "/prices-api/?signin=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?signin=not_member"); // No session is issued … assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); // … and none is cleared either: the only `Set-Cookie` is the pending @@ -736,7 +736,7 @@ async fn an_unreadable_age_parameter_signs_the_visitor_in_without_a_key() { let reply = sign_in_against(&app_with_keys_and(&discord, &gateway, settings)).await; - assert_eq!(reply.location(), "/prices-api/"); + assert_eq!(reply.location(), "/api/prices-api/"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!(discord.member_calls(), 1, "membership is still proved"); assert_eq!( @@ -761,7 +761,7 @@ async fn an_unreadable_guild_parameter_still_refuses_sign_in() { let reply = sign_in_against(&app_with_keys_and(&discord, &gateway, settings)).await; - assert_eq!(reply.location(), "/prices-api/?signin=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?signin=unknown"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!(discord.member_calls(), 0); } @@ -780,7 +780,7 @@ async fn a_control_plane_failure_at_sign_in_lands_plain_with_a_session() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/"); + assert_eq!(reply.location(), "/api/prices-api/"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -807,7 +807,7 @@ async fn a_too_young_account_signs_in_but_gets_no_key() { assert!( reply .location() - .starts_with("/prices-api/?issue=too_young&wait_secs="), + .starts_with("/api/prices-api/?issue=too_young&wait_secs="), "{}", reply.location() ); @@ -839,7 +839,7 @@ async fn a_revoked_account_signs_in_and_lands_plain() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/"); + assert_eq!(reply.location(), "/api/prices-api/"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -859,7 +859,7 @@ async fn a_non_member_gets_no_key_either() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?signin=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?signin=not_member"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!(gateway.with(|s| s.list_calls), 0); assert_eq!(gateway.with(|s| s.create_calls), 0); @@ -896,7 +896,7 @@ async fn a_non_member_cannot_sign_in() { let reply = sign_in_against(&open).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?signin=not_member"); assert!( reply.cookie(cookies::SESSION_COOKIE).is_none(), "a refused sign-in must not leave a session behind" @@ -927,7 +927,7 @@ async fn a_member_still_in_screening_is_refused_as_pending_rules() { let reply = sign_in_against(&app_against(&mock)).await; - assert_eq!(reply.location(), "/prices-api/?signin=pending_rules"); + assert_eq!(reply.location(), "/api/prices-api/?signin=pending_rules"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert!(reply.clears(cookies::PENDING_COOKIE)); } @@ -955,7 +955,7 @@ async fn an_unanswerable_membership_question_refuses_without_accusing() { let reply = sign_in_against(&app_against(&mock)).await; - assert_eq!(reply.location(), "/prices-api/?signin=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?signin=unknown"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); } } @@ -996,7 +996,7 @@ async fn a_deployment_with_no_eligibility_settings_refuses_sign_in() { let reply = sign_in_against(&unwired).await; - assert_eq!(reply.location(), "/prices-api/?signin=not_open"); + assert_eq!(reply.location(), "/api/prices-api/?signin=not_open"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!( mock.member_calls(), @@ -1114,7 +1114,7 @@ async fn a_mismatched_state_is_rejected_and_issues_no_session() { // tab. What the refusal is made of has not changed — no session, no // exchange, the pending cookie untouched — only what the visitor is shown. assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); // And it leaves the browser's pending login alone — see // `an_unverifiable_callback_cannot_cancel_someone_elses_sign_in`. @@ -1135,7 +1135,7 @@ async fn a_callback_with_no_pending_cookie_is_rejected() { let reply = fetch(&open, &format!("{CALLBACK_PATH}?code=c&state={state}"), &[]).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!(mock.exchanges(), 0); } @@ -1159,7 +1159,7 @@ async fn replaying_a_callback_url_after_the_cookie_is_cleared_is_rejected() { // The replay: same URL, and the browser no longer holds the cookie. let second = fetch(&open, &uri, &[]).await; assert_eq!(second.status, StatusCode::SEE_OTHER); - assert_eq!(second.location(), "/prices-api/?signin=failed"); + assert_eq!(second.location(), "/api/prices-api/?signin=failed"); assert!(second.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!( mock.exchanges(), @@ -1187,7 +1187,7 @@ async fn a_forged_state_signed_with_another_key_is_rejected() { ) .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert_eq!(mock.exchanges(), 0); } @@ -1211,7 +1211,7 @@ async fn a_self_signed_pair_is_rejected() { ) .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert_eq!(mock.exchanges(), 0); } @@ -1239,7 +1239,7 @@ async fn a_cancelled_sign_in_returns_to_the_portal_saying_so() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=cancelled"); + assert_eq!(reply.location(), "/api/prices-api/?signin=cancelled"); assert!(reply.clears(cookies::PENDING_COOKIE)); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); } @@ -1345,7 +1345,7 @@ async fn a_verified_callback_always_drops_the_pending_cookie() { ) .await; assert_eq!(upstream.status, StatusCode::SEE_OTHER); - assert_eq!(upstream.location(), "/prices-api/?signin=failed"); + assert_eq!(upstream.location(), "/api/prices-api/?signin=failed"); assert!(upstream.clears(cookies::PENDING_COOKIE)); } @@ -1398,18 +1398,18 @@ async fn only_access_denied_is_reported_as_a_cancellation() { let open = signed_in_app(true); for (error, expected) in [ - ("access_denied", "/prices-api/?signin=cancelled"), - ("invalid_scope", "/prices-api/?signin=failed"), - ("server_error", "/prices-api/?signin=failed"), - ("temporarily_unavailable", "/prices-api/?signin=failed"), - ("invalid_request", "/prices-api/?signin=failed"), - ("unauthorized_client", "/prices-api/?signin=failed"), + ("access_denied", "/api/prices-api/?signin=cancelled"), + ("invalid_scope", "/api/prices-api/?signin=failed"), + ("server_error", "/api/prices-api/?signin=failed"), + ("temporarily_unavailable", "/api/prices-api/?signin=failed"), + ("invalid_request", "/api/prices-api/?signin=failed"), + ("unauthorized_client", "/api/prices-api/?signin=failed"), // Not in RFC 6749 §4.1.2.1 at all. Anything unrecognised is a failure, // not a cancellation — defaulting the other way is how a new Discord // error code would silently become "cancelled". ( "something_discord_invented_later", - "/prices-api/?signin=failed", + "/api/prices-api/?signin=failed", ), ] { let started = start_login(&open).await; @@ -1452,7 +1452,7 @@ async fn the_error_value_never_reaches_the_redirect_target() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(!reply.location().contains("evil")); assert!(reply.headers.get("x-injected").is_none()); } @@ -1527,7 +1527,11 @@ async fn a_grant_that_is_not_exactly_the_two_scopes_is_refused() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER, "scope={drifted}"); - assert_eq!(reply.location(), "/prices-api/?signin=failed", "{drifted}"); + assert_eq!( + reply.location(), + "/api/prices-api/?signin=failed", + "{drifted}" + ); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none(), "{drifted}"); } @@ -1542,7 +1546,7 @@ async fn a_grant_that_is_not_exactly_the_two_scopes_is_refused() { ) .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_ne!(reply.location(), "/prices-api/?signin=failed"); + assert_ne!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); } @@ -1570,7 +1574,7 @@ async fn a_failed_token_exchange_lands_with_no_upstream_detail() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); let body = String::from_utf8(reply.body.clone()).unwrap(); assert!(!body.contains("upstream said no"), "{body}"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); @@ -1733,7 +1737,7 @@ async fn an_open_portal_whose_load_fails_lands_on_a_retryable_failure() { let login = fetch(&router, LOGIN_PATH, &[]).await; assert_eq!(login.status, StatusCode::SEE_OTHER); - assert_eq!(login.location(), "/prices-api/?signin=failed"); + assert_eq!(login.location(), "/api/prices-api/?signin=failed"); // `/auth/me` cannot check a cookie without the signing key, and "nobody // is signed in" would be false for a visitor who is: a `503` the page @@ -1762,7 +1766,7 @@ async fn an_open_portal_loaded_without_credentials_lands_on_the_closed_card() { let login = fetch(&router, LOGIN_PATH, &[]).await; assert_eq!(login.status, StatusCode::SEE_OTHER); - assert_eq!(login.location(), "/prices-api/?signin=not_open"); + assert_eq!(login.location(), "/api/prices-api/?signin=not_open"); // No credentials means no sessions: a truthful "signed out". let me = fetch(&router, ME_PATH, &[]).await; @@ -1850,7 +1854,7 @@ async fn an_issue_callback_on_a_failed_load_lands_on_issue_failed() { for sent in [with_cookie, without] { let reply = fetch(&failing, &callback, sent).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert!( reply.headers.get(header::SET_COOKIE).is_none(), "an unverified callback must leave the pending cookie alone" @@ -1878,7 +1882,7 @@ async fn a_sign_in_callback_on_a_failed_load_lands_on_signin_failed() { ) .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.headers.get(header::SET_COOKIE).is_none()); assert_eq!(mock.exchanges(), 0); } @@ -1904,7 +1908,7 @@ async fn a_callback_whose_load_fits_the_allowance_still_exchanges_the_code() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_ne!(reply.location(), "/prices-api/?signin=failed"); + assert_ne!(reply.location(), "/api/prices-api/?signin=failed"); assert_eq!(mock.exchanges(), 1, "the token exchange must run"); } @@ -1931,7 +1935,7 @@ async fn a_callback_that_spent_its_allowance_loading_lands_on_signin_failed() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.clears(cookies::PENDING_COOKIE)); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!(mock.exchanges(), 0, "the token exchange must not start"); @@ -1957,7 +1961,7 @@ async fn an_issue_callback_that_spent_its_allowance_loading_lands_on_issue_faile .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert!(reply.clears(cookies::PENDING_COOKIE)); assert_eq!(mock.exchanges(), 0, "the token exchange must not start"); } @@ -2021,7 +2025,7 @@ async fn a_round_trip_lands_on_the_configured_web_origin() { assert_eq!(reply.status, StatusCode::SEE_OTHER); assert_eq!( reply.location(), - "https://sorobanscan.example/prices-api/?signin=cancelled" + "https://sorobanscan.example/api/prices-api/?signin=cancelled" ); // The cookie work is unchanged by the host: still dropped on this host. assert!(reply.set_cookies().iter().any(|c| c.contains("Max-Age=0"))); diff --git a/packages/prices-api/tests/portal_issue.rs b/packages/prices-api/tests/portal_issue.rs index 10ade41f..08b8be6c 100644 --- a/packages/prices-api/tests/portal_issue.rs +++ b/packages/prices-api/tests/portal_issue.rs @@ -126,7 +126,7 @@ async fn a_member_in_good_standing_gets_a_key_and_lands_on_issue_ok() { let reply = issue_round_trip(&app).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert!( reply.cookie(cookies::SESSION_COOKIE).is_some(), "the round-trip proves identity, so it signs the visitor in" @@ -171,11 +171,11 @@ async fn a_second_round_trip_returns_the_same_key() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!( @@ -205,13 +205,13 @@ async fn two_users_get_two_different_keys() { issue_round_trip(&issue_app(&mine, &gateway)) .await .location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!( issue_round_trip(&issue_app(&theirs, &gateway)) .await .location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let names: Vec = gateway.with(|s| s.keys.iter().map(|k| k.name.clone()).collect()); @@ -229,8 +229,8 @@ async fn two_simultaneous_round_trips_leave_exactly_one_key() { let app = issue_app(&discord, &gateway); let (a, b) = tokio::join!(issue_round_trip(&app), issue_round_trip(&app)); - assert_eq!(a.location(), "/prices-api/?issue=ok"); - assert_eq!(b.location(), "/prices-api/?issue=ok"); + assert_eq!(a.location(), "/api/prices-api/?issue=ok"); + assert_eq!(b.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.named(&key_name()).len()), @@ -265,7 +265,7 @@ async fn a_non_member_is_refused_and_no_key_is_created() { let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?issue=not_member"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!( @@ -295,7 +295,7 @@ async fn an_unknown_guild_code_is_also_refused_as_not_a_member() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?issue=not_member"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -313,7 +313,7 @@ async fn a_404_with_an_unrecognised_code_is_unknown_not_an_accusation() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -341,7 +341,7 @@ async fn a_429_or_5xx_from_discord_refuses_without_claiming_non_membership() { let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!( reply.location(), - "/prices-api/?issue=unknown", + "/api/prices-api/?issue=unknown", "a {status} must be 'could not verify', never 'not a member'" ); assert_eq!(gateway.with(|s| s.create_calls), 0, "{status}"); @@ -366,7 +366,7 @@ async fn a_pending_member_is_refused_as_pending_rules() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=pending_rules"); + assert_eq!(reply.location(), "/api/prices-api/?issue=pending_rules"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -386,7 +386,7 @@ async fn an_absent_pending_field_does_not_silently_pass() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -403,7 +403,7 @@ async fn a_malformed_member_body_is_unknown() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -446,7 +446,11 @@ async fn a_grant_missing_the_member_scope_lands_on_denied_before_any_member_call let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER, "scope={drifted}"); - assert_eq!(reply.location(), "/prices-api/?issue=denied", "{drifted}"); + assert_eq!( + reply.location(), + "/api/prices-api/?issue=denied", + "{drifted}" + ); // Not the sign-in arm's error page: nothing renderable, no way back. assert!(reply.body.is_empty(), "{drifted}"); assert_eq!(discord.member_calls(), 0, "{drifted}"); @@ -467,7 +471,7 @@ async fn a_failed_token_exchange_on_an_issue_lands_on_unknown() { let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert!(!String::from_utf8_lossy(&reply.body).contains("upstream said no")); assert_eq!(discord.member_calls(), 0); assert_eq!(gateway.with(|s| s.create_calls), 0); @@ -495,7 +499,7 @@ async fn a_failed_identity_read_on_an_issue_lands_on_unknown() { let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert_eq!(discord.member_calls(), 1, "the membership call did happen"); // No identity means no key: the reconciler needs a `sub` to name one. assert_eq!(gateway.with(|s| s.create_calls), 0); @@ -557,7 +561,7 @@ async fn an_account_below_the_threshold_is_refused_with_the_time_remaining() { let (base, wait) = location .split_once("&wait_secs=") .unwrap_or_else(|| panic!("no wait_secs in {location}")); - assert_eq!(base, "/prices-api/?issue=too_young"); + assert_eq!(base, "/api/prices-api/?issue=too_young"); let wait: u64 = wait.parse().expect("wait_secs must be digits"); // ~180s remain; generous bounds absorb test-runner latency. assert!((150..=181).contains(&wait), "wait_secs={wait}"); @@ -588,7 +592,7 @@ async fn the_threshold_value_decides_the_verdict_at_action_time() { assert!( strict .location() - .starts_with("/prices-api/?issue=too_young"), + .starts_with("/api/prices-api/?issue=too_young"), "{}", strict.location() ); @@ -603,7 +607,7 @@ async fn the_threshold_value_decides_the_verdict_at_action_time() { ) .await; let lax = issue_round_trip(&issue_app_with(&lax_discord, &gateway, GUILD_ID, "0")).await; - assert_eq!(lax.location(), "/prices-api/?issue=ok"); + assert_eq!(lax.location(), "/api/prices-api/?issue=ok"); } // --------------------------------------------------------------------------- @@ -639,7 +643,7 @@ async fn a_session_for_someone_else_is_replaced_by_the_re_auth_identity() { ) .await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); let cookie = reply.cookie(cookies::SESSION_COOKIE).unwrap(); let session = Session::decode(SIGNING_KEY.as_bytes(), &cookie, state_token::now_secs()) .expect("the fresh session must verify"); @@ -681,7 +685,7 @@ async fn a_successful_issue_evicts_the_cached_no_key() { assert_eq!(before.json()["code"], "no_key"); let issued = issue_round_trip(&app).await; - assert_eq!(issued.location(), "/prices-api/?issue=ok"); + assert_eq!(issued.location(), "/api/prices-api/?issue=ok"); // Well inside the 60s TTL, so only the eviction can explain this. let after = call_path(app, "GET", USAGE_PATH, Some(&session)).await; @@ -717,7 +721,7 @@ async fn a_successful_issue_evicts_the_cached_no_plan() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| assert_eq!(s.attach_calls, 1, "the sign-in attached the key")); @@ -795,7 +799,7 @@ async fn an_unreadable_eligibility_parameter_is_unknown() { let reply = issue_round_trip(&app).await; assert_eq!( reply.location(), - "/prices-api/?issue=unknown", + "/api/prices-api/?issue=unknown", "guild={guild:?} age={age:?}" ); assert_eq!(gateway.with(|s| s.create_calls), 0); @@ -821,7 +825,7 @@ async fn a_control_plane_failure_after_eligibility_lands_on_issue_failed_not_unk gateway.with(|s| s.fail_list = true); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert_eq!(discord.member_calls(), 1, "eligibility ran and passed"); // Still signed in: identity and membership both proved. assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); @@ -836,7 +840,7 @@ async fn a_missing_usage_plan_is_issue_failed_without_minting_more_keys() { gateway.with(|s| s.attach_always_404 = true); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert_eq!(gateway.with(|s| s.create_calls), 1); } @@ -852,7 +856,7 @@ async fn a_stale_listing_bounds_the_attempts_and_lands_on_failed() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert_eq!( gateway.with(|s| s.list_calls), 2, @@ -874,7 +878,7 @@ async fn an_adopted_key_is_put_on_the_free_plan() { let orphan = gateway.with(|s| s.seed(&key_name(), 1_000)); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.plan_keys.clone()), vec![(PLAN_ID.to_string(), orphan.clone())], @@ -903,7 +907,7 @@ async fn duplicates_converge_on_the_earliest_and_the_losers_are_deleted() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); let mut deleted = gateway.with(|s| s.deleted.clone()); deleted.sort(); @@ -934,7 +938,7 @@ async fn the_reconciler_pages_get_api_keys_to_exhaustion_before_ranking() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.list_calls), 3, @@ -961,7 +965,7 @@ async fn a_key_that_vanishes_before_the_attach_is_not_a_dead_end() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); let survivors: Vec = gateway.with(|s| s.named(&key_name()).iter().map(|k| k.id.clone()).collect()); @@ -991,7 +995,7 @@ async fn a_key_the_listing_has_not_caught_up_with_is_not_orphaned() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.keys.len()), 1, @@ -1017,7 +1021,7 @@ async fn a_duplicate_that_will_not_delete_does_not_withhold_the_key() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.keys.len()), 2, diff --git a/packages/prices-api/tests/portal_keys_logs.rs b/packages/prices-api/tests/portal_keys_logs.rs index 77423245..b61a7c26 100644 --- a/packages/prices-api/tests/portal_keys_logs.rs +++ b/packages/prices-api/tests/portal_keys_logs.rs @@ -115,7 +115,7 @@ async fn no_key_value_ever_reaches_the_logs() { Some(eligibility(GUILD_ID, "5")), ); let issued = issue_round_trip(&app).await; - assert_eq!(issued.location(), "/prices-api/?issue=ok"); + assert_eq!(issued.location(), "/api/prices-api/?issue=ok"); let body = reveal(&mock, USER_ID).await.json(); let text = String::from_utf8_lossy(&logs.0.lock().unwrap()).to_string(); diff --git a/packages/prices-api/tests/portal_lazy_load.rs b/packages/prices-api/tests/portal_lazy_load.rs index 784f5d7d..6bff6469 100644 --- a/packages/prices-api/tests/portal_lazy_load.rs +++ b/packages/prices-api/tests/portal_lazy_load.rs @@ -388,7 +388,7 @@ async fn a_callback_on_a_failed_load_that_claims_no_action_lands_on_signin_faile let reply = get(&router, "/api/auth/callback?code=c&state=s").await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.headers.get(header::SET_COOKIE).is_none()); } @@ -405,8 +405,11 @@ async fn a_callback_on_a_failed_load_that_claims_no_action_lands_on_signin_faile #[tokio::test] async fn login_on_a_failed_load_lands_on_a_retryable_failure() { for (uri, landing) in [ - ("/api/auth/login", "/prices-api/?signin=failed"), - ("/api/auth/login?action=issue", "/prices-api/?issue=failed"), + ("/api/auth/login", "/api/prices-api/?signin=failed"), + ( + "/api/auth/login?action=issue", + "/api/prices-api/?issue=failed", + ), ] { let (sources, _) = scripted(vec![Step::Fail], Duration::ZERO); let router = router(true, sources); diff --git a/packages/prices-api/tests/portal_rework.rs b/packages/prices-api/tests/portal_rework.rs index 04cf42d1..8694989e 100644 --- a/packages/prices-api/tests/portal_rework.rs +++ b/packages/prices-api/tests/portal_rework.rs @@ -305,7 +305,7 @@ async fn a_partial_revocation_is_reported_as_partial() { // 1st. assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!( gateway.with(|s| s.create_calls), @@ -372,7 +372,7 @@ async fn the_revocation_instant_comes_from_the_control_plane_not_our_clock() { // rolled, so a key is due immediately. assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); } @@ -393,7 +393,7 @@ async fn an_undeletable_revoked_record_does_not_block_the_re_issue() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { let live: Vec<_> = s.keys.iter().filter(|k| k.enabled).collect(); @@ -551,7 +551,7 @@ async fn an_issue_after_a_revoke_in_the_same_period_is_capped_with_the_date() { assert_eq!(reply.status, StatusCode::SEE_OTHER); assert_eq!( reply.location(), - format!("/prices-api/?issue=capped&next_eligible_at={expected_date}") + format!("/api/prices-api/?issue=capped&next_eligible_at={expected_date}") ); assert_eq!( discord.member_calls(), @@ -584,7 +584,7 @@ async fn revoked_on_the_3rd_refuses_until_the_1st_and_issues_once_it_has_passed( let app = app_with_discord(&discord, &gateway); assert_eq!( issue_round_trip(&app).await.location(), - format!("/prices-api/?issue=capped&next_eligible_at={expected_date}") + format!("/api/prices-api/?issue=capped&next_eligible_at={expected_date}") ); assert_eq!(gateway.with(|s| s.create_calls), 0); let revealed = reveal_via(&app).await; @@ -597,7 +597,7 @@ async fn revoked_on_the_3rd_refuses_until_the_1st_and_issues_once_it_has_passed( let app = app_with_discord(&discord, &gateway); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!( @@ -713,7 +713,7 @@ async fn an_undated_duplicate_does_not_lock_the_owner_out_forever() { assert_eq!(revealed.json()["code"], "no_key"); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!(gateway.with(|s| s.create_calls), 1); } @@ -732,7 +732,7 @@ async fn the_latest_revocation_governs_the_cap() { issue_round_trip(&app) .await .location() - .starts_with("/prices-api/?issue=capped") + .starts_with("/api/prices-api/?issue=capped") ); assert_eq!(gateway.with(|s| s.create_calls), 0); assert_eq!(gateway.with(|s| s.deleted.len()), 0); @@ -755,7 +755,7 @@ async fn membership_is_still_checked_before_the_cap() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=not_member" + "/api/prices-api/?issue=not_member" ); assert_eq!(gateway.with(|s| s.list_calls), 0); } @@ -775,7 +775,7 @@ async fn a_live_key_beside_a_revoked_one_is_the_current_key() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!(s.deleted, vec![dead.clone()]); @@ -796,7 +796,7 @@ async fn a_live_key_is_adopted_as_before_the_cap_existed() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!(gateway.with(|s| s.create_calls), 0); assert_eq!(reveal_via(&app).await.json()["key_id"], key); @@ -814,7 +814,7 @@ async fn the_full_cycle_issue_revoke_wait_reissue() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let first = gateway.with(|s| s.keys[0].clone()); @@ -823,7 +823,7 @@ async fn the_full_cycle_issue_revoke_wait_reissue() { issue_round_trip(&app) .await .location() - .starts_with("/prices-api/?issue=capped") + .starts_with("/api/prices-api/?issue=capped") ); // The 1st arrives. @@ -832,7 +832,7 @@ async fn the_full_cycle_issue_revoke_wait_reissue() { }); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { @@ -931,7 +931,7 @@ async fn the_reveal_and_the_issue_agree_on_the_cap_with_mixed_period_revocations assert_eq!( issue_round_trip(&app).await.location(), - format!("/prices-api/?issue=capped&next_eligible_at={expected_date}") + format!("/api/prices-api/?issue=capped&next_eligible_at={expected_date}") ); // And the revoke's idempotent answer names the same instant. @@ -1011,7 +1011,7 @@ async fn a_stale_listing_after_the_roll_does_not_rank_the_deleted_record() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!(s.keys.len(), 1); @@ -1037,7 +1037,7 @@ async fn a_lost_create_response_is_not_retried_into_duplicates() { let app = app_with_discord(&discord, &gateway); let reply = issue_round_trip(&app).await; - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert_eq!( gateway.with(|s| s.create_calls), 1, @@ -1046,7 +1046,7 @@ async fn a_lost_create_response_is_not_retried_into_duplicates() { // The next press adopts the key that landed, as before. assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!(gateway.with(|s| s.create_calls), 1); } @@ -1166,7 +1166,7 @@ async fn rework_rolled_on(plan: StoredPlan) -> (MockGateway, Router, String, Str let app = app_with_discord(&discord, &gateway); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let new = gateway.with(|s| { assert_eq!(s.keys.len(), 1, "the revocation record is gone"); @@ -1286,7 +1286,7 @@ async fn a_crash_between_create_and_attach_retries_onto_the_previous_keys_plan() assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=failed" + "/api/prices-api/?issue=failed" ); let new = gateway.with(|s| { assert_eq!(s.keys.len(), 2, "the new key exists beside the revoked one"); @@ -1305,7 +1305,7 @@ async fn a_crash_between_create_and_attach_retries_onto_the_previous_keys_plan() assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert!( @@ -1347,7 +1347,7 @@ async fn a_live_key_already_on_a_plan_gets_no_attach() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok", + "/api/prices-api/?issue=ok", "{plan_id}" ); gateway.with(|s| { @@ -1389,7 +1389,7 @@ async fn a_rework_keeps_the_paid_plan_when_a_newer_revoked_record_is_on_free() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok", + "/api/prices-api/?issue=ok", "{newer_on:?}" ); gateway.with(|s| { @@ -1436,7 +1436,7 @@ async fn between_two_non_free_revoked_plans_the_newest_revocation_wins() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { let new = s @@ -1482,7 +1482,7 @@ async fn a_rework_whose_new_key_is_not_listed_yet_still_lands_on_the_paid_plan() assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let new = gateway.with(|s| { let new = s @@ -1509,7 +1509,7 @@ async fn a_rework_whose_new_key_is_not_listed_yet_still_lands_on_the_paid_plan() assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!( @@ -1552,7 +1552,7 @@ async fn an_attach_refused_for_another_plan_on_the_stage_keeps_that_plan() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!(s.attach_calls, 1, "the lagging lookup led to one attach"); @@ -1590,7 +1590,7 @@ async fn a_conflict_on_the_same_plan_is_settled_and_the_sweep_still_runs() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!(s.attach_calls, 1); @@ -1623,7 +1623,7 @@ async fn a_refused_attach_whose_plan_never_shows_is_not_handed_out() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=failed" + "/api/prices-api/?issue=failed" ); gateway.with(|s| { assert!(s.attach_calls >= 1); @@ -1659,7 +1659,7 @@ async fn after_a_second_rework_usage_reports_the_key_revoked_last() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let new = gateway.with(|s| { assert!( diff --git a/packages/prices-clickhouse/schema/init.sql b/packages/prices-clickhouse/schema/init.sql index 74399235..68342739 100644 --- a/packages/prices-clickhouse/schema/init.sql +++ b/packages/prices-clickhouse/schema/init.sql @@ -413,6 +413,33 @@ ENGINE = ReplacingMergeTree(fetched_at) ORDER BY (contract_address) SETTINGS index_granularity = 8192; +---------------------------------------------------------------------- +-- `decimals()` of each pure Soroban token an AMM swap has touched (task 0329). +-- The ingest scales each swap leg's raw i128 amount by its token's decimals. +-- Before this table it assumed 7 for every token, the classic convention, and +-- priced every other token off by 10^(7 - decimals). A classic identity is 7 +-- by protocol and needs no row. A SAC the ingest does not yet know as one (its +-- classic asset not seen yet, task 0242) is held as a contract like any token: +-- it is resolved over RPC and gets a row, with 7. +-- +-- Written by the ingest itself (live processor, events-backfill, sdex-backfill) +-- the first time a swap names a token it has no row for: one Soroban RPC +-- `decimals()` call, then this row. Never refreshed: decimals are fixed at +-- deploy, the same reasoning as `asset_symbol`. Only answers are stored; a +-- token whose call failed has no row and is asked again later. +-- +-- Read in full at cold start. The live processor's init FAILS if the table is +-- absent, so apply this before deploying an ingest that reads it. +---------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS prices.asset_decimals ( + contract_address String, + decimals UInt8, + fetched_at DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(fetched_at) +ORDER BY (contract_address) +SETTINGS index_granularity = 8192; + ---------------------------------------------------------------------- -- Oracle reference prices (§3.4). ReplacingMergeTree, monthly partitions. -- Written by the Oracle Fetcher Lambda in production; the backfill writes diff --git a/packages/prices-clickhouse/src/lib.rs b/packages/prices-clickhouse/src/lib.rs index c6fe2b1b..fe584f95 100644 --- a/packages/prices-clickhouse/src/lib.rs +++ b/packages/prices-clickhouse/src/lib.rs @@ -212,6 +212,10 @@ pub const DEFAULT_USER: &str = "default"; /// The `prices` logical store. Every table in `schema/init.sql` lives here. pub const PROD_DATABASE: &str = "prices"; +/// BE's database on the shared cluster, holding `soroban_events` and +/// `soroban_contracts`. The one definition for every reader of BE's tables. +pub const BE_DATABASE: &str = "default"; + /// Canonical mainnet issuer of USDC (Circle). **Load-bearing join key** for the /// USD-close path: the backfill interns the USDC identity under this issuer, the /// enrichment peg tier and `resolve_reference_ids` match `prices.assets` on it, @@ -247,6 +251,29 @@ pub const USDT_ISSUER: &str = "GCQTGZQQ5G4PTM2GL7CDIFKUBIPEC52BROAQIAPW53XBRJVN6 /// precondition 3 is the check that confirms it before the pass runs. pub const USDC_ORACLE_EPOCH_S: u32 = 1_773_237_600; +/// 2021-02-01T00:00:00Z: before it Stellar has no USDC, so candles are priced +/// from an imported XLM/USD rate (task 0331). A bucket edge on every tier, so +/// `timestamp <` it means the whole bucket lies before it. +pub const PRE_USDC_END_S: u32 = 1_612_137_600; + +/// `usd_rate.reference_asset` of the imported pre-2021 XLM/USD series (task +/// 0331). Both its readers filter on it, so they read the same rows. +pub const XLM_RATE_SOURCE: &str = "third-party"; + +/// `prices.assets` rows of a classic dollar token of any issuer, canonical +/// USDC/USDT included; keyed on code/issuer/contract. +pub const DOLLAR_TOKEN_PRED: &str = + "contract_address = '' AND issuer_address != '' AND asset_code IN ('USD', 'USDT', 'USDC')"; + +/// Dollar tokens that lost their peg, excluded by identity (task 0331). +#[rustfmt::skip] +pub const OFF_PEG_DOLLAR_TOKENS: [(&str, &str); 4] = [ + ("USDT", "GC2PJ6J37QM3DOQ63DVGCXTRC5QOKAJUBQO2XKBXOJCWTCOCP75KWUDX"), + ("USD", "GD3LHBHY6NYJXETU6YO5OYI37F3X6L6C46ZWKBFBT35RBRKACDVMRF5K"), + ("USDT", "GC3YRTJQUXVQMG5STS7MSWVF3OF3Z5MVF6NEWZLKNQAJZX5UUFLMCFMA"), + ("USD", "GA5O7NTUIKYDPZSV3MNMIRFO2LCV5RJBNO7TNN7Y3MDO2IXRBSWDEO3T"), +]; + /// ClickHouse client configuration, sourced from environment with local-dev /// defaults. #[derive(Debug, Clone)] @@ -472,6 +499,14 @@ mod tests { ); } + #[test] + fn pre_usdc_end_is_2021_02_01t00_00_00z() { + assert_eq!( + i64::from(PRE_USDC_END_S), + days_from_civil(2021, 2, 1) * 86_400 + ); + } + #[test] fn split_statements_drops_line_comments_and_empty_chunks() { let sql = "-- top\n\ @@ -512,8 +547,9 @@ mod tests { // (−1 = 42: task 0256 removed `discovery_state` with the ledger scan // that was its only reader and writer.) // (+1 = 43: task 0274's `current_prices.price_basis` ALTER, same shape.) + // (+1 = 44: task 0329's `asset_decimals` table.) let stmts = split_statements(INIT_SQL); - assert_eq!(stmts.len(), 43, "got {}", stmts.len()); + assert_eq!(stmts.len(), 44, "got {}", stmts.len()); } /// The single `CREATE TABLE … IF NOT EXISTS (` statement of `sql`. diff --git a/packages/prices-ingest-core/Cargo.toml b/packages/prices-ingest-core/Cargo.toml index d898ffa5..54b7c795 100644 --- a/packages/prices-ingest-core/Cargo.toml +++ b/packages/prices-ingest-core/Cargo.toml @@ -11,6 +11,14 @@ path = "src/lib.rs" [dependencies] stellar-xdr = { workspace = true } stellar-strkey = { workspace = true } +# Task 0329: `decimals()` over Soroban RPC `simulateTransaction` (moved here +# from asset-discovery's symbol stage, which now shares it). Same feature list +# as asset-discovery and oracle-worker. +base64 = "0.22" +reqwest = { version = "0.12", default-features = false, features = [ + "rustls-tls-webpki-roots", + "json", +] } sha2 = { workspace = true } xdr-parser = { workspace = true } clickhouse = { workspace = true } @@ -34,3 +42,5 @@ ledger-processor = { path = "../ledger-processor" } [dev-dependencies] # task 0078: reuse the shared Phoenix xyk swap fixture in the classify-seam test. phoenix-extractor = { path = "../phoenix-extractor", features = ["test-fixtures"] } +# task 0335: `decode.rs` tests pack a ledger the way Galexie writes it. +zstd = "0.13" diff --git a/packages/prices-ingest-core/src/bucket.rs b/packages/prices-ingest-core/src/bucket.rs index c0ad6a2e..0d5657ad 100644 --- a/packages/prices-ingest-core/src/bucket.rs +++ b/packages/prices-ingest-core/src/bucket.rs @@ -843,7 +843,7 @@ mod tests { /// The fixture is the shape that actually reaches this: a Soroban AMM fill /// whose raw i128 amounts PASS the rounding bound (both above 1000, product /// of the shifted factors far above 10^6) yet whose quotient is enormous — - /// 1001 units in against 7.9e28 out. After `AMM_AMOUNT_SCALE` the price is + /// 1001 units in against 7.9e28 out. Scaled at 7 decimals the price is /// ~7.9e25, well past the 10^24 an integer part may reach, and so is the /// vwap derived from the same two volumes. Clamping the volume sums alone /// would leave `vwap` to be saturated by `decimal_to_i128` at `i128::MAX` diff --git a/packages/prices-ingest-core/src/canonical.rs b/packages/prices-ingest-core/src/canonical.rs index 02eae991..0ca76b3f 100644 --- a/packages/prices-ingest-core/src/canonical.rs +++ b/packages/prices-ingest-core/src/canonical.rs @@ -65,6 +65,10 @@ impl AssetIdentity { /// symbol resolves to the same identity used as a trade quote. pub(crate) use prices_clickhouse::{USDC_ISSUER, USDT_ISSUER}; +/// Decimals of every classic asset, and so of every SAC: fixed by the protocol +/// (stroops). A pure Soroban token has its own, read from `decimals()`. +pub const CLASSIC_DECIMALS: u32 = 7; + /// Mainnet (Public) network passphrase. A SAC contract id is **network-scoped**: /// it is `sha256(HashIdPreimage::ContractId { network_id, asset })`, and /// `network_id = sha256(passphrase)`. So the passphrase decides which network's @@ -118,6 +122,23 @@ fn identity_to_asset(identity: &AssetIdentity) -> Option { } } +/// A SEP-11 asset string as a SAC event's last topic carries it: `native`, or +/// `CODE:ISSUER` with a 1–12 char alphanumeric code and a valid G-strkey. +fn parse_sep11(s: &str) -> Option { + if s == "native" { + return Some(AssetIdentity::Native); + } + let (code, issuer) = s.split_once(':')?; + if code.is_empty() || code.len() > 12 || !code.bytes().all(|b| b.is_ascii_alphanumeric()) { + return None; + } + stellar_strkey::ed25519::PublicKey::from_string(issuer).ok()?; + Some(AssetIdentity::Credit { + code: code.to_string(), + issuer: issuer.to_string(), + }) +} + /// The deterministic Stellar Asset Contract (SAC) address for a classic asset on /// the given network: `C…`-strkey of `sha256(HashIdPreimage::ContractId{network, /// ContractIdPreimage::Asset(asset)})`. This is the standard offline derivation — @@ -168,6 +189,13 @@ pub struct AssetRegistry { /// intern order. Replaces the id watermark (task 0139): a set of /// identities says "new" without assuming ids are handed out in order. pending: Vec, + /// Contracts BE flags `is_sac` (task 0242). One of these with no entry in + /// `sac_index` is never minted as a `Contract` identity. + sac_candidates: HashSet, + /// `decimals()` of each pure Soroban token resolved so far, by contract + /// address (`prices.asset_decimals`, task 0329). Classic identities need + /// no entry — see [`CLASSIC_DECIMALS`]. + decimals: HashMap, } impl AssetRegistry { @@ -180,6 +208,8 @@ impl AssetRegistry { network_id: mainnet_network_id(), sac_index: HashMap::new(), pending: Vec::new(), + sac_candidates: HashSet::new(), + decimals: HashMap::new(), }; // Pre-seed the canonical quote SACs so an AMM-via-SAC USDC/USDT/XLM // collapses even before that asset's first classic (SDEX) sighting in the @@ -232,6 +262,40 @@ impl AssetRegistry { self.sac_index.get(contract_addr).cloned() } + /// Map `contract` to the SEP-11 asset it names (`native` | `CODE:G…`), but + /// only when `contract` IS that asset's SAC (task 0242). Interns nothing. + pub fn learn_sac(&mut self, contract: &str, sep11: &str) -> bool { + let Some(identity) = parse_sep11(sep11) else { + return false; + }; + if self.sac_index.get(contract) == Some(&identity) { + return true; + } + let derived = identity_to_asset(&identity).and_then(|a| sac_address(&a, &self.network_id)); + if derived.as_deref() != Some(contract) { + return false; + } + self.sac_index.insert(contract.to_string(), identity); + true + } + + /// The contracts BE flags `is_sac` (task 0242), replacing any earlier set. + /// + /// A snapshot: the live processor sets it once per cold start and never + /// refreshes it. A SAC deployed while a container stays warm is no + /// candidate there, so a trade of it with no in-band proof mints a + /// `Contract` identity. The probe's `SacContractIdentities` counts it (a + /// known gap, heal runbook §8). + pub fn set_sac_candidates(&mut self, contracts: HashSet) { + self.sac_candidates = contracts; + } + + /// True for an `is_sac` contract this registry cannot resolve: its trades + /// are skipped, never minted as a `Contract` identity (task 0242 D2). + pub fn is_unproven_sac(&self, contract: &str) -> bool { + self.sac_candidates.contains(contract) && !self.sac_index.contains_key(contract) + } + /// The SAC contract address that wraps a classic identity (Native / Credit), /// or `None` for a pure `Contract` identity (no classic underlying). Persisted /// onto `prices.assets.sac_address` so a read-time consumer can resolve a @@ -242,6 +306,22 @@ impl AssetRegistry { identity_to_asset(identity).and_then(|asset| sac_address(&asset, &self.network_id)) } + /// The decimals an AMM leg in `identity` is scaled by, or `None` for a pure + /// Soroban token whose `decimals()` is not known yet. Never a guess: the + /// fixed 7 this replaced priced every non-7-decimal token off by + /// 10^(7 − decimals) (task 0329). + pub fn decimals_of(&self, identity: &AssetIdentity) -> Option { + match identity { + AssetIdentity::Contract(addr) => self.decimals.get(addr).copied(), + AssetIdentity::Native | AssetIdentity::Credit { .. } => Some(CLASSIC_DECIMALS), + } + } + + /// Record a Soroban token's resolved `decimals()`. + pub fn set_decimals(&mut self, contract: String, decimals: u32) { + self.decimals.insert(contract, decimals); + } + pub fn assets(&self) -> impl Iterator { self.known.iter() } @@ -417,6 +497,69 @@ mod tests { ); } + // Task 0242 fixture: XCR and its SAC, public on-chain identifiers. + const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; + const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + + fn xcr() -> AssetIdentity { + AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + } + } + + #[test] + fn learn_sac_accepts_only_the_sac_of_the_named_asset() { + let mut reg = AssetRegistry::from_existing(vec![]); + assert_eq!(reg.sac_address_of(&xcr()).as_deref(), Some(XCR_SAC)); + let known = reg.assets().count(); + + assert!(reg.learn_sac(XCR_SAC, &format!("XCR:{XCR_ISSUER}"))); + assert_eq!(reg.resolve_sac(XCR_SAC), Some(xcr())); + assert!( + reg.learn_sac(XCR_SAC, &format!("XCR:{XCR_ISSUER}")), + "idempotent" + ); + assert_eq!(reg.assets().count(), known, "interns nothing"); + assert!(pending(®).is_empty()); + + assert!(reg.learn_sac(NATIVE_SAC, "native")); + assert!(!reg.learn_sac(XCR_SAC, "native")); + } + + #[test] + fn learn_sac_rejects_impostors_and_malformed_assets() { + let usdc = format!("USDC:{USDC_ISSUER}"); + let impostor = stellar_strkey::Contract([7; 32]).to_string(); + let cases = [ + (impostor.as_str(), usdc.clone()), + (XCR_SAC, usdc), + (XCR_SAC, String::new()), + (XCR_SAC, "XCR".to_string()), + (XCR_SAC, "XCR:".to_string()), + (XCR_SAC, format!(":{XCR_ISSUER}")), + (XCR_SAC, "XCR:GNOTAKEY".to_string()), + (XCR_SAC, format!("ABCDEFGHIJKLM:{XCR_ISSUER}")), + (XCR_SAC, format!("XC-R:{XCR_ISSUER}")), + (XCR_SAC, "NATIVE".to_string()), + ]; + for (contract, sep11) in cases { + let mut reg = AssetRegistry::from_existing(vec![]); + assert!(!reg.learn_sac(contract, &sep11), "{contract} {sep11:?}"); + assert_eq!(reg.resolve_sac(contract), None, "{contract} {sep11:?}"); + } + } + + #[test] + fn an_sac_candidate_is_unproven_until_learned() { + let mut reg = AssetRegistry::from_existing(vec![]); + reg.set_sac_candidates(HashSet::from([XCR_SAC.to_string()])); + assert!(reg.is_unproven_sac(XCR_SAC)); + assert!(!reg.is_unproven_sac(NATIVE_SAC), "not a candidate"); + reg.learn_sac(XCR_SAC, &format!("XCR:{XCR_ISSUER}")); + assert!(!reg.is_unproven_sac(XCR_SAC)); + } + fn pending(reg: &AssetRegistry) -> Vec { reg.pending_new().cloned().collect() } diff --git a/packages/prices-ingest-core/src/decimals.rs b/packages/prices-ingest-core/src/decimals.rs new file mode 100644 index 00000000..97de8c4c --- /dev/null +++ b/packages/prices-ingest-core/src/decimals.rs @@ -0,0 +1,319 @@ +//! Soroban token decimals for the AMM path (task 0329). +//! +//! A swap event carries each leg as a raw i128 in its token's own decimals. +//! Classic assets and their SACs are 7 by protocol; a pure Soroban token's +//! decimals are whatever its SEP-41 `decimals()` returns. Decoding is +//! synchronous and never guesses, so a trade whose token is unknown is dropped +//! and reported ([`crate::LedgerSoroban::missing_decimals`]). +//! [`decode_resolving`] is the step every caller runs around a decode: ask the +//! resolver, persist what it learned to `prices.asset_decimals`, record it in +//! the [`AssetRegistry`], and decode the ledger again. + +use std::collections::{HashMap, HashSet}; +use std::time::{Duration, Instant}; + +use serde::Serialize; +use stellar_xdr::ScVal; + +use crate::canonical::AssetRegistry; +use crate::soroban::LedgerSoroban; +use crate::soroban_rpc::{Simulated, http_client, rpc_url_from_env, simulate}; + +/// The largest scale `rust_decimal` holds. A token claiming more cannot be +/// represented, so it is treated as unresolved rather than truncated. +pub const MAX_DECIMALS: u32 = 28; + +/// How long the live processor leaves a contract that gave no answer at all (a +/// Transient failure) before asking again. Bounds the RPC cost of an outage for +/// a token that trades in every ledger, without parking a blip for a warm +/// container's whole lifetime. +pub const LIVE_RETRY_AFTER: Duration = Duration::from_secs(600); + +/// A backfill's inline retries of a Transient failure, in milliseconds. Ten +/// minutes of wall clock is days of ledgers to a backfill, so it does not park: +/// it retries on the spot and then asks again on the token's next trade. +pub const BACKFILL_BACKOFF_MS: [u64; 3] = [1_000, 5_000, 20_000]; + +/// What one `decimals()` call told us. +#[derive(Debug, PartialEq, Eq)] +pub enum Decimals { + Known(u8), + /// The contract answered and the answer is not a usable scale: it has no + /// `decimals()`, errored, returned something other than a `u32`, or more + /// than [`MAX_DECIMALS`]. Repeats on every call. + Absent, + /// No answer about the contract (transport, JSON-RPC error, archived state). + Transient, + /// The node has not reached the ledger being decoded, so a just-deployed + /// token may not exist for it yet. Asked again on the token's next trade. + Behind, +} + +/// One `prices.asset_decimals` row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, clickhouse::Row)] +pub struct DecimalsRow { + pub contract_address: String, + pub decimals: u8, +} + +impl DecimalsRow { + /// Record this row in `assets`, after it is durably written. + pub fn record(&self, assets: &mut AssetRegistry) { + assets.set_decimals(self.contract_address.clone(), self.decimals.into()); + } +} + +/// What [`decode_resolving`] asks for missing decimals: [`DecimalsResolver`] in +/// production, a fake in tests. +#[allow(async_fn_in_trait)] // only ever used through generics, never as `dyn` +pub trait ResolveDecimals { + /// Answers for the contracts in `missing`, met while decoding `ledger`. + async fn resolve(&mut self, missing: &[String], ledger: u32) -> Vec; +} + +/// Decode once. If the decode dropped trades for unknown decimals, resolve +/// them, `persist` the answers, record them in `assets` — only after `persist` +/// succeeds — and decode again. Decoding is deterministic and the first result +/// is discarded whole, so nothing is counted twice. Whatever is still in +/// `trades_missing_decimals` afterwards is volume the caller did not price. +pub async fn decode_resolving( + resolver: &mut impl ResolveDecimals, + assets: &mut AssetRegistry, + ledger: u32, + mut decode: impl FnMut(&mut AssetRegistry) -> LedgerSoroban, + persist: impl AsyncFnOnce(&[DecimalsRow]) -> Result<(), E>, +) -> Result { + let out = decode(assets); + if out.missing_decimals.is_empty() { + return Ok(out); + } + let rows = resolver.resolve(&out.missing_decimals, ledger).await; + if rows.is_empty() { + return Ok(out); + } + persist(&rows).await?; + for row in &rows { + row.record(assets); + } + Ok(decode(assets)) +} + +/// Asks Soroban RPC for the decimals a decode reported missing. +pub struct DecimalsResolver { + http: reqwest::Client, + rpc_url: String, + /// How long a Transient failure parks a contract. + retry_after: Duration, + /// Inline retries of a Transient failure before parking it. + backoff_ms: &'static [u64], + /// Contracts whose answer was [`Decimals::Absent`]. Never asked again by + /// this process: a fact does not change, and a cold start asks once more. + absent: HashSet, + /// When each contract last gave a [`Decimals::Transient`] failure. + failed: HashMap, +} + +impl DecimalsResolver { + /// For the live processor: no inline retries, a Transient failure parks the + /// contract for [`LIVE_RETRY_AFTER`]. + pub fn live(rpc_url: String) -> Self { + Self::with(rpc_url, LIVE_RETRY_AFTER, &[]) + } + + /// For a backfill: [`BACKFILL_BACKOFF_MS`] inline, then no parking. + pub fn backfill(rpc_url: String) -> Self { + Self::with(rpc_url, Duration::ZERO, &BACKFILL_BACKOFF_MS) + } + + /// [`live`](Self::live) against `SOROBAN_RPC_URL`, or the public default. + pub fn live_from_env() -> Self { + Self::live(rpc_url_from_env()) + } + + /// [`backfill`](Self::backfill) against `SOROBAN_RPC_URL`, or the public default. + pub fn backfill_from_env() -> Self { + Self::backfill(rpc_url_from_env()) + } + + fn with(rpc_url: String, retry_after: Duration, backoff_ms: &'static [u64]) -> Self { + Self { + http: http_client(), + rpc_url, + retry_after, + backoff_ms, + absent: HashSet::new(), + failed: HashMap::new(), + } + } + + /// Resolve each distinct contract in `missing` that is neither known Absent + /// nor parked after a Transient failure. Returns the answers, which the + /// caller persists and then [`record`](DecimalsRow::record)s. Empty means + /// nothing new was learned, so decoding again would price nothing more. + /// + /// The calls run concurrently, so a ledger naming N new tokens costs one + /// RPC timeout ([`crate::soroban_rpc::RPC_TIMEOUT_SECS`]) at worst, not N. + pub async fn resolve(&mut self, missing: &[String], ledger: u32) -> Vec { + let mut asked = HashSet::new(); + let mut calls = tokio::task::JoinSet::new(); + for contract in missing { + let parked = self.absent.contains(contract) + || self + .failed + .get(contract) + .is_some_and(|at| at.elapsed() < self.retry_after); + if parked || !asked.insert(contract) { + continue; + } + let (http, rpc_url, contract) = + (self.http.clone(), self.rpc_url.clone(), contract.clone()); + let backoff_ms = self.backoff_ms; + calls.spawn(async move { + let mut answer = decimals_of(&http, &rpc_url, &contract, ledger).await; + for ms in backoff_ms { + if answer != Decimals::Transient { + break; + } + tokio::time::sleep(Duration::from_millis(*ms)).await; + answer = decimals_of(&http, &rpc_url, &contract, ledger).await; + } + (contract, answer) + }); + } + + let mut rows = Vec::new(); + while let Some(joined) = calls.join_next().await { + // A panicked call answered nothing; the token is asked again on its + // next trade. + let Ok((contract, answer)) = joined else { + continue; + }; + match answer { + Decimals::Known(decimals) => { + tracing::info!(contract, decimals, "resolved soroban token decimals"); + self.failed.remove(&contract); + rows.push(DecimalsRow { + contract_address: contract, + decimals, + }); + } + Decimals::Absent => { + tracing::warn!( + contract, + "decimals() has no usable answer — this token's AMM trades are dropped; \ + not asked again until the process restarts" + ); + self.absent.insert(contract); + } + Decimals::Transient => { + tracing::warn!( + contract, + retry_after_secs = self.retry_after.as_secs(), + "decimals() got no answer — this token's AMM trades are dropped until it resolves" + ); + self.failed.insert(contract, Instant::now()); + } + // Not parked: the node catches up within a ledger or two. + Decimals::Behind => { + tracing::info!( + contract, + ledger, + "rpc node behind; decimals() asked again next trade" + ); + } + } + } + rows.sort_by(|a, b| a.contract_address.cmp(&b.contract_address)); + rows + } +} + +impl ResolveDecimals for DecimalsResolver { + async fn resolve(&mut self, missing: &[String], ledger: u32) -> Vec { + DecimalsResolver::resolve(self, missing, ledger).await + } +} + +/// `decimals()` of `contract` as of `ledger`, classified. +pub async fn decimals_of( + http: &reqwest::Client, + rpc_url: &str, + contract: &str, + ledger: u32, +) -> Decimals { + classify( + simulate(http, rpc_url, contract, "decimals", Some(ledger)).await, + contract, + ) +} + +/// SEP-41 declares `decimals() -> u32`; anything else, or more than +/// [`MAX_DECIMALS`], is not a scale we can apply. +fn classify(simulated: Simulated, contract: &str) -> Decimals { + match simulated { + Simulated::Value(ScVal::U32(d)) if d <= MAX_DECIMALS => Decimals::Known(d as u8), + Simulated::Value(other) => { + tracing::warn!(contract, value = ?other, "decimals() returned no usable scale"); + Decimals::Absent + } + Simulated::Absent => Decimals::Absent, + Simulated::Transient => Decimals::Transient, + Simulated::Behind => Decimals::Behind, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SOLVBTC: &str = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN"; + + #[test] + fn only_a_u32_within_the_decimal_scale_is_known() { + let v = |v| classify(Simulated::Value(v), "C"); + assert_eq!(v(ScVal::U32(8)), Decimals::Known(8)); + assert_eq!(v(ScVal::U32(0)), Decimals::Known(0)); + assert_eq!(v(ScVal::U32(28)), Decimals::Known(28)); + assert_eq!(v(ScVal::U32(29)), Decimals::Absent); + assert_eq!(v(ScVal::I32(7)), Decimals::Absent); + assert_eq!(v(ScVal::Void), Decimals::Absent); + // The simulation's own verdict passes through: a fact stays a fact, + // no answer stays retryable. + assert_eq!(classify(Simulated::Absent, "C"), Decimals::Absent); + assert_eq!(classify(Simulated::Transient, "C"), Decimals::Transient); + assert_eq!(classify(Simulated::Behind, "C"), Decimals::Behind); + } + + #[tokio::test] + async fn an_absent_contract_is_never_asked_again() { + // Port 1 would answer Transient; a contract already known Absent must + // not reach it at all. + let mut r = DecimalsResolver::live("http://127.0.0.1:1/".to_string()); + let c = SOLVBTC.to_string(); + r.absent.insert(c.clone()); + assert!(r.resolve(std::slice::from_ref(&c), 1).await.is_empty()); + assert!( + !r.failed.contains_key(&c), + "not asked, so no Transient recorded" + ); + } + + #[tokio::test] + async fn a_failed_contract_is_parked_for_its_retry_window() { + // Port 1 refuses at once, so each attempt is a fast Transient. + let mut r = DecimalsResolver::live("http://127.0.0.1:1/".to_string()); + let c = SOLVBTC.to_string(); + assert!(r.resolve(&[c.clone(), c.clone()], 1).await.is_empty()); + let first = r.failed[&c]; + assert!(r.resolve(std::slice::from_ref(&c), 1).await.is_empty()); + assert_eq!(r.failed[&c], first, "not asked again inside the window"); + + // A backfill's window is zero: the next trade asks again. + r.retry_after = Duration::ZERO; + assert!(r.resolve(std::slice::from_ref(&c), 1).await.is_empty()); + assert!( + r.failed[&c] > first, + "asked again once the window has passed" + ); + } +} diff --git a/packages/prices-ingest-core/src/decode.rs b/packages/prices-ingest-core/src/decode.rs index 8ce07c19..0db3a186 100644 --- a/packages/prices-ingest-core/src/decode.rs +++ b/packages/prices-ingest-core/src/decode.rs @@ -43,3 +43,85 @@ pub fn ledger_close_time(lcm: &LedgerCloseMeta) -> i64 { LedgerCloseMeta::V2(v) => v.ledger_header.header.scp_value.close_time.0 as i64, } } + +#[cfg(test)] +mod tests { + use stellar_xdr::{ + Hash, LedgerCloseMetaBatch, LedgerCloseMetaV2, LedgerCloseValueSignature, LedgerHeader, + LedgerHeaderHistoryEntry, Limits, NodeId, PublicKey, Signature, StellarValue, + StellarValueExt, StellarValueProposedMsValue, StellarValueSignedMsValue, TimePoint, + TimePointMs, Uint256, WriteXdr, + }; + + use super::*; + + const SEQ: u32 = 70_000_000; + const CLOSE_TIME: u64 = 1_793_000_000; + + fn signature() -> LedgerCloseValueSignature { + LedgerCloseValueSignature { + node_id: NodeId(PublicKey::PublicKeyTypeEd25519(Uint256([1; 32]))), + signature: Signature(vec![2; 64].try_into().unwrap()), + } + } + + /// One ledger with `ext`, packed as Galexie writes it: a zstd + /// `LedgerCloseMetaBatch`. + fn object(ext: StellarValueExt) -> Vec { + let lcm = LedgerCloseMeta::V2(LedgerCloseMetaV2 { + ledger_header: LedgerHeaderHistoryEntry { + header: LedgerHeader { + ledger_seq: SEQ, + scp_value: StellarValue { + close_time: TimePoint(CLOSE_TIME), + ext, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }); + let batch = LedgerCloseMetaBatch { + start_sequence: SEQ, + end_sequence: SEQ, + ledger_close_metas: vec![lcm].try_into().unwrap(), + }; + let xdr = batch.to_xdr(Limits::none()).unwrap(); + zstd::encode_all(&xdr[..], 0).unwrap() + } + + /// Protocol 30 (CAP-0088) puts one of the two millisecond arms on every + /// ledger after the vote; `stellar-xdr` 28 rejects both. `close_time` + /// stays whole seconds (`closeTime == closeTimeMs / 1000`). + #[test] + fn protocol_30_millisecond_arms_decode_with_close_time_in_seconds() { + let close_time_ms = TimePointMs(CLOSE_TIME * 1000 + 250); + let cases = [ + ( + "SignedMs", + StellarValueExt::SignedMs(StellarValueSignedMsValue { + close_time_ms: close_time_ms.clone(), + lc_value_signature: signature(), + }), + ), + ( + "EmptyTxSetMs", + StellarValueExt::EmptyTxSetMs(StellarValueProposedMsValue { + close_time_ms, + tx_set_hash: Hash([3; 32]), + previous_ledger_hash: Hash([4; 32]), + previous_ledger_version: 29, + lc_value_signature: signature(), + }), + ), + ]; + for (arm, ext) in cases { + let lcms = decode_object(&object(ext)).unwrap_or_else(|e| panic!("{arm}: {e}")); + assert_eq!(lcms.len(), 1, "{arm}"); + assert_eq!(ledger_sequence(&lcms[0]), SEQ, "{arm}"); + assert_eq!(ledger_close_time(&lcms[0]), CLOSE_TIME as i64, "{arm}"); + } + } +} diff --git a/packages/prices-ingest-core/src/lib.rs b/packages/prices-ingest-core/src/lib.rs index 5b659e4f..7609bafb 100644 --- a/packages/prices-ingest-core/src/lib.rs +++ b/packages/prices-ingest-core/src/lib.rs @@ -13,10 +13,15 @@ //! Layers, in pipeline order: //! - [`filter`] — classic SDEX trades from `LedgerCloseMeta` operation results. //! - [`soroban`] — Soroban AMM trades + oracle samples from contract events. +//! - [`soroban_rpc`] — read-only token calls (`decimals()`, `symbol()`) over +//! RPC `simulateTransaction`. //! - [`static_pools`] — the committed list of factory-less pools //! ([`STATIC_POOLS`], task 0300) merged into the AMM registries. //! - [`canonical`] — asset identity, the [`AssetRegistry`] of known and newly //! seen identities, and `(base, quote)` canonicalisation. +//! - [`sac_proof`] — the BE reads behind the SAC resolver (task 0242). +//! - [`decimals`] — Soroban token decimals: the RPC resolver the AMM path's +//! callers use when a decode reports a token it cannot scale. //! - [`price`] / [`tick`] — per-trade price + the [`TradeTick`] the bucketer eats. //! - [`bucket`] — 1-minute OHLCV accumulation ([`CandleAccumulator`]). //! - [`writer`] — the transport-agnostic ClickHouse [`OhlcvWriter`] (works with a @@ -26,20 +31,24 @@ pub mod bucket; pub mod canonical; +pub mod decimals; pub mod decode; pub mod error; pub mod filter; pub mod price; pub mod registry_io; pub mod retry; +pub mod sac_proof; pub mod safe_log; pub mod soroban; +pub mod soroban_rpc; pub mod static_pools; pub mod tick; pub mod writer; pub use bucket::{CandleAccumulator, OhlcvCandle}; pub use canonical::{AssetIdentity, AssetRegistry, CanonicalPair, canonicalise}; +pub use decimals::{DecimalsResolver, DecimalsRow, ResolveDecimals, decode_resolving}; pub use decode::{decode_object, ledger_close_time, ledger_sequence}; pub use error::IngestError; pub use filter::{ @@ -52,6 +61,10 @@ pub use price::{ }; pub use registry_io::PoolRegistryRow; pub use retry::{DEFAULT_BACKOFF_MS, retry_with_backoff}; +pub use sac_proof::{ + BE_DATABASE, PRICES_DATABASE, SacPreload, SacProofRow, apply_sac_proofs, load_sac_contracts, + load_sac_proofs, preload_sac_resolver, +}; pub use safe_log::safe_response_token; pub use soroban::{ LedgerSoroban, RawSorobanEvent, Registries, UnresolvedPoolSwap, learn_factory_event, diff --git a/packages/prices-ingest-core/src/price.rs b/packages/prices-ingest-core/src/price.rs index 81000e97..a3138e56 100644 --- a/packages/prices-ingest-core/src/price.rs +++ b/packages/prices-ingest-core/src/price.rs @@ -164,7 +164,7 @@ pub fn price_forming_i64(amount_sold: i64, amount_bought: i64) -> bool { /// [`rounding_bound_holds`] for a Soroban AMM fill's RAW i128 amounts. /// /// Must be called on the amounts as the event carries them, in each token's own -/// decimals — BEFORE `soroban::AMM_AMOUNT_SCALE` converts them to `Decimal`. +/// decimals — BEFORE they are scaled by those decimals into `Decimal`. /// After that conversion the integer unit the bound reasons about is gone, and /// every fill looks equally precise. pub fn price_forming_i128(amount_in: i128, amount_out: i128) -> bool { @@ -340,7 +340,7 @@ mod tests { } /// Soroban amounts are raw i128 in each token's own decimals — classified - /// BEFORE `AMM_AMOUNT_SCALE`, because after scaling the integer information + /// BEFORE scaling, because after scaling the integer information /// the bound reads is gone. #[test] fn the_i128_wrapper_widens_without_losing_the_verdict() { diff --git a/packages/prices-ingest-core/src/sac_proof.rs b/packages/prices-ingest-core/src/sac_proof.rs new file mode 100644 index 00000000..d673e323 --- /dev/null +++ b/packages/prices-ingest-core/src/sac_proof.rs @@ -0,0 +1,225 @@ +//! BE reads behind the SAC resolver (task 0242): which contracts BE flags as a +//! Stellar Asset Contract, and one proof event per flagged contract our +//! registry cannot resolve yet. A flagged contract no proof resolves is +//! skipped, not minted as a `Contract` identity (D2). +//! +//! No `SETTINGS` in any statement here: a read that crosses the client's GET +//! length is sent as POST with `readonly=1`, which rejects per-query settings +//! (code 164; see events-backfill `source.rs`). + +use std::collections::HashSet; + +use crate::canonical::AssetRegistry; +use crate::error::IngestError; + +/// BE's database, holding `soroban_contracts` and `soroban_events`. +pub const BE_DATABASE: &str = prices_clickhouse::BE_DATABASE; +/// Ours, holding `assets`. +pub const PRICES_DATABASE: &str = prices_clickhouse::PROD_DATABASE; + +/// `^[A-Za-z_][A-Za-z0-9_]*$`: a bare identifier, safe to splice unquoted. +fn identifier(s: &str) -> Result<&str, IngestError> { + let mut chars = s.chars(); + let ok = matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); + if ok { + Ok(s) + } else { + Err(IngestError::Precondition(format!( + "not a database identifier: {s:?}" + ))) + } +} + +/// Every contract BE flags `is_sac`, once each. +/// +/// No `FINAL`, on purpose. `soroban_contracts` is a +/// `ReplacingMergeTree(wasm_uploaded_at_ledger)`, so a contract BE re-inserts +/// with `is_sac = false` stays in this set until BE's parts merge. That errs +/// toward D2: its trades are skipped and counted, never minted. `FINAL` would +/// merge BE's whole table on every live cold start, a read that fails Init +/// (PC3), to shorten a case that is rare because `is_sac` is fixed at deploy. +pub(crate) fn sac_contracts_sql(be_db: &str) -> Result { + let be = identifier(be_db)?; + Ok(format!( + "SELECT DISTINCT contract_id FROM {be}.soroban_contracts WHERE is_sac" + )) +} + +/// Load the `is_sac` contract set (about 4,000 addresses on production). +pub async fn load_sac_contracts( + client: &clickhouse::Client, + be_db: &str, +) -> Result, IngestError> { + let rows = client + .query(&sac_contracts_sql(be_db)?) + .fetch_all::() + .await?; + Ok(rows.into_iter().collect()) +} + +/// One SAC-signature event per `is_sac` contract whose address is no +/// `sac_address` in `.assets`: the contract and its event's last topic +/// (the SEP-11 asset). Without the "not resolvable" filter the scan times out +/// on production (0242 RESEARCH M16b). +pub(crate) fn sac_proof_preload_sql(be_db: &str, prices_db: &str) -> Result { + let be = identifier(be_db)?; + let prices = identifier(prices_db)?; + Ok(format!( + "SELECT sc.contract_id AS contract_id, \ + JSONExtractString(x.topics_xdr, JSONLength(x.topics_xdr), 'value') AS sep11 \ + FROM (SELECT contract_id, topics_xdr FROM {be}.soroban_events \ + WHERE contract_id IN (SELECT id FROM {be}.soroban_contracts \ + WHERE is_sac AND contract_id NOT IN \ + (SELECT sac_address FROM {prices}.assets FINAL WHERE sac_address != '')) \ + AND signature IN ('transfer', 'mint', 'burn', 'clawback') \ + LIMIT 1 BY contract_id) x \ + INNER JOIN (SELECT id, any(contract_id) AS contract_id FROM {be}.soroban_contracts \ + WHERE is_sac GROUP BY id) sc ON sc.id = x.contract_id" + )) +} + +/// A row of [`sac_proof_preload_sql`], fields in SELECT order. +#[derive(Debug, Clone, PartialEq, Eq, clickhouse::Row, serde::Deserialize)] +pub struct SacProofRow { + pub contract_id: String, + pub sep11: String, +} + +/// Load the proofs events-backfill feeds to `learn_sac` before its first chunk +/// (about 1,100 rows in 2.6 s on production). +pub async fn load_sac_proofs( + client: &clickhouse::Client, + be_db: &str, + prices_db: &str, +) -> Result, IngestError> { + Ok(client + .query(&sac_proof_preload_sql(be_db, prices_db)?) + .fetch_all::() + .await?) +} + +/// Feed every proof through `learn_sac`: (verified, rejected). +pub fn apply_sac_proofs(assets: &mut AssetRegistry, rows: &[SacProofRow]) -> (u64, u64) { + let verified = rows + .iter() + .filter(|r| assets.learn_sac(&r.contract_id, &r.sep11)) + .count() as u64; + (verified, rows.len() as u64 - verified) +} + +/// What [`preload_sac_resolver`] armed a registry with. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SacPreload { + /// BE's `is_sac` contracts, now the registry's candidates. + pub candidates: usize, + /// Proof rows read. + pub proofs: usize, + /// Proofs `learn_sac` accepted, and the ones it did not. + pub verified: u64, + pub rejected: u64, +} + +/// events-backfill's run-start step (task 0242 D1): BE's `is_sac` set as the +/// registry's candidates, then one proof per SAC `assets` cannot resolve, +/// through `learn_sac`. That path streams pool events only, so this is how it +/// resolves a SAC whose classic is not in `assets` yet. +pub async fn preload_sac_resolver( + client: &clickhouse::Client, + be_db: &str, + prices_db: &str, + assets: &mut AssetRegistry, +) -> Result { + let candidates = load_sac_contracts(client, be_db).await?; + let n = candidates.len(); + assets.set_sac_candidates(candidates); + let proofs = load_sac_proofs(client, be_db, prices_db).await?; + let (verified, rejected) = apply_sac_proofs(assets, &proofs); + Ok(SacPreload { + candidates: n, + proofs: proofs.len(), + verified, + rejected, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sac_contracts_sql_reads_bes_is_sac_set_without_settings() { + let sql = sac_contracts_sql("default").unwrap(); + assert!(sql.contains("FROM default.soroban_contracts"), "{sql}"); + assert!(sql.contains("WHERE is_sac"), "{sql}"); + assert!(!sql.contains("SETTINGS"), "{sql}"); + assert!(!sql.contains("FINAL"), "{sql}"); + } + + #[test] + fn a_non_identifier_database_is_refused() { + for name in ["", "1db", "default; DROP", "de.fault", "db`"] { + assert!( + matches!(sac_contracts_sql(name), Err(IngestError::Precondition(_))), + "{name:?}" + ); + } + assert!(sac_contracts_sql("it_0242_be").is_ok()); + for (be, prices) in [ + ("default; DROP", "prices"), + ("default", "pri ces"), + ("", ""), + ] { + assert!( + matches!( + sac_proof_preload_sql(be, prices), + Err(IngestError::Precondition(_)) + ), + "{be:?} {prices:?}" + ); + } + } + + #[test] + fn sac_proof_preload_sql_reads_one_proof_per_unresolvable_sac_without_settings() { + let sql = sac_proof_preload_sql("default", "prices").unwrap(); + for part in [ + "LIMIT 1 BY contract_id", + "signature IN ('transfer', 'mint', 'burn', 'clawback')", + "NOT IN (SELECT sac_address FROM prices.assets FINAL WHERE sac_address != '')", + "default.soroban_events", + "default.soroban_contracts", + "JSONLength(", + ] { + assert!(sql.contains(part), "{part} in {sql}"); + } + assert!(!sql.contains("SETTINGS"), "{sql}"); + } + + const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; + const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + + #[test] + fn apply_sac_proofs_learns_only_what_verifies() { + let xcr = crate::AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + }; + let mut assets = AssetRegistry::from_existing(vec![]); + assert_eq!(assets.sac_address_of(&xcr).as_deref(), Some(XCR_SAC)); + let impostor = stellar_strkey::Contract([7; 32]).to_string(); + let rows = [ + SacProofRow { + contract_id: XCR_SAC.to_string(), + sep11: format!("XCR:{XCR_ISSUER}"), + }, + SacProofRow { + contract_id: impostor.clone(), + sep11: format!("XCR:{XCR_ISSUER}"), + }, + ]; + assert_eq!(apply_sac_proofs(&mut assets, &rows), (1, 1)); + assert_eq!(assets.resolve_sac(XCR_SAC), Some(xcr)); + assert_eq!(assets.resolve_sac(&impostor), None); + } +} diff --git a/packages/prices-ingest-core/src/soroban.rs b/packages/prices-ingest-core/src/soroban.rs index 7181884c..e63a8256 100644 --- a/packages/prices-ingest-core/src/soroban.rs +++ b/packages/prices-ingest-core/src/soroban.rs @@ -21,8 +21,8 @@ use extractors_core::{SorobanEventRow, TaggedValue, Venue, VenueRegistry}; use ledger_processor::dispatch::{PairRegistries, dispatch}; use phoenix_extractor::PhoenixPoolRegistry; use soroswap_extractor::SoroswapPoolRegistry; -use xdr_parser::extract_events; -use xdr_parser::types::EventSource; +use xdr_parser::LedgerEvents; +use xdr_parser::types::ExtractedEvent; // USDT_ISSUER is deliberately NOT imported here: task 0172 removed the `USDT` // arm from `reflector_key_to_identity`, so this module must not resolve that @@ -32,10 +32,6 @@ use crate::price::{price_forming_i128, price_survives_column_scale}; use crate::tick::TradeTick; use crate::writer::OracleSample; -/// AMM token amounts are treated as 7-decimal (Stellar SAC convention). Token -/// decimals vary; this is a documented sizing-measurement approximation. -const AMM_AMOUNT_SCALE: u32 = 7; - /// Per-run registries, grown incrementally from in-window factory events. /// /// Oracle assets are NOT kept here: they are resolved through the same @@ -195,6 +191,19 @@ pub struct LedgerSoroban { /// re-read ledger and in callers that never read the count, so it only /// logs at debug level. pub unregistered_pool_contracts: Vec, + /// One entry per trade skipped because a leg is a contract BE flags + /// `is_sac` that no SAC proof resolved, holding that leg's address (task + /// 0242 D2). Minting it as a `Contract` identity would split its asset. + pub sac_unproven: Vec, + /// The Soroban token behind every AMM trade dropped because its + /// `decimals()` is not known yet (task 0329): one entry per unknown leg of + /// each dropped trade. A caller that can reach RPC resolves these and + /// decodes the ledger again. + pub missing_decimals: Vec, + /// How many AMM trades were dropped for that reason — trades, not legs. + /// Whatever is still counted after the caller's second decode is lost + /// volume. + pub trades_missing_decimals: u32, } fn collect_tx_metas(lcm: &LedgerCloseMeta) -> Vec<&TransactionMeta> { @@ -301,6 +310,37 @@ fn signature(topics: &Value) -> Option<&str> { topic_symbol(topics, 0) } +/// The SEP-11 asset a SAC names in its own `transfer`/`mint`/`burn`/`clawback` +/// event: the last topic, when it is a string (task 0242). Only a claim until +/// [`AssetRegistry::learn_sac`] checks it against the emitter. +fn sac_proof(topics: &Value) -> Option<&str> { + if !matches!( + signature(topics), + Some("transfer" | "mint" | "burn" | "clawback") + ) { + return None; + } + let last = topics.as_array()?.last()?.as_object()?; + match last.get("type")?.as_str()? { + "string" | "str" => last.get("value")?.as_str(), + _ => None, + } +} + +/// Feed every SAC proof among one transaction's events to the registry, before +/// its trades are classified, so event order inside the transaction does not +/// matter (task 0242 D1). +fn learn_sacs<'a>( + events: impl IntoIterator, + assets: &mut AssetRegistry, +) { + for (contract_id, topics) in events { + if let Some(sep11) = sac_proof(topics) { + assets.learn_sac(contract_id, sep11); + } + } +} + /// Process all Soroban events in one ledger. pub fn process_ledger( lcm: &LedgerCloseMeta, @@ -309,43 +349,50 @@ pub fn process_ledger( ) -> LedgerSoroban { let (ledger_seq, closed_at) = ledger_header(lcm); let tx_metas = collect_tx_metas(lcm); + let ledger_events = LedgerEvents::new(ledger_seq, closed_at, &tx_metas); let mut out = LedgerSoroban::default(); - for (tx_index, tx_meta) in tx_metas.iter().enumerate() { + for tx_index in 0..tx_metas.len() { let tx_id = format!("{ledger_seq}:{tx_index}"); - let events = extract_events(tx_meta, &tx_id, ledger_seq, closed_at); + // Consensus events only; the diagnostic mirrors come back in + // `.diagnostic`, unread. An event's position here is its per-tx + // `event_index`: tx-level events first, then per-op. + let events = ledger_events.extract(tx_index, &tx_id).events; // Grouped AMM event rows by emitting contract, order preserved. let mut amm_groups: HashMap> = HashMap::new(); - for ev in &events { - if ev.source == EventSource::Diagnostic { - continue; // drop diagnostic mirrors - } - let Some(contract_id) = ev.contract_id.clone() else { + for (event_index, ev) in events.iter().enumerate() { + let Some(contract_id) = ev.body.contract_id.clone() else { continue; }; - let sig = signature(&ev.topics); + let sig = signature(&ev.body.topics); match sig { Some("REFLECTOR") => decode_reflector(ev, assets, &mut out), Some("REDSTONE") => decode_redstone(ev, &mut out), _ => { // Factory events grow the registry; pool events are queued. - learn_factory(&ev.topics, &ev.data, reg); + learn_factory(&ev.body.topics, &ev.body.data, reg); let row = SorobanEventRow { contract_id: contract_id.clone(), transaction_id: tx_id.clone(), ledger_sequence: ledger_seq as u64, - event_index: ev.event_index, - topics: topics_to_tagged(&ev.topics), - data: json_to_tagged(&ev.data), + event_index: event_index as u32, + topics: topics_to_tagged(&ev.body.topics), + data: json_to_tagged(&ev.body.data), }; amm_groups.entry(contract_id).or_default().push(row); } } } + learn_sacs( + events + .iter() + .filter_map(|ev| Some((ev.body.contract_id.as_deref()?, &ev.body.topics))), + assets, + ); classify_amm_groups( amm_groups, tx_index as u16, @@ -469,6 +516,10 @@ pub fn process_soroban_event_rows( .all(|e| e.transaction_index == transaction_index), "events of transaction {tx_id} disagree on transaction_index" ); + // No `learn_sacs` here, unlike `process_ledger`: events-backfill streams + // pool events only, so a SAC's own transfer/mint/burn/clawback never + // reaches this seam. Its proofs come from `preload_sac_resolver` at run + // start, before the first chunk (`execute`'s test pins the order). classify_amm_groups( amm_groups, transaction_index, @@ -565,9 +616,25 @@ fn classify_amm_groups( match dispatch(&rows, ®.venue, ®.phoenix, reg.pair_registries()) { Ok(trades) => { for t in trades { - if let Some(tick) = amm_trade_to_tick(&t, transaction_index, closed_at, assets) + // Before `canonicalise`, so a skipped trade interns nothing. + if let Some(sac) = [&t.token_in, &t.token_out] + .into_iter() + .find(|c| assets.is_unproven_sac(c)) { + out.sac_unproven.push(sac.clone()); + continue; + } + let unknown_legs = out.missing_decimals.len(); + if let Some(tick) = amm_trade_to_tick( + &t, + transaction_index, + closed_at, + assets, + &mut out.missing_decimals, + ) { out.amm_ticks.push((source, tick)); + } else if out.missing_decimals.len() > unknown_legs { + out.trades_missing_decimals += 1; } } } @@ -818,7 +885,8 @@ fn first_address(v: &Value) -> Option { /// Resolve an AMM token (contract address) to a canonical `AssetIdentity`: the /// underlying classic asset if the address is a known SAC (§12.4), else the -/// `Contract` identity for a pure Soroban token. +/// `Contract` identity for a pure Soroban token. Only tokens BE does not flag +/// `is_sac` reach the fallback: an unproven SAC is skipped earlier (task 0242). fn resolve_amm_token(contract_addr: &str, assets: &AssetRegistry) -> AssetIdentity { assets .resolve_sac(contract_addr) @@ -826,11 +894,15 @@ fn resolve_amm_token(contract_addr: &str, assets: &AssetRegistry) -> AssetIdenti } /// Convert a venue `TradeRow` into a `TradeTick` for the candle accumulator. +/// +/// `None` also when a leg's decimals are unknown; that leg's contract is then +/// pushed onto `missing_decimals`. fn amm_trade_to_tick( trade: &extractors_core::TradeRow, transaction_index: u16, closed_at: i64, assets: &mut AssetRegistry, + missing_decimals: &mut Vec, ) -> Option { // Collapse a SAC token onto its underlying classic identity (§12.4) so // AMM-via-SAC and SDEX-classic share one asset_id; a pure Soroban token keeps @@ -848,24 +920,46 @@ fn amm_trade_to_tick( return None; } + // Task 0329: each leg is scaled by its own token's decimals. A pure Soroban + // token whose `decimals()` is not known yet gets no tick at all — never the + // fixed 7 this replaced, which put SolvBTC (8) at a tenth of its price and + // every 18-decimal token 10^11 too low. Checked before `canonicalise`, so a + // dropped trade interns nothing. + let (Some(decimals_in), Some(decimals_out)) = + (assets.decimals_of(&sold), assets.decimals_of(&bought)) + else { + for leg in [&sold, &bought] { + if let AssetIdentity::Contract(addr) = leg + && assets.decimals_of(leg).is_none() + { + missing_decimals.push(addr.clone()); + } + } + return None; + }; + let pair = canonicalise(&sold, &bought, assets); // Classified on the RAW i128 amounts, in each token's own decimals, BEFORE - // the scaling below (task 0286, ADR 0287 §1): `AMM_AMOUNT_SCALE` turns them - // into a Decimal, and after that the integer unit the rounding bound reasons - // about is gone — every fill would look equally precise. + // the scaling below (task 0286, ADR 0287 §1): scaling turns them into a + // Decimal, and after that the integer unit the rounding bound reasons about + // is gone — every fill would look equally precise. let bound_holds = price_forming_i128(trade.amount_in, trade.amount_out); - let amount_in = Decimal::try_from_i128_with_scale(trade.amount_in, AMM_AMOUNT_SCALE).ok()?; - let amount_out = Decimal::try_from_i128_with_scale(trade.amount_out, AMM_AMOUNT_SCALE).ok()?; + let amount_in = Decimal::try_from_i128_with_scale(trade.amount_in, decimals_in).ok()?; + let amount_out = Decimal::try_from_i128_with_scale(trade.amount_out, decimals_out).ok()?; if amount_in.is_zero() || amount_out.is_zero() { return None; } + // Checked: with per-token scales the quotient carries a factor of up to + // 10^28 (a 28-decimal leg against a 0-decimal one), past `Decimal::MAX`, + // and the plain `/` panics on overflow. A price no `Decimal` can hold is + // not one the candle columns could store either, so the trade is dropped. let (price, volume_base, volume_quote) = if pair.inverted { - (amount_in / amount_out, amount_out, amount_in) + (amount_in.checked_div(amount_out)?, amount_out, amount_in) } else { - (amount_out / amount_in, amount_in, amount_out) + (amount_out.checked_div(amount_in)?, amount_in, amount_out) }; // The bound clears a fill whose two legs are both enormous and says nothing @@ -892,12 +986,9 @@ fn amm_trade_to_tick( /// Decode a REFLECTOR `update` event: data.update_data is a vec of /// [asset_key, price_i128@1e14] pairs; topic[2] is the u64 ms timestamp. -fn decode_reflector( - ev: &xdr_parser::types::ExtractedEvent, - assets: &mut AssetRegistry, - out: &mut LedgerSoroban, -) { +fn decode_reflector(ev: &ExtractedEvent, assets: &mut AssetRegistry, out: &mut LedgerSoroban) { let ts_ms = ev + .body .topics .as_array() .and_then(|t| t.get(2)) @@ -907,7 +998,7 @@ fn decode_reflector( .unwrap_or((ev.created_at.max(0) as u64) * 1000); let timestamp = (ts_ms / 1000) as u32; - let TaggedValue::Map(m) = json_to_tagged(&ev.data) else { + let TaggedValue::Map(m) = json_to_tagged(&ev.body.data) else { return; }; let Some((_, update_data)) = m.iter().find(|(k, _)| k.as_str() == Some("update_data")) else { @@ -959,8 +1050,9 @@ fn decode_reflector( /// row. That id is functionally dead anyway (price_usd = 0, oracle_name = /// 'redstone'; never read by the `reflector` ASOF join), and the raw payload is /// preserved for the byte-footprint measurement. -fn decode_redstone(ev: &xdr_parser::types::ExtractedEvent, out: &mut LedgerSoroban) { +fn decode_redstone(ev: &ExtractedEvent, out: &mut LedgerSoroban) { let raw = ev + .body .data .as_object() .and_then(|o| o.get("value")) @@ -1003,6 +1095,37 @@ mod tests { } } + #[test] + fn sac_proof_reads_a_string_last_topic_of_a_sac_signature() { + // Task 0242: transfer/mint/burn/clawback carry the SEP-11 asset last. + let topics = |sig: &str, last: Value| { + json!([ + {"type":"sym","value":sig}, + {"type":"address","value":"GFROM"}, + last + ]) + }; + let s = |v: &str| json!({"type":"string","value":v}); + let cases: [(&str, Value, Option<&str>); 8] = [ + ("transfer", s("XCR:G"), Some("XCR:G")), + ("mint", s("native"), Some("native")), + ("burn", s("XCR:G"), Some("XCR:G")), + ( + "clawback", + json!({"type":"str","value":"XCR:G"}), + Some("XCR:G"), + ), + ("swap", s("XCR:G"), None), + ("approve", s("XCR:G"), None), + ("transfer", json!({"type":"address","value":"GTO"}), None), + ("transfer", json!({"type":"sym","value":"XCR"}), None), + ]; + for (sig, last, want) in cases { + assert_eq!(sac_proof(&topics(sig, last.clone())), want, "{sig} {last}"); + } + assert_eq!(sac_proof(&json!([])), None, "empty topics"); + } + #[test] fn signature_reads_topic0_symbol() { let topics = json!([{"type":"sym","value":"REFLECTOR"},{"type":"sym","value":"update"}]); @@ -1628,6 +1751,7 @@ mod tests { .register(POOL.to_string(), T0.to_string(), T1.to_string()); let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(T1.to_string(), 7); let mut out = LedgerSoroban::default(); classify_amm_groups(groups, 0, ®, &mut assets, SEQ, CLOSED_AT, &mut out); @@ -1825,6 +1949,7 @@ mod tests { .register(POOL.to_string(), T0.to_string(), T1.to_string()); let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(T1.to_string(), 7); let mut out = LedgerSoroban::default(); process_soroban_event_rows(SEQ, CLOSED_AT, &[swap], &mut reg, &mut assets, &mut out); @@ -2032,6 +2157,8 @@ mod tests { ); let mut assets = AssetRegistry::from_existing(vec![]); + // Pinned at 7 so the fixture keeps the scale it was written for. + assets.set_decimals(T1.to_string(), 7); let tick = amm_trade_to_tick( &extractors_core::TradeRow { venue: Venue::Soroswap, @@ -2049,6 +2176,7 @@ mod tests { 0, 1_700_000_000, &mut assets, + &mut Vec::new(), ) .expect("the swap still produces a tick"); // T0 is XLM's SAC: the tick carries the collapsed identity (D6). @@ -2104,16 +2232,30 @@ mod tests { } let mut assets = AssetRegistry::from_existing(vec![]); + // Pinned at 7 so the fixture keeps the scale it was written for. + assets.set_decimals(T1.to_string(), 7); assert!(price_forming_i128(2_000, AMOUNT_OUT)); - let at_floor = amm_trade_to_tick(&trade(2_000, AMOUNT_OUT), 0, 1_700_000_000, &mut assets) - .expect("the swap prices"); + let at_floor = amm_trade_to_tick( + &trade(2_000, AMOUNT_OUT), + 0, + 1_700_000_000, + &mut assets, + &mut Vec::new(), + ) + .expect("the swap prices"); assert_eq!(at_floor.price, Decimal::new(1, 12), "the fixture is 1e-12"); assert!(at_floor.price_forming, "the floor itself is a price"); assert!(price_forming_i128(1_980, AMOUNT_OUT)); - let under = amm_trade_to_tick(&trade(1_980, AMOUNT_OUT), 0, 1_700_000_000, &mut assets) - .expect("the swap still produces a tick"); + let under = amm_trade_to_tick( + &trade(1_980, AMOUNT_OUT), + 0, + 1_700_000_000, + &mut assets, + &mut Vec::new(), + ) + .expect("the swap still produces a tick"); assert_eq!(under.price, Decimal::new(99, 14), "the fixture is 9.9e-13"); assert!( !under.price_forming, @@ -2127,7 +2269,7 @@ mod tests { /// Task 0286 / ADR 0287 §1 — an AMM fill is classified on the RAW i128 /// amounts the swap event carries, in each token's own decimals, BEFORE - /// `AMM_AMOUNT_SCALE` turns them into `Decimal`s. Once scaled they are + /// scaling turns them into `Decimal`s. Once scaled they are /// fractions and every fill looks equally precise, so the bound has to be /// evaluated on the way in or not at all. #[test] @@ -2156,15 +2298,23 @@ mod tests { } let mut assets = AssetRegistry::from_existing(vec![]); + // Pinned at 7 so the fixture keeps the scale it was written for. + assets.set_decimals(T1.to_string(), 7); - let ordinary = amm_trade_to_tick(&trade(1_000_000, 914_145), 4, CLOSED_AT, &mut assets) - .expect("an ordinary swap prices"); + let ordinary = amm_trade_to_tick( + &trade(1_000_000, 914_145), + 4, + CLOSED_AT, + &mut assets, + &mut Vec::new(), + ) + .expect("an ordinary swap prices"); assert!( ordinary.price_forming, "a million units against ~914k clears the bound by orders of magnitude" ); - let dust = amm_trade_to_tick(&trade(34, 5), 4, CLOSED_AT, &mut assets) + let dust = amm_trade_to_tick(&trade(34, 5), 4, CLOSED_AT, &mut assets, &mut Vec::new()) .expect("a dust swap still produces a tick"); assert!( !dust.price_forming, @@ -2172,7 +2322,7 @@ mod tests { ); assert_eq!( dust.volume_base + dust.volume_quote, - Decimal::try_from_i128_with_scale(39, AMM_AMOUNT_SCALE).unwrap(), + Decimal::try_from_i128_with_scale(39, 7).unwrap(), "the dust fill keeps its volumes: it is a real trade" ); @@ -2227,12 +2377,19 @@ mod tests { reg } + /// SEAM_T0 is XLM's SAC; SEAM_T1 is a pure Soroban token, pinned at 7. + fn seam_assets() -> AssetRegistry { + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(SEAM_T1.to_string(), 7); + assets + } + /// The events-backfill seam carries BE's apply order onto the tick, so a /// repriced candle orders its fills exactly like the live path does. #[test] fn the_seam_carries_the_groups_apply_order_onto_its_ticks() { let mut reg = seam_registries(); - let mut assets = AssetRegistry::from_existing(vec![]); + let mut assets = seam_assets(); let mut out = LedgerSoroban::default(); process_soroban_event_rows( SEAM_SEQ, @@ -2257,7 +2414,7 @@ mod tests { #[should_panic(expected = "disagree on transaction_index")] fn a_group_disagreeing_on_its_apply_order_is_refused_in_debug() { let mut reg = seam_registries(); - let mut assets = AssetRegistry::from_existing(vec![]); + let mut assets = seam_assets(); let mut out = LedgerSoroban::default(); process_soroban_event_rows( SEAM_SEQ, @@ -2294,9 +2451,339 @@ mod tests { let mut assets = AssetRegistry::from_existing(vec![]); let before = assets.assets().count(); - assert!(amm_trade_to_tick(&trade, 0, 1_700_000_000, &mut assets).is_none()); + assert!( + amm_trade_to_tick(&trade, 0, 1_700_000_000, &mut assets, &mut Vec::new()).is_none() + ); assert_eq!(assets.assets().count(), before, "nothing interned"); } + + // ---- task 0329: each leg in its own token's decimals -------------------- + + /// SolvBTC, the token SBE reported at a tenth of its price. + const SOLVBTC: &str = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN"; + + fn usdc() -> AssetIdentity { + AssetIdentity::Credit { + code: "USDC".to_string(), + issuer: USDC_ISSUER.to_string(), + } + } + + /// One whole `token` sold for `usdc_raw` stroops of USDC, through its SAC. + fn sell_for_usdc( + assets: &AssetRegistry, + token: &str, + token_raw: i128, + usdc_raw: i128, + ) -> extractors_core::TradeRow { + extractors_core::TradeRow { + venue: Venue::Aquarius, + contract_id: "CDE57N6XTUPBKYYDGQMXX7E7SLNOLFY3JEQB4MULSMR2AKTSAENGX2HC".to_string(), + transaction_id: "tx".to_string(), + ledger_sequence: 100, + first_event_index: 0, + token_in: token.to_string(), + token_out: assets.sac_address_of(&usdc()).expect("USDC has a SAC"), + amount_in: token_raw, + amount_out: usdc_raw, + fee: None, + trader: None, + } + } + + /// The defect SBE reported: amounts were scaled at a fixed 7, so a token + /// with d decimals priced at 10^(7 − d) of market. Each case sells exactly + /// one whole token for a USDC (7-decimal) amount and must read back that + /// amount as the price and one token as the base volume. + #[test] + fn each_leg_is_scaled_by_its_own_tokens_decimals() { + for (decimals, usdc_per_token) in [(8u32, 86_055i128), (18, 1), (6, 2), (7, 41)] { + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(SOLVBTC.to_string(), decimals); + let one_token = 10i128.pow(decimals); + let trade = sell_for_usdc(&assets, SOLVBTC, one_token, usdc_per_token * 10_000_000); + + let tick = amm_trade_to_tick(&trade, 0, 1_700_000_000, &mut assets, &mut Vec::new()) + .unwrap_or_else(|| panic!("{decimals}-decimal leg prices")); + + assert_eq!( + (&tick.base, &tick.quote), + (&AssetIdentity::Contract(SOLVBTC.to_string()), &usdc()), + "the token is the base, USDC the quote" + ); + assert_eq!( + tick.price, + Decimal::from(usdc_per_token), + "{decimals} decimals" + ); + assert_eq!(tick.volume_base, Decimal::ONE, "{decimals} decimals"); + assert_eq!( + tick.volume_quote, + Decimal::from(usdc_per_token), + "{decimals} decimals" + ); + } + } + + /// The mirror of the case above: USDC sold for the token, so the pair is + /// inverted (`amount_in / amount_out`, base volume from `amount_out`). A + /// swapped `decimals_in` / `decimals_out` would hide in this branch. + #[test] + fn the_inverted_branch_scales_each_leg_by_its_own_decimals() { + for (decimals, usdc_per_token) in [(8u32, 86_055i128), (18, 1), (6, 2)] { + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(SOLVBTC.to_string(), decimals); + let sell = sell_for_usdc(&assets, SOLVBTC, 0, 0); + let buy = extractors_core::TradeRow { + token_in: sell.token_out.clone(), + token_out: SOLVBTC.to_string(), + amount_in: usdc_per_token * 10_000_000, + amount_out: 10i128.pow(decimals), + ..sell + }; + + let tick = amm_trade_to_tick(&buy, 0, 1_700_000_000, &mut assets, &mut Vec::new()) + .unwrap_or_else(|| panic!("{decimals}-decimal leg prices")); + + assert_eq!(tick.base, AssetIdentity::Contract(SOLVBTC.to_string())); + assert_eq!( + tick.price, + Decimal::from(usdc_per_token), + "{decimals} decimals" + ); + assert_eq!(tick.volume_base, Decimal::ONE, "{decimals} decimals"); + assert_eq!( + tick.volume_quote, + Decimal::from(usdc_per_token), + "{decimals} decimals" + ); + } + } + + /// Review of PR #395: per-token scales can push the quotient past + /// `Decimal::MAX`. One raw unit of a 28-decimal token for 9.07e7 stroops is + /// ~9.07e28 XLM per token; the plain `/` panicked there and halted ingest + /// on that ledger for good. It must drop the trade instead. + #[test] + fn a_price_past_decimal_max_drops_the_trade_instead_of_panicking() { + const DUST28: &str = "CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO"; + const WHOLE0: &str = "CB7OOP3VSAWBZOOTOG2YEFANVU45GVWYUUM5HI32DKLHVKUDOFVQ37XP"; + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(DUST28.to_string(), 28); + assets.set_decimals(WHOLE0.to_string(), 0); + let xlm_sac = assets + .sac_address_of(&AssetIdentity::Native) + .expect("XLM has a SAC"); + let swap = |token_in: &str, token_out: &str, amount_out: i128| extractors_core::TradeRow { + venue: Venue::Soroswap, + contract_id: "CDBBBNMCWRMWEIFHUD5BXBCRTW6QM33ZEXIOBGKKQNDSH3WEF7WVBGMI".to_string(), + transaction_id: "tx".to_string(), + ledger_sequence: 100, + first_event_index: 0, + token_in: token_in.to_string(), + token_out: token_out.to_string(), + amount_in: 1, + amount_out, + fee: None, + trader: None, + }; + + // (28, 7): one raw unit in, ~9.07 XLM out. + let t = swap(DUST28, &xlm_sac, 90_700_000); + assert!(amm_trade_to_tick(&t, 0, 1_700_000_000, &mut assets, &mut Vec::new()).is_none()); + // (28, 0), both directions. Which leg lands in the denominator is the + // pair's canonical order, so only the call returning is asserted. + for (token_in, token_out) in [(DUST28, WHOLE0), (WHOLE0, DUST28)] { + let t = swap(token_in, token_out, 9); + let _ = amm_trade_to_tick(&t, 0, 1_700_000_000, &mut assets, &mut Vec::new()); + } + } + + // ---- decode_resolving: the step every caller runs around a decode ------- + + /// Answers every request with `answer` and records what it was asked. + struct FakeResolver { + answer: Vec, + asked: Vec<(Vec, u32)>, + } + + impl crate::decimals::ResolveDecimals for FakeResolver { + async fn resolve(&mut self, missing: &[String], ledger: u32) -> Vec { + self.asked.push((missing.to_vec(), ledger)); + self.answer.clone() + } + } + + fn decode_seam(reg: &mut Registries, assets: &mut AssetRegistry) -> LedgerSoroban { + let mut out = LedgerSoroban::default(); + process_soroban_event_rows( + SEAM_SEQ, + SEAM_CLOSED_AT, + &[seam_swap("tx-a", 0, 0)], + reg, + assets, + &mut out, + ); + out + } + + #[tokio::test] + async fn a_resolved_token_is_persisted_then_recorded_then_priced() { + let mut reg = seam_registries(); + let mut assets = AssetRegistry::from_existing(vec![]); + let row = crate::DecimalsRow { + contract_address: SEAM_T1.to_string(), + decimals: 8, + }; + let mut fake = FakeResolver { + answer: vec![row.clone()], + asked: Vec::new(), + }; + let mut persisted = Vec::new(); + + let out = crate::decimals::decode_resolving( + &mut fake, + &mut assets, + SEAM_SEQ, + |assets| decode_seam(&mut reg, assets), + async |rows: &[crate::DecimalsRow]| { + persisted.extend_from_slice(rows); + Ok::<(), ()>(()) + }, + ) + .await + .unwrap(); + + assert_eq!(fake.asked, vec![(vec![SEAM_T1.to_string()], SEAM_SEQ)]); + assert_eq!(persisted, vec![row]); + assert_eq!( + assets.decimals_of(&AssetIdentity::Contract(SEAM_T1.to_string())), + Some(8) + ); + assert_eq!(out.amm_ticks.len(), 1, "the second decode prices the swap"); + assert_eq!(out.trades_missing_decimals, 0); + } + + #[tokio::test] + async fn an_unresolved_token_is_counted_and_nothing_is_persisted() { + let mut reg = seam_registries(); + let mut assets = AssetRegistry::from_existing(vec![]); + let mut fake = FakeResolver { + answer: Vec::new(), + asked: Vec::new(), + }; + let mut persisted: Vec = Vec::new(); + + let out = crate::decimals::decode_resolving( + &mut fake, + &mut assets, + SEAM_SEQ, + |assets| decode_seam(&mut reg, assets), + async |rows: &[crate::DecimalsRow]| { + persisted.extend_from_slice(rows); + Ok::<(), ()>(()) + }, + ) + .await + .unwrap(); + + assert_eq!(fake.asked.len(), 1); + assert!(persisted.is_empty()); + assert!(out.amm_ticks.is_empty()); + assert_eq!(out.trades_missing_decimals, 1, "the lost trade is counted"); + } + + #[tokio::test] + async fn a_failed_persist_records_nothing() { + let mut reg = seam_registries(); + let mut assets = AssetRegistry::from_existing(vec![]); + let mut fake = FakeResolver { + answer: vec![crate::DecimalsRow { + contract_address: SEAM_T1.to_string(), + decimals: 8, + }], + asked: Vec::new(), + }; + + let got = crate::decimals::decode_resolving( + &mut fake, + &mut assets, + SEAM_SEQ, + |assets| decode_seam(&mut reg, assets), + async |_: &[crate::DecimalsRow]| Err("insert failed"), + ) + .await; + + assert_eq!(got.err(), Some("insert failed")); + assert_eq!( + assets.decimals_of(&AssetIdentity::Contract(SEAM_T1.to_string())), + None, + "recorded only after a durable write, so the next run asks again" + ); + } + + /// A pure Soroban token whose decimals are unknown is never guessed at 7: + /// the trade yields no tick, interns nothing, and every unknown leg is + /// reported so the caller can resolve it. + #[test] + fn a_leg_with_unknown_decimals_prices_nothing_and_is_reported() { + let mut assets = AssetRegistry::from_existing(vec![]); + let before = assets.assets().count(); + let mut missing = Vec::new(); + let trade = sell_for_usdc(&assets, SOLVBTC, 100_000_000, 860_550_000_000); + + assert!(amm_trade_to_tick(&trade, 0, 1_700_000_000, &mut assets, &mut missing).is_none()); + assert_eq!(missing, vec![SOLVBTC.to_string()], "only the unknown leg"); + assert_eq!(assets.assets().count(), before, "nothing interned"); + + // Two unknown legs: both reported, so one resolve pass covers the trade. + let other = "CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO"; + let both = extractors_core::TradeRow { + token_out: other.to_string(), + ..trade + }; + missing.clear(); + assert!(amm_trade_to_tick(&both, 0, 1_700_000_000, &mut assets, &mut missing).is_none()); + assert_eq!(missing, vec![SOLVBTC.to_string(), other.to_string()]); + } + + /// The contract the callers rely on: a ledger decoded before a token's + /// decimals are known reports the token and prices nothing; decoded again + /// once they are set, it prices. + #[test] + fn a_ledger_decoded_again_after_its_decimals_resolve_prices() { + let mut reg = seam_registries(); + let mut assets = AssetRegistry::from_existing(vec![]); + let mut out = LedgerSoroban::default(); + let events = [seam_swap("tx-a", 0, 0)]; + + process_soroban_event_rows( + SEAM_SEQ, + SEAM_CLOSED_AT, + &events, + &mut reg, + &mut assets, + &mut out, + ); + assert!(out.amm_ticks.is_empty()); + assert_eq!(out.missing_decimals, vec![SEAM_T1.to_string()]); + assert_eq!(out.trades_missing_decimals, 1); + assert!(out.unresolved.is_empty(), "a known pool is not unresolved"); + + assets.set_decimals(SEAM_T1.to_string(), 7); + let mut out = LedgerSoroban::default(); + process_soroban_event_rows( + SEAM_SEQ, + SEAM_CLOSED_AT, + &events, + &mut reg, + &mut assets, + &mut out, + ); + assert_eq!(out.amm_ticks.len(), 1); + assert!(out.missing_decimals.is_empty()); + assert_eq!(out.trades_missing_decimals, 0); + } } /// Task 0300: the Comet venue, pinned on real production payloads. diff --git a/packages/prices-ingest-core/src/soroban/comet_tests.rs b/packages/prices-ingest-core/src/soroban/comet_tests.rs index cc4b2630..954593df 100644 --- a/packages/prices-ingest-core/src/soroban/comet_tests.rs +++ b/packages/prices-ingest-core/src/soroban/comet_tests.rs @@ -218,6 +218,7 @@ fn an_aquarius_self_trade_yields_no_tick_and_no_error() { let mut reg = Registries::new(); reg.venue.insert(AQUA.to_string(), Venue::Aquarius); let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(TOKEN.to_string(), 7); let out = run( 62_078_348, std::slice::from_ref(&trade), @@ -571,3 +572,193 @@ fn the_static_list_routes_comet_from_an_empty_registry() { assert_eq!((&tick.base, &tick.quote), (&blnd(), &usdc())); assert!(out.unregistered_pool_events.is_empty()); } + +/// Blend v2.1's Comet pool (task 0334), wasm `d735c339…` (CometDEX v1.1.0). +const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; + +/// `v1_1_blnd_to_usdc` — pool `CASYAKK3…`, ledger 64,875,167, tx idx 199, +/// op 0, event 8. BLND → USDC, 16805005302 in / 94368304 out. +fn v1_1_blnd_to_usdc() -> RawSorobanEvent { + RawSorobanEvent { + contract_id: COMET_V1_1.to_string(), + ..comet_event( + 64_875_167, + 199, + 8, + json!([{"type": "sym", "value": "POOL"}, {"type": "sym", "value": "swap"}]), + json!({"type": "map", "value": [{"key": {"type": "sym", "value": "caller"}, "value": {"type": "address", "value": "GDCRZPZYBZ24RHRO3WBPJGFDL7NDFKUQBS3ZDB6YGBJB3TGKMFYBQ3LD"}}, {"key": {"type": "sym", "value": "token_amount_in"}, "value": {"type": "i128", "value": "16805005302"}}, {"key": {"type": "sym", "value": "token_amount_out"}, "value": {"type": "i128", "value": "94368304"}}, {"key": {"type": "sym", "value": "token_in"}, "value": {"type": "address", "value": "CD25MNVTZDL4Y3XBCPCJXGXATV5WUHHOWMYFF4YBEGU5FCPGMYTVG5JY"}}, {"key": {"type": "sym", "value": "token_out"}, "value": {"type": "address", "value": "CCW67TSZV3SSS2HXMBQ5JFGCKJNXKZM7UQUWUZPUTHXSTZLEO7SJMI75"}}]}), + ) + } +} + +/// Task 0334: the v1.1 pool's real swap, unregistered before and routed by +/// the static list after, from an empty registry. +#[test] +fn the_static_list_routes_the_blend_v2_1_comet_pool() { + let mut assets = seeded_assets(); + let out = run( + 64_875_167, + &[v1_1_blnd_to_usdc()], + &mut Registries::new(), + &mut assets, + ); + assert!(out.amm_ticks.is_empty()); + assert_eq!(out.unregistered_pool_events, vec![("comet", 1)]); + assert_eq!( + out.unregistered_pool_contracts, + vec![COMET_V1_1.to_string()] + ); + + let mut reg = Registries::new(); + reg.merge_static_pools(); + let out = run(64_875_167, &[v1_1_blnd_to_usdc()], &mut reg, &mut assets); + assert_eq!(out.amm_ticks.len(), 1); + let (source, tick) = &out.amm_ticks[0]; + assert_eq!(*source, "comet"); + assert_eq!((&tick.base, &tick.quote), (&blnd(), &usdc())); + // BLND → USDC: price = USDC out / BLND in (≈ 0.005616). + assert_eq!(tick.price, d(94_368_304) / d(16_805_005_302)); + assert_eq!(tick.volume_base, d(16_805_005_302)); + assert!(tick.price_forming); + assert!(out.unregistered_pool_events.is_empty()); + assert!(out.dispatch_errors.is_empty()); +} + +// ---- task 0337: three more factory-less Comet v1.1 pools ------------------- +// +// Fixtures from StellarExpert's copy of each event (`/contract//events`), +// decoded from its `bodyXdr`; ledger and tx application order are the event +// id's TOID (`ledger << 32 | tx << 12 | op`), event index its suffix. + +const BLNT_USDC: &str = "CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3"; +const BLNT_XLM: &str = "CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC"; +const BLND_USDC_SMALL: &str = "CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ"; +/// BLNT's issuer; its SAC is `CCBNWP5L…`. +const BLNT_ISSUER: &str = "GAP2ACGH3O6TRFAERGM4USJ6XWU3U2BFIK224JPV6NI3CUNEEEGTBTFB"; + +fn blnt() -> AssetIdentity { + AssetIdentity::Credit { + code: "BLNT".to_string(), + issuer: BLNT_ISSUER.to_string(), + } +} + +/// One `POOL/swap` of `pool`, its payload map built from the five fields. +fn v1_1_swap( + pool: &str, + (ledger, tx_index, event_index): (u32, u16, u32), + caller: &str, + (token_in, amount_in): (&str, &str), + (token_out, amount_out): (&str, &str), +) -> RawSorobanEvent { + RawSorobanEvent { + contract_id: pool.to_string(), + ..comet_event( + ledger, + tx_index, + event_index, + json!([{"type": "sym", "value": "POOL"}, {"type": "sym", "value": "swap"}]), + json!({"type": "map", "value": [{"key": {"type": "sym", "value": "caller"}, "value": {"type": "address", "value": caller}}, {"key": {"type": "sym", "value": "token_amount_in"}, "value": {"type": "i128", "value": amount_in}}, {"key": {"type": "sym", "value": "token_amount_out"}, "value": {"type": "i128", "value": amount_out}}, {"key": {"type": "sym", "value": "token_in"}, "value": {"type": "address", "value": token_in}}, {"key": {"type": "sym", "value": "token_out"}, "value": {"type": "address", "value": token_out}}]}), + ) + } +} + +const USDC_SAC: &str = "CCW67TSZV3SSS2HXMBQ5JFGCKJNXKZM7UQUWUZPUTHXSTZLEO7SJMI75"; +const BLND_SAC: &str = "CD25MNVTZDL4Y3XBCPCJXGXATV5WUHHOWMYFF4YBEGU5FCPGMYTVG5JY"; +const BLNT_SAC: &str = "CCBNWP5L7ZSXA3DBWCMFAVKFVRW3ZHWO7B5J557W42O64DDZNPJP7VDS"; +const XLM_SAC: &str = "CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA"; +/// The router behind every BLNT swap so far (USDC → BLNT → XLM in one tx). +const BLNT_ROUTER: &str = "CB2DT56GJPXZG7SXGPJOW3F3WNF6RQ7RT5HZLAD2XW4TJ7NUN2YWUZLW"; + +/// Event `279761757207027713-0001` — `CDQMIKIH…`, ledger 65,137,110, tx 262, +/// event 1. USDC → BLNT, 500000000 in / 103337956008 out. +fn blnt_usdc_swap() -> RawSorobanEvent { + v1_1_swap( + BLNT_USDC, + (65_137_110, 262, 1), + BLNT_ROUTER, + (USDC_SAC, "500000000"), + (BLNT_SAC, "103337956008"), + ) +} + +/// Event `279761757207027713-0005` — `CA5HX3D4…`, the same transaction's +/// second hop. BLNT → XLM, 103337956008 in / 3711330029 out. +fn blnt_xlm_swap() -> RawSorobanEvent { + v1_1_swap( + BLNT_XLM, + (65_137_110, 262, 5), + BLNT_ROUTER, + (BLNT_SAC, "103337956008"), + (XLM_SAC, "3711330029"), + ) +} + +/// Event `279599050962505729-0009` — `CCNDNFA7…`, ledger 65,099,227, tx 641, +/// event 9, the pool's first swap. BLND → USDC, 42983166 in / 231757 out. +fn blnd_usdc_small_swap() -> RawSorobanEvent { + v1_1_swap( + BLND_USDC_SMALL, + (65_099_227, 641, 9), + "CDYYYMHP3RKMPDY7VARR5J65RBLUFBHMARTHZM5NZ774D4CMOV7VHRR2", + (BLND_SAC, "42983166"), + (USDC_SAC, "231757"), + ) +} + +/// Run `event` from an empty registry, then from one with the static list +/// merged: unregistered before, exactly one `"comet"` tick after. +fn routed_by_the_static_list(event: RawSorobanEvent) -> TradeTick { + let mut assets = AssetRegistry::from_existing(vec![usdc(), blnd(), blnt()]); + let ledger = event.ledger_sequence; + let pool = event.contract_id.clone(); + + let out = run( + ledger, + std::slice::from_ref(&event), + &mut Registries::new(), + &mut assets, + ); + assert!(out.amm_ticks.is_empty()); + assert_eq!(out.unregistered_pool_events, vec![("comet", 1)]); + assert_eq!(out.unregistered_pool_contracts, vec![pool]); + + let mut reg = Registries::new(); + reg.merge_static_pools(); + let out = run(ledger, &[event], &mut reg, &mut assets); + assert!(out.unregistered_pool_events.is_empty()); + assert!(out.dispatch_errors.is_empty()); + assert!(out.unresolved.is_empty()); + assert_eq!(out.amm_ticks.len(), 1); + let (source, tick) = out.amm_ticks.into_iter().next().unwrap(); + assert_eq!(source, "comet"); + assert!(tick.price_forming); + tick +} + +#[test] +fn the_static_list_routes_the_blnt_usdc_pool() { + let tick = routed_by_the_static_list(blnt_usdc_swap()); + assert_eq!((&tick.base, &tick.quote), (&blnt(), &usdc())); + // USDC → BLNT: price = USDC in / BLNT out (≈ 0.00484). + assert_eq!(tick.price, d(500_000_000) / d(103_337_956_008)); + assert_eq!(tick.volume_base, d(103_337_956_008)); +} + +#[test] +fn the_static_list_routes_the_blnt_xlm_pool() { + let tick = routed_by_the_static_list(blnt_xlm_swap()); + assert_eq!((&tick.base, &tick.quote), (&blnt(), &AssetIdentity::Native)); + // BLNT → XLM: price = XLM out / BLNT in (≈ 0.0359). + assert_eq!(tick.price, d(3_711_330_029) / d(103_337_956_008)); + assert_eq!(tick.volume_base, d(103_337_956_008)); +} + +#[test] +fn the_static_list_routes_the_small_blnd_usdc_pool() { + let tick = routed_by_the_static_list(blnd_usdc_small_swap()); + assert_eq!((&tick.base, &tick.quote), (&blnd(), &usdc())); + // BLND → USDC: price = USDC out / BLND in (≈ 0.00539). + assert_eq!(tick.price, d(231_757) / d(42_983_166)); + assert_eq!(tick.volume_base, d(42_983_166)); +} diff --git a/packages/prices-ingest-core/src/soroban_rpc.rs b/packages/prices-ingest-core/src/soroban_rpc.rs new file mode 100644 index 00000000..02b23de8 --- /dev/null +++ b/packages/prices-ingest-core/src/soroban_rpc.rs @@ -0,0 +1,358 @@ +//! Read-only calls to a Soroban token contract over RPC `simulateTransaction`. +//! +//! Two readers share this: asset-discovery's `symbol()` stage (task 0210) and +//! the ingest's `decimals()` lookup (task 0329). Both need the same envelope and +//! the same answer to one question — did the simulation actually evaluate the +//! contract? — so that boundary lives here once rather than in each caller. + +use base64::Engine; +use stellar_xdr::{ + ContractId, Hash, HostFunction, InvokeContractArgs, InvokeHostFunctionOp, Limits, Memo, + MuxedAccount, Operation, OperationBody, Preconditions, ReadXdr, ScAddress, ScSymbol, ScVal, + SequenceNumber, Transaction, TransactionEnvelope, TransactionExt, TransactionV1Envelope, + Uint256, VecM, WriteXdr, +}; + +/// Default public Soroban RPC endpoint (overridable via `SOROBAN_RPC_URL`). +/// Needs no auth — same endpoint and same reasoning as `oracle-worker`. +pub const DEFAULT_SOROBAN_RPC: &str = "https://mainnet.sorobanrpc.com"; + +/// Per-request RPC timeout. Each caller's time bound is this times the number +/// of calls it allows itself per run. +pub const RPC_TIMEOUT_SECS: u64 = 5; + +/// `SOROBAN_RPC_URL`, or [`DEFAULT_SOROBAN_RPC`]. +pub fn rpc_url_from_env() -> String { + std::env::var("SOROBAN_RPC_URL").unwrap_or_else(|_| DEFAULT_SOROBAN_RPC.to_string()) +} + +/// An HTTP client bounded by [`RPC_TIMEOUT_SECS`]. +/// +/// # Panics +/// +/// If the client cannot be built — in practice a TLS backend that failed to +/// initialise, which is an environment fault at cold start, identical on every +/// invocation. Deliberately not a silent fallback to `Client::default()`: that +/// client carries **no timeout**, so every caller's time bound would quietly +/// become unbounded. A loud failure at startup beats an outage with no obvious +/// cause. +pub fn http_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(RPC_TIMEOUT_SECS)) + .build() + .expect("build an HTTP client with a timeout") +} + +/// Base64 `TransactionEnvelope` invoking the zero-argument `func()` on +/// `contract`, for a read-only `simulateTransaction` (source account / fee / seq +/// are not checked by simulation). +/// +/// `None` when `contract` is not a well-formed C-strkey. +pub fn build_simulate_envelope(contract: &str, func: &str) -> Option { + let contract_hash = stellar_strkey::Contract::from_string(contract).ok()?.0; + let invoke = InvokeContractArgs { + contract_address: ScAddress::Contract(ContractId(Hash(contract_hash))), + function_name: ScSymbol(func.try_into().ok()?), + args: VecM::default(), + }; + let op = Operation { + source_account: None, + body: OperationBody::InvokeHostFunction(InvokeHostFunctionOp { + host_function: HostFunction::InvokeContract(invoke), + auth: VecM::default(), + }), + }; + let tx = Transaction { + source_account: MuxedAccount::Ed25519(Uint256([0u8; 32])), + fee: 0, + seq_num: SequenceNumber(0), + cond: Preconditions::None, + memo: Memo::None, + operations: vec![op].try_into().ok()?, + ext: TransactionExt::V0, + }; + let envelope = TransactionEnvelope::Tx(TransactionV1Envelope { + tx, + signatures: VecM::default(), + }); + let xdr = envelope.to_xdr(Limits::none()).ok()?; + Some(base64::engine::general_purpose::STANDARD.encode(xdr)) +} + +/// What one simulated call produced. +/// +/// The Absent/Transient split is load-bearing for every caller: Absent is a +/// fact about the contract that will repeat, Transient means we learned +/// nothing and must ask again. +#[derive(Debug, PartialEq)] +pub enum Simulated { + /// The contract evaluated the call and returned this value. + Value(ScVal), + /// The contract evaluated the call and errored, or returned undecodable + /// bytes, or the address is not a contract at all. Repeats on every call. + Absent, + /// No answer about the contract: transport failure, JSON-RPC error, + /// archived state, an empty body. Ask again later. + Transient, + /// The contract errored, but the node has not yet reached the ledger the + /// caller asked about, so a contract deployed since may simply be invisible + /// to it. Ask again on the next ledger. + Behind, +} + +#[derive(serde::Deserialize)] +struct RpcResponse { + result: Option, + /// JSON-RPC 2.0 reports failure *here*, with HTTP 200 — which is how public + /// endpoints signal quota exhaustion, a disabled method, and a node that is + /// not synced. Without this field such a body deserialises to + /// `result: None` and would read as a fact about the contract. + #[serde(default)] + error: Option, +} +#[derive(serde::Deserialize)] +struct RpcResult { + #[serde(default)] + results: Vec, + #[serde(default)] + error: Option, + /// Set when the contract's instance or Wasm is archived but restorable — + /// a live contract, not one that lacks the function. + #[serde(default, rename = "restorePreamble")] + restore_preamble: Option, + /// The newest ledger the node had applied when it simulated. + #[serde(default, rename = "latestLedger")] + latest_ledger: Option, +} +#[derive(serde::Deserialize)] +struct RpcInvokeResult { + xdr: String, +} + +/// Call the zero-argument `func()` on `contract` by simulation and classify +/// the result. Never returns an error: every failure is Absent, Transient or +/// Behind, which is exactly the decision a caller has to make. +/// +/// `as_of` is the ledger at which the caller needs the contract to exist (the +/// one it is decoding). A contract error from a node that has not reached it is +/// [`Simulated::Behind`], not a fact. `None` skips that check. +pub async fn simulate( + http: &reqwest::Client, + rpc_url: &str, + contract: &str, + func: &str, + as_of: Option, +) -> Simulated { + let Some(envelope) = build_simulate_envelope(contract, func) else { + tracing::warn!(contract, func, "not a well-formed contract address"); + return Simulated::Absent; + }; + let body = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "simulateTransaction", + "params": { "transaction": envelope }, + }); + + // A transport-level failure — connection, timeout, non-2xx (a 429 or 5xx + // from the public endpoint lands here), unparseable body — is systemic, not + // a fact about this contract. `error_for_status` is what keeps a + // rate-limited run from recording every contract it touches as Absent. + let resp = match http + .post(rpc_url) + .json(&body) + .send() + .await + .and_then(|r| r.error_for_status()) + { + Ok(r) => r, + Err(err) => { + tracing::warn!(contract, func, error = %err, "simulate rpc failed"); + return Simulated::Transient; + } + }; + match resp.json::().await { + Ok(r) => classify(r, contract, func, as_of), + Err(err) => { + tracing::warn!(contract, func, error = %err, "simulate rpc body unparseable"); + Simulated::Transient + } + } +} + +/// Decide what a parsed simulate response means for this contract. +/// +/// The line between the two failure arms is **whether the simulation actually +/// evaluated the contract**. If it did, its outcome is a fact that will repeat, +/// so it is Absent. If we never got that far — a JSON-RPC error, archived +/// state, an empty body — we learned nothing, so it is Transient. A wrong +/// Transient costs a repeated call; a wrong Absent can make a caller give up +/// on a contract for good. +fn classify(resp: RpcResponse, contract: &str, func: &str, as_of: Option) -> Simulated { + // Arrives with HTTP 200, so `error_for_status` cannot see it. + if let Some(err) = resp.error { + tracing::warn!(contract, func, error = %err, "simulate returned a json-rpc error"); + return Simulated::Transient; + } + let Some(result) = resp.result else { + tracing::warn!(contract, func, "simulate returned neither result nor error"); + return Simulated::Transient; + }; + if result.restore_preamble.is_some() { + tracing::warn!(contract, func, "simulate needs a state restore"); + return Simulated::Transient; + } + // The simulation ran and the contract itself errored — deterministic, so a + // fact. This is the one arm that stays permanent, unless the node is still + // behind the ledger the caller is decoding: a contract deployed in between + // does not exist for it yet (task 0329 review). + if let Some(err) = result.error { + if let (Some(needed), Some(tip)) = (as_of, result.latest_ledger) + && tip < needed + { + tracing::debug!( + contract, + func, + tip, + needed, + "node behind the ledger being decoded" + ); + return Simulated::Behind; + } + tracing::debug!(contract, func, error = %err, "contract errored in simulation"); + return Simulated::Absent; + } + let Some(first) = result.results.into_iter().next() else { + tracing::warn!(contract, func, "simulated without a result or an error"); + return Simulated::Transient; + }; + match base64::engine::general_purpose::STANDARD + .decode(first.xdr) + .ok() + .and_then(|bytes| ScVal::from_xdr(&bytes, Limits::none()).ok()) + { + Some(v) => Simulated::Value(v), + // It answered, and the answer does not decode. A fact, and otherwise an + // invisible one, so name the contract. + None => { + tracing::warn!(contract, func, "simulate result is not a decodable ScVal"); + Simulated::Absent + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn classify_json(body: &str) -> Simulated { + classify_json_as_of(body, None) + } + + fn classify_json_as_of(body: &str, as_of: Option) -> Simulated { + classify( + serde_json::from_str(body).expect("test body parses"), + "C_TEST", + "symbol", + as_of, + ) + } + + #[test] + fn a_json_rpc_error_retries_instead_of_being_absent() { + // JSON-RPC reports this with HTTP 200, so `error_for_status` cannot see + // it. Reading it as Absent would turn an RPC outage into a fact about + // every contract the run touched. + let out = classify_json( + r#"{"jsonrpc":"2.0","id":1,"error":{"code":-32601,"message":"method not enabled"}}"#, + ); + assert_eq!(out, Simulated::Transient); + } + + #[test] + fn an_empty_or_archived_response_retries() { + // Neither result nor error: we learned nothing about the contract. + assert_eq!( + classify_json(r#"{"jsonrpc":"2.0","id":1}"#), + Simulated::Transient + ); + // Archived-but-restorable state is a live contract. + assert_eq!( + classify_json(r#"{"result":{"results":[],"restorePreamble":{"minResourceFee":"1"}}}"#), + Simulated::Transient, + ); + // Simulated clean, returned nothing at all — not a fact either. + assert_eq!( + classify_json(r#"{"result":{"results":[]}}"#), + Simulated::Transient + ); + } + + #[test] + fn a_contract_error_is_the_one_permanent_arm() { + assert_eq!( + classify_json(r#"{"result":{"results":[],"error":"HostError: missing symbol"}}"#), + Simulated::Absent, + ); + } + + #[test] + fn a_contract_error_from_a_node_behind_the_ledger_is_not_a_fact() { + let body = r#"{"result":{"results":[],"error":"contract not found","latestLedger":100}}"#; + // The node is at 100 and the caller decodes 101: the contract may have + // been deployed in 101. + assert_eq!(classify_json_as_of(body, Some(101)), Simulated::Behind); + // At or past the ledger, the error is a fact again. + assert_eq!(classify_json_as_of(body, Some(100)), Simulated::Absent); + assert_eq!(classify_json_as_of(body, None), Simulated::Absent); + // A node that does not report its tip cannot be judged behind. + let no_tip = r#"{"result":{"results":[],"error":"contract not found"}}"#; + assert_eq!(classify_json_as_of(no_tip, Some(101)), Simulated::Absent); + } + + #[test] + fn a_returned_value_is_decoded() { + let xdr = base64::engine::general_purpose::STANDARD + .encode(ScVal::U32(8).to_xdr(Limits::none()).unwrap()); + let body = format!(r#"{{"result":{{"results":[{{"xdr":"{xdr}"}}]}}}}"#); + assert_eq!(classify_json(&body), Simulated::Value(ScVal::U32(8))); + // Bytes that are not an ScVal: it answered, with garbage. + assert_eq!( + classify_json(r#"{"result":{"results":[{"xdr":"AAAA//8="}]}}"#), + Simulated::Absent + ); + } + + #[test] + fn builds_an_envelope_that_decodes_back_to_the_call() { + // A locally-derived valid C-strkey: self-contained, no network, and it + // cannot rot the way a pasted mainnet address can. + let contract = stellar_strkey::Contract([7u8; 32]).to_string(); + let b64 = build_simulate_envelope(&contract, "decimals").expect("valid contract address"); + let raw = base64::engine::general_purpose::STANDARD + .decode(&b64) + .expect("valid base64"); + let env = TransactionEnvelope::from_xdr(&raw, Limits::none()).expect("valid xdr"); + let TransactionEnvelope::Tx(v1) = env else { + panic!("expected a v1 envelope") + }; + let OperationBody::InvokeHostFunction(op) = &v1.tx.operations[0].body else { + panic!("expected InvokeHostFunction") + }; + let HostFunction::InvokeContract(args) = &op.host_function else { + panic!("expected InvokeContract") + }; + assert_eq!(args.function_name.0.to_utf8_string().unwrap(), "decimals"); + assert_eq!(args.args.len(), 0, "the call takes no arguments"); + assert_eq!( + args.contract_address, + ScAddress::Contract(ContractId(Hash([7u8; 32]))) + ); + } + + #[test] + fn rejects_a_malformed_contract_address() { + assert_eq!(build_simulate_envelope("not-a-contract", "symbol"), None); + assert_eq!(build_simulate_envelope("", "symbol"), None); + } +} diff --git a/packages/prices-ingest-core/src/static_pools.rs b/packages/prices-ingest-core/src/static_pools.rs index bb4549b0..7d856906 100644 --- a/packages/prices-ingest-core/src/static_pools.rs +++ b/packages/prices-ingest-core/src/static_pools.rs @@ -18,12 +18,42 @@ use crate::soroban::Registries; /// issuer, with wasm `8abc28913035c07411ed5d134e6bfeab4723d97ddd4d1a22a0605d35c94d1a36`. /// It has no factory and emits no init / new-pool event (task 0300). /// -/// Its sister `CB3A6LUP…` (same wasm) is deliberately absent: it has emitted -/// no event, ever. -pub const STATIC_POOLS: &[(&str, Venue)] = &[( - "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM", - Venue::Comet, -)]; +/// - `CASYAKK3…UMIHE` — Blend v2.1's Comet BLND/USDC pool, listed in +/// `blend-capital/docs-v2` `mainnet-deployments.md`. Deployed 2026-09-30 +/// with wasm `d735c3395f59510172cf5cf838823a1389b97cfec6bf24e580e9bd77d2b3e687`, +/// CometDEX release v1.1.0, which rejects self-swaps (task 0334). +/// +/// - `CDQMIKIH…C7Y3` (BLNT/USDC) and `CA5HX3D4…3FLC` (BLNT/XLM) — Comet v1.1 +/// pools deployed 2026-10-25 by `GDEA2WMO…` for the BLNT token launch, same +/// wasm `d735c339…`. First swaps at ledger 65,137,110 (task 0337). +/// +/// - `CCNDNFA7…7QFZ` — a small Comet v1.1 BLND/USDC pool deployed 2026-09-23 +/// by `GCRKRXU2…`, same wasm. First swap at ledger 65,099,227 (task 0337). +/// +/// Its sister `CB3A6LUP…` (same wasm as `CAS3FL6T…`) is deliberately absent: +/// it has emitted no event, ever. +pub const STATIC_POOLS: &[(&str, Venue)] = &[ + ( + "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM", + Venue::Comet, + ), + ( + "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE", + Venue::Comet, + ), + ( + "CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3", + Venue::Comet, + ), + ( + "CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC", + Venue::Comet, + ), + ( + "CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ", + Venue::Comet, + ), +]; impl Registries { /// Route every [`STATIC_POOLS`] entry to its venue. The committed list @@ -59,10 +89,20 @@ mod tests { use crate::registry_io::PoolRegistryRow; const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; + const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; + const BLNT_USDC: &str = "CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3"; + const BLNT_XLM: &str = "CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC"; + const BLND_USDC_SMALL: &str = "CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ"; + const ALL: [&str; 5] = [COMET, COMET_V1_1, BLNT_USDC, BLNT_XLM, BLND_USDC_SMALL]; #[test] - fn the_list_holds_exactly_the_comet_pool() { - assert_eq!(STATIC_POOLS, &[(COMET, Venue::Comet)]); + fn the_list_holds_exactly_the_five_comet_pools() { + let listed: Vec<(&str, Venue)> = STATIC_POOLS + .iter() + .map(|(id, v)| (*id, v.clone())) + .collect(); + let expected: Vec<(&str, Venue)> = ALL.iter().map(|id| (*id, Venue::Comet)).collect(); + assert_eq!(listed, expected); } #[test] @@ -79,28 +119,32 @@ mod tests { fn merging_routes_the_pools_and_touches_nothing_else() { let mut reg = Registries::new(); reg.merge_static_pools(); - assert_eq!(reg.venue.len(), 1); - assert_eq!(reg.venue.get(COMET), Some(&Venue::Comet)); + assert_eq!(reg.venue.len(), ALL.len()); + for id in ALL { + assert_eq!(reg.venue.get(id), Some(&Venue::Comet), "{id}"); + } assert_eq!(reg.pool_count(), 0, "no pair or phoenix entry"); reg.merge_static_pools(); - assert_eq!(reg.venue.len(), 1, "idempotent"); + assert_eq!(reg.venue.len(), ALL.len(), "idempotent"); } #[test] - fn a_merged_pool_flattens_to_one_venue_only_row() { + fn each_merged_pool_flattens_to_one_venue_only_row() { let mut reg = Registries::new(); reg.merge_static_pools(); - assert_eq!( - reg.to_pool_rows(), - vec![PoolRegistryRow { - contract_id: COMET.to_string(), - venue: "comet".to_string(), - token0: String::new(), - token1: String::new(), - pool_type: 0, - wasm_hash: String::new(), - }] - ); + let mut rows = reg.to_pool_rows(); + rows.sort_by(|a, b| a.contract_id.cmp(&b.contract_id)); + let row = |id: &str| PoolRegistryRow { + contract_id: id.to_string(), + venue: "comet".to_string(), + token0: String::new(), + token1: String::new(), + pool_type: 0, + wasm_hash: String::new(), + }; + let mut expected: Vec = ALL.iter().map(|id| row(id)).collect(); + expected.sort_by(|a, b| a.contract_id.cmp(&b.contract_id)); + assert_eq!(rows, expected); } } diff --git a/packages/prices-ingest-core/src/writer.rs b/packages/prices-ingest-core/src/writer.rs index 42a8bd56..2f66f933 100644 --- a/packages/prices-ingest-core/src/writer.rs +++ b/packages/prices-ingest-core/src/writer.rs @@ -18,6 +18,7 @@ use tracing::info; use crate::bucket::OhlcvCandle; use crate::canonical::{AssetIdentity, AssetRegistry}; +use crate::decimals::DecimalsRow; use crate::error::IngestError; use crate::price::CANDLE_PRICE_SCALE; use crate::registry_io::PoolRegistryRow; @@ -105,6 +106,37 @@ impl OhlcvWriter { Ok(assets) } + /// Load every resolved Soroban token decimals from `prices.asset_decimals` + /// into `registry` (task 0329), so a cold start scales known tokens + /// without an RPC call. The table must exist: an ingest that cannot read + /// it fails init rather than dropping every pure Soroban token's trades. + pub async fn load_decimals(&self, registry: &mut AssetRegistry) -> Result<(), IngestError> { + let rows = self + .client + .query("SELECT contract_address, decimals FROM prices.asset_decimals FINAL") + .fetch_all::<(String, u8)>() + .await?; + info!(tokens = rows.len(), "loaded soroban token decimals"); + for (contract, decimals) in rows { + registry.set_decimals(contract, decimals.into()); + } + Ok(()) + } + + /// Persist decimals a run resolved over RPC to `prices.asset_decimals` + /// (task 0329). A no-op on an empty slice. + pub async fn write_decimals(&self, rows: &[DecimalsRow]) -> Result<(), IngestError> { + if rows.is_empty() { + return Ok(()); + } + let mut insert = self.client.insert("prices.asset_decimals")?; + for row in rows { + insert.write(row).await?; + } + insert.end().await?; + Ok(()) + } + /// Rehydrate the discovered AMM pool [`Registries`] from `prices.pool_registry` /// so a consumer resolves pools created before its processing window instead of /// re-deriving from Soroban activation (task 0053 decision #4). Returns empty diff --git a/packages/prices-ingest-core/tests/decimals_it.rs b/packages/prices-ingest-core/tests/decimals_it.rs new file mode 100644 index 00000000..0e055aa6 --- /dev/null +++ b/packages/prices-ingest-core/tests/decimals_it.rs @@ -0,0 +1,59 @@ +//! Task 0329 — `prices.asset_decimals` round-trips through the writer. +//! +//! tools/scripts/ignored-tests.sh # all of them: CI runs exactly this on every Rust PR +//! cargo test -p prices-ingest-core --test decimals_it -- --ignored --test-threads=1 +//! +//! The writer hardcodes `prices.asset_decimals`, so this resets the real +//! `prices` database's table. Never run against a shared cluster. + +use clickhouse::Client; +use prices_ingest_core::{AssetIdentity, AssetRegistry, DecimalsRow, OhlcvWriter}; + +fn ch_url() -> String { + std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".to_string()) +} + +const SOLVBTC: &str = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN"; +const BNUSD: &str = "CCT4ZYIYZ3TUO2AWQFEOFGBZ6HQP3GW5TA37CK7CRZVFRDXYTHTYX7KP"; + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn written_decimals_load_back_into_the_registry() { + let c = Client::default().with_url(ch_url()); + prices_clickhouse::apply_sql(&c, prices_clickhouse::INIT_SQL) + .await + .expect("apply init schema"); + c.query("TRUNCATE TABLE prices.asset_decimals") + .execute() + .await + .unwrap(); + + let writer = OhlcvWriter::new(c.clone()); + let row = |contract: &str, decimals| DecimalsRow { + contract_address: contract.to_string(), + decimals, + }; + writer + .write_decimals(&[row(SOLVBTC, 8), row(BNUSD, 18)]) + .await + .unwrap(); + // A re-resolve writes the same row again; FINAL collapses it. + writer.write_decimals(&[row(SOLVBTC, 8)]).await.unwrap(); + writer.write_decimals(&[]).await.unwrap(); + + let mut assets = AssetRegistry::from_existing(vec![]); + writer.load_decimals(&mut assets).await.unwrap(); + let of = |c: &str| assets.decimals_of(&AssetIdentity::Contract(c.to_string())); + assert_eq!(of(SOLVBTC), Some(8)); + assert_eq!(of(BNUSD), Some(18)); + assert_eq!( + of("CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO"), + None, + "a token with no row stays unknown" + ); + + c.query("TRUNCATE TABLE prices.asset_decimals") + .execute() + .await + .unwrap(); +} diff --git a/packages/prices-ingest-core/tests/decimals_rpc_it.rs b/packages/prices-ingest-core/tests/decimals_rpc_it.rs new file mode 100644 index 00000000..f3c0e979 --- /dev/null +++ b/packages/prices-ingest-core/tests/decimals_rpc_it.rs @@ -0,0 +1,51 @@ +//! Live Soroban RPC check for the task-0329 `decimals()` lookup. Gated +//! `#[ignore]` because it talks to the public mainnet endpoint: +//! +//! cargo test -p prices-ingest-core --test decimals_rpc_it -- --ignored +//! +//! Recorded in the `#[ignore]` inventory (tools/scripts/ignored-tests.sh) as a +//! public-network test and never run by CI: third-party uptime must not gate a +//! PR. The parser is pinned offline in `decimals.rs` and `soroban_rpc.rs`; this +//! proves the two ends meet against real tokens. + +use prices_ingest_core::decimals::{Decimals, decimals_of}; +use prices_ingest_core::soroban_rpc::{http_client, rpc_url_from_env}; + +#[tokio::test] +#[ignore = "requires public network — third-party uptime; never gates a PR"] +async fn reads_the_decimals_sbe_checked_on_chain() { + // The tokens SBE reported mispriced (task 0329), with the decimals they + // read by calling decimals() themselves. + for (name, contract, want) in [ + ( + "SolvBTC", + "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN", + 8, + ), + ( + "XAUM", + "CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO", + 9, + ), + ( + "XRP", + "CB7OOP3VSAWBZOOTOG2YEFANVU45GVWYUUM5HI32DKLHVKUDOFVQ37XP", + 6, + ), + ] { + let got = decimals_of(&http_client(), &rpc_url_from_env(), contract, 1).await; + assert_eq!(got, Decimals::Known(want), "{name} ({contract})"); + } +} + +#[tokio::test] +#[ignore = "requires public network — third-party uptime; never gates a PR"] +async fn a_contract_that_was_never_deployed_is_absent() { + let never_deployed = stellar_strkey::Contract([3u8; 32]).to_string(); + let got = decimals_of(&http_client(), &rpc_url_from_env(), &never_deployed, 1).await; + assert_eq!( + got, + Decimals::Absent, + "a fact about the contract, not a retry" + ); +} diff --git a/packages/prices-ingest-core/tests/sac_proof_it.rs b/packages/prices-ingest-core/tests/sac_proof_it.rs new file mode 100644 index 00000000..319eb8d9 --- /dev/null +++ b/packages/prices-ingest-core/tests/sac_proof_it.rs @@ -0,0 +1,274 @@ +//! Task 0242 — the two BE reads behind the SAC resolver: BE's `is_sac` set and +//! the events-backfill proof preload, run as rendered against real tables. +//! +//! tools/scripts/ignored-tests.sh # all of them: CI runs exactly this on every Rust PR +//! cargo test -p prices-ingest-core --test sac_proof_it -- --ignored --test-threads=1 +//! +//! Each test DROPs and CREATEs scratch databases `it_0242__be` (BE's +//! `soroban_events` / `soroban_contracts`, DDL as in coverage_sweep_it.rs) and +//! `it_0242__prices` (our `init.sql`). The real `default` and `prices` +//! are never touched. Never run against a shared cluster. + +use std::collections::HashSet; + +use clickhouse::Client; +use prices_clickhouse::USDC_ISSUER; +use prices_clickhouse::asset_id::fixture::{AssetFixture, assets_insert}; +use prices_ingest_core::{ + AssetIdentity, AssetRegistry, SacPreload, SacProofRow, apply_sac_proofs, load_sac_contracts, + load_sac_proofs, preload_sac_resolver, +}; + +const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; +const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + +fn ch_url() -> String { + std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".to_string()) +} + +fn admin() -> Client { + Client::default().with_url(ch_url()) +} + +async fn exec(sql: &str) { + admin().query(sql).execute().await.expect(sql); +} + +fn xcr() -> AssetIdentity { + AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + } +} + +fn usdc() -> AssetIdentity { + AssetIdentity::Credit { + code: "USDC".to_string(), + issuer: USDC_ISSUER.to_string(), + } +} + +fn strkey(n: u8) -> String { + stellar_strkey::Contract([n; 32]).to_string() +} + +/// `[transfer, from, to, ]` in BE's typed-JSON shape. +fn topics(signature: &str, last: &str) -> String { + let from = stellar_strkey::ed25519::PublicKey([1; 32]).to_string(); + format!( + r#"[{{"type":"sym","value":"{signature}"}},{{"type":"address","value":"{from}"}},{{"type":"address","value":"{XCR_SAC}"}},{{"type":"string","value":"{last}"}}]"# + ) +} + +struct Fixture { + be: String, + prices: String, +} + +impl Fixture { + /// Fresh scratch databases, and the contracts every case shares: + /// - 1 XCR's SAC, is_sac, held as two RMT versions; classic absent from + /// `assets`; an `approve` and two `transfer`s. + /// - 2 USDC's SAC, is_sac; its classic is in `assets` with `sac_address`. + /// - 3 a non-SAC contract emitting `transfer` with XCR's SEP-11 topic. + /// - 4 an is_sac contract with no events. + async fn new(case: &str) -> Self { + let f = Fixture { + be: format!("it_0242_{case}_be"), + prices: format!("it_0242_{case}_prices"), + }; + for db in [&f.be, &f.prices] { + exec(&format!("DROP DATABASE IF EXISTS {db}")).await; + exec(&format!("CREATE DATABASE {db}")).await; + } + let init = prices_clickhouse::INIT_SQL + .replace("prices.", &format!("{}.", f.prices)) + .replace( + "IF NOT EXISTS prices", + &format!("IF NOT EXISTS {}", f.prices), + ); + prices_clickhouse::apply_sql(&admin(), &init) + .await + .expect("init schema"); + exec(&format!( + "CREATE TABLE {}.soroban_events ( + contract_id Int64, + ledger_sequence Int64, + transaction_index UInt32, + operation_index UInt16, + event_index UInt32, + application_order Int16, + event_type Int16, + signature LowCardinality(Nullable(String)), + topics_xdr String CODEC(ZSTD(3)), + data_xdr String CODEC(ZSTD(3)) + ) + ENGINE = ReplacingMergeTree + PARTITION BY intDiv(ledger_sequence, 500000) + ORDER BY (contract_id, ledger_sequence, transaction_index, operation_index, event_index)", + f.be + )) + .await; + exec(&format!( + "CREATE TABLE {}.soroban_contracts ( + id Int64, + contract_id String, + wasm_hash Nullable(FixedString(32)), + wasm_uploaded_at_ledger Int64 DEFAULT 0, + deployer_id Nullable(Int64), + deployed_at_ledger Nullable(Int64), + contract_type Nullable(Int16), + is_sac Bool + ) + ENGINE = ReplacingMergeTree(wasm_uploaded_at_ledger) + ORDER BY (contract_id)", + f.be + )) + .await; + + let usdc_sac = AssetRegistry::from_existing(vec![]) + .sac_address_of(&usdc()) + .expect("USDC derives a SAC"); + let (genuine, empty) = (strkey(3), strkey(4)); + // Two inserts, two parts: the RMT versions stay unmerged. + for (id, contract, version, is_sac) in [ + (1, XCR_SAC, 1, true), + (1, XCR_SAC, 2, true), + (2, usdc_sac.as_str(), 1, true), + (3, genuine.as_str(), 1, false), + (4, empty.as_str(), 1, true), + ] { + f.contract(id, contract, version, is_sac).await; + } + let xcr_sep11 = format!("XCR:{XCR_ISSUER}"); + f.event(1, 100, "approve", "NOT-A-PROOF").await; + f.event(1, 101, "transfer", &xcr_sep11).await; + f.event(1, 102, "transfer", &xcr_sep11).await; + f.event(2, 100, "transfer", &format!("USDC:{USDC_ISSUER}")) + .await; + f.event(3, 100, "transfer", &xcr_sep11).await; + + exec(&assets_insert( + &f.prices, + &[AssetFixture::new("USDC", "classic", USDC_ISSUER, "").with_sac(&usdc_sac)], + )) + .await; + f + } + + async fn contract(&self, id: i64, contract: &str, version: i64, is_sac: bool) { + exec(&format!( + "INSERT INTO {}.soroban_contracts (id, contract_id, wasm_uploaded_at_ledger, is_sac) \ + VALUES ({id}, '{contract}', {version}, {is_sac})", + self.be + )) + .await; + } + + async fn event(&self, id: i64, ledger: i64, signature: &str, last: &str) { + exec(&format!( + "INSERT INTO {}.soroban_events \ + (contract_id, ledger_sequence, transaction_index, operation_index, event_index, application_order, event_type, signature, topics_xdr, data_xdr) \ + VALUES ({id}, {ledger}, 0, 0, 0, 0, 1, '{signature}', '{}', '{{\"type\":\"i128\",\"value\":\"1\"}}')", + self.be, + topics(signature, last), + )) + .await; + } + + async fn drop(self) { + for db in [&self.be, &self.prices] { + exec(&format!("DROP DATABASE IF EXISTS {db}")).await; + } + } +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_is_sac_set_holds_each_flagged_contract_once() { + let f = Fixture::new("sacs").await; + let got = load_sac_contracts(&admin(), &f.be).await.unwrap(); + let usdc_sac = AssetRegistry::from_existing(vec![]) + .sac_address_of(&usdc()) + .unwrap(); + assert_eq!( + got, + HashSet::from([XCR_SAC.to_string(), usdc_sac, strkey(4)]), + "every is_sac contract once, the non-SAC one never" + ); + f.drop().await; +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_preload_returns_one_proof_per_sac_the_registry_cannot_resolve() { + let f = Fixture::new("preload").await; + let rows = load_sac_proofs(&admin(), &f.be, &f.prices).await.unwrap(); + assert_eq!( + rows, + [SacProofRow { + contract_id: XCR_SAC.to_string(), + sep11: format!("XCR:{XCR_ISSUER}"), + }], + "USDC is resolvable from assets, the non-SAC is no candidate, the eventless SAC has no proof" + ); + + let mut assets = AssetRegistry::from_existing(vec![]); + assert_eq!(assets.sac_address_of(&xcr()).as_deref(), Some(XCR_SAC)); + assert_eq!(apply_sac_proofs(&mut assets, &rows), (1, 0)); + assert_eq!(assets.resolve_sac(XCR_SAC), Some(xcr())); + f.drop().await; +} + +/// events-backfill's run-start step, end to end: the candidates are BE's +/// `is_sac` set, and the one provable SAC `assets` cannot resolve is learnt. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_run_start_preload_arms_candidates_and_learns_the_proofs() { + let f = Fixture::new("arm").await; + let mut assets = AssetRegistry::from_existing(vec![]); + let got = preload_sac_resolver(&admin(), &f.be, &f.prices, &mut assets) + .await + .unwrap(); + assert_eq!( + got, + SacPreload { + candidates: 3, + proofs: 1, + verified: 1, + rejected: 0, + } + ); + assert_eq!(assets.resolve_sac(XCR_SAC), Some(xcr())); + assert!(!assets.is_unproven_sac(XCR_SAC)); + assert!(assets.is_unproven_sac(&strkey(4)), "eventless SAC: skipped"); + assert!(!assets.is_unproven_sac(&strkey(3)), "not a SAC: a token"); + f.drop().await; +} + +/// IN-04: the `is_sac` read has no `FINAL`. A contract BE corrects to +/// `is_sac = false` stays a candidate (skipped, never minted) until BE's parts +/// merge, and leaves the set once they have. Merges are stopped so the two +/// versions cannot merge before the first read. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn an_unmerged_is_sac_correction_stays_a_candidate_until_merged() { + let f = Fixture::new("flip").await; + let table = format!("{}.soroban_contracts", f.be); + exec(&format!("SYSTEM STOP MERGES {table}")).await; + let flipped = strkey(5); + f.contract(5, &flipped, 1, true).await; + f.contract(5, &flipped, 1, false).await; + let got = load_sac_contracts(&admin(), &f.be).await.unwrap(); + assert!( + got.contains(&flipped), + "unmerged: either version may answer" + ); + + exec(&format!("SYSTEM START MERGES {table}")).await; + exec(&format!("OPTIMIZE TABLE {table} FINAL")).await; + let got = load_sac_contracts(&admin(), &f.be).await.unwrap(); + assert!(!got.contains(&flipped), "merged: the last insert wins"); + assert!(got.contains(XCR_SAC)); + f.drop().await; +} diff --git a/packages/prices-ledger-processor/src/bin/cli.rs b/packages/prices-ledger-processor/src/bin/cli.rs index 917adcd9..cc800159 100644 --- a/packages/prices-ledger-processor/src/bin/cli.rs +++ b/packages/prices-ledger-processor/src/bin/cli.rs @@ -83,6 +83,10 @@ async fn main() -> Result<(), Box> { } else { let sink = ClickHouseSink::plaintext(&args.clickhouse_url); sink.preflight().await?; + // `load_registry`, not the Lambda's `load_ingest_registry`, by design: a + // local ClickHouse has no BE `default.soroban_contracts`, so this registry + // has no `is_sac` candidates and still mints an unproven SAC as a + // `Contract` identity (task 0242 D2 applies to the Lambda only). let registry = sink.load_registry().await?; let pool_registry = sink.load_pool_registry().await?; let reconciler = Reconciler::new(fetcher, cursor, sink, registry, pool_registry); diff --git a/packages/prices-ledger-processor/src/main.rs b/packages/prices-ledger-processor/src/main.rs index efab6fc3..cc631c8d 100644 --- a/packages/prices-ledger-processor/src/main.rs +++ b/packages/prices-ledger-processor/src/main.rs @@ -67,16 +67,23 @@ async fn main() -> Result<(), Error> { let sink = sink?; let fetcher = S3Fetcher::new(aws_sdk_s3::Client::new(&aws_cfg), &bucket); - // Three independent ClickHouse reads on the cold path — the durable cursor - // (task 0064) plus the two registries — joined so they overlap instead of - // running serially (same reasoning as the fetcher+sink join above). All use - // the sink's shared client; clients are cheap to clone and pool-backed, so - // concurrent queries are safe. + // Four independent ClickHouse reads on the cold path — the durable cursor + // (task 0064), the two registries and BE's SAC set — joined so they overlap + // instead of running serially (same reasoning as the fetcher+sink join + // above). All use the sink's shared client; clients are cheap to clone and + // pool-backed, so concurrent queries are safe. `load_ingest_registry` joins + // the asset registry with BE's SAC set itself. // // - cursor.read(): last processed ledger from `prices.ingest_cursor`. Durable, // so the loop resumes across execution-environment recycles instead of // rewinding to INITIAL_CURSOR every cold start (the freeze this task fixes). - // - load_registry: the identities already in `prices.assets`. + // - load_ingest_registry: the identities already in `prices.assets`, armed + // with the contracts BE flags `is_sac` in `default.soroban_contracts`, read + // with prices_writer's SELECT on that table (task 0242). A flagged token no + // SAC proof resolves is skipped, not minted. A failed BE read does NOT fail + // Init: ingest runs without candidates and every invocation publishes + // `SacCandidatesUnavailable` until a cold start reads the set again. + // `tests/sac_identity.rs` pins this call. // - load_pool_registry: discovered AMM pool classification from // `prices.pool_registry` (task 0078). The processor writes back only the // pools it learns from factory events (task 0291; `prices_writer` holds @@ -89,10 +96,10 @@ async fn main() -> Result<(), Error> { let cursor = ClickHouseCursor::new(sink.client().clone(), CURSOR_ID); let (cursor_state, registry, pool_registry) = tokio::join!( cursor.read(), - sink.load_registry(), + sink.load_ingest_registry(), sink.load_pool_registry(), ); - let registry = registry?; + let (registry, sac_candidates_loaded) = registry?; let pool_registry = pool_registry?; // Seed only on a genuinely empty table (true first run); any OTHER read error @@ -184,7 +191,17 @@ async fn main() -> Result<(), Error> { let r = reconciler.clone(); let cw = cw.clone(); let env_name = env_name.clone(); - async move { handler(event, r, max_iterations, cw, env_name).await } + async move { + handler( + event, + r, + max_iterations, + cw, + env_name, + sac_candidates_loaded, + ) + .await + } })) .await } @@ -195,6 +212,7 @@ async fn handler( max_iterations: usize, cw: Arc, env_name: Arc, + sac_candidates_loaded: bool, ) -> Result { let (payload, _ctx) = event.into_parts(); let mut batch_item_failures = Vec::new(); @@ -235,6 +253,17 @@ async fn handler( m.extend(metrics::unregistered_pool_event_metrics( stats.unregistered_pool_events.values().sum(), )); + // Task 0242 — trades skipped on a SAC no proof resolved. + m.extend(metrics::sac_unproven_metrics(stats.sac_unproven_skipped)); + // Task 0242 — this container runs without BE's `is_sac` set. + m.extend(metrics::sac_candidates_unavailable_metrics( + !sac_candidates_loaded, + )); + // Task 0329 — trades dropped because a token's decimals did not + // resolve. Silent otherwise: the run succeeds without them. + m.extend(metrics::trades_missing_decimals_metrics( + stats.trades_missing_decimals, + )); if let Err(e) = metrics::publish(&cw, &env_name, &m).await { warn!(error = %e, "cloudwatch metric publish failed (non-fatal)"); } diff --git a/packages/prices-ledger-processor/src/metrics.rs b/packages/prices-ledger-processor/src/metrics.rs index 6aade4f1..789afbcc 100644 --- a/packages/prices-ledger-processor/src/metrics.rs +++ b/packages/prices-ledger-processor/src/metrics.rs @@ -63,6 +63,28 @@ pub const FORCED_PARTIAL_FLUSH: &str = "ForcedPartialFlushes"; /// factory, or a factory event missed before a cold start. pub const UNREGISTERED_POOL_EVENTS: &str = "UnregisteredPoolEvents"; +/// Trades a reconcile run skipped because a leg is a contract BE flags `is_sac` +/// that no SAC proof in its transaction resolved (task 0242 D2). Minting it as +/// a `Contract` identity would split one asset in two, so the trade is dropped. +/// +/// Emitted ONLY when non-zero, so the alarm on it is `>= 1` over +/// `NOT_BREACHING`. Non-zero means a SAC traded without a transfer naming its +/// asset in the same transaction, a shape the resolver does not cover. +pub const SAC_UNPROVEN_SKIPPED: &str = "SacUnprovenSkipped"; + +/// One per run of a container whose cold start could not read BE's `is_sac` +/// set (task 0242). Such a container mints an unproven SAC as a `Contract` +/// identity instead of skipping it, until its next cold start. +/// +/// Emitted ONLY while the set is missing, so the alarm on it is `>= 1` over +/// `NOT_BREACHING`. +pub const SAC_CANDIDATES_UNAVAILABLE: &str = "SacCandidatesUnavailable"; +/// AMM trades a reconcile run dropped because a leg's Soroban token decimals +/// did not resolve over RPC (task 0329) — the RPC was down, or the token has no +/// usable `decimals()`. Emitted ONLY when non-zero, like +/// [`UNREGISTERED_POOL_EVENTS`]. +pub const TRADES_MISSING_DECIMALS: &str = "TradesMissingDecimals"; + /// `PutMetricData` accepts at most 150 entries in a datum's `Values` array, so /// a run with more INSERTs than that spills into further datums of the same /// metric rather than being truncated (or, worse, aggregated back into @@ -170,6 +192,46 @@ pub fn unregistered_pool_event_metrics(total: u64) -> Vec { }] } +/// The unproven-SAC datapoint for one run (task 0242), or nothing when the run +/// skipped none. Same absent-while-healthy rule as +/// [`forced_partial_flush_metrics`]. +pub fn sac_unproven_metrics(total: u64) -> Vec { + if total == 0 { + return Vec::new(); + } + vec![Metric { + name: SAC_UNPROVEN_SKIPPED, + unit: Unit::Count, + values: vec![total as f64], + }] +} + +/// The dropped-for-decimals datapoint for one run (task 0329), or nothing when +/// the run dropped none. +pub fn trades_missing_decimals_metrics(total: u64) -> Vec { + if total == 0 { + return Vec::new(); + } + vec![Metric { + name: TRADES_MISSING_DECIMALS, + unit: Unit::Count, + values: vec![total as f64], + }] +} + +/// The missing-candidates datapoint for one run (task 0242), or nothing when +/// the cold start read BE's `is_sac` set. +pub fn sac_candidates_unavailable_metrics(unavailable: bool) -> Vec { + if !unavailable { + return Vec::new(); + } + vec![Metric { + name: SAC_CANDIDATES_UNAVAILABLE, + unit: Unit::Count, + values: vec![1.0], + }] +} + /// Publish `metrics` to CloudWatch under [`METRIC_NAMESPACE`], tagged with an /// `Environment` dimension. One `PutMetricData` call for the whole batch. /// @@ -242,6 +304,16 @@ mod tests { assert!(unregistered_pool_event_metrics(0).is_empty()); } + /// Task 0329: same absent-while-healthy contract. + #[test] + fn trades_missing_decimals_publish_only_when_some_were_dropped() { + assert!(trades_missing_decimals_metrics(0).is_empty()); + let m = trades_missing_decimals_metrics(3); + assert_eq!(m.len(), 1); + assert_eq!(m[0].name, TRADES_MISSING_DECIMALS); + assert_eq!(m[0].values, vec![3.0]); + } + #[test] fn dropped_unregistered_trades_publish_one_count_of_the_total() { let m = unregistered_pool_event_metrics(7); @@ -251,6 +323,28 @@ mod tests { assert_eq!(m[0].values, vec![7.0]); } + /// Task 0242: absent while healthy, one count of the total otherwise. + #[test] + fn unproven_sac_skips_publish_only_when_non_zero() { + assert!(sac_unproven_metrics(0).is_empty()); + let m = sac_unproven_metrics(3); + assert_eq!(m.len(), 1); + assert_eq!(m[0].name, SAC_UNPROVEN_SKIPPED); + assert_eq!(m[0].unit, Unit::Count); + assert_eq!(m[0].values, vec![3.0]); + } + + /// Task 0242: absent while the set is loaded, one count per run otherwise. + #[test] + fn missing_sac_candidates_publish_one_per_run() { + assert!(sac_candidates_unavailable_metrics(false).is_empty()); + let m = sac_candidates_unavailable_metrics(true); + assert_eq!(m.len(), 1); + assert_eq!(m[0].name, SAC_CANDIDATES_UNAVAILABLE); + assert_eq!(m[0].unit, Unit::Count); + assert_eq!(m[0].values, vec![1.0]); + } + #[test] fn a_forced_run_publishes_exactly_one_count_of_one() { let m = forced_partial_flush_metrics(true); diff --git a/packages/prices-ledger-processor/src/reconcile.rs b/packages/prices-ledger-processor/src/reconcile.rs index 6d57c99c..4bd0dbbe 100644 --- a/packages/prices-ledger-processor/src/reconcile.rs +++ b/packages/prices-ledger-processor/src/reconcile.rs @@ -63,8 +63,9 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::Instant; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, OfferLookupCounts, OracleSample, PoolRegistryRow, Registries, - decode_object, extract_trades_with_counts, ledger_close_time, ledger_sequence, process_ledger, + AssetRegistry, CandleAccumulator, DecimalsResolver, DecimalsRow, OfferLookupCounts, + OracleSample, PoolRegistryRow, Registries, decode_object, decode_resolving, + extract_trades_with_counts, ledger_close_time, ledger_sequence, process_ledger, raw_trade_to_tick, }; use tokio::sync::Mutex; @@ -116,6 +117,14 @@ pub struct RunStats { /// missing from the registry — dropped, not priced (task 0291). Counted over /// the minutes this run WROTE, like `offer_lookups`. Empty when healthy. pub unregistered_pool_events: BTreeMap<&'static str, u64>, + /// Trades skipped because a leg is an `is_sac` contract no SAC proof + /// resolved (task 0242 D2). Counted over the minutes this run WROTE, like + /// `unregistered_pool_events`. 0 when healthy. + pub sac_unproven_skipped: u64, + /// AMM trades dropped because a leg's token decimals did not resolve over + /// RPC, even after the second decode (task 0329). Counted over the minutes + /// this run WROTE, like `unregistered_pool_events`. 0 when healthy. + pub trades_missing_decimals: u64, /// Candle-INSERT latency for this run, or `None` when the run wrote no /// candles at all. `None` rather than a zeroed struct so an idle run /// publishes no `ClickHouseWriteLatencyMs` datapoint instead of a 0 ms one @@ -137,6 +146,10 @@ pub struct ProcessingState { /// here only *after* the write succeeds, so a failed write is retried next /// run. Init: everything loaded at cold start is already in the table. pub persisted_pools: HashMap, + /// Resolves Soroban token decimals a decode reports missing (task 0329). + /// Warm across invocations, like the registries, so a token that will not + /// resolve is not re-asked on every ledger. + pub decimals: DecimalsResolver, } /// The cold-start [`ProcessingState`] built from what was loaded. @@ -160,6 +173,7 @@ fn initial_state(assets: AssetRegistry, mut registries: Registries) -> Processin assets, registries, persisted_pools, + decimals: DecimalsResolver::live_from_env(), } } @@ -191,6 +205,13 @@ where } } + /// Replace the decimals resolver (task 0329). Tests point it at an address + /// that refuses connections, so no test can reach a public RPC. + pub fn with_decimals_resolver(mut self, resolver: DecimalsResolver) -> Self { + self.state.get_mut().decimals = resolver; + self + } + /// One reconcile run for a LONG-RUNNING caller (the Lambda). Minutes the /// run did not see the end of are held back for the next run. pub async fn run(&self, max_iterations: usize) -> Result { @@ -256,6 +277,12 @@ where // minute closes. let mut ledger_unregistered: Vec<(u32, &'static str, u32)> = Vec::new(); let mut ledger_unregistered_contracts: Vec<(u32, String)> = Vec::new(); + // (minute_start, contract) per trade skipped on an unproven SAC (task + // 0242), per ledger for the same reason. + let mut ledger_sac_unproven: Vec<(u32, String)> = Vec::new(); + // (minute_start, trades, contracts) dropped because a token's decimals + // did not resolve (task 0329). Per ledger for the same reason as above. + let mut ledger_missing_decimals: Vec<(u32, u32, Vec)> = Vec::new(); for _ in 0..max_iterations { let next = current + 1; @@ -278,8 +305,18 @@ where for trade in trades { sdex.merge(raw_trade_to_tick(&trade, &mut state.assets)); } - // Soroban AMM trades + oracle samples. - let sob = process_ledger(lcm, &mut state.registries, &mut state.assets); + // Soroban AMM trades + oracle samples. Task 0329: a trade naming a + // token whose decimals are not known yet is dropped, not guessed; + // `decode_resolving` resolves it, persists it, and decodes again. + let registries = &mut state.registries; + let sob = decode_resolving( + &mut state.decimals, + &mut state.assets, + ledger_sequence(lcm), + |assets| process_ledger(lcm, registries, assets), + async |rows: &[DecimalsRow]| self.sink.write_decimals(rows).await, + ) + .await?; for (source, tick) in sob.amm_ticks { amm.entry(source).or_default().merge(tick); } @@ -300,6 +337,14 @@ where .iter() .map(|c| (minute, c.clone())), ); + ledger_sac_unproven.extend(sob.sac_unproven.into_iter().map(|c| (minute, c))); + if sob.trades_missing_decimals > 0 { + ledger_missing_decimals.push(( + minute, + sob.trades_missing_decimals, + sob.missing_decimals, + )); + } obj_max = obj_max.max(seq); } // Flag only this object's HIGHEST ledger as a valid cursor landing @@ -326,6 +371,8 @@ where rows_emitted: 0, pools_persisted: 0, unregistered_pool_events: BTreeMap::new(), + sac_unproven_skipped: 0, + trades_missing_decimals: 0, // Nothing was persisted, so no INSERT happened: no datapoint. ch_write: None, }); @@ -486,6 +533,8 @@ where // Pools are dimension rows written above, like the assets. pools_persisted, unregistered_pool_events: BTreeMap::new(), + sac_unproven_skipped: 0, + trades_missing_decimals: 0, ch_write: None, }); }; @@ -566,6 +615,38 @@ where "dropped trades from pools missing from prices.pool_registry (task 0291)" ); } + let sac_unproven: Vec<&str> = ledger_sac_unproven + .iter() + .filter(|(minute, _)| *minute < flush_boundary) + .map(|(_, c)| c.as_str()) + .collect(); + if !sac_unproven.is_empty() { + let contracts: BTreeSet<&str> = sac_unproven.iter().copied().collect(); + tracing::warn!( + skipped = sac_unproven.len(), + ?contracts, + "skipped trades on is_sac contracts with no SAC proof (task 0242)" + ); + } + let flushed_missing_decimals: Vec<&(u32, u32, Vec)> = ledger_missing_decimals + .iter() + .filter(|(minute, _, _)| *minute < flush_boundary) + .collect(); + let trades_missing_decimals: u64 = flushed_missing_decimals + .iter() + .map(|(_, n, _)| *n as u64) + .sum(); + if trades_missing_decimals > 0 { + let contracts: BTreeSet<&str> = flushed_missing_decimals + .iter() + .flat_map(|(_, _, c)| c.iter().map(String::as_str)) + .collect(); + tracing::warn!( + trades_missing_decimals, + ?contracts, + "dropped AMM trades whose token decimals did not resolve (task 0329)" + ); + } info!( start, end = current, @@ -590,6 +671,8 @@ where rows_emitted, pools_persisted, unregistered_pool_events, + sac_unproven_skipped: sac_unproven.len() as u64, + trades_missing_decimals, ch_write: (!ch_write.samples_ms.is_empty()).then_some(ch_write), }) } diff --git a/packages/prices-ledger-processor/src/sink/mod.rs b/packages/prices-ledger-processor/src/sink/mod.rs index c435da3d..217a6218 100644 --- a/packages/prices-ledger-processor/src/sink/mod.rs +++ b/packages/prices-ledger-processor/src/sink/mod.rs @@ -8,13 +8,52 @@ //! shared Hetzner cluster over mTLS via the task-0052 client. Tests use the //! in-memory [`CountingSink`]. +use std::collections::HashSet; use std::future::Future; use prices_ingest_core::{ - AssetRegistry, DEFAULT_BACKOFF_MS, OhlcvCandle, OhlcvWriter, OracleSample, PoolRegistryRow, - Registries, retry_with_backoff, + AssetRegistry, DEFAULT_BACKOFF_MS, DecimalsRow, OhlcvCandle, OhlcvWriter, OracleSample, + PoolRegistryRow, Registries, retry_with_backoff, }; +/// `registry` with `sac_contracts` as its `is_sac` candidates (task 0242 D2): +/// the step [`ClickHouseSink::load_ingest_registry`] adds to a plain load. +pub fn with_sac_candidates( + mut registry: AssetRegistry, + sac_contracts: HashSet, +) -> AssetRegistry { + tracing::info!( + sac_contracts = sac_contracts.len(), + "loaded the is_sac contract set" + ); + registry.set_sac_candidates(sac_contracts); + registry +} + +/// [`with_sac_candidates`] when BE's `is_sac` set was read, else the registry +/// as loaded, with no candidates; the flag says which (task 0242 PC3, +/// reversed in review). An unreadable BE table must not stop live ingest: with +/// no candidates an unproven SAC mints a `Contract` identity, which the probe's +/// `SacContractIdentities` counts and the 0242 heal removes, while SDEX, oracle +/// and AMM candles keep flowing. The caller publishes +/// `SacCandidatesUnavailable` while the flag is false. +pub fn arm_sac_candidates( + registry: AssetRegistry, + sac_contracts: Result, SinkError>, +) -> (AssetRegistry, bool) { + match sac_contracts { + Ok(set) => (with_sac_candidates(registry, set), true), + Err(e) => { + tracing::warn!( + error = %e, + "is_sac contract set unreadable: no SAC candidates until the next cold start, \ + an unproven SAC mints a Contract identity (task 0242)" + ); + (registry, false) + } + } +} + #[derive(Debug, thiserror::Error)] pub enum SinkError { #[error("sink write failed: {0}")] @@ -52,6 +91,16 @@ pub trait CandleSink { &self, rows: &[PoolRegistryRow], ) -> impl Future> + Send; + + /// Persist Soroban token decimals a run resolved over RPC (task 0329). A + /// no-op by default: a sink that drops them only costs an RPC call per + /// token at the next cold start, because the registry keeps them warm. + fn write_decimals( + &self, + _rows: &[DecimalsRow], + ) -> impl Future> + Send { + async { Ok(()) } + } } /// ClickHouse sink backed by the shared [`OhlcvWriter`]. Works against either a @@ -101,7 +150,38 @@ impl ClickHouseSink { /// identity (task 0139), so live and backfill ids agree by construction. pub async fn load_registry(&self) -> Result { let existing = self.writer.load_assets().await.map_err(redact)?; - Ok(AssetRegistry::from_existing(existing)) + let mut registry = AssetRegistry::from_existing(existing); + // Task 0329: the AMM path scales a pure Soroban token's legs by its + // `decimals()`; loading the resolved ones here saves an RPC call each. + self.writer + .load_decimals(&mut registry) + .await + .map_err(redact)?; + Ok(registry) + } + + /// The contracts BE flags `is_sac` (task 0242), for + /// [`AssetRegistry::set_sac_candidates`]. Separate from [`load_registry`]: + /// the local CLI has no BE tables, so `prices-cli` never calls it and still + /// mints an unproven SAC as a `Contract` identity. Local fixtures only. + /// + /// [`load_registry`]: ClickHouseSink::load_registry + pub async fn load_sac_contracts(&self) -> Result, SinkError> { + prices_ingest_core::load_sac_contracts(self.client(), prices_ingest_core::BE_DATABASE) + .await + .map_err(redact) + } + + /// The registry the live processor ingests with: [`load_registry`] armed + /// with BE's `is_sac` set by [`arm_sac_candidates`], the two reads joined, + /// and whether the set was read. Only the `prices.assets` read fails the + /// cold start; an unreadable BE set is a warning and a metric (task 0242). + /// + /// [`load_registry`]: ClickHouseSink::load_registry + pub async fn load_ingest_registry(&self) -> Result<(AssetRegistry, bool), SinkError> { + let (registry, sac_contracts) = + tokio::join!(self.load_registry(), self.load_sac_contracts()); + Ok(arm_sac_candidates(registry?, sac_contracts)) } /// Preload the discovered AMM pool registry from `prices.pool_registry` so the @@ -162,6 +242,17 @@ impl CandleSink for ClickHouseSink { .await .map(|_| ()) } + + async fn write_decimals(&self, rows: &[DecimalsRow]) -> Result<(), SinkError> { + // Idempotent (RMT on contract_address) → retried like the other writes. + retry_with_backoff( + &DEFAULT_BACKOFF_MS, + |_| true, + || async { self.writer.write_decimals(rows).await.map_err(redact) }, + ) + .await + .map(|_| ()) + } } /// Map an ingest error into a sink error. `IngestError`'s `Display` is already diff --git a/packages/prices-ledger-processor/tests/pool_registry_persist.rs b/packages/prices-ledger-processor/tests/pool_registry_persist.rs index bf1be7e2..22b73eff 100644 --- a/packages/prices-ledger-processor/tests/pool_registry_persist.rs +++ b/packages/prices-ledger-processor/tests/pool_registry_persist.rs @@ -16,7 +16,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use prices_ingest_core::{ - AssetRegistry, OhlcvCandle, OracleSample, PoolRegistryRow, Registries, process_ledger, + AssetRegistry, DecimalsResolver, OhlcvCandle, OracleSample, PoolRegistryRow, Registries, + process_ledger, }; use prices_ledger_processor::{ cursor::{Cursor, StubFileCursor}, @@ -237,7 +238,8 @@ async fn harness( sink.clone(), AssetRegistry::from_existing(Vec::new()), registries, - ); + ) + .with_decimals_resolver(DecimalsResolver::live("http://127.0.0.1:1/".to_string())); (reconciler, sink) } diff --git a/packages/prices-ledger-processor/tests/reconcile_drain.rs b/packages/prices-ledger-processor/tests/reconcile_drain.rs index a7ce7828..0ad991ec 100644 --- a/packages/prices-ledger-processor/tests/reconcile_drain.rs +++ b/packages/prices-ledger-processor/tests/reconcile_drain.rs @@ -27,8 +27,8 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use prices_ingest_core::{ - AssetIdentity, AssetRegistry, OfferLookupCounts, OhlcvCandle, OracleSample, PoolRegistryRow, - Registries, decode_object, extract_trades_with_counts, + AssetIdentity, AssetRegistry, DecimalsResolver, OfferLookupCounts, OhlcvCandle, OracleSample, + PoolRegistryRow, Registries, decode_object, extract_trades_with_counts, }; use prices_ledger_processor::{ cursor::{Cursor, StubFileCursor}, @@ -194,7 +194,8 @@ async fn harness(dir: &std::path::Path) -> (Harness, MemoryFetcher, RecordingSin sink.clone(), AssetRegistry::from_existing(Vec::new()), Registries::new(), - ); + ) + .with_decimals_resolver(DecimalsResolver::live("http://127.0.0.1:1/".to_string())); (reconciler, fetcher, sink) } diff --git a/packages/prices-ledger-processor/tests/reconcile_e2e.rs b/packages/prices-ledger-processor/tests/reconcile_e2e.rs index 6d87760c..6bf03625 100644 --- a/packages/prices-ledger-processor/tests/reconcile_e2e.rs +++ b/packages/prices-ledger-processor/tests/reconcile_e2e.rs @@ -14,7 +14,9 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use prices_ingest_core::{AssetRegistry, OhlcvCandle, OracleSample, PoolRegistryRow, Registries}; +use prices_ingest_core::{ + AssetRegistry, DecimalsResolver, OhlcvCandle, OracleSample, PoolRegistryRow, Registries, +}; use prices_ledger_processor::{ cursor::{Cursor, StubFileCursor}, object_fetcher::LocalDiskFetcher, @@ -58,6 +60,7 @@ fn reconciler( AssetRegistry::from_existing(Vec::new()), Registries::new(), ) + .with_decimals_resolver(DecimalsResolver::live("http://127.0.0.1:1/".to_string())) } /// Fault-injecting sink: fails the first `write_new_assets` call, succeeds after. @@ -117,7 +120,8 @@ async fn assets_from_a_failed_run_are_written_on_the_next_run() { sink.clone(), AssetRegistry::from_existing(Vec::new()), Registries::new(), - ); + ) + .with_decimals_resolver(DecimalsResolver::live("http://127.0.0.1:1/".to_string())); // Run 1: interns new assets, then the (first) asset write fails → the run // errors and the cursor is never advanced (the doorbell would redeliver). diff --git a/packages/prices-ledger-processor/tests/sac_identity.rs b/packages/prices-ledger-processor/tests/sac_identity.rs new file mode 100644 index 00000000..1c379cfd --- /dev/null +++ b/packages/prices-ledger-processor/tests/sac_identity.rs @@ -0,0 +1,422 @@ +//! Task 0242: a SAC never becomes a second asset identity on the live path. +//! +//! A SAC's own `transfer`/`mint`/`burn`/`clawback` names its asset (SEP-11) in +//! the last topic. `process_ledger` checks that claim against the emitter's +//! address and resolves the SAC to its classic identity, wherever the event sits +//! in the transaction. A contract BE flags `is_sac` that no proof resolves is +//! skipped and counted, never minted as a `Contract` identity. +//! +//! The ledgers are synthetic, like `pool_registry_persist.rs`, so these run +//! everywhere. + +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +use extractors_core::Venue; +use prices_ingest_core::{ + AssetIdentity, AssetRegistry, LedgerSoroban, OhlcvCandle, OracleSample, PoolRegistryRow, + Registries, process_ledger, +}; +use prices_ledger_processor::{ + cursor::{Cursor, StubFileCursor}, + galexie_key::ledger_s3_key, + object_fetcher::{FetchError, ObjectFetcher}, + reconcile::Reconciler, + sink::{CandleSink, SinkError, arm_sac_candidates, with_sac_candidates}, +}; +use stellar_xdr::{ + ContractEvent, ContractEventBody, ContractEventType, ContractEventV0, ContractId, Hash, + Int128Parts, LedgerCloseMeta, LedgerCloseMetaBatch, LedgerCloseMetaV2, Limits, OperationMetaV2, + ScAddress, ScString, ScSymbol, ScVal, TimePoint, TransactionMeta, TransactionMetaV4, + TransactionResultMetaV1, WriteXdr, +}; +use tempfile::tempdir; + +const FIRST: u64 = 70_000_000; +const BASE_MINUTE: u64 = 1_800_000_000 / 60 * 60; + +// Public on-chain identifiers: XCR, its SAC, and the XLM SAC. +const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; +const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; +const XLM_SAC: &str = "CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA"; + +const POOL: [u8; 32] = [0x55; 32]; +const IMPOSTOR: [u8; 32] = [0x77; 32]; +const GENUINE_TOKEN: [u8; 32] = [0x22; 32]; + +fn xcr() -> AssetIdentity { + AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + } +} + +fn xcr_sep11() -> String { + format!("XCR:{XCR_ISSUER}") +} + +fn strkey(bytes: [u8; 32]) -> String { + ContractId(Hash(bytes)).to_string() +} + +fn cid(strkey: &str) -> ContractId { + ContractId::from_str(strkey).unwrap() +} + +fn addr(strkey: &str) -> ScVal { + ScVal::Address(ScAddress::Contract(cid(strkey))) +} + +fn sym(s: &str) -> ScVal { + ScVal::Symbol(ScSymbol(s.try_into().unwrap())) +} + +fn i128v(lo: u64) -> ScVal { + ScVal::I128(Int128Parts { hi: 0, lo }) +} + +fn event(emitter: &str, topics: Vec, data: ScVal) -> ContractEvent { + ContractEvent { + ext: Default::default(), + contract_id: Some(cid(emitter)), + type_: ContractEventType::Contract, + body: ContractEventBody::V0(ContractEventV0 { + topics: topics.try_into().unwrap(), + data, + }), + } +} + +/// A SAC-shaped `transfer` from `emitter`, naming `sep11` in its last topic. +fn sac_transfer(emitter: &str, sep11: &str) -> ContractEvent { + event( + emitter, + vec![ + sym("transfer"), + addr(&strkey([0x66; 32])), + addr(&strkey(POOL)), + ScVal::String(ScString(sep11.try_into().unwrap())), + ], + i128v(10_000_000), + ) +} + +/// An Aquarius pool `trade` on [`POOL`]: `token_in` sold for `token_out`. +fn trade(token_in: &str, token_out: &str) -> ContractEvent { + event( + &strkey(POOL), + vec![ + sym("trade"), + addr(token_in), + addr(token_out), + addr(&strkey([0x66; 32])), + ], + ScVal::Vec(Some( + vec![i128v(10_000_000), i128v(20_000_000), i128v(3_000)] + .try_into() + .unwrap(), + )), + ) +} + +/// One ledger whose single transaction carries `events`, in order. +fn lcm_with(seq: u64, close_time: u64, events: Vec) -> LedgerCloseMeta { + let mut v2 = LedgerCloseMetaV2::default(); + v2.ledger_header.header.ledger_seq = seq as u32; + v2.ledger_header.header.scp_value.close_time = TimePoint(close_time); + if !events.is_empty() { + let op = OperationMetaV2 { + events: events.try_into().unwrap(), + ..Default::default() + }; + let meta = TransactionMetaV4 { + operations: vec![op].try_into().unwrap(), + ..Default::default() + }; + v2.tx_processing = vec![TransactionResultMetaV1 { + tx_apply_processing: TransactionMeta::V4(meta), + ..Default::default() + }] + .try_into() + .unwrap(); + } + LedgerCloseMeta::V2(v2) +} + +/// The same ledger, encoded as Galexie writes it. +fn ledger_with(seq: u64, close_time: u64, events: Vec) -> Vec { + let batch = LedgerCloseMetaBatch { + start_sequence: seq as u32, + end_sequence: seq as u32, + ledger_close_metas: vec![lcm_with(seq, close_time, events)].try_into().unwrap(), + }; + let xdr = batch.to_xdr(Limits::none()).unwrap(); + zstd::encode_all(&xdr[..], 0).unwrap() +} + +fn pool_registries() -> Registries { + let mut reg = Registries::new(); + reg.venue.insert(strkey(POOL), Venue::Aquarius); + reg +} + +/// Through the production step, so every case here runs the live arming. +fn with_candidates(assets: AssetRegistry) -> AssetRegistry { + with_sac_candidates(assets, HashSet::from([XCR_SAC.to_string()])) +} + +fn run(events: Vec, assets: &mut AssetRegistry) -> LedgerSoroban { + process_ledger( + &lcm_with(FIRST, BASE_MINUTE, events), + &mut pool_registries(), + assets, + ) +} + +fn pending(assets: &AssetRegistry) -> Vec { + assets.pending_new().cloned().collect() +} + +fn holds_contract(assets: &AssetRegistry, addr: &str) -> bool { + assets + .assets() + .any(|a| *a == AssetIdentity::Contract(addr.to_string())) +} + +/// `main.rs` needs the `lambda` feature and AWS, so no test runs it. Its cold +/// start must build the registry through `load_ingest_registry`: a plain +/// `load_registry` leaves the candidate set empty, and every unproven SAC would +/// mint a `Contract` identity again with all other tests green. +#[test] +fn the_lambda_cold_start_arms_the_sac_candidates() { + let main = include_str!("../src/main.rs"); + assert!(main.contains("sink.load_ingest_registry()")); + assert!( + !main.contains(".load_registry()"), + "a registry without candidates" + ); + let sink = include_str!("../src/sink/mod.rs"); + assert!(sink.contains("Ok(arm_sac_candidates(registry?, sac_contracts))")); + assert!(main.contains("sac_candidates_unavailable_metrics(")); +} + +/// Review of 0242 (PC3 reversed): an unreadable `is_sac` set does not stop +/// ingest. The flag drops, the SAC set is empty, and an unproven SAC trades +/// as a `Contract` identity, which the probe counts and the heal removes. +#[test] +fn an_unreadable_sac_set_ingests_without_candidates() { + let (mut assets, loaded) = arm_sac_candidates( + AssetRegistry::from_existing(vec![]), + Err(SinkError::Write("Code: 60. Unknown table".to_string())), + ); + assert!(!loaded); + // Minted as a `Contract`, so its decimals resolve like any Soroban token's. + assets.set_decimals(XCR_SAC.to_string(), 7); + let out = run(vec![trade(XLM_SAC, XCR_SAC)], &mut assets); + assert!(out.sac_unproven.is_empty(), "not a candidate, not skipped"); + assert_eq!(out.amm_ticks.len(), 1, "the trade still prices"); + assert!(holds_contract(&assets, XCR_SAC), "minted, for the probe"); + + let (assets, loaded) = arm_sac_candidates( + AssetRegistry::from_existing(vec![]), + Ok(HashSet::from([XCR_SAC.to_string()])), + ); + assert!(loaded); + assert!(assets.is_unproven_sac(XCR_SAC)); +} + +/// The fixture constants are pinned by our own derivation, not trusted. +#[test] +fn the_fixture_sacs_are_the_derived_ones() { + let assets = AssetRegistry::from_existing(vec![]); + assert_eq!(assets.sac_address_of(&xcr()).as_deref(), Some(XCR_SAC)); + assert_eq!( + assets.sac_address_of(&AssetIdentity::Native).as_deref(), + Some(XLM_SAC) + ); +} + +/// T2 and T4: the proof resolves the SAC whether it comes before or after the +/// trade in the transaction, on an empty registry. +#[test] +fn a_sac_proven_in_its_transaction_trades_as_its_classic_in_either_order() { + let cases = [ + ( + "proof first", + vec![sac_transfer(XCR_SAC, &xcr_sep11()), trade(XLM_SAC, XCR_SAC)], + ), + ( + "proof after", + vec![trade(XLM_SAC, XCR_SAC), sac_transfer(XCR_SAC, &xcr_sep11())], + ), + ]; + for (name, events) in cases { + for candidates in [false, true] { + let mut assets = AssetRegistry::from_existing(vec![]); + if candidates { + assets = with_candidates(assets); + } + let out = run(events.clone(), &mut assets); + assert_eq!(out.amm_ticks.len(), 1, "{name}"); + let (source, tick) = &out.amm_ticks[0]; + assert_eq!(*source, "aquarius", "{name}"); + assert_eq!(tick.base, xcr(), "{name}"); + assert_eq!(tick.quote, AssetIdentity::Native, "{name}"); + assert!(out.sac_unproven.is_empty(), "{name}"); + assert!(pending(&assets).contains(&xcr()), "{name}"); + assert!(!holds_contract(&assets, XCR_SAC), "{name}: no Contract"); + } + } +} + +/// T3: one identity whichever path sees the asset first. +#[test] +fn classic_and_sac_meet_on_one_identity_in_either_order() { + // (a) The SDEX path interns the classic; the SAC then resolves through it, + // with no proof in the transaction. + let mut assets = with_candidates(AssetRegistry::from_existing(vec![])); + assert!(assets.intern(&xcr())); + let out = run(vec![trade(XLM_SAC, XCR_SAC)], &mut assets); + assert_eq!(out.amm_ticks.len(), 1); + assert_eq!(out.amm_ticks[0].1.base, xcr()); + assert!(out.sac_unproven.is_empty()); + assert_eq!(pending(&assets).iter().filter(|a| **a == xcr()).count(), 1); + assert!(!holds_contract(&assets, XCR_SAC)); + + // (b) The SAC trades first with its proof; the classic is then known. + let mut assets = with_candidates(AssetRegistry::from_existing(vec![])); + let out = run( + vec![sac_transfer(XCR_SAC, &xcr_sep11()), trade(XLM_SAC, XCR_SAC)], + &mut assets, + ); + assert_eq!(out.amm_ticks[0].1.base, xcr()); + assert!(!assets.intern(&xcr()), "already interned by the trade"); + assert_eq!(pending(&assets).iter().filter(|a| **a == xcr()).count(), 1); + assert!(!holds_contract(&assets, XCR_SAC)); +} + +/// T6-live and the impostor: an `is_sac` leg with no valid proof is skipped, +/// counted, and interns nothing. +#[test] +fn an_unproven_sac_trade_is_skipped_and_interns_nothing() { + let impostor = strkey(IMPOSTOR); + let cases = [ + ("no proof", vec![trade(XLM_SAC, XCR_SAC)]), + ( + "impostor proof", + vec![ + sac_transfer(&impostor, &xcr_sep11()), + trade(XLM_SAC, XCR_SAC), + ], + ), + ]; + for (name, events) in cases { + let mut assets = with_candidates(AssetRegistry::from_existing(vec![])); + let out = run(events, &mut assets); + assert!(out.amm_ticks.is_empty(), "{name}"); + assert_eq!(out.sac_unproven, vec![XCR_SAC.to_string()], "{name}"); + assert!(pending(&assets).is_empty(), "{name}: nothing interned"); + assert_eq!(assets.resolve_sac(XCR_SAC), None, "{name}"); + assert_eq!(assets.resolve_sac(&impostor), None, "{name}"); + } +} + +/// D2: a token BE does not flag `is_sac` keeps its `Contract` identity. +#[test] +fn a_genuine_soroban_token_keeps_its_contract_identity() { + let token = strkey(GENUINE_TOKEN); + let mut assets = with_candidates(AssetRegistry::from_existing(vec![])); + assets.set_decimals(token.clone(), 7); + let out = run(vec![trade(XLM_SAC, &token)], &mut assets); + assert_eq!(out.amm_ticks.len(), 1); + assert_eq!( + out.amm_ticks[0].1.base, + AssetIdentity::Contract(token.clone()) + ); + assert!(out.sac_unproven.is_empty()); + assert!(holds_contract(&assets, &token)); +} + +#[derive(Clone, Default)] +struct MemoryFetcher { + objects: Arc>>>, +} + +impl MemoryFetcher { + fn publish(&self, seq: u64, bytes: Vec) { + self.objects + .lock() + .unwrap() + .insert(ledger_s3_key(seq as i64), bytes); + } +} + +impl ObjectFetcher for MemoryFetcher { + async fn fetch(&self, key: &str) -> Result>, FetchError> { + Ok(self.objects.lock().unwrap().get(key).cloned()) + } +} + +#[derive(Clone, Default)] +struct NullSink; + +impl CandleSink for NullSink { + async fn write_candles(&self, _: &[OhlcvCandle], _: &str) -> Result<(), SinkError> { + Ok(()) + } + + async fn write_oracle(&self, _: &[OracleSample]) -> Result<(), SinkError> { + Ok(()) + } + + async fn write_new_assets(&self, _: &AssetRegistry) -> Result<(), SinkError> { + Ok(()) + } + + async fn write_pool_rows(&self, _: &[PoolRegistryRow]) -> Result<(), SinkError> { + Ok(()) + } +} + +/// T6-live through the `Reconciler`: the count follows the WRITTEN minute, as +/// `UnregisteredPoolEvents` does, so a held-back ledger is not counted twice. +#[tokio::test] +async fn unproven_sac_trades_are_counted_once_when_their_minute_is_written() { + let dir = tempdir().unwrap(); + let cursor = StubFileCursor::new(dir.path().join("cursor.txt")); + cursor.write(FIRST - 1).await.unwrap(); + let fetcher = MemoryFetcher::default(); + let reconciler = Reconciler::new( + fetcher.clone(), + cursor, + NullSink, + with_candidates(AssetRegistry::from_existing(vec![])), + pool_registries(), + ); + let unproven = || vec![trade(XLM_SAC, XCR_SAC)]; + + for (seq, close) in [(FIRST, BASE_MINUTE), (FIRST + 1, BASE_MINUTE + 5)] { + fetcher.publish(seq, ledger_with(seq, close, unproven())); + let held = reconciler.run(32).await.unwrap(); + assert_eq!(held.end_cursor, FIRST - 1, "the minute is still open"); + assert_eq!(held.sac_unproven_skipped, 0); + } + + // The next minute's ledger is decoded but held back: 2, not 3. + fetcher.publish( + FIRST + 2, + ledger_with(FIRST + 2, BASE_MINUTE + 60, unproven()), + ); + let written = reconciler.run(32).await.unwrap(); + assert_eq!(written.end_cursor, FIRST + 1); + assert_eq!(written.sac_unproven_skipped, 2); + + fetcher.publish(FIRST + 3, ledger_with(FIRST + 3, BASE_MINUTE + 120, vec![])); + let next = reconciler.run(32).await.unwrap(); + assert_eq!(next.end_cursor, FIRST + 2); + assert_eq!( + next.sac_unproven_skipped, 1, + "only the minute this run wrote" + ); +} diff --git a/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs b/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs index e413611b..f02eff5d 100644 --- a/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs +++ b/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs @@ -18,9 +18,13 @@ //! whose `asset_id` or `quote_asset_id` has no `assets` row. This is the //! post-window detector for a writer out of step with the registry, e.g. an //! old binary still writing counter ids after the swap (BRIEF §3.7). +//! 3. `SacContractIdentities` = contract rows of `assets FINAL` whose address +//! is a SAC: some row's `sac_address`, or a contract BE flags `is_sac` +//! (task 0242). A SAC is a facet of its classic asset, never an identity of +//! its own. Baseline 36 until the 0242 heal, then 0. //! -//! The two reads are independent: a failure in one must not suppress the -//! other's datum. A read that saw nothing is refused, never published as a +//! The reads are independent: a failure in one must not suppress another's +//! datum. A read that saw nothing is refused, never published as a //! healthy zero (same rule as [`crate::zero_invariants`]): the alarms are //! `NOT_BREACHING` on missing data, so an empty registry or an empty window //! must surface as a failed check. @@ -36,6 +40,12 @@ pub const ASSET_ID_COLLISIONS_METRIC: &str = "AssetIdCollisions"; /// CloudWatch metric: recent candles naming an id that has no `assets` row. pub const ASSET_ID_ORPHAN_CANDLES_METRIC: &str = "AssetIdOrphanCandles"; +/// CloudWatch metric: contract identities in `assets` that are SACs. +pub const SAC_CONTRACT_IDENTITIES_METRIC: &str = "SacContractIdentities"; + +/// BE's database, holding `soroban_contracts` (prices_writer has SELECT on it). +pub const BE_DATABASE: &str = prices_clickhouse::BE_DATABASE; + /// The tier every writer of candles writes; the coarse tiers derive from it. pub const ORPHAN_CANDLE_TABLE: &str = "price_ohlcv_1m"; @@ -84,6 +94,36 @@ pub fn orphan_candles_query() -> String { ) } +/// One reading of the contract rows of `assets FINAL` against the SAC sets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clickhouse::Row, serde::Deserialize)] +pub struct SacContractCounts { + /// Contract rows whose address is a SAC. + pub sac_rows: u64, + /// Contract rows examined. Zero means the scan measured nothing. + pub scanned: u64, + /// Contracts BE flags `is_sac`. Zero means the second test measured nothing. + pub be_sacs: u64, +} + +/// The SAC-identity read. A contract row is a SAC when its address is the +/// `sac_address` of a classic row, or when BE flags it `is_sac` (a SAC whose +/// classic we never held). `uniqExact`, because `soroban_contracts` is a +/// ReplacingMergeTree read without `FINAL`; `ifNull`, because a scalar +/// subquery is `Nullable` and RowBinary would misread it as a `u64`. +pub fn sac_contract_identities_query(be_db: &str) -> String { + format!( + "SELECT \ + countIf(contract_address IN (SELECT sac_address FROM assets WHERE sac_address != '') \ + OR contract_address IN (SELECT contract_id FROM {be_db}.soroban_contracts WHERE is_sac)) \ + AS sac_rows, \ + count() AS scanned, \ + ifNull((SELECT uniqExact(contract_id) FROM {be_db}.soroban_contracts WHERE is_sac), 0) \ + AS be_sacs \ + FROM assets FINAL \ + WHERE contract_address != ''" + ) +} + /// Why a reading was not published: the read measured nothing, so its zero /// would be a false healthy datum. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -92,6 +132,10 @@ pub enum UniquenessRefusal { EmptyRegistry, /// The candle window held no row. EmptyWindow, + /// `assets FINAL` held no contract row. + NoContractRows, + /// BE's `soroban_contracts` held no `is_sac` contract. + EmptyBeSacSet, } impl std::fmt::Display for UniquenessRefusal { @@ -110,6 +154,17 @@ impl std::fmt::Display for UniquenessRefusal { zero over a scan that measured nothing", ORPHAN_LOOKBACK_SECONDS, ORPHAN_CANDLE_TABLE ), + Self::NoContractRows => write!( + f, + "sac-contract-identities unreadable: assets FINAL holds no contract row, so a \ + count of 0 would measure nothing; the registry is empty or unreadable to this user" + ), + Self::EmptyBeSacSet => write!( + f, + "sac-contract-identities unreadable: {BE_DATABASE}.soroban_contracts holds no \ + is_sac contract, so SACs whose classic we never held go uncounted; BE's table \ + is empty or unreadable to this user" + ), } } } @@ -140,6 +195,22 @@ pub fn orphan_candles_metric( }) } +/// Shape a SAC-identity reading into its datum, or refuse it. +pub fn sac_contract_identities_metric( + counts: &SacContractCounts, +) -> Result { + if counts.scanned == 0 { + return Err(UniquenessRefusal::NoContractRows); + } + if counts.be_sacs == 0 { + return Err(UniquenessRefusal::EmptyBeSacSet); + } + Ok(SanityMetric { + name: SAC_CONTRACT_IDENTITIES_METRIC, + value: counts.sac_rows as f64, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -225,5 +296,59 @@ mod tests { assert!(window.contains("unreadable"), "{window}"); assert!(window.contains("price_ohlcv_1m"), "{window}"); assert!(window.contains("7200"), "{window}"); + let contracts = UniquenessRefusal::NoContractRows.to_string(); + assert!(contracts.contains("unreadable"), "{contracts}"); + assert!(contracts.contains("no contract row"), "{contracts}"); + let be = UniquenessRefusal::EmptyBeSacSet.to_string(); + assert!(be.contains("unreadable"), "{be}"); + assert!(be.contains("default.soroban_contracts"), "{be}"); + } + + #[test] + fn the_sac_query_counts_contract_rows_against_both_sac_sets() { + let sql = sac_contract_identities_query(BE_DATABASE); + for part in [ + "FROM assets FINAL", + "contract_address != ''", + "contract_address IN (SELECT sac_address FROM assets WHERE sac_address != '')", + "contract_address IN (SELECT contract_id FROM default.soroban_contracts WHERE is_sac)", + "ifNull((SELECT uniqExact(contract_id) FROM default.soroban_contracts WHERE is_sac), 0)", + "AS be_sacs", + "AS sac_rows", + "count() AS scanned", + ] { + assert!(sql.contains(part), "{part}: {sql}"); + } + assert!(!sql.contains("SETTINGS"), "{sql}"); + } + + #[test] + fn sac_contract_identities_are_published_or_refused() { + let cases = [ + ((0, 5, 10), Ok(0.0), "no SAC among the contracts"), + ((36, 59, 4_042), Ok(36.0), "production on 2026-10-05"), + ( + (0, 0, 10), + Err(UniquenessRefusal::NoContractRows), + "no contract row", + ), + ( + (0, 5, 0), + Err(UniquenessRefusal::EmptyBeSacSet), + "empty BE SAC set", + ), + ]; + for ((sac_rows, scanned, be_sacs), want, why) in cases { + let got = sac_contract_identities_metric(&SacContractCounts { + sac_rows, + scanned, + be_sacs, + }); + assert_eq!( + got.map(|m| (m.name, m.value)), + want.map(|v| (SAC_CONTRACT_IDENTITIES_METRIC, v)), + "{why}" + ); + } } } diff --git a/packages/rollup-freshness-probe/src/lib.rs b/packages/rollup-freshness-probe/src/lib.rs index de13b29a..5101189b 100644 --- a/packages/rollup-freshness-probe/src/lib.rs +++ b/packages/rollup-freshness-probe/src/lib.rs @@ -75,6 +75,12 @@ pub mod zero_invariants; /// recent candles naming an id `assets` does not hold. See /// [`asset_id_uniqueness`] for why each refuses an empty read. pub mod asset_id_uniqueness; +/// Stored candles that break the OHLC band or carry a price `<= 0`, on all seven +/// tiers (task 0236). Asserted at the source, with zero tolerance, because task +/// 0229's read-path clamp repairs a crossed candle on its way out of `/ohlcv` — +/// so the API can no longer show one. See [`ohlc_band`] for the per-tier window +/// and the refusal rule. +pub mod ohlc_band; /// `current_prices` writer liveness (task 0243). Rides in the same invocation /// and publishes under the same [`METRIC_NAME`] with `Table = current_prices` — diff --git a/packages/rollup-freshness-probe/src/main.rs b/packages/rollup-freshness-probe/src/main.rs index 5b6b102c..78f0a735 100644 --- a/packages/rollup-freshness-probe/src/main.rs +++ b/packages/rollup-freshness-probe/src/main.rs @@ -18,8 +18,9 @@ async fn main() -> Result<(), lambda_runtime::Error> { use lambda_runtime::{LambdaEvent, run, service_fn}; use rollup_freshness_probe::asset_id_uniqueness::{ - AssetIdCounts, OrphanCandleCounts, collisions_metric, collisions_query, - orphan_candles_metric, orphan_candles_query, + AssetIdCounts, BE_DATABASE, OrphanCandleCounts, SacContractCounts, collisions_metric, + collisions_query, orphan_candles_metric, orphan_candles_query, + sac_contract_identities_metric, sac_contract_identities_query, }; use rollup_freshness_probe::current_prices::{ CurrentPricesAge, current_prices_age_query, current_prices_metric, @@ -29,6 +30,10 @@ async fn main() -> Result<(), lambda_runtime::Error> { MV_DRIFT_CRITICAL_METRIC, MV_DRIFT_METRIC, describe, drift_metrics, publish_drift, visible_objects_query, }; + use rollup_freshness_probe::ohlc_band::{ + OHLC_BAND_TIERS, OhlcBandCounts, ohlc_band_detail, ohlc_band_metric, ohlc_band_queries, + ohlc_band_totals, + }; use rollup_freshness_probe::reconcile_mismatch::{ MISMATCH_MIN_READ_SECS, MISMATCH_RESERVE, MismatchCount, mismatch_metric, mismatch_queries, mismatch_read_bound, publish_mismatch, @@ -56,7 +61,8 @@ async fn main() -> Result<(), lambda_runtime::Error> { // secret/endpoint surfaces on the first query (the invocation fails and the // probe's error alarm fires) — no separate `SELECT 1` liveness probe is // needed. The `ingestion` mTLS identity (prices_writer) has SELECT on - // prices.*, which is all this probe needs. + // prices.*, and on `default.soroban_contracts`, which the SAC-identity + // read joins (task 0242). // // ⚠️ This comment used to claim the probe touches no `system.*` table. Since // task 0204 gap 3 it reads `system.tables`, and that is fine — `system.tables` @@ -74,6 +80,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { let zero_invariant_query = Arc::new(zero_invariant_query()); let collisions_query = Arc::new(collisions_query()); let orphan_candles_query = Arc::new(orphan_candles_query()); + let sac_contract_identities_query = Arc::new(sac_contract_identities_query(BE_DATABASE)); let refresh_waits_query = Arc::new(refresh_waits_query("prices")); // The mismatch reads are the only ones here whose cost grows with a week // of the child tier (the 15m read scans seven days of `_1m`: 0.7–4.0 s on @@ -86,6 +93,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { // deadline, and a read that would get under 2 s is skipped and recorded as // a failure — see `reconcile_mismatch` (review WR-04). let mismatch_queries = Arc::new(mismatch_queries()); + let ohlc_band_queries = Arc::new(ohlc_band_queries()); let aws_cfg = aws_config::defaults(aws_config::BehaviorVersion::latest()) .load() @@ -103,8 +111,10 @@ async fn main() -> Result<(), lambda_runtime::Error> { let zero_invariant_query = zero_invariant_query.clone(); let collisions_query = collisions_query.clone(); let orphan_candles_query = orphan_candles_query.clone(); + let sac_contract_identities_query = sac_contract_identities_query.clone(); let refresh_waits_query = refresh_waits_query.clone(); let mismatch_queries = mismatch_queries.clone(); + let ohlc_band_queries = ohlc_band_queries.clone(); let environment = environment.clone(); let deadline = event.context.deadline(); async move { @@ -361,7 +371,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { // ---- 5. The zero sentinel's stored-data invariants (ADR 0292) --- // - // Late on purpose (only 5b and the mismatch reads, 6, come after it). It is the one unscoped read here: `timestamp` is the + // Late on purpose (only 5b, 5c and the mismatch reads, 6, come after it). It is the one unscoped read here: `timestamp` is the // fourth sort-key column, so the 48 h window prunes only to the monthly // partition, which is then merged `FINAL` across every pair. Measured on // production 2026-09-18 it is cheap (0.04 s, 650k rows read) — but it @@ -398,9 +408,9 @@ async fn main() -> Result<(), lambda_runtime::Error> { Err(e) => failures.push(format!("zero-invariants read: {e}")), } - // ---- 5b. Asset-id uniqueness (task 0139) ---------------------- + // ---- 5b. Asset-id uniqueness (tasks 0139, 0242) --------------- // - // Two independent reads, after the zero invariants for the same + // Three independent reads, after the zero invariants for the same // reason: they grow with the data (the registry, and the live tip // of `_1m` against it). A refusal is an empty read, recorded as a // failure rather than published as a healthy 0. @@ -446,6 +456,114 @@ async fn main() -> Result<(), lambda_runtime::Error> { Err(e) => failures.push(format!("asset-id-orphans read: {e}")), } + let mut sac_counts: Option = None; + match ch + .query(&sac_contract_identities_query) + .fetch_one::() + .await + { + Ok(counts) => { + sac_counts = Some(counts); + match sac_contract_identities_metric(&counts) { + Ok(metric) => { + if let Err(e) = + publish_sanity(&cw, &environment, std::slice::from_ref(&metric)) + .await + { + failures.push(format!("sac-contract-identities publish: {e}")); + } + } + Err(refusal) => { + failures.push(format!("sac-contract-identities: {refusal}")) + } + } + } + Err(e) => failures.push(format!("sac-contract-identities read: {e}")), + } + + // ---- 5c. Stored-candle OHLC band + positive prices, all seven tiers (task 0236) --- + // + // Late, for the reason block 5 gives: these reads grow with the + // tables. Before the mismatch reads (6), which stay last because + // their budget is what is left of the invocation. Measured on + // production 2026-10-06 the seven reads together are cheap: 0.29 s + // of server time, 3.7 M rows read. + // + // Seven INDEPENDENT reads, not one `UNION ALL`: a statement fails as a + // unit, so one dropped tier or lost grant would hide the other six + // counts, and the alarm is `NOT_BREACHING` on missing data. A failed + // tier is recorded and the others still count. + // + // Run in parallel, each under the bound block 6 uses: `min(10 s, + // time left - 5 s reserve)`, server-side and client-side. A stalled + // tier (a `FINAL` during a merge, a busy box during a re-ingest) + // is then a recorded failure, not a hard timeout that loses block 6 + // and the summary line. The block costs the slowest read, not the sum. + // + // Independent of block 5, although a priced `_1m` row with + // `close = 0` fires both: that overlap is expected (see `ohlc_band`). + let mut band_readings: Vec<(&'static str, OhlcBandCounts)> = Vec::new(); + let remaining = deadline + .duration_since(std::time::SystemTime::now()) + .unwrap_or_default(); + match mismatch_read_bound(remaining) { + None => failures.push(format!( + "ohlc-band skipped: {:.1} s left of the invocation, \ + under the {} s reserve + {MISMATCH_MIN_READ_SECS} s a read needs", + remaining.as_secs_f64(), + MISMATCH_RESERVE.as_secs(), + )), + Some(bound) => { + let mut reads = tokio::task::JoinSet::new(); + for (table, sql) in ohlc_band_queries.iter() { + let bounded = prices_clickhouse::with_execution_bound((*ch).clone(), bound); + let (table, sql) = (*table, sql.clone()); + reads.spawn(async move { + let read = tokio::time::timeout( + std::time::Duration::from_secs(bound + 2), + bounded.query(&sql).fetch_one::(), + ) + .await; + (table, read) + }); + } + while let Some(joined) = reads.join_next().await { + match joined { + Ok((table, Ok(Ok(counts)))) => band_readings.push((table, counts)), + Ok((table, Ok(Err(e)))) => failures + .push(format!("ohlc-band read {table} (bound {bound} s): {e}")), + Ok((table, Err(_))) => failures.push(format!( + "ohlc-band read {table}: no answer within {} s", + bound + 2 + )), + Err(e) => failures.push(format!("ohlc-band read task: {e}")), + } + } + // Completion order is arbitrary; log in tier order. + band_readings.sort_by_key(|(t, _)| { + OHLC_BAND_TIERS.iter().position(|(o, _)| o == t) + }); + } + } + match ohlc_band_metric(&band_readings) { + Ok(metric) => { + if let Err(e) = + publish_sanity(&cw, &environment, std::slice::from_ref(&metric)).await + { + failures.push(format!("ohlc-band publish: {e}")); + } + } + Err(refusal) => failures.push(format!("ohlc-band: {refusal}")), + } + // Totals only for a complete run: a partial sum logged as a number + // would read as the whole chain. The detail names each tier. + let band_totals = ohlc_band_totals(&band_readings); + let band_scanned_1m = band_readings + .iter() + .find(|(t, _)| *t == OHLC_BAND_TIERS[0].0) + .map(|(_, c)| c.scanned); + let band_detail = ohlc_band_detail(&band_readings); + // ---- 6. Coarse buckets disagreeing with their source (0203) --- // // The new tail, for the reason the zero invariants were last: these @@ -548,6 +666,12 @@ async fn main() -> Result<(), lambda_runtime::Error> { mv_drift_critical = %reading(drift_critical, drift_missing), mv_drift = %reading(drift_count, drift_missing), mv_visible_objects = %reading(visible_objects, FAILED), + ohlc_band_violations = %reading(band_totals.map(|t| t.0), FAILED), + ohlc_band_band = %reading(band_totals.map(|t| t.1), FAILED), + ohlc_band_nonpositive = %reading(band_totals.map(|t| t.2), FAILED), + ohlc_band_scanned_1m = %reading(band_scanned_1m, FAILED), + ohlc_band_tiers_read = band_readings.len(), + ohlc_band_detail = %band_detail, mv_detail = %drift_detail, mv_refresh_waiting = %reading(refresh_waiting, refresh_missing), mv_refresh_disabled = %reading(refresh_disabled, refresh_missing), @@ -592,6 +716,21 @@ async fn main() -> Result<(), lambda_runtime::Error> { "ids": id_counts.map(|c| c.ids), "orphan_candles": orphan_counts.map(|c| c.orphans), "orphan_scanned": orphan_counts.map(|c| c.scanned), + "sac_contract_rows": sac_counts.map(|c| c.sac_rows), + "sac_contract_scanned": sac_counts.map(|c| c.scanned), + "be_sac_contracts": sac_counts.map(|c| c.be_sacs), + }, + "ohlc_band": { + "violations": band_totals.map(|t| t.0), + "tiers": band_readings + .iter() + .map(|(t, c)| serde_json::json!({ + "tier": t, + "band": c.band, + "nonpositive": c.nonpositive, + "scanned": c.scanned, + })) + .collect::>(), }, "mv_drift": { "critical": drift_critical, diff --git a/packages/rollup-freshness-probe/src/ohlc_band.rs b/packages/rollup-freshness-probe/src/ohlc_band.rs new file mode 100644 index 00000000..d4bcb7da --- /dev/null +++ b/packages/rollup-freshness-probe/src/ohlc_band.rs @@ -0,0 +1,503 @@ +//! Stored candles that are internally inconsistent, on all seven tiers (task 0236). +//! +//! A candle's four prices obey two shapes: `low` and `high` bracket the open and +//! the close, and every price is positive. Since task 0229 the `/ohlcv` read path +//! CLAMPS a crossed candle into shape on its way out (ADR-0011 §3), which is the +//! right thing for a read API to do — and it removed the only place anyone would +//! have noticed a corrupt stored row. This check moves that detector to the +//! SOURCE: it reads the stored rows, not what the API serves. +//! +//! # The predicate — exact, with zero tolerance +//! +//! A priced row (`pf_trade_count > 0`) is a violation when +//! +//! ```text +//! low > least(open, close) OR high < greatest(open, close) OR low > high +//! OR open <= 0 OR high <= 0 OR low <= 0 OR close <= 0 +//! ``` +//! +//! There is no epsilon. The stored decimals are produced by `min`/`max`/ +//! `argMin`/`argMax` over the same fills (and, up the chain, over the tier +//! below), which COPY a value bit for bit — nothing is computed, so nothing can +//! round. Any crossing in storage is a real defect, and the comparisons are plain +//! `Decimal` against integer literals: no `toFloat`, no float path at all. +//! +//! The predicate is split into two **disjoint** classes, so their sum counts ROWS: +//! +//! - `band` — the three band arms, over rows whose four prices are ALL `> 0`; +//! - `nonpositive` — any of the four prices `<= 0`. +//! +//! Their union is exactly the predicate above. Without the positive gate a row +//! like `open = 0, low = 3` would fire both `low > least(open, close)` and +//! `open <= 0` and count twice. `low > high` follows logically from the other two +//! band arms (if `low > high` then either `low > least(o, c)`, or +//! `least(o, c) >= low > high`, and then `greatest(o, c) > high`); it is kept as +//! the literal invariant the reader expects to find. +//! +//! # `pf_trade_count > 0` means priced on every tier +//! +//! On a coarse tier `pf_trade_count` is a plain `sum` over the children, while +//! the four prices come only from children with `pf_trade_count > 0 AND close +//! >= 1e-12`. The two agree because the ingest counts a fill as price-forming +//! only when its price clears that same floor +//! (`prices_ingest_core::price::price_survives_column_scale`, pinned to +//! `prices_clickhouse::PRICE_FLOOR_LITERAL`). So no `_1m` row has +//! `pf_trade_count > 0` with a close below the floor, and a coarse row with +//! `pf_trade_count > 0` always has a child that lends it a price. On prod, from +//! 2026-09-23 to 2026-10-06, no row on any tier broke this. If one does, the +//! ingest and rollup floors have drifted apart (the 0286 review WR-03 defect), +//! and the `nonpositive` count reports it. +//! +//! # The window is per tier, on bucket START +//! +//! `timestamp >= now() - 2 days - ` (`_1m` plain 2 days). A +//! bucket of length `L` starting at `s` overlaps the last two days iff +//! `s > now - 2d - L`. A flat 2-day window would NEVER see a `_1w` or `_1M` +//! bucket for most of the week or month — its start is older than two days. The +//! widening is by one bucket, and since weeks are Monday-aligned and month +//! subtraction clamps to the month's end, `now - 1 MONTH` is always at or before +//! the current month's start: the window always reaches the current bucket. +//! +//! ⚠️ **Bucket time, not write time** — as with [`crate::zero_invariants`], the +//! tables carry no insert-time column. A backfill, pre-roll or operator +//! `INSERT … SELECT` that writes buckets older than the window is NOT covered; +//! a historical rewrite needs its own assertion over the partitions it touched. +//! +//! ⚠️ **A scheduled assertion, not a ClickHouse `CHECK` constraint** — for the +//! reason ADR 0292 §6 gives: a violated CHECK fails the whole insert, so one bad +//! row would stall a refreshable MV's tier and turn a data defect into a +//! freshness incident. +//! +//! # Overlap and multiplication — both expected +//! +//! - A priced `_1m` row with `close = 0` is also [`crate::zero_invariants`] +//! invariant 3, so both alarms fire. `low = 0` beside a positive close is +//! caught ONLY here — the shape the prod baseline was mostly made of. +//! - The metric sums TIER-ROWS. The rollups carry a bad minute into every coarse +//! bucket above it, so one defect can count up to seven times. Read the per-tier +//! detail: a violation on a coarse tier with a clean `_1m` is a ROLLUP defect, +//! not an ingest one. +//! +//! # Seven reads, not one `UNION ALL` +//! +//! One statement fails as a unit: a dropped or renamed tier, or a lost grant, +//! would take every other tier's count down with it, and the alarm is +//! `NOT_BREACHING` on missing data. Seven independent reads keep the six that +//! work. +//! +//! # When a reading is refused +//! +//! A positive sum is always published — it can never be a false OK. A ZERO is +//! published only when all seven tiers were read AND the `_1m` window was +//! non-empty; otherwise [`OhlcBandRefusal`] says why, and the probe's `-errors` +//! alarm carries it. Coarse tiers with nothing in their window are fine. +//! +//! # Baseline +//! +//! Measured on production 2026-09-28: band violations 0 on every tier, every +//! source, since 2015. Priced rows with a price `<= 0` are all exact zeros and +//! all older than the task 0286 rollout (2026-09-22 12:00 UTC); 0 after it. That +//! legacy residue is outside this window by construction and is re-ingested by +//! 0286 phase 3. A healthy reading is therefore exactly **0**. + +use crate::usd_sanity::SanityMetric; + +/// CloudWatch metric: stored candles in the per-tier windows that break the band +/// or carry a price `<= 0`, summed over all seven tiers. +pub const OHLC_BAND_METRIC: &str = "CandleBandViolations"; + +/// Two days, the same base window as +/// [`crate::zero_invariants::ZERO_INVARIANT_LOOKBACK_SECONDS`]; the coarse tiers +/// widen it by one bucket (see [`OHLC_BAND_TIERS`]). +pub const OHLC_BAND_LOOKBACK_SECONDS: i64 = 2 * 86_400; + +/// `(table, widening)` per tier, in [`crate::ROLLUP_TIERS`] order (a unit test +/// pins the two together). The widening is one bucket length, as a ClickHouse +/// `INTERVAL` literal; `_1m` is not widened. +pub const OHLC_BAND_TIERS: [(&str, Option<&str>); 7] = [ + ("price_ohlcv_1m", None), + ("price_ohlcv_15m", Some("15 MINUTE")), + ("price_ohlcv_1h", Some("1 HOUR")), + ("price_ohlcv_4h", Some("4 HOUR")), + ("price_ohlcv_1d", Some("1 DAY")), + ("price_ohlcv_1w", Some("1 WEEK")), + ("price_ohlcv_1M", Some("1 MONTH")), +]; + +/// One tier's reading. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clickhouse::Row, serde::Deserialize)] +pub struct OhlcBandCounts { + /// Priced rows with four positive prices that break the band. + pub band: u64, + /// Priced rows with any of the four prices `<= 0`. Disjoint from `band`. + pub nonpositive: u64, + /// Rows examined. Zero on `_1m` means the scan measured nothing. + pub scanned: u64, +} + +impl OhlcBandCounts { + /// A reading from its three counts, for tests and fixtures. + pub const fn new(band: u64, nonpositive: u64, scanned: u64) -> Self { + Self { + band, + nonpositive, + scanned, + } + } + + /// Rows breaking either shape. The classes are disjoint, so this is a row + /// count, not a count of arms. + pub fn violations(&self) -> u64 { + self.band + self.nonpositive + } +} + +/// The seven assertion queries, `(table, sql)` in [`OHLC_BAND_TIERS`] order. +/// +/// `FINAL`, because a corrected row is re-inserted at a higher `version` and the +/// alarm must stop once the data is fixed. Takes no argument: the SQL is built +/// only from `'static` constants, so no caller-supplied string reaches `format!`. +pub fn ohlc_band_queries() -> Vec<(&'static str, String)> { + OHLC_BAND_TIERS + .iter() + .map(|&(table, widen)| { + let widening = widen + .map(|w| format!(" - INTERVAL {w}")) + .unwrap_or_default(); + let sql = format!( + "SELECT \ + countIf(pf_trade_count > 0 AND open > 0 AND high > 0 AND low > 0 AND close > 0 \ + AND (low > least(open, close) OR high < greatest(open, close) OR low > high)) \ + AS band, \ + countIf(pf_trade_count > 0 AND (open <= 0 OR high <= 0 OR low <= 0 OR close <= 0)) \ + AS nonpositive, \ + count() AS scanned \ + FROM {table} FINAL \ + WHERE timestamp >= now() - INTERVAL {lookback} SECOND{widening}", + lookback = OHLC_BAND_LOOKBACK_SECONDS, + ); + (table, sql) + }) + .collect() +} + +/// Why a zero was not published. +/// +/// Its own type, NOT [`crate::usd_sanity::SanityRefusal`], for the reason +/// [`crate::zero_invariants::ZeroInvariantRefusal`] gives: that one blames the +/// USDT identity, and this check resolves no quote leg. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OhlcBandRefusal { + /// The `_1m` window held no candle at all. + EmptyScan, + /// At least one tier's read failed in this run. + Incomplete, +} + +impl std::fmt::Display for OhlcBandRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::EmptyScan => write!( + f, + "no candle of any pair was found in the last {} s of {} — ingestion has \ + stopped or the table is empty (this check is not scoped to a quote leg); \ + refusing to publish a healthy zero over a scan that measured nothing", + OHLC_BAND_LOOKBACK_SECONDS, OHLC_BAND_TIERS[0].0 + ), + Self::Incomplete => write!( + f, + "fewer than {} tiers were read in this run (each failed tier read is \ + reported alongside) — a zero would claim tiers nobody measured", + OHLC_BAND_TIERS.len() + ), + } + } +} + +impl std::error::Error for OhlcBandRefusal {} + +/// `(violations, band, nonpositive)` summed over the run, or `None` unless every +/// tier of [`OHLC_BAND_TIERS`] was read exactly once. The one completeness rule +/// the metric and the log share: a duplicate or unknown table name is not a +/// complete run either. +pub fn ohlc_band_totals(readings: &[(&'static str, OhlcBandCounts)]) -> Option<(u64, u64, u64)> { + let complete = readings.len() == OHLC_BAND_TIERS.len() + && OHLC_BAND_TIERS + .iter() + .all(|(t, _)| readings.iter().filter(|(r, _)| r == t).count() == 1); + complete.then(|| { + readings.iter().fold((0, 0, 0), |(v, b, n), (_, c)| { + (v + c.violations(), b + c.band, n + c.nonpositive) + }) + }) +} + +/// Shape the readings into the one datum to publish, or refuse. +/// +/// In order: a positive sum is always published (never a false OK, even with a +/// tier missing); a zero without a complete run ([`ohlc_band_totals`]) is +/// [`OhlcBandRefusal::Incomplete`]; a zero over an empty `_1m` window is +/// [`OhlcBandRefusal::EmptyScan`]; otherwise a healthy zero. +pub fn ohlc_band_metric( + readings: &[(&'static str, OhlcBandCounts)], +) -> Result { + let total: u64 = readings.iter().map(|(_, c)| c.violations()).sum(); + if total > 0 { + return Ok(SanityMetric { + name: OHLC_BAND_METRIC, + value: total as f64, + }); + } + if ohlc_band_totals(readings).is_none() { + return Err(OhlcBandRefusal::Incomplete); + } + let scanned_1m = readings + .iter() + .find(|(t, _)| *t == OHLC_BAND_TIERS[0].0) + .map(|(_, c)| c.scanned); + if scanned_1m == Some(0) { + return Err(OhlcBandRefusal::EmptyScan); + } + Ok(SanityMetric { + name: OHLC_BAND_METRIC, + value: 0.0, + }) +} + +/// One `
band= nonpositive= scanned=` segment per reading, +/// joined by `; ` — the per-tier breakdown the alarm text points at. +pub fn ohlc_band_detail(readings: &[(&'static str, OhlcBandCounts)]) -> String { + readings + .iter() + .map(|(t, c)| { + format!( + "{t} band={} nonpositive={} scanned={}", + c.band, c.nonpositive, c.scanned + ) + }) + .collect::>() + .join("; ") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ROLLUP_TIERS; + + /// Seven readings, one per tier, all the given counts. + fn all_tiers(c: OhlcBandCounts) -> Vec<(&'static str, OhlcBandCounts)> { + OHLC_BAND_TIERS.iter().map(|(t, _)| (*t, c)).collect() + } + + #[test] + fn every_tier_query_asserts_every_predicate_arm() { + let queries = ohlc_band_queries(); + assert_eq!(queries.len(), 7); + for (table, sql) in &queries { + for arm in [ + "low > least(open, close)", + "high < greatest(open, close)", + "low > high", + "open <= 0 OR high <= 0 OR low <= 0 OR close <= 0", + ] { + assert!(sql.contains(arm), "{table} is missing `{arm}`: {sql}"); + } + } + } + + /// The two classes are disjoint: `band` only over four positive prices, so + /// a row breaking both shapes (open = 0 with a positive low) counts once, + /// and both only over priced rows. + #[test] + fn the_classes_are_disjoint_and_both_gated_on_a_priced_row() { + for (table, sql) in ohlc_band_queries() { + assert!( + sql.contains( + "countIf(pf_trade_count > 0 AND open > 0 AND high > 0 AND low > 0 AND close > 0 \ + AND (low > least(open, close) OR high < greatest(open, close) OR low > high)) \ + AS band" + ), + "{table}: {sql}" + ); + assert!( + sql.contains( + "countIf(pf_trade_count > 0 AND (open <= 0 OR high <= 0 OR low <= 0 OR close <= 0)) \ + AS nonpositive" + ), + "{table}: {sql}" + ); + assert!(sql.contains("count() AS scanned"), "{table}: {sql}"); + assert!( + !sql.contains("toFloat"), + "exact Decimal, no float path: {sql}" + ); + } + } + + /// BRIEF point 4: the window is on bucket START, widened by one bucket per + /// tier. One assertion per tier, each pinned to the END of the statement. + #[test] + fn each_tier_reads_its_table_final_with_its_own_bucket_window() { + let expected = [ + ( + "price_ohlcv_1m", + "timestamp >= now() - INTERVAL 172800 SECOND", + ), + ( + "price_ohlcv_15m", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 15 MINUTE", + ), + ( + "price_ohlcv_1h", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 1 HOUR", + ), + ( + "price_ohlcv_4h", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 4 HOUR", + ), + ( + "price_ohlcv_1d", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 1 DAY", + ), + ( + "price_ohlcv_1w", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 1 WEEK", + ), + ( + "price_ohlcv_1M", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 1 MONTH", + ), + ]; + let queries = ohlc_band_queries(); + assert_eq!(queries.len(), expected.len()); + for ((table, sql), (want_table, want_window)) in queries.iter().zip(expected) { + assert_eq!(*table, want_table); + assert!( + sql.contains(&format!("FROM {want_table} FINAL WHERE")), + "{table} must read FINAL: {sql}" + ); + assert!( + sql.ends_with(&format!("WHERE {want_window}")), + "{table} window: {sql}" + ); + } + } + + /// An eighth tier added to `ROLLUP_TIERS` must not go unwatched here. + #[test] + fn the_tiers_are_exactly_the_rollup_tiers_in_order() { + let ours: Vec<&str> = OHLC_BAND_TIERS.iter().map(|(t, _)| *t).collect(); + let rollup: Vec<&str> = ROLLUP_TIERS.iter().map(|t| t.table).collect(); + assert_eq!(ours, rollup); + } + + #[test] + fn the_metric_is_the_sum_of_both_classes_over_every_tier() { + let readings: Vec<(&'static str, OhlcBandCounts)> = OHLC_BAND_TIERS + .iter() + .enumerate() + .map(|(i, (t, _))| (*t, OhlcBandCounts::new(i as u64, 2 * i as u64, 100))) + .collect(); + // band 0+1+..+6 = 21, nonpositive 2*21 = 42 + let metric = ohlc_band_metric(&readings).unwrap(); + assert_eq!(metric.name, OHLC_BAND_METRIC); + assert_eq!(metric.name, "CandleBandViolations"); + assert_eq!(metric.value, 63.0); + assert_eq!(OhlcBandCounts::new(2, 3, 9).violations(), 5); + } + + #[test] + fn an_empty_1m_scan_is_refused_not_published_as_healthy() { + let refusal = ohlc_band_metric(&all_tiers(OhlcBandCounts::new(0, 0, 0))).unwrap_err(); + assert_eq!(refusal, OhlcBandRefusal::EmptyScan); + } + + #[test] + fn a_zero_over_a_missing_tier_is_refused_as_incomplete() { + let mut readings = all_tiers(OhlcBandCounts::new(0, 0, 10)); + readings.remove(3); + assert_eq!( + ohlc_band_metric(&readings).unwrap_err(), + OhlcBandRefusal::Incomplete + ); + } + + /// Seven readings are not a complete run when one tier is read twice or an + /// unknown table stands in for a real one. The metric and the log share + /// this rule through `ohlc_band_totals`. + #[test] + fn a_duplicate_or_unknown_tier_is_not_a_complete_run() { + let complete = all_tiers(OhlcBandCounts::new(1, 2, 10)); + assert_eq!(ohlc_band_totals(&complete), Some((21, 7, 14))); + + let mut duplicate = complete.clone(); + duplicate[6].0 = duplicate[5].0; + let mut unknown = complete.clone(); + unknown[6].0 = "price_ohlcv_2h"; + for readings in [duplicate, unknown] { + assert_eq!(ohlc_band_totals(&readings), None); + let zero: Vec<_> = readings + .iter() + .map(|(t, _)| (*t, OhlcBandCounts::new(0, 0, 10))) + .collect(); + assert_eq!( + ohlc_band_metric(&zero).unwrap_err(), + OhlcBandRefusal::Incomplete + ); + } + } + + /// A positive value is never a false OK, so a failed tier elsewhere does + /// not hide the violations the other six found. + #[test] + fn a_positive_sum_is_published_even_with_a_tier_missing() { + let mut readings = all_tiers(OhlcBandCounts::new(0, 0, 10)); + readings.remove(6); + readings[2].1 = OhlcBandCounts::new(1, 2, 10); + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 3.0); + } + + #[test] + fn empty_coarse_tiers_beside_a_scanned_1m_publish_a_healthy_zero() { + let mut readings = all_tiers(OhlcBandCounts::new(0, 0, 0)); + readings[0].1 = OhlcBandCounts::new(0, 0, 500); + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 0.0); + } + + #[test] + fn a_coarse_violation_is_published_even_when_1m_scanned_nothing() { + let mut readings = all_tiers(OhlcBandCounts::new(0, 0, 0)); + readings[5].1 = OhlcBandCounts::new(2, 0, 4); + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 2.0); + } + + #[test] + fn the_empty_scan_refusal_names_the_table_and_not_a_quote_leg() { + let message = OhlcBandRefusal::EmptyScan.to_string(); + assert!(message.contains("price_ohlcv_1m"), "{message}"); + assert!(message.contains("ingestion"), "{message}"); + assert!(!message.contains("USDT"), "{message}"); + assert!(!message.contains("0139"), "{message}"); + } + + #[test] + fn the_incomplete_refusal_says_a_tier_read_failed_in_this_run() { + let message = OhlcBandRefusal::Incomplete.to_string(); + assert!(message.contains("tier"), "{message}"); + assert!(message.contains("read"), "{message}"); + assert!(message.contains("this run"), "{message}"); + } + + #[test] + fn the_detail_names_every_reading() { + let detail = ohlc_band_detail(&[ + ("price_ohlcv_1m", OhlcBandCounts::new(1, 2, 6)), + ("price_ohlcv_1w", OhlcBandCounts::new(0, 0, 0)), + ]); + assert_eq!( + detail, + "price_ohlcv_1m band=1 nonpositive=2 scanned=6; \ + price_ohlcv_1w band=0 nonpositive=0 scanned=0" + ); + } +} diff --git a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs index 27659200..e2e21c68 100644 --- a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs +++ b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs @@ -1929,3 +1929,487 @@ async fn an_empty_registry_or_window_is_refused_as_unreadable() { drop_scratch_db(db).await; } + +// ---- Task 0242: SAC contract identities ------------------------------------- +// +// A scratch prices database plus a scratch BE database holding only +// `soroban_contracts` (DDL as coverage_sweep_it.rs), read through the +// production query. + +const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; +const XCR: AssetFixture = AssetFixture::new( + "XCR", + "classic", + "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7", + "", +) +.with_sac(XCR_SAC); +const XCR_AS_CONTRACT: AssetFixture = AssetFixture::new("", "soroban", "", XCR_SAC); +/// An is_sac contract whose classic `assets` never held. +const ORPHAN_SAC: &str = "CORPHANSACXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"; +const ORPHAN_SAC_ROW: AssetFixture = AssetFixture::new("", "soroban", "", ORPHAN_SAC); +const TOKEN: &str = "CGENUINETOKENXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"; +const TOKEN_ROW: AssetFixture = AssetFixture::new("", "soroban", "", TOKEN); + +async fn scratch_be_db(db: &str) { + let admin = Client::default().with_url(ch_url()); + exec(&admin, &format!("DROP DATABASE IF EXISTS {db}")).await; + exec(&admin, &format!("CREATE DATABASE {db}")).await; + exec( + &admin, + &format!( + "CREATE TABLE {db}.soroban_contracts ( + id Int64, + contract_id String, + wasm_hash Nullable(FixedString(32)), + wasm_uploaded_at_ledger Int64 DEFAULT 0, + deployer_id Nullable(Int64), + deployed_at_ledger Nullable(Int64), + contract_type Nullable(Int16), + is_sac Bool + ) + ENGINE = ReplacingMergeTree(wasm_uploaded_at_ledger) + ORDER BY (contract_id)" + ), + ) + .await; +} + +async fn be_contract(be: &str, contract: &str, is_sac: bool) { + let admin = Client::default().with_url(ch_url()); + exec( + &admin, + &format!( + "INSERT INTO {be}.soroban_contracts (id, contract_id, is_sac) \ + VALUES (1, '{contract}', {is_sac})" + ), + ) + .await; +} + +async fn read_sac_contracts( + c: &Client, + be: &str, +) -> rollup_freshness_probe::asset_id_uniqueness::SacContractCounts { + c.query(&rollup_freshness_probe::asset_id_uniqueness::sac_contract_identities_query(be)) + .fetch_one() + .await + .expect("the SAC-identity query executes and deserializes") +} + +/// A contract row is counted through either disjunct: XCR's SAC through the +/// classic row's `sac_address`, a SAC with no classic through BE's `is_sac`. +/// A genuine token is scanned, not counted. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn sac_contract_identities_counts_both_disjuncts_and_not_a_token() { + use rollup_freshness_probe::asset_id_uniqueness::sac_contract_identities_metric; + + let (db, be) = ("it_probe_0242_sac", "it_probe_0242_sac_be"); + let c = scratch_db(db).await; + scratch_be_db(be).await; + // XCR_SAC is deliberately absent from BE: the first disjunct alone counts it. + // ORPHAN_SAC on two unmerged versions still counts once in be_sacs. + be_contract(be, ORPHAN_SAC, true).await; + be_contract(be, ORPHAN_SAC, true).await; + be_contract(be, TOKEN, false).await; + + let cases = [ + (XCR, (0, 0), "classic only: no contract row"), + (XCR_AS_CONTRACT, (1, 1), "XCR's SAC via sac_address"), + (ORPHAN_SAC_ROW, (2, 2), "a SAC with no classic via is_sac"), + (TOKEN_ROW, (2, 3), "a genuine token is not a SAC"), + ]; + for (row, (sac_rows, scanned), why) in cases { + exec(&c, &assets_insert(db, &[row])).await; + let counts = read_sac_contracts(&c, be).await; + assert_eq!( + (counts.sac_rows, counts.scanned), + (sac_rows, scanned), + "{why}" + ); + assert_eq!(counts.be_sacs, 1, "{why}"); + if scanned > 0 { + assert_eq!( + sac_contract_identities_metric(&counts).unwrap().value, + sac_rows as f64, + "{why}" + ); + } + } + + drop_scratch_db(db).await; + drop_scratch_db(be).await; +} + +/// No contract row, or no is_sac contract in BE, is unreadable: refused rather +/// than published as a healthy 0. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn sac_contract_identities_refuses_an_empty_scan_or_be_set() { + use rollup_freshness_probe::asset_id_uniqueness::{ + UniquenessRefusal, sac_contract_identities_metric, + }; + + let (db, be) = ("it_probe_0242_sac_empty", "it_probe_0242_sac_empty_be"); + let c = scratch_db(db).await; + scratch_be_db(be).await; + be_contract(be, ORPHAN_SAC, true).await; + exec(&c, &assets_insert(db, &[XCR])).await; + assert_eq!( + sac_contract_identities_metric(&read_sac_contracts(&c, be).await), + Err(UniquenessRefusal::NoContractRows) + ); + + scratch_be_db(be).await; + be_contract(be, TOKEN, false).await; + exec(&c, &assets_insert(db, &[TOKEN_ROW])).await; + assert_eq!( + sac_contract_identities_metric(&read_sac_contracts(&c, be).await), + Err(UniquenessRefusal::EmptyBeSacSet) + ); + + drop_scratch_db(db).await; + drop_scratch_db(be).await; +} + +// ---- task 0236: stored candles outside the OHLC band, on all seven tiers ---- + +/// Clear every tier of `OHLC_BAND_TIERS`, in its order: `_1m` FIRST, then the +/// coarse tiers finest to coarsest. The rollup MVs run in this database: +/// clearing a source before its target stops a refresh landing between the two +/// from re-feeding the tier just cleared. Its own helper — [`reset_sanity_tables`] +/// is shared and clears only two tiers. +async fn reset_ohlc_band_tables(c: &Client) { + for (table, _) in rollup_freshness_probe::ohlc_band::OHLC_BAND_TIERS { + exec(c, &format!("TRUNCATE TABLE prices.{table}")).await; + } +} + +/// One candle on any tier with its four prices spelled out, each as an exact +/// `Decimal` literal — no Float64 path, so a fixture one ulp off the band is +/// stored as written. Every other column as in [`insert_invariant_row`]: +/// `pf_trade_count` explicit, never left to its DEFAULT. +/// +/// `ts` is a unix timestamp fixed by the caller, so a repair at a higher +/// `version` lands on the same key as the row it supersedes. +async fn insert_ohlc_row( + c: &Client, + table: &str, + ts: u32, + asset_id: u32, + (open, high, low, close): (&str, &str, &str, &str), + pf: u32, + version: u32, +) { + exec( + c, + &format!( + "INSERT INTO prices.{table} \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, \ + version, pf_trade_count, pf_volume, pf_price_volume) \ + SELECT toDateTime({ts}), {asset_id}, 2, 'sdex', \ + toDecimal128('{open}', 14), toDecimal128('{high}', 14), \ + toDecimal128('{low}', 14), toDecimal128('{close}', 14), \ + 1, 1, 0, 0, 1, 3, {version}, {pf}, {pf}, {pf}" + ), + ) + .await; +} + +/// Run every production query — `ohlc_band_queries()` itself, never a copy. +async fn read_ohlc_band( + c: &Client, +) -> Vec<( + &'static str, + rollup_freshness_probe::ohlc_band::OhlcBandCounts, +)> { + let mut readings = Vec::new(); + for (table, sql) in rollup_freshness_probe::ohlc_band::ohlc_band_queries() { + let counts = c + .query(&sql) + .fetch_one() + .await + .unwrap_or_else(|e| panic!("the ohlc-band query for {table} executes: {e}")); + readings.push((table, counts)); + } + readings +} + +/// A server-side unix timestamp for `now() - `, computed ONCE so every +/// fixture of a test shares it. +async fn ts_ago(c: &Client, interval: &str) -> u32 { + c.query(&format!("SELECT toUnixTimestamp(now() - {interval})")) + .fetch_one() + .await + .unwrap() +} + +fn reading( + readings: &[( + &'static str, + rollup_freshness_probe::ohlc_band::OhlcBandCounts, + )], + table: &str, +) -> rollup_freshness_probe::ohlc_band::OhlcBandCounts { + readings + .iter() + .find(|(t, _)| *t == table) + .unwrap_or_else(|| panic!("{table} was read")) + .1 +} + +/// The seven queries must **execute and deserialize** on the production build, +/// and count exactly the rows outside the band — none of the healthy shapes a +/// careless predicate flags: a single-trade candle (O=H=L=C), a dust-only +/// candle (all four 0 with `pf_trade_count = 0`, correct per ADR 0287), and a +/// priced candle still pending enrichment (`close_usd = 0`). The row breaking +/// BOTH shapes (`open = 0` with a positive low) counts ONCE — RED if the band +/// class loses its four-positive-prices gate. +/// +/// `_1m` fixtures sit at `now() - 3 HOUR`: inside the probe's 2-day window, +/// outside the 1m→15m MV's 2-hour window, so no refresh copies them upward +/// mid-test and every coarse tier reads exactly zero. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_counts_only_rows_outside_the_band_on_every_tier() { + use rollup_freshness_probe::ohlc_band::{OHLC_BAND_TIERS, OhlcBandCounts, ohlc_band_metric}; + + let c = client().await; + reset_ohlc_band_tables(&c).await; + let ts = ts_ago(&c, "INTERVAL 3 HOUR").await; + let m = "price_ohlcv_1m"; + + insert_ohlc_row(&c, m, ts, 20, ("5", "5", "5", "5"), 1, 1).await; // single trade + insert_ohlc_row(&c, m, ts, 21, ("0", "0", "0", "0"), 0, 1).await; // dust only + insert_ohlc_row(&c, m, ts, 22, ("4", "6", "3", "5"), 3, 1).await; // pending enrichment + insert_ohlc_row(&c, m, ts, 23, ("5", "4.5", "3", "5"), 2, 1).await; // ⛔ high < close + insert_ohlc_row(&c, m, ts, 24, ("4", "6", "0", "5"), 2, 1).await; // ⛔ low = 0 + insert_ohlc_row(&c, m, ts, 25, ("0", "6", "3", "5"), 2, 1).await; // ⛔ open = 0, low > open + + let readings = read_ohlc_band(&c).await; + assert_eq!(readings.len(), OHLC_BAND_TIERS.len()); + assert_eq!( + reading(&readings, m), + OhlcBandCounts::new(1, 2, 6), + "_1m: one band violation, two non-positive rows (the double-shaped one counted once)" + ); + for (table, counts) in &readings[1..] { + assert_eq!( + *counts, + OhlcBandCounts::new(0, 0, 0), + "{table} holds nothing" + ); + } + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 3.0); + + reset_ohlc_band_tables(&c).await; +} + +/// Seed the one healthy `_1m` row every arm test needs so the scan is not +/// refused as empty, and return the shared `now() - 3 HOUR` timestamp. +async fn seed_healthy_1m(c: &Client) -> u32 { + let ts = ts_ago(c, "INTERVAL 3 HOUR").await; + insert_ohlc_row(c, "price_ohlcv_1m", ts, 30, ("5", "5", "5", "5"), 1, 1).await; + ts +} + +/// The arm `high < greatest(open, close)`, and ONLY that arm: `o4 h4.5 l3 c5` +/// has `low (3) <= least (4)` and `low <= high`. RED without the arm. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_catches_a_high_below_the_close() { + let c = client().await; + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + insert_ohlc_row(&c, "price_ohlcv_1m", ts, 31, ("4", "4.5", "3", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 2), + "high 4.5 below close 5 is outside the band" + ); + + reset_ohlc_band_tables(&c).await; +} + +/// The arm `low > least(open, close)`, and ONLY that arm: `o3 h6 l4 c5` has +/// `high (6) >= greatest (5)` and `low <= high`. RED without the arm. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_catches_a_low_above_the_open() { + let c = client().await; + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + insert_ohlc_row(&c, "price_ohlcv_1m", ts, 32, ("3", "6", "4", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 2), + "low 4 above open 3 is outside the band" + ); + + reset_ohlc_band_tables(&c).await; +} + +/// `low > high` — `o5 h4 l6 c5` — counted ONCE although it fires all three band +/// arms (countIf counts rows). The arm is implied by the other two (if +/// `low > high`, either `low > least(o, c)`, or `least(o, c) >= low > high` and +/// then `greatest(o, c) > high`), so this goes RED only with all three band arms +/// removed; it pins that the literal invariant is caught and counted once. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_catches_a_low_above_the_high_once() { + let c = client().await; + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + insert_ohlc_row(&c, "price_ohlcv_1m", ts, 33, ("5", "4", "6", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 2), + "a low above the high is one bad row" + ); + + reset_ohlc_band_tables(&c).await; +} + +/// `low = 0` beside a positive close — the shape most of the prod baseline was +/// made of, and the one `zero_invariants` cannot see (its invariant 3 reads +/// `close = 0`). Non-positive, not band: the classes are disjoint. RED without +/// `low <= 0`. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_catches_a_zero_low_beside_a_positive_close() { + let c = client().await; + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + insert_ohlc_row(&c, "price_ohlcv_1m", ts, 34, ("4", "6", "0", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(0, 1, 2), + "a priced low of 0 is a non-positive price" + ); + + reset_ohlc_band_tables(&c).await; +} + +/// BRIEF point 4, the regression it exists for: a `_1w` bucket starting 5 days +/// ago and a `_1M` bucket starting 12 days ago both overlap the last two days +/// once each is widened by its own length, and a FLAT 2-day window never sees +/// them. RED with the widening removed. Only the terminal sinks `_1w`/`_1M` +/// are used (fed daily from `_1d`, which is empty here), and both sit days away +/// from either window edge. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_reads_each_coarse_tier_over_its_own_bucket_window() { + use rollup_freshness_probe::ohlc_band::ohlc_band_metric; + + let c = client().await; + reset_ohlc_band_tables(&c).await; + seed_healthy_1m(&c).await; + let week = ts_ago(&c, "INTERVAL 5 DAY").await; + let month = ts_ago(&c, "INTERVAL 12 DAY").await; + insert_ohlc_row(&c, "price_ohlcv_1w", week, 35, ("5", "4.5", "3", "5"), 2, 1).await; + insert_ohlc_row(&c, "price_ohlcv_1M", month, 36, ("3", "6", "4", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1w"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 1), + "a week bucket older than 2 days but inside its widened window" + ); + assert_eq!( + reading(&readings, "price_ohlcv_1M"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 1), + "a month bucket older than 2 days but inside its widened window" + ); + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(0, 0, 1) + ); + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 2.0); + + reset_ohlc_band_tables(&c).await; +} + +/// The window has an edge. A `_1m` violation an hour beyond 2 days and a `_1w` +/// violation an hour beyond `2 days + 1 week` are NOT counted — nor scanned. +/// RED with the WHERE removed (the legacy pre-0286 residue would page forever). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_ignores_violations_outside_the_window() { + let c = client().await; + reset_ohlc_band_tables(&c).await; + seed_healthy_1m(&c).await; + let old_minute = ts_ago(&c, "INTERVAL 2 DAY - INTERVAL 1 HOUR").await; + let old_week = ts_ago(&c, "INTERVAL 2 DAY - INTERVAL 1 WEEK - INTERVAL 1 HOUR").await; + insert_ohlc_row( + &c, + "price_ohlcv_1m", + old_minute, + 37, + ("5", "4.5", "3", "5"), + 2, + 1, + ) + .await; + insert_ohlc_row( + &c, + "price_ohlcv_1w", + old_week, + 38, + ("4", "6", "0", "5"), + 2, + 1, + ) + .await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(0, 0, 1), + "only the healthy seed is inside the 1m window" + ); + assert_eq!( + reading(&readings, "price_ohlcv_1w"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(0, 0, 0), + "a week bucket starting before now - 2d - 1w is outside" + ); + for (table, counts) in &readings { + assert_eq!(counts.violations(), 0, "{table}"); + } + + reset_ohlc_band_tables(&c).await; +} + +/// The alarm must stop once the data is fixed: a bad row re-inserted good at a +/// higher `version` on the same key is superseded, so `FINAL` neither counts +/// nor scans the bad version. The unrepaired neighbour still counts. RED +/// without `FINAL` (the superseded version is read as a second row). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn a_repaired_candle_stops_counting_in_the_ohlc_band_scan() { + let c = client().await; + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + let m = "price_ohlcv_1m"; + insert_ohlc_row(&c, m, ts, 40, ("5", "4.5", "3", "5"), 2, 1).await; // bad + insert_ohlc_row(&c, m, ts, 40, ("4", "6", "3", "5"), 2, 2).await; // its repair + insert_ohlc_row(&c, m, ts, 41, ("5", "4.5", "3", "5"), 2, 1).await; // left bad + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, m), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 3), + "the superseded version is neither counted nor scanned" + ); + + reset_ohlc_band_tables(&c).await; +} diff --git a/packages/sdex-backfill/src/cli.rs b/packages/sdex-backfill/src/cli.rs index b8f582a6..39b76ba8 100644 --- a/packages/sdex-backfill/src/cli.rs +++ b/packages/sdex-backfill/src/cli.rs @@ -67,7 +67,7 @@ pub struct Cli { pub clickhouse_url: String, /// Caddy host fronting the Hetzner CH, for `--transport hetzner` (e.g. - /// ch.sorobanscan.rumblefish.dev). The client connects to `https://{domain}`. + /// ch.sorobanscan.com). The client connects to `https://{domain}`. #[arg(long, env = "CH_DOMAIN")] pub ch_domain: Option, diff --git a/packages/sdex-backfill/src/ingest.rs b/packages/sdex-backfill/src/ingest.rs index 3ce7c2f9..43c3ca48 100644 --- a/packages/sdex-backfill/src/ingest.rs +++ b/packages/sdex-backfill/src/ingest.rs @@ -5,8 +5,9 @@ use std::time::{Duration, Instant}; use tracing::{info, warn}; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, OhlcvCandle, Registries, TradeTick, UnresolvedPoolSwap, - extract_trades, ledger_sequence, offer_lookup_counts, process_ledger, raw_trade_to_tick, + AssetRegistry, CandleAccumulator, DecimalsResolver, DecimalsRow, OhlcvCandle, Registries, + TradeTick, UnresolvedPoolSwap, decode_resolving, extract_trades, ledger_sequence, + offer_lookup_counts, process_ledger, raw_trade_to_tick, }; use crate::error::BackfillError; @@ -32,6 +33,10 @@ pub struct PartitionStats { pub skipped: usize, pub trade_ticks: usize, pub amm_ticks: usize, + /// AMM trades dropped because a token's decimals did not resolve over RPC + /// (task 0329). Their ledgers are still marked done, so a non-zero total + /// means a range to reprice. + pub trades_missing_decimals: usize, pub oracle_rows: usize, pub candles_written: usize, pub total_bytes: u64, @@ -333,6 +338,7 @@ pub async fn index_partition( completed: &HashSet, registry: &mut AssetRegistry, reg: &mut Registries, + decimals: &mut DecimalsResolver, mode: ExtractMode, accs: &mut RunAccumulators, end: PartitionEnd, @@ -401,7 +407,17 @@ pub async fn index_partition( // Only in Combined mode — pre-Soroban ledgers carry no Soroban // events, so SdexOnly skips the decode entirely. if mode == ExtractMode::Combined { - let sob = process_ledger(lcm, reg, registry); + // Task 0329: a trade naming a token whose decimals are unknown + // is dropped, not guessed — the same step as the live processor. + let sob = decode_resolving( + decimals, + registry, + ledger_sequence(lcm), + |assets| process_ledger(lcm, reg, assets), + async |rows: &[DecimalsRow]| sink.write_decimals(rows).await, + ) + .await?; + stats.trades_missing_decimals += sob.trades_missing_decimals as usize; for (source, tick) in sob.amm_ticks { accs.merge_amm(source, tick); stats.amm_ticks += 1; @@ -470,6 +486,7 @@ pub async fn index_partition( skipped = stats.skipped, trade_ticks = stats.trade_ticks, amm_ticks = stats.amm_ticks, + trades_missing_decimals = stats.trades_missing_decimals, oracle_rows = stats.oracle_rows, candles = stats.candles_written, bytes = stats.total_bytes, diff --git a/packages/sdex-backfill/src/run.rs b/packages/sdex-backfill/src/run.rs index 6aef8925..aa4bc6fb 100644 --- a/packages/sdex-backfill/src/run.rs +++ b/packages/sdex-backfill/src/run.rs @@ -5,7 +5,9 @@ use tokio::process::Command; use tokio::task::JoinHandle; use tracing::{info, warn}; -use prices_ingest_core::{AssetRegistry, Registries, UnresolvedPool, UnresolvedPoolSwap}; +use prices_ingest_core::{ + AssetRegistry, DecimalsResolver, Registries, UnresolvedPool, UnresolvedPoolSwap, +}; use crate::error::BackfillError; use crate::ingest::{ @@ -146,6 +148,12 @@ pub async fn execute( let existing_assets = sink.load_assets().await?; let mut registry = AssetRegistry::from_existing(existing_assets); + // Task 0329: only the AMM path scales by token decimals, so an SDEX-only + // run neither reads `prices.asset_decimals` nor needs it to exist. + if mode == ExtractMode::Combined { + sink.load_decimals(&mut registry).await?; + } + let mut decimals = DecimalsResolver::backfill_from_env(); // Venue / pool registries. Preloaded from the persisted `pool_registry` // artifact (decision #4) so a window starting after activation still // resolves earlier-created pools; empty on a fresh full run. Then grown @@ -193,6 +201,7 @@ pub async fn execute( &completed, &mut registry, &mut reg, + &mut decimals, mode, &mut accs, partition_end, @@ -203,6 +212,7 @@ pub async fn execute( totals.skipped += stats.skipped; totals.trade_ticks += stats.trade_ticks; totals.amm_ticks += stats.amm_ticks; + totals.trades_missing_decimals += stats.trades_missing_decimals; totals.oracle_rows += stats.oracle_rows; totals.candles_written += stats.candles_written; totals.total_bytes += stats.total_bytes; @@ -534,6 +544,13 @@ fn print_run_summary( println!("ledgers already in DB: {}", totals.skipped); println!("SDEX trade ticks: {}", totals.trade_ticks); println!("AMM trade ticks: {}", totals.amm_ticks); + // Always printed, 0 included. Non-zero: a token's decimals() did not + // resolve, its trades are NOT in the candles, and their ledgers are marked + // done anyway — reprice that range (task 0329). + println!( + "trades dropped (decimals): {}", + totals.trades_missing_decimals + ); println!("oracle rows: {}", totals.oracle_rows); println!("price_ohlcv_1m rows: {}", totals.candles_written); println!("total bytes downloaded: {}", totals.total_bytes); diff --git a/packages/sdex-backfill/src/sink.rs b/packages/sdex-backfill/src/sink.rs index 0c2cf1bf..f9bd099a 100644 --- a/packages/sdex-backfill/src/sink.rs +++ b/packages/sdex-backfill/src/sink.rs @@ -13,8 +13,8 @@ use std::collections::HashSet; use clickhouse::Row; use prices_ingest_core::canonical::AssetIdentity; use prices_ingest_core::{ - AssetRegistry, DEFAULT_BACKOFF_MS, OhlcvCandle, OhlcvWriter, Registries, UnresolvedPool, - retry_with_backoff, + AssetRegistry, DEFAULT_BACKOFF_MS, DecimalsRow, OhlcvCandle, OhlcvWriter, Registries, + UnresolvedPool, retry_with_backoff, }; use serde::{Deserialize, Serialize}; use tracing::info; @@ -72,6 +72,11 @@ impl Sink { Ok(self.writer.load_assets().await?) } + /// Resolved Soroban token decimals into `registry` (task 0329). + pub async fn load_decimals(&self, registry: &mut AssetRegistry) -> Result<(), BackfillError> { + Ok(self.writer.load_decimals(registry).await?) + } + // All `prices.*` writes below are idempotent (ReplacingMergeTree keyed by // `version`), so a retried INSERT can only replace, never duplicate. That // lets the sink retry every failure as transient (`|_| true`) — a bounded @@ -116,6 +121,11 @@ impl Sink { .await } + pub async fn write_decimals(&self, rows: &[DecimalsRow]) -> Result<(), BackfillError> { + self.retry_write(|| async { self.writer.write_decimals(rows).await }) + .await + } + pub async fn write_unresolved_pools( &self, pools: &[UnresolvedPool], diff --git a/skills/stellar-prices-api/SKILL.md b/skills/stellar-prices-api/SKILL.md index 8d9ba916..42df3381 100644 --- a/skills/stellar-prices-api/SKILL.md +++ b/skills/stellar-prices-api/SKILL.md @@ -59,7 +59,7 @@ KEY=${STELLAR_PRICES_API_KEY:-$(sed -n 's/^\(export \)*STELLAR_PRICES_API_KEY=// **If the key is missing, stop and ask the user for it.** Do not call the API without a key, and do not guess or invent prices. Tell the user how to get one: -1. Open https://sorobanscan.rumblefish.dev/prices-api/?utm_source=stellar-skill +1. Open https://sorobanscan.com/api/prices-api/?utm_source=stellar-skill 2. Click **Sign in with Discord**. The account must be a member of the [Stellar Developers Discord](https://discord.gg/stellardev), must have passed the server's membership screening, and must be older than 5 minutes. @@ -89,12 +89,12 @@ Handle the key like a secret: ## Basics -| | | -| -------------------- | --------------------------------------------------------------------------- | -| Base URL | `https://prices-api.sorobanscan.rumblefish.dev/v1` | -| Auth header | `x-api-key: `. `Authorization: Bearer` is rejected with 403. | -| Spec (no key needed) | `https://prices-api.sorobanscan.rumblefish.dev/api-docs-json` (OpenAPI 3.1) | -| Human docs | https://sorobanscan.rumblefish.dev/prices-api/docs | +| | | +| -------------------- | ----------------------------------------------------------------- | +| Base URL | `https://prices-api.sorobanscan.com/v1` | +| Auth header | `x-api-key: `. `Authorization: Bearer` is rejected with 403. | +| Spec (no key needed) | `https://prices-api.sorobanscan.com/api-docs-json` (OpenAPI 3.1) | +| Human docs | https://sorobanscan.com/api/prices-api/docs | The spec describes every parameter and field in detail. Fetch it whenever something is not covered here. @@ -159,7 +159,7 @@ options send the `stellar-prices-skill/1` user agent and make an HTTP error visible (`curl: (22) … 403`) instead of letting `jq` print nulls. ```bash -API=https://prices-api.sorobanscan.rumblefish.dev/v1 +API=https://prices-api.sorobanscan.com/v1 KEY=${STELLAR_PRICES_API_KEY:-$(sed -n 's/^\(export \)*STELLAR_PRICES_API_KEY=//p' .env 2>/dev/null | head -1 | tr -d "\"'\r")} H=(-sS --fail-with-body -H "x-api-key: $KEY" -A "stellar-prices-skill/1") USDC=USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN diff --git a/tools/scripts/__pycache__/reingest_0286.cpython-315.pyc b/tools/scripts/__pycache__/reingest_0286.cpython-315.pyc new file mode 100644 index 00000000..93103db0 Binary files /dev/null and b/tools/scripts/__pycache__/reingest_0286.cpython-315.pyc differ diff --git a/tools/scripts/generate-lore-board.test.mjs b/tools/scripts/generate-lore-board.test.mjs index 30edd9a9..e2f772a8 100644 --- a/tools/scripts/generate-lore-board.test.mjs +++ b/tools/scripts/generate-lore-board.test.mjs @@ -1,9 +1,11 @@ -// Tests for getAssignee in generate-lore-board.mjs (task 0327). +// Tests for getAssignee in generate-lore-board.mjs (task 0327) and its use in +// board.html (task 0328). // // Run: node --test tools/scripts/generate-lore-board.test.mjs import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; import { getAssignee } from './generate-lore-board.mjs'; @@ -95,3 +97,14 @@ for (const { name, dir = 'active', task, want } of CASES) { assert.equal(getAssignee({ _dir: dir, ...task }), want); }); } + +// The page must show what board.json carries, not re-derive it (task 0328). +test('board.html reads the assignee from board.json', () => { + const html = readFileSync( + new URL('../../lore/board.html', import.meta.url), + 'utf-8', + ); + const fn = html.match(/function getTaskAssignee\(t\) \{([\s\S]*?)\n\s*\}/); + assert.ok(fn, 'getTaskAssignee not found in board.html'); + assert.equal(fn[1].trim(), 'return t.assignee || null;'); +}); diff --git a/tools/scripts/reingest_0286.py b/tools/scripts/reingest_0286.py index c9610307..9026712e 100755 --- a/tools/scripts/reingest_0286.py +++ b/tools/scripts/reingest_0286.py @@ -540,8 +540,20 @@ def gate(name, passed, detail): COLLIDING_IDS = "(SELECT new_id FROM prices.asset_id_map_0139 WHERE status = 'colliding')" MAP_RESTRICT = f" AND asset_id IN {COPIED_IDS} AND quote_asset_id IN {COPIED_IDS}" -SUMS = """SELECT source, count(), sum(trade_count), sum(volume_base), sum(volume_quote) -FROM prices.price_ohlcv_{tier} FINAL WHERE toYYYYMM(timestamp) = {m}{restrict} +# Task 0329: the AMM path now scales a pure Soroban token's legs by its own +# decimals, so a re-ingested month's volume moves BY DESIGN on every candle that +# touches a contract token (an 18-decimal token's base volume falls by 10^11, a +# 6-decimal one's rises tenfold). Volumes are therefore compared on the other +# candles only; trades are still compared on all of them. The subquery reads +# ids in whatever width `prices.assets` carries, so it holds on either side of +# the 0139 swap. +CONTRACT_IDS = "(SELECT asset_id FROM prices.assets WHERE contract_address != '')" +NO_CONTRACT = f"asset_id NOT IN {CONTRACT_IDS} AND quote_asset_id NOT IN {CONTRACT_IDS}" +VOL_SCOPE = "no-contract-0329" # recorded on `before`, so an older reading is never compared + +SUMS = f"""SELECT source, count(), sum(trade_count), + sumIf(volume_base, {NO_CONTRACT}), sumIf(volume_quote, {NO_CONTRACT}) +FROM prices.price_ohlcv_{{tier}} FINAL WHERE toYYYYMM(timestamp) = {{m}}{{restrict}} GROUP BY source ORDER BY source""" OUTSIDE = f"""SELECT source, @@ -674,7 +686,12 @@ def require_transport_flag(a): "(run it on the CH host as `default`) or --amm wait + amm-done.") -def amm_summary(text, logfile): +# events-backfill's task-0242 line (run.rs UNPROVEN_SAC_LABEL). Absent from a +# pre-0242 binary, which passes (D4: phase 3 is not held for the code). +UNPROVEN_SAC_LABEL = "unproven sac swaps:" + + +def amm_summary(text, logfile, *, dry_run, unproven_line_seen=False): """Read events-backfill's closing summary, or Stop naming what is missing. Called after the --dry-run pass as well as the write, so a binary whose @@ -682,21 +699,56 @@ def amm_summary(text, logfile): Python AttributeError after the month's AMM candles are already in. `negative apply order:` replaced `events with no apply order:` in task 0304; a binary without it predates that fix and cannot read BE's events anyway. + + `unproven sac swaps:` (task 0242) may be absent: a binary built before 0242 + prints none, and phase 3 runs on one by design, the 0242 seed resolving its + SACs (D4, PC6). Absent after an earlier month of this state printed it + (`unproven_line_seen`), the host binary went back to a pre-0242 build: Stop. + `trades dropped (decimals):` came with task 0329; a binary without it still + scales every Soroban token at 7 decimals and must not write a month. """ found = {} - for key, label in (("fallbacks", "negative apply order:"), ("dropped", "swaps dropped (unresolved):")): + for key, label in (("fallbacks", "negative apply order:"), ("dropped", "swaps dropped (unresolved):"), + ("decimals", "trades dropped (decimals):")): hit = re.search(rf"^{re.escape(label)}\s*(\d+)", text, re.M) if not hit: raise Stop(f"events-backfill printed no `{label}` line — " - f"see {logfile}. A binary without it predates task 0304: rebuild events-backfill " - "from develop (on the CH host for --amm ssh)") + f"see {logfile}. A binary without it predates task 0304 or 0329: rebuild " + "events-backfill from develop (on the CH host for --amm ssh)") found[key] = int(hit.group(1)) + hit = re.search(rf"^{re.escape(UNPROVEN_SAC_LABEL)}\s*(\d+)", text, re.M) + found["unproven_sacs"] = int(hit.group(1)) if hit else None + if found["unproven_sacs"] is None and unproven_line_seen: + raise Stop(f"events-backfill printed no `{UNPROVEN_SAC_LABEL}` line, but an earlier month of this " + "state did: the binary on the host is now one built before task 0242, which mints an " + f"unproven SAC as a second identity — see {logfile}. Put the 0242 build back " + "(docs/runbooks/0242-sac-identity-heal.md §4e) and `run` again") + if found["unproven_sacs"]: + m = Path(logfile).parent.name + state = (f"This was the --dry-run pass: no AMM candle was written, but {m}'s 1m partition is " + "already dropped and holds SDEX-only candles. Fix it today and `run` again, or " + f"`rollback {m}`" if dry_run else + f"This was the write pass: {m}'s AMM candles are in without those swaps. `rollback {m}`") + raise Stop(f"events-backfill skipped {found['unproven_sacs']} swaps because their SAC could not be " + f"proven (task 0242 D2) — the contracts are in the WARN in {logfile}. {state}: " + "docs/runbooks/0242-sac-identity-heal.md §4c") return found +def unproven_line_seen(st, m): + """Whether a month of this state other than `m` recorded `unproven sac swaps:` (task 0242).""" + return any(v.get("unproven_sacs") is not None for k, v in st.d["months"].items() if k != str(m)) + + def record_amm_summary(ms, m, summary): """Keep the write pass's figures on the month and note the ones that need reading.""" ms["fallbacks"], ms["dropped"] = summary["fallbacks"], summary["dropped"] + ms["unproven_sacs"] = summary["unproven_sacs"] + if ms["unproven_sacs"] is None: + note(f"{m}: events-backfill printed no `{UNPROVEN_SAC_LABEL}` line — a binary built before task 0242, " + "so its SACs resolve only through assets.sac_address (the 0242 seed), and an unseeded one " + "would mint a second identity (heal runbook §5a counts it)") + ms["decimals"] = summary["decimals"] if ms["dropped"]: note(f"{m}: {ms['dropped']} swaps dropped for unregistered pools — see prices.unresolved_pools") if ms["fallbacks"]: @@ -825,6 +877,7 @@ def done(): if not b: # cleanup dropped whole 1m months on 2026-07-18; the coarse copy is the survivor b, ms["ref"] = sums(ch, "1h", m, post), "1h" ms["before"], ms["ids"] = b, "u64" if post else "u32" + ms["vol_scope"] = VOL_SCOPE if post: ms["outside_before"] = outside(ch, ms["ref"], m) done() @@ -889,6 +942,7 @@ def done(): if step("amm"): ms["fallbacks"] = "-" + ms.pop("decimals", None) # only a parsed summary sets it; never carry a stale count if soroban and not ch.dry: base = f"~/events-backfill --start {S} --end {E} --clickhouse-url http://localhost:8123 --verbose" if a.amm == "mtls": @@ -915,7 +969,8 @@ def done(): code, text = stream(cmd, env, mdir / "events-backfill.log") if code != 0 or "=== events-backfill complete ===" not in text: raise Stop(f"events-backfill {' '.join(flag)} exit {code} — see {mdir}/events-backfill.log") - summary = amm_summary(text, mdir / "events-backfill.log") + summary = amm_summary(text, mdir / "events-backfill.log", dry_run=bool(flag), + unproven_line_seen=unproven_line_seen(st, m)) record_amm_summary(ms, m, summary) elif a.amm == "ssh": for flag in (" --dry-run", ""): @@ -924,17 +979,19 @@ def done(): mdir / "events-backfill.log", stdin_text=pw) if code != 0 or "=== events-backfill complete ===" not in text: raise Stop(f"events-backfill{flag} exit {code} — see {mdir}/events-backfill.log") - summary = amm_summary(text, mdir / "events-backfill.log") + summary = amm_summary(text, mdir / "events-backfill.log", dry_run=bool(flag), + unproven_line_seen=unproven_line_seen(st, m)) record_amm_summary(ms, m, summary) else: marker = mdir / "amm.done" note(f"{m}: waiting for the host run — read -rs CH_PW; CLICKHOUSE_PASSWORD=\"$CH_PW\" {base}" - f" then: reingest_0286.py amm-done {m} --fallbacks ") + f" then: reingest_0286.py amm-done {m} --fallbacks " + "--decimals ") while not marker.exists(): DASH["step"] = "waiting for amm-done" RERENDER() time.sleep(15) - ms["fallbacks"] = int(marker.read_text().strip() or 0) + ms["fallbacks"], ms["decimals"] = read_amm_done(marker) done() if step("reconcile"): @@ -951,10 +1008,20 @@ def done(): "did not copy, back under their own identities — information, not reconciled") rank, amm_bits = 0, [] # 0 OK, 1 FINDING, 2 DEFECT damaged = m in LIVE_LOSS_MONTHS + scoped = ms.get("vol_scope") == VOL_SCOPE + if not scoped: + rank = 1 + note(f"{m}: its before volumes were read before task 0329 took contract-token candles out of " + "the comparison — volumes are NOT compared for this month, trades are") + if ms.get("decimals"): + rank = 2 + note(f"{m}: {ms['decimals']} AMM trades dropped because a token's decimals() did not resolve " + "— they are NOT in the candles; re-run the month once it resolves (task 0329)") for src in sorted(set(ms["before"]) | set(after)): b = ms["before"].get(src, {"trades": 0, "vb": "0", "vq": "0"}) x = after.get(src, {"trades": 0, "vb": "0", "vq": "0"}) - less = x["trades"] < b["trades"] or Decimal(x["vb"]) < Decimal(b["vb"]) or Decimal(x["vq"]) < Decimal(b["vq"]) + less = x["trades"] < b["trades"] or scoped and ( + Decimal(x["vb"]) < Decimal(b["vb"]) or Decimal(x["vq"]) < Decimal(b["vq"])) same = (x["trades"], Decimal(x["vb"]), Decimal(x["vq"])) == (b["trades"], Decimal(b["vb"]), Decimal(b["vq"])) gain = (x["trades"] - b["trades"]) / b["trades"] * 100 if b["trades"] else 0.0 if less: @@ -1067,7 +1134,17 @@ def cmd_status(a, ch, st): def cmd_amm_done(a, ch, st): (st.dir / str(a.month)).mkdir(exist_ok=True) - (st.dir / str(a.month) / "amm.done").write_text(str(a.fallbacks)) + (st.dir / str(a.month) / "amm.done").write_text(f"{a.fallbacks} {a.decimals}") + + +def read_amm_done(marker): + """(fallbacks, decimals) from an `amm.done`. A marker without the decimals + count predates task 0329 and is refused rather than read as zero.""" + parts = marker.read_text().split() + if len(parts) < 2: + raise Stop(f"{marker} carries no `trades dropped (decimals)` count — re-record it with " + "`amm-done --fallbacks N --decimals N` from the run's summary") + return int(parts[0]), int(parts[1]) def cmd_finish(a, ch, st): @@ -1096,8 +1173,9 @@ def cmd_finish(a, ch, st): print(f"acceptance 2 — OHLC order on {t}: {bad} violations", flush=True) print(""" Not automatic, by design (runbook §7b-7e, §8): - 7b the enrichment worker now re-prices the whole history on its own (hours). Capture - POST_RUN_0228_VERSION_BEFORE_1W / _1M BEFORE it starts. + 7b drain 1m with coarse-repair from this machine (days); the hourly Lambda alone takes + months. Then 7b-2, coarse-repair on 15m..1M. Capture + POST_RUN_0228_VERSION_BEFORE_1W / _1M BEFORE either starts. 7c cargo test -p enrichment-worker --test post_run_0228_it -- --ignored 7e the USDT-peg read (USDT by identity, runbook §7e): peg_written = 0, pivot_written > 0, on 1m and 1h 8.3 XLM/USDC 1d closes on the seven dust days within 5 % of Bitstamp @@ -1155,6 +1233,8 @@ def build_parser(): p.add_argument("--yes", action="store_true") p.add_argument("--accept", action="append", help="YYYYMM whose failed reconciliation is understood and recorded") p.add_argument("--fallbacks", type=int, default=0) + p.add_argument("--decimals", type=int, + help="amm-done: the run's `trades dropped (decimals)` line (task 0329)") p.add_argument("--ack-phase1-measured", action="store_true") p.add_argument("--ack-0285", action="store_true") p.add_argument("--ack-0300-binary", action="store_true", @@ -1173,7 +1253,7 @@ def build_parser(): p.add_argument("--state-dir", default="~/reingest-0286") p.add_argument("--repo", default=subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True).stdout.strip() or ".") - p.add_argument("--ch-url", default="https://ch.sorobanscan.rumblefish.dev") + p.add_argument("--ch-url", default="https://ch.sorobanscan.com") p.add_argument("--database", default="prices") # Defaults are the bundles ON the campaign machine (fishuser-hero), per the # 2026-09-21 decision: prices_admin, NOT dev_shared. The admin cert is the @@ -1199,6 +1279,8 @@ def main(): a = p.parse_args() if a.command in ("amm-done", "rollback", "release") and not a.month: p.error(f"{a.command} needs a month") + if a.command == "amm-done" and a.decimals is None: + p.error("amm-done needs --decimals: the run's `trades dropped (decimals)` line (task 0329)") if a.min_excluded_rows and not a.second_pass: p.error("--min-excluded-rows applies to --second-pass only") global LOG diff --git a/tools/scripts/reingest_0286_post0139_test.py b/tools/scripts/reingest_0286_post0139_test.py index 8113d203..d2f26ef2 100644 --- a/tools/scripts/reingest_0286_post0139_test.py +++ b/tools/scripts/reingest_0286_post0139_test.py @@ -250,12 +250,13 @@ def setUp(self): def tearDown(self): self.tmp.cleanup() - def run_reconcile(self, after_copied, ids="u64", width="UInt64", accept=()): + def run_reconcile(self, after_copied, ids="u64", width="UInt64", accept=(), **month): st = state([202201], self.tmp.name) ms = st.month(202201) ms.update(step=r.STEPS.index("reconcile"), ref="1m", ids=ids, before={"sdex": {"candles": 9, "trades": 1000, "vb": "40", "vq": "50"}}, - outside_before={}) + outside_before={}, vol_scope=r.VOL_SCOPE) + ms.update(month) def sums(sql): return after_copied if RESTRICT in sql else SUMS_ALL @@ -306,6 +307,48 @@ def test_before_on_the_old_width_stops_unless_accepted(self): self.assertIsNone(err) self.assertEqual(ms["result"]["verdict"], "OK") + # ---- task 0329: per-token decimals move contract-token volumes by design ---- + + def test_volumes_leave_out_contract_token_candles_and_trades_do_not(self): + sql = " ".join(r.SUMS.format(tier="1m", m=202201, restrict="").split()) + self.assertIn("sum(trade_count)", sql) + no_contract = ("asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '') " + "AND quote_asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '')") + self.assertIn(f"sumIf(volume_base, {no_contract})", sql) + self.assertIn(f"sumIf(volume_quote, {no_contract})", sql) + + def test_lower_scoped_volume_is_still_a_defect(self): + ms, _, err = self.run_reconcile([["sdex", "9", "1000", "39", "50"]]) + self.assertEqual(ms["result"]["verdict"], "DEFECT") + self.assertRegex(str(err), "reconciliation failed") + + def test_a_before_read_without_the_scope_compares_trades_not_volumes(self): + ms, _, err = self.run_reconcile([["sdex", "9", "1000", "39", "50"]], vol_scope=None) + self.assertIsNone(err) + self.assertEqual(ms["result"]["verdict"], "FINDING") + self.assertTrue(any("volumes are NOT compared" in n for n in r.DASH["notes"])) + + def test_trades_dropped_for_decimals_are_a_defect(self): + ms, _, err = self.run_reconcile(SUMS_COPIED, decimals=3) + self.assertEqual(ms["result"]["verdict"], "DEFECT") + self.assertTrue(any("3 AMM trades dropped" in n for n in r.DASH["notes"])) + + def test_the_summary_needs_the_decimals_line(self): + text = ("negative apply order: 0\nswaps dropped (unresolved):0\n" + "trades dropped (decimals): 2\n") + self.assertEqual(r.amm_summary(text, "log", dry_run=False), {"fallbacks": 0, "dropped": 0, "decimals": 2, "unproven_sacs": None}) + with self.assertRaisesRegex(r.Stop, "trades dropped"): + r.amm_summary(text.replace("trades dropped (decimals): 2\n", ""), "log", dry_run=False) + + def test_amm_done_carries_the_decimals_count(self): + st = state([202201], self.tmp.name) + r.cmd_amm_done(args("amm-done", "202201", "--fallbacks", "1", "--decimals", "40"), None, st) + marker = Path(self.tmp.name) / "202201" / "amm.done" + self.assertEqual(r.read_amm_done(marker), (1, 40)) + marker.write_text("0") # written before task 0329 + with self.assertRaisesRegex(r.Stop, "decimals"): + r.read_amm_done(marker) + def test_pre0139_reconciles_unrestricted(self): ms, ch, _ = self.run_reconcile(SUMS_COPIED, ids="u32", width="UInt32") self.assertEqual(ms["result"]["verdict"], "DEFECT") # +10 % on the unrestricted sums diff --git a/tools/scripts/reingest_0286_sac_test.py b/tools/scripts/reingest_0286_sac_test.py new file mode 100644 index 00000000..9eac77a9 --- /dev/null +++ b/tools/scripts/reingest_0286_sac_test.py @@ -0,0 +1,93 @@ +"""reingest_0286.py reads events-backfill's task-0242 `unproven sac swaps:` line. + +python3 -m unittest discover -s tools/scripts -p 'reingest_0286_sac_test.py' +""" + +import importlib.util +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +spec = importlib.util.spec_from_file_location("reingest_0286", HERE / "reingest_0286.py") +r = importlib.util.module_from_spec(spec) +spec.loader.exec_module(r) + +LOG = Path("/m/202609/events-backfill.log") + + +def summary(unproven=None, fallbacks=True): + """events-backfill's closing summary; `unproven=None` is a pre-0242 binary.""" + lines = ["=== events-backfill complete ===", + "swaps dropped (unresolved):0", + "swaps failed dispatch: 0", + "trades dropped (decimals): 0"] + if fallbacks: + lines.append("negative apply order: 0") + if unproven is not None: + lines.append(f"unproven sac swaps: {unproven}") + return "\n".join(lines) + "\n" + + +class UnprovenSacLine(unittest.TestCase): + def test_the_line_is_read_and_only_non_zero_stops(self): + for unproven, expected in ((0, 0), (None, None)): + with self.subTest(unproven=unproven): + found = r.amm_summary(summary(unproven), LOG, dry_run=True) + self.assertEqual(found["unproven_sacs"], expected) + self.assertEqual((found["fallbacks"], found["dropped"]), (0, 0)) + + def test_a_non_zero_count_stops_naming_it_and_what_the_month_holds(self): + # The dry run comes after drop_1m and sdex: never "nothing was written". + for dry_run, says in ((True, ("--dry-run pass", "no AMM candle was written", + "202609's 1m partition is already dropped", "SDEX-only", + "Fix it today", "`rollback 202609`")), + (False, ("write pass", "AMM candles are in without those swaps", + "`rollback 202609`"))): + with self.subTest(dry_run=dry_run), self.assertRaises(r.Stop) as cm: + r.amm_summary(summary(3), LOG, dry_run=dry_run) + msg = str(cm.exception) + for part in ("3 swaps", "task 0242 D2", str(LOG), + "docs/runbooks/0242-sac-identity-heal.md §4c") + says: + self.assertIn(part, msg) + self.assertNotIn("nothing was written", msg) + + def test_a_missing_line_stops_only_after_an_earlier_month_printed_it(self): + self.assertIsNone(r.amm_summary(summary(None), LOG, dry_run=True)["unproven_sacs"]) + with self.assertRaises(r.Stop) as cm: + r.amm_summary(summary(None), LOG, dry_run=True, unproven_line_seen=True) + msg = str(cm.exception) + for part in ("earlier month", "built before task 0242", str(LOG), "§4e"): + self.assertIn(part, msg) + self.assertEqual(r.amm_summary(summary(0), LOG, dry_run=True, unproven_line_seen=True) + ["unproven_sacs"], 0) + + def test_the_month_records_the_line_and_the_seen_test_reads_other_months(self): + class St: + d = {"months": {"202405": {"step": 9}}} + notes = [] + orig, r.note = r.note, notes.append + try: + ms = St.d["months"]["202405"] + r.record_amm_summary(ms, 202405, {"fallbacks": 0, "dropped": 0, "unproven_sacs": None, "decimals": 0}) + self.assertIsNone(ms["unproven_sacs"]) + self.assertTrue(any("built before task 0242" in n for n in notes), notes) + self.assertFalse(r.unproven_line_seen(St, 202406)) + r.record_amm_summary(ms, 202405, {"fallbacks": 0, "dropped": 0, "unproven_sacs": 0, "decimals": 0}) + self.assertTrue(r.unproven_line_seen(St, 202406)) + self.assertFalse(r.unproven_line_seen(St, 202405), "the month itself does not count") + finally: + r.note = orig + + def test_a_missing_required_line_still_stops(self): + with self.assertRaises(r.Stop) as cm: + r.amm_summary(summary(0, fallbacks=False), LOG, dry_run=True) + self.assertIn("negative apply order:", str(cm.exception)) + + def test_the_label_is_the_one_events_backfill_prints(self): + run_rs = (REPO / "packages/events-backfill/src/run.rs").read_text() + self.assertIn(f'UNPROVEN_SAC_LABEL: &str = "{r.UNPROVEN_SAC_LABEL}"', run_rs) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/scripts/verify-xdr-protocol-gap.mjs b/tools/scripts/verify-xdr-protocol-gap.mjs index 4ffa96ff..4a01fe13 100755 --- a/tools/scripts/verify-xdr-protocol-gap.mjs +++ b/tools/scripts/verify-xdr-protocol-gap.mjs @@ -61,11 +61,11 @@ // crates.io's newest `stellar-xdr` was 28.0.1 — the issue asked for a bump // nobody could make. So a LAGGING reading also asks crates.io: // -// WAITING no published stable major reaches core_supported. The protocol is +// WAITING no published stable major in (ours, core_supported]. The protocol is // announced, there is nothing to bump to yet. NOT fatal, even under // --watch: a red run and an issue are for something someone can // act on, and here nobody can. The report says so and exits 0. -// LAGGING a stable major that reaches core_supported is on crates.io. Bump +// LAGGING a stable major in (ours, core_supported] is on crates.io. Bump // now (after BE's xdr-parser). Fatal under --watch. // // BEHIND asks crates.io the same way (task 0325): mainnet on a protocol with no @@ -74,6 +74,13 @@ // that DOES change XDR and is voted before its crate ships stays green here; // that stall shows as frozen candles, which the rollup-freshness and // ledger-processor-no-invocations alarms catch within the hour. +// +// "A crate for protocol N" means a major ABOVE ours and NOT ABOVE N (task +// 0330). Upstream skipped 29: crates.io went 28.0.1 → 30.0.0, and 30.0.0 +// carries protocol 30's XDR. Reading only max_stable_version (30 ≥ 29) made +// the watch report BEHIND for a protocol 28 already decodes. Each protocol is +// asked separately: mainnet on crate-less 29 with core on 30 and a crate 30 +// published is LAGGING. // ponytail: compares protocol numbers; comparing the crate's XDR commit with // core's (`src/protocol-curr/xdr`) is the exact check if this misfires again. // @@ -199,20 +206,37 @@ const publishedCrate = async () => { }); if (!response.ok) throw new Error(`HTTP ${response.status}`); - const { crate } = await response.json(); + const { crate, versions } = await response.json(); const stable = crate?.max_stable_version; - const major = Number(String(stable).match(/^(\d+)\.\d+\.\d+$/)?.[1]); - if (!Number.isInteger(major)) - throw new Error(`unexpected crate document: max_stable_version=${stable}`); + if (!/^\d+\.\d+\.\d+$/.test(String(stable)) || !Array.isArray(versions)) + throw new Error( + `unexpected crate document: max_stable_version=${stable}, ` + + `versions=${Array.isArray(versions) ? versions.length : versions}`, + ); const newest = crate.max_version; return { stable, - major, preRelease: newest && newest !== stable ? newest : null, + releases: versions + .filter((v) => !v.yanked && /^\d+\.\d+\.\d+$/.test(v.num)) + .map((v) => v.num), }; }; +/** + * The newest stable release a bump for `protocol` could use: a major above + * our pin and NOT above `protocol`. A later major carries a later protocol's + * XDR (task 0330: upstream skipped 29, and 30.0.0 is protocol 30, not 29). + */ +const bumpTarget = (crate, protocol) => + crate.releases + .filter((v) => { + const major = Number(v.split('.')[0]); + return major > pinned && major <= protocol; + }) + .sort((a, b) => b.localeCompare(a, 'en', { numeric: true }))[0]; + // -------------------------------------------------------------------- run -- const pinned = pinnedMajor(); @@ -282,9 +306,14 @@ if (behind || pinned < network.supported) { process.exit(0); } + // Asked per protocol: mainnet can sit on one with no crate (29) while core + // already supports the next one, which has a crate (30). That is LAGGING. + const forMainnet = behind && bumpTarget(crate, network.current); + const forCore = bumpTarget(crate, network.supported); + // The `— WAITING: no stellar-xdr` wording is matched by master's // xdr-protocol-watch.yml to close an open issue as "not actionable yet". - if (crate.major < needed) { + if (!forMainnet && !forCore) { console.log( [ behind @@ -294,13 +323,21 @@ if (behind || pinned < network.supported) { ` ${summary} | newest on crates.io ${crate.stable}` + (crate.preRelease ? ` (pre-release ${crate.preRelease})` : ''), '', + ...(Number(crate.stable.split('.')[0]) > needed + ? [ + `stellar-xdr ${crate.stable} carries a later protocol's XDR, so it is not`, + `a bump target for ${needed}.`, + '', + ] + : []), ...(behind ? [ 'Mainnet has voted, but there is no crate to bump to: either the', 'protocol changed no XDR (29 kept the XDR of 28) or the crate is late.', 'A real decode wall shows as frozen candles — the rollup-freshness and', 'ledger-processor-no-invocations alarms. The watch fails and opens an', - `issue the day a stable stellar-xdr ${needed}.x appears on crates.io.`, + `issue the day a stable stellar-xdr ${needed}.x appears on crates.io,`, + 'or core supports a protocol whose crate is published.', ] : [ 'Nothing to do yet: there is no crate to bump to, and mainnet has not', @@ -312,11 +349,11 @@ if (behind || pinned < network.supported) { process.exit(0); } - if (behind) + if (forMainnet) die( `error: stellar-xdr ${pinned} is BEHIND mainnet protocol ${network.current}.`, '', - ` ${summary} | newest on crates.io ${crate.stable}`, + ` ${summary} | newest on crates.io ${crate.stable} | bump to ${forMainnet}`, '', 'This is the proto27 condition: the decode wall is live or one ledger', 'away, and a stalled ledger-processor drains its queue and logs nothing.', @@ -331,8 +368,8 @@ if (behind || pinned < network.supported) { '', ` ${summary} | newest on crates.io ${crate.stable}`, '', - `stellar-xdr ${crate.stable} is published — the bump is possible now.`, - 'Mainnet has not voted yet, so nothing is broken — this is the lead time.', + `stellar-xdr ${forCore} is published — the bump is possible now.`, + `Mainnet has not voted ${network.supported} yet, so nothing is broken — this is the lead time.`, 'Open the bump before the vote. BE must bump xdr-parser first, and check', "the crate's XDR commit, not only its version number.", ]; diff --git a/tools/scripts/verify-xdr-protocol-gap.test.mjs b/tools/scripts/verify-xdr-protocol-gap.test.mjs index 6591d372..2704b480 100644 --- a/tools/scripts/verify-xdr-protocol-gap.test.mjs +++ b/tools/scripts/verify-xdr-protocol-gap.test.mjs @@ -1,5 +1,5 @@ // Tier matrix for tools/scripts/verify-xdr-protocol-gap.mjs (tasks 0319, -// 0325), run against a local mock of Horizon and crates.io so it needs no +// 0325, 0330), run against a local mock of Horizon and crates.io so it needs no // network. Every case is relative to the real pin in Cargo.toml, so a // stellar-xdr bump needs no edit here. // @@ -45,6 +45,7 @@ const net = (current, supported) => ({ }); const published = (stable, max = stable) => ({ crate: { max_stable_version: stable, max_version: max }, + versions: [...new Set([max, stable])].map((num) => ({ num, yanked: false })), }); // Async, not spawnSync: the mock lives in this process and must keep serving. @@ -78,6 +79,11 @@ const cases = [ ['BEHIND, crate published, watch', net(p + 1, p + 1), published(`${p + 1}.0.0`), true, 1, 'is BEHIND mainnet protocol'], ['BEHIND, crate published, PR', net(p + 1, p + 1), published(`${p + 1}.0.0`), false, 1, 'is BEHIND mainnet protocol'], ['BEHIND, crate for mainnet but not core', net(p + 1, p + 2), published(`${p + 1}.0.0`), true, 1, 'is BEHIND mainnet protocol'], + ['voted, only a later crate (#398), watch', net(p + 1, p + 1), published(`${p + 2}.0.0`), true, 0, `— WAITING: no stellar-xdr ${p + 1}`], + ['voted, only a later crate (#398), PR', net(p + 1, p + 1), published(`${p + 2}.0.0`), false, 0, 'not\na bump target'], + ['before the vote, only a later crate', net(p, p + 1), published(`${p + 2}.0.0`), true, 0, '— WAITING: no stellar-xdr'], + ['voted without a crate, core next with one', net(p + 1, p + 2), published(`${p + 2}.0.0`), true, 1, `${p + 2}.0.0 is published — the bump is possible now`], + ['crates.io lists no versions, watch', net(p + 1, p + 1), { crate: { max_stable_version: `${p + 1}.0.0` } }, true, 1, 'error: cannot read'], ['voted, crates.io down, watch', net(p + 1, p + 1), null, true, 1, 'error: cannot read'], ['voted, crates.io down, PR', net(p + 1, p + 1), null, false, 0, 'crate availability not checked'], ['Horizon down, watch', null, published(`${p}.0.1`), true, 1, 'error: cannot reach'], diff --git a/web/portal/index.html b/web/portal/index.html index 8690510f..d81fd0a7 100644 --- a/web/portal/index.html +++ b/web/portal/index.html @@ -45,15 +45,15 @@ Stellar Prices API — API keys