From 7da65ce88f8df2cd473d959ca7dfe10895a2cd1c Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:06:16 +0200 Subject: [PATCH 001/144] fix(lore-0242): resolve a SAC from its in-band proof and never mint an unproven one A SAC leg was minted as a Contract identity whenever its classic asset had not been interned yet, giving one token two asset_ids. learn_sac accepts a contract -> SEP-11 mapping only when the sha256 SAC derivation reproduces the address. The live path feeds it from the transaction's own transfer/mint/burn/clawback events before the swaps are classified. A trade whose leg BE flags is_sac but carries no proof is skipped, interns nothing and is counted as SacUnprovenSkipped. The is_sac set is a fourth cold-start read and fails Init on error. --- packages/prices-ingest-core/src/canonical.rs | 112 +++++ packages/prices-ingest-core/src/lib.rs | 3 + packages/prices-ingest-core/src/sac_proof.rs | 82 ++++ packages/prices-ingest-core/src/soroban.rs | 90 +++- .../prices-ledger-processor/src/bin/cli.rs | 4 + packages/prices-ledger-processor/src/main.rs | 21 +- .../prices-ledger-processor/src/metrics.rs | 34 ++ .../prices-ledger-processor/src/reconcile.rs | 24 ++ .../prices-ledger-processor/src/sink/mod.rs | 39 ++ .../tests/sac_identity.rs | 395 ++++++++++++++++++ 10 files changed, 796 insertions(+), 8 deletions(-) create mode 100644 packages/prices-ingest-core/src/sac_proof.rs create mode 100644 packages/prices-ledger-processor/tests/sac_identity.rs diff --git a/packages/prices-ingest-core/src/canonical.rs b/packages/prices-ingest-core/src/canonical.rs index 02eae991..35191757 100644 --- a/packages/prices-ingest-core/src/canonical.rs +++ b/packages/prices-ingest-core/src/canonical.rs @@ -118,6 +118,23 @@ fn identity_to_asset(identity: &AssetIdentity) -> Option { } } +/// A SEP-11 asset string as a SAC event's last topic carries it: `native`, or +/// `CODE:ISSUER` with a 1–12 char alphanumeric code and a valid G-strkey. +fn parse_sep11(s: &str) -> Option { + if s == "native" { + return Some(AssetIdentity::Native); + } + let (code, issuer) = s.split_once(':')?; + if code.is_empty() || code.len() > 12 || !code.bytes().all(|b| b.is_ascii_alphanumeric()) { + return None; + } + stellar_strkey::ed25519::PublicKey::from_string(issuer).ok()?; + Some(AssetIdentity::Credit { + code: code.to_string(), + issuer: issuer.to_string(), + }) +} + /// The deterministic Stellar Asset Contract (SAC) address for a classic asset on /// the given network: `C…`-strkey of `sha256(HashIdPreimage::ContractId{network, /// ContractIdPreimage::Asset(asset)})`. This is the standard offline derivation — @@ -168,6 +185,9 @@ pub struct AssetRegistry { /// intern order. Replaces the id watermark (task 0139): a set of /// identities says "new" without assuming ids are handed out in order. pending: Vec, + /// Contracts BE flags `is_sac` (task 0242). One of these with no entry in + /// `sac_index` is never minted as a `Contract` identity. + sac_candidates: HashSet, } impl AssetRegistry { @@ -180,6 +200,7 @@ impl AssetRegistry { network_id: mainnet_network_id(), sac_index: HashMap::new(), pending: Vec::new(), + sac_candidates: HashSet::new(), }; // Pre-seed the canonical quote SACs so an AMM-via-SAC USDC/USDT/XLM // collapses even before that asset's first classic (SDEX) sighting in the @@ -232,6 +253,34 @@ impl AssetRegistry { self.sac_index.get(contract_addr).cloned() } + /// Map `contract` to the SEP-11 asset it names (`native` | `CODE:G…`), but + /// only when `contract` IS that asset's SAC (task 0242). Interns nothing. + pub fn learn_sac(&mut self, contract: &str, sep11: &str) -> bool { + let Some(identity) = parse_sep11(sep11) else { + return false; + }; + if self.sac_index.get(contract) == Some(&identity) { + return true; + } + let derived = identity_to_asset(&identity).and_then(|a| sac_address(&a, &self.network_id)); + if derived.as_deref() != Some(contract) { + return false; + } + self.sac_index.insert(contract.to_string(), identity); + true + } + + /// The contracts BE flags `is_sac` (task 0242), replacing any earlier set. + pub fn set_sac_candidates(&mut self, contracts: HashSet) { + self.sac_candidates = contracts; + } + + /// True for an `is_sac` contract this registry cannot resolve: its trades + /// are skipped, never minted as a `Contract` identity (task 0242 D2). + pub fn is_unproven_sac(&self, contract: &str) -> bool { + self.sac_candidates.contains(contract) && !self.sac_index.contains_key(contract) + } + /// The SAC contract address that wraps a classic identity (Native / Credit), /// or `None` for a pure `Contract` identity (no classic underlying). Persisted /// onto `prices.assets.sac_address` so a read-time consumer can resolve a @@ -417,6 +466,69 @@ mod tests { ); } + // Task 0242 fixture: XCR and its SAC, public on-chain identifiers. + const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; + const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + + fn xcr() -> AssetIdentity { + AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + } + } + + #[test] + fn learn_sac_accepts_only_the_sac_of_the_named_asset() { + let mut reg = AssetRegistry::from_existing(vec![]); + assert_eq!(reg.sac_address_of(&xcr()).as_deref(), Some(XCR_SAC)); + let known = reg.assets().count(); + + assert!(reg.learn_sac(XCR_SAC, &format!("XCR:{XCR_ISSUER}"))); + assert_eq!(reg.resolve_sac(XCR_SAC), Some(xcr())); + assert!( + reg.learn_sac(XCR_SAC, &format!("XCR:{XCR_ISSUER}")), + "idempotent" + ); + assert_eq!(reg.assets().count(), known, "interns nothing"); + assert!(pending(®).is_empty()); + + assert!(reg.learn_sac(NATIVE_SAC, "native")); + assert!(!reg.learn_sac(XCR_SAC, "native")); + } + + #[test] + fn learn_sac_rejects_impostors_and_malformed_assets() { + let usdc = format!("USDC:{USDC_ISSUER}"); + let impostor = stellar_strkey::Contract([7; 32]).to_string(); + let cases = [ + (impostor.as_str(), usdc.clone()), + (XCR_SAC, usdc), + (XCR_SAC, String::new()), + (XCR_SAC, "XCR".to_string()), + (XCR_SAC, "XCR:".to_string()), + (XCR_SAC, format!(":{XCR_ISSUER}")), + (XCR_SAC, "XCR:GNOTAKEY".to_string()), + (XCR_SAC, format!("ABCDEFGHIJKLM:{XCR_ISSUER}")), + (XCR_SAC, format!("XC-R:{XCR_ISSUER}")), + (XCR_SAC, "NATIVE".to_string()), + ]; + for (contract, sep11) in cases { + let mut reg = AssetRegistry::from_existing(vec![]); + assert!(!reg.learn_sac(contract, &sep11), "{contract} {sep11:?}"); + assert_eq!(reg.resolve_sac(contract), None, "{contract} {sep11:?}"); + } + } + + #[test] + fn an_sac_candidate_is_unproven_until_learned() { + let mut reg = AssetRegistry::from_existing(vec![]); + reg.set_sac_candidates(HashSet::from([XCR_SAC.to_string()])); + assert!(reg.is_unproven_sac(XCR_SAC)); + assert!(!reg.is_unproven_sac(NATIVE_SAC), "not a candidate"); + reg.learn_sac(XCR_SAC, &format!("XCR:{XCR_ISSUER}")); + assert!(!reg.is_unproven_sac(XCR_SAC)); + } + fn pending(reg: &AssetRegistry) -> Vec { reg.pending_new().cloned().collect() } diff --git a/packages/prices-ingest-core/src/lib.rs b/packages/prices-ingest-core/src/lib.rs index 5b659e4f..54370c1b 100644 --- a/packages/prices-ingest-core/src/lib.rs +++ b/packages/prices-ingest-core/src/lib.rs @@ -17,6 +17,7 @@ //! ([`STATIC_POOLS`], task 0300) merged into the AMM registries. //! - [`canonical`] — asset identity, the [`AssetRegistry`] of known and newly //! seen identities, and `(base, quote)` canonicalisation. +//! - [`sac_proof`] — the BE reads behind the SAC resolver (task 0242). //! - [`price`] / [`tick`] — per-trade price + the [`TradeTick`] the bucketer eats. //! - [`bucket`] — 1-minute OHLCV accumulation ([`CandleAccumulator`]). //! - [`writer`] — the transport-agnostic ClickHouse [`OhlcvWriter`] (works with a @@ -32,6 +33,7 @@ pub mod filter; pub mod price; pub mod registry_io; pub mod retry; +pub mod sac_proof; pub mod safe_log; pub mod soroban; pub mod static_pools; @@ -52,6 +54,7 @@ pub use price::{ }; pub use registry_io::PoolRegistryRow; pub use retry::{DEFAULT_BACKOFF_MS, retry_with_backoff}; +pub use sac_proof::{BE_DATABASE, load_sac_contracts}; pub use safe_log::safe_response_token; pub use soroban::{ LedgerSoroban, RawSorobanEvent, Registries, UnresolvedPoolSwap, learn_factory_event, diff --git a/packages/prices-ingest-core/src/sac_proof.rs b/packages/prices-ingest-core/src/sac_proof.rs new file mode 100644 index 00000000..d443e029 --- /dev/null +++ b/packages/prices-ingest-core/src/sac_proof.rs @@ -0,0 +1,82 @@ +//! BE reads behind the SAC resolver (task 0242): which contracts BE flags as a +//! Stellar Asset Contract. A flagged contract no proof resolves is skipped, not +//! minted as a `Contract` identity (D2). +//! +//! No `SETTINGS` in any statement here: a read that crosses the client's GET +//! length is sent as POST with `readonly=1`, which rejects per-query settings +//! (code 164; see events-backfill `source.rs`). + +use std::collections::HashSet; + +use crate::error::IngestError; + +/// BE's database, holding `soroban_contracts` and `soroban_events`. +pub const BE_DATABASE: &str = "default"; +/// Ours, holding `assets`. +pub const PRICES_DATABASE: &str = prices_clickhouse::PROD_DATABASE; + +/// `^[A-Za-z_][A-Za-z0-9_]*$`: a bare identifier, safe to splice unquoted. +fn identifier(s: &str) -> Result<&str, IngestError> { + let mut chars = s.chars(); + let ok = matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); + if ok { + Ok(s) + } else { + Err(IngestError::Precondition(format!( + "not a database identifier: {s:?}" + ))) + } +} + +/// Every contract BE flags `is_sac`, once each. +/// +/// No `FINAL`, on purpose. `soroban_contracts` is a +/// `ReplacingMergeTree(wasm_uploaded_at_ledger)`, so a contract BE re-inserts +/// with `is_sac = false` stays in this set until BE's parts merge. That errs +/// toward D2: its trades are skipped and counted, never minted. `FINAL` would +/// merge BE's whole table on every live cold start, a read that fails Init +/// (PC3), to shorten a case that is rare because `is_sac` is fixed at deploy. +pub(crate) fn sac_contracts_sql(be_db: &str) -> Result { + let be = identifier(be_db)?; + Ok(format!( + "SELECT DISTINCT contract_id FROM {be}.soroban_contracts WHERE is_sac" + )) +} + +/// Load the `is_sac` contract set (about 4,000 addresses on production). +pub async fn load_sac_contracts( + client: &clickhouse::Client, + be_db: &str, +) -> Result, IngestError> { + let rows = client + .query(&sac_contracts_sql(be_db)?) + .fetch_all::() + .await?; + Ok(rows.into_iter().collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sac_contracts_sql_reads_bes_is_sac_set_without_settings() { + let sql = sac_contracts_sql("default").unwrap(); + assert!(sql.contains("FROM default.soroban_contracts"), "{sql}"); + assert!(sql.contains("WHERE is_sac"), "{sql}"); + assert!(!sql.contains("SETTINGS"), "{sql}"); + assert!(!sql.contains("FINAL"), "{sql}"); + } + + #[test] + fn a_non_identifier_database_is_refused() { + for name in ["", "1db", "default; DROP", "de.fault", "db`"] { + assert!( + matches!(sac_contracts_sql(name), Err(IngestError::Precondition(_))), + "{name:?}" + ); + } + assert!(sac_contracts_sql("it_0242_be").is_ok()); + } +} diff --git a/packages/prices-ingest-core/src/soroban.rs b/packages/prices-ingest-core/src/soroban.rs index 7181884c..505bdfaf 100644 --- a/packages/prices-ingest-core/src/soroban.rs +++ b/packages/prices-ingest-core/src/soroban.rs @@ -195,6 +195,10 @@ pub struct LedgerSoroban { /// re-read ledger and in callers that never read the count, so it only /// logs at debug level. pub unregistered_pool_contracts: Vec, + /// One entry per trade skipped because a leg is a contract BE flags + /// `is_sac` that no SAC proof resolved, holding that leg's address (task + /// 0242 D2). Minting it as a `Contract` identity would split its asset. + pub sac_unproven: Vec, } fn collect_tx_metas(lcm: &LedgerCloseMeta) -> Vec<&TransactionMeta> { @@ -301,6 +305,37 @@ fn signature(topics: &Value) -> Option<&str> { topic_symbol(topics, 0) } +/// The SEP-11 asset a SAC names in its own `transfer`/`mint`/`burn`/`clawback` +/// event: the last topic, when it is a string (task 0242). Only a claim until +/// [`AssetRegistry::learn_sac`] checks it against the emitter. +fn sac_proof(topics: &Value) -> Option<&str> { + if !matches!( + signature(topics), + Some("transfer" | "mint" | "burn" | "clawback") + ) { + return None; + } + let last = topics.as_array()?.last()?.as_object()?; + match last.get("type")?.as_str()? { + "string" | "str" => last.get("value")?.as_str(), + _ => None, + } +} + +/// Feed every SAC proof among one transaction's events to the registry, before +/// its trades are classified, so event order inside the transaction does not +/// matter (task 0242 D1). +fn learn_sacs<'a>( + events: impl IntoIterator, + assets: &mut AssetRegistry, +) { + for (contract_id, topics) in events { + if let Some(sep11) = sac_proof(topics) { + assets.learn_sac(contract_id, sep11); + } + } +} + /// Process all Soroban events in one ledger. pub fn process_ledger( lcm: &LedgerCloseMeta, @@ -346,6 +381,13 @@ pub fn process_ledger( } } + learn_sacs( + events + .iter() + .filter(|ev| ev.source != EventSource::Diagnostic) + .filter_map(|ev| Some((ev.contract_id.as_deref()?, &ev.topics))), + assets, + ); classify_amm_groups( amm_groups, tx_index as u16, @@ -469,6 +511,12 @@ pub fn process_soroban_event_rows( .all(|e| e.transaction_index == transaction_index), "events of transaction {tx_id} disagree on transaction_index" ); + learn_sacs( + events[tx_start..tx_end] + .iter() + .map(|ev| (ev.contract_id.as_str(), &ev.topics)), + assets, + ); classify_amm_groups( amm_groups, transaction_index, @@ -565,6 +613,14 @@ fn classify_amm_groups( match dispatch(&rows, ®.venue, ®.phoenix, reg.pair_registries()) { Ok(trades) => { for t in trades { + // Before `canonicalise`, so a skipped trade interns nothing. + if let Some(sac) = [&t.token_in, &t.token_out] + .into_iter() + .find(|c| assets.is_unproven_sac(c)) + { + out.sac_unproven.push(sac.clone()); + continue; + } if let Some(tick) = amm_trade_to_tick(&t, transaction_index, closed_at, assets) { out.amm_ticks.push((source, tick)); @@ -818,7 +874,8 @@ fn first_address(v: &Value) -> Option { /// Resolve an AMM token (contract address) to a canonical `AssetIdentity`: the /// underlying classic asset if the address is a known SAC (§12.4), else the -/// `Contract` identity for a pure Soroban token. +/// `Contract` identity for a pure Soroban token. Only tokens BE does not flag +/// `is_sac` reach the fallback: an unproven SAC is skipped earlier (task 0242). fn resolve_amm_token(contract_addr: &str, assets: &AssetRegistry) -> AssetIdentity { assets .resolve_sac(contract_addr) @@ -1003,6 +1060,37 @@ mod tests { } } + #[test] + fn sac_proof_reads_a_string_last_topic_of_a_sac_signature() { + // Task 0242: transfer/mint/burn/clawback carry the SEP-11 asset last. + let topics = |sig: &str, last: Value| { + json!([ + {"type":"sym","value":sig}, + {"type":"address","value":"GFROM"}, + last + ]) + }; + let s = |v: &str| json!({"type":"string","value":v}); + let cases: [(&str, Value, Option<&str>); 8] = [ + ("transfer", s("XCR:G"), Some("XCR:G")), + ("mint", s("native"), Some("native")), + ("burn", s("XCR:G"), Some("XCR:G")), + ( + "clawback", + json!({"type":"str","value":"XCR:G"}), + Some("XCR:G"), + ), + ("swap", s("XCR:G"), None), + ("approve", s("XCR:G"), None), + ("transfer", json!({"type":"address","value":"GTO"}), None), + ("transfer", json!({"type":"sym","value":"XCR"}), None), + ]; + for (sig, last, want) in cases { + assert_eq!(sac_proof(&topics(sig, last.clone())), want, "{sig} {last}"); + } + assert_eq!(sac_proof(&json!([])), None, "empty topics"); + } + #[test] fn signature_reads_topic0_symbol() { let topics = json!([{"type":"sym","value":"REFLECTOR"},{"type":"sym","value":"update"}]); diff --git a/packages/prices-ledger-processor/src/bin/cli.rs b/packages/prices-ledger-processor/src/bin/cli.rs index 917adcd9..cc800159 100644 --- a/packages/prices-ledger-processor/src/bin/cli.rs +++ b/packages/prices-ledger-processor/src/bin/cli.rs @@ -83,6 +83,10 @@ async fn main() -> Result<(), Box> { } else { let sink = ClickHouseSink::plaintext(&args.clickhouse_url); sink.preflight().await?; + // `load_registry`, not the Lambda's `load_ingest_registry`, by design: a + // local ClickHouse has no BE `default.soroban_contracts`, so this registry + // has no `is_sac` candidates and still mints an unproven SAC as a + // `Contract` identity (task 0242 D2 applies to the Lambda only). let registry = sink.load_registry().await?; let pool_registry = sink.load_pool_registry().await?; let reconciler = Reconciler::new(fetcher, cursor, sink, registry, pool_registry); diff --git a/packages/prices-ledger-processor/src/main.rs b/packages/prices-ledger-processor/src/main.rs index efab6fc3..8d6209fb 100644 --- a/packages/prices-ledger-processor/src/main.rs +++ b/packages/prices-ledger-processor/src/main.rs @@ -67,16 +67,21 @@ async fn main() -> Result<(), Error> { let sink = sink?; let fetcher = S3Fetcher::new(aws_sdk_s3::Client::new(&aws_cfg), &bucket); - // Three independent ClickHouse reads on the cold path — the durable cursor - // (task 0064) plus the two registries — joined so they overlap instead of - // running serially (same reasoning as the fetcher+sink join above). All use - // the sink's shared client; clients are cheap to clone and pool-backed, so - // concurrent queries are safe. + // Four independent ClickHouse reads on the cold path — the durable cursor + // (task 0064), the two registries and BE's SAC set — joined so they overlap + // instead of running serially (same reasoning as the fetcher+sink join + // above). All use the sink's shared client; clients are cheap to clone and + // pool-backed, so concurrent queries are safe. `load_ingest_registry` joins + // the asset registry with BE's SAC set itself. // // - cursor.read(): last processed ledger from `prices.ingest_cursor`. Durable, // so the loop resumes across execution-environment recycles instead of // rewinding to INITIAL_CURSOR every cold start (the freeze this task fixes). - // - load_registry: the identities already in `prices.assets`. + // - load_ingest_registry: the identities already in `prices.assets`, armed + // with the contracts BE flags `is_sac` in `default.soroban_contracts`, read + // with prices_writer's SELECT on that table (task 0242). A flagged token no + // SAC proof resolves is skipped, not minted, so a failed read fails Init: an + // empty set would mint (D2). `tests/sac_identity.rs` pins this call. // - load_pool_registry: discovered AMM pool classification from // `prices.pool_registry` (task 0078). The processor writes back only the // pools it learns from factory events (task 0291; `prices_writer` holds @@ -89,7 +94,7 @@ async fn main() -> Result<(), Error> { let cursor = ClickHouseCursor::new(sink.client().clone(), CURSOR_ID); let (cursor_state, registry, pool_registry) = tokio::join!( cursor.read(), - sink.load_registry(), + sink.load_ingest_registry(), sink.load_pool_registry(), ); let registry = registry?; @@ -235,6 +240,8 @@ async fn handler( m.extend(metrics::unregistered_pool_event_metrics( stats.unregistered_pool_events.values().sum(), )); + // Task 0242 — trades skipped on a SAC no proof resolved. + m.extend(metrics::sac_unproven_metrics(stats.sac_unproven_skipped)); if let Err(e) = metrics::publish(&cw, &env_name, &m).await { warn!(error = %e, "cloudwatch metric publish failed (non-fatal)"); } diff --git a/packages/prices-ledger-processor/src/metrics.rs b/packages/prices-ledger-processor/src/metrics.rs index 6aade4f1..712be106 100644 --- a/packages/prices-ledger-processor/src/metrics.rs +++ b/packages/prices-ledger-processor/src/metrics.rs @@ -63,6 +63,15 @@ pub const FORCED_PARTIAL_FLUSH: &str = "ForcedPartialFlushes"; /// factory, or a factory event missed before a cold start. pub const UNREGISTERED_POOL_EVENTS: &str = "UnregisteredPoolEvents"; +/// Trades a reconcile run skipped because a leg is a contract BE flags `is_sac` +/// that no SAC proof in its transaction resolved (task 0242 D2). Minting it as +/// a `Contract` identity would split one asset in two, so the trade is dropped. +/// +/// Emitted ONLY when non-zero, so the alarm on it is `>= 1` over +/// `NOT_BREACHING`. Non-zero means a SAC traded without a transfer naming its +/// asset in the same transaction, a shape the resolver does not cover. +pub const SAC_UNPROVEN_SKIPPED: &str = "SacUnprovenSkipped"; + /// `PutMetricData` accepts at most 150 entries in a datum's `Values` array, so /// a run with more INSERTs than that spills into further datums of the same /// metric rather than being truncated (or, worse, aggregated back into @@ -170,6 +179,20 @@ pub fn unregistered_pool_event_metrics(total: u64) -> Vec { }] } +/// The unproven-SAC datapoint for one run (task 0242), or nothing when the run +/// skipped none. Same absent-while-healthy rule as +/// [`forced_partial_flush_metrics`]. +pub fn sac_unproven_metrics(total: u64) -> Vec { + if total == 0 { + return Vec::new(); + } + vec![Metric { + name: SAC_UNPROVEN_SKIPPED, + unit: Unit::Count, + values: vec![total as f64], + }] +} + /// Publish `metrics` to CloudWatch under [`METRIC_NAMESPACE`], tagged with an /// `Environment` dimension. One `PutMetricData` call for the whole batch. /// @@ -251,6 +274,17 @@ mod tests { assert_eq!(m[0].values, vec![7.0]); } + /// Task 0242: absent while healthy, one count of the total otherwise. + #[test] + fn unproven_sac_skips_publish_only_when_non_zero() { + assert!(sac_unproven_metrics(0).is_empty()); + let m = sac_unproven_metrics(3); + assert_eq!(m.len(), 1); + assert_eq!(m[0].name, SAC_UNPROVEN_SKIPPED); + assert_eq!(m[0].unit, Unit::Count); + assert_eq!(m[0].values, vec![3.0]); + } + #[test] fn a_forced_run_publishes_exactly_one_count_of_one() { let m = forced_partial_flush_metrics(true); diff --git a/packages/prices-ledger-processor/src/reconcile.rs b/packages/prices-ledger-processor/src/reconcile.rs index 6d57c99c..2a3e41a1 100644 --- a/packages/prices-ledger-processor/src/reconcile.rs +++ b/packages/prices-ledger-processor/src/reconcile.rs @@ -116,6 +116,10 @@ pub struct RunStats { /// missing from the registry — dropped, not priced (task 0291). Counted over /// the minutes this run WROTE, like `offer_lookups`. Empty when healthy. pub unregistered_pool_events: BTreeMap<&'static str, u64>, + /// Trades skipped because a leg is an `is_sac` contract no SAC proof + /// resolved (task 0242 D2). Counted over the minutes this run WROTE, like + /// `unregistered_pool_events`. 0 when healthy. + pub sac_unproven_skipped: u64, /// Candle-INSERT latency for this run, or `None` when the run wrote no /// candles at all. `None` rather than a zeroed struct so an idle run /// publishes no `ClickHouseWriteLatencyMs` datapoint instead of a 0 ms one @@ -256,6 +260,9 @@ where // minute closes. let mut ledger_unregistered: Vec<(u32, &'static str, u32)> = Vec::new(); let mut ledger_unregistered_contracts: Vec<(u32, String)> = Vec::new(); + // (minute_start, contract) per trade skipped on an unproven SAC (task + // 0242), per ledger for the same reason. + let mut ledger_sac_unproven: Vec<(u32, String)> = Vec::new(); for _ in 0..max_iterations { let next = current + 1; @@ -300,6 +307,7 @@ where .iter() .map(|c| (minute, c.clone())), ); + ledger_sac_unproven.extend(sob.sac_unproven.into_iter().map(|c| (minute, c))); obj_max = obj_max.max(seq); } // Flag only this object's HIGHEST ledger as a valid cursor landing @@ -326,6 +334,7 @@ where rows_emitted: 0, pools_persisted: 0, unregistered_pool_events: BTreeMap::new(), + sac_unproven_skipped: 0, // Nothing was persisted, so no INSERT happened: no datapoint. ch_write: None, }); @@ -486,6 +495,7 @@ where // Pools are dimension rows written above, like the assets. pools_persisted, unregistered_pool_events: BTreeMap::new(), + sac_unproven_skipped: 0, ch_write: None, }); }; @@ -566,6 +576,19 @@ where "dropped trades from pools missing from prices.pool_registry (task 0291)" ); } + let sac_unproven: Vec<&str> = ledger_sac_unproven + .iter() + .filter(|(minute, _)| *minute < flush_boundary) + .map(|(_, c)| c.as_str()) + .collect(); + if !sac_unproven.is_empty() { + let contracts: BTreeSet<&str> = sac_unproven.iter().copied().collect(); + tracing::warn!( + skipped = sac_unproven.len(), + ?contracts, + "skipped trades on is_sac contracts with no SAC proof (task 0242)" + ); + } info!( start, end = current, @@ -590,6 +613,7 @@ where rows_emitted, pools_persisted, unregistered_pool_events, + sac_unproven_skipped: sac_unproven.len() as u64, ch_write: (!ch_write.samples_ms.is_empty()).then_some(ch_write), }) } diff --git a/packages/prices-ledger-processor/src/sink/mod.rs b/packages/prices-ledger-processor/src/sink/mod.rs index c435da3d..05e3f24a 100644 --- a/packages/prices-ledger-processor/src/sink/mod.rs +++ b/packages/prices-ledger-processor/src/sink/mod.rs @@ -8,6 +8,7 @@ //! shared Hetzner cluster over mTLS via the task-0052 client. Tests use the //! in-memory [`CountingSink`]. +use std::collections::HashSet; use std::future::Future; use prices_ingest_core::{ @@ -15,6 +16,20 @@ use prices_ingest_core::{ Registries, retry_with_backoff, }; +/// `registry` with `sac_contracts` as its `is_sac` candidates (task 0242 D2): +/// the step [`ClickHouseSink::load_ingest_registry`] adds to a plain load. +pub fn with_sac_candidates( + mut registry: AssetRegistry, + sac_contracts: HashSet, +) -> AssetRegistry { + tracing::info!( + sac_contracts = sac_contracts.len(), + "loaded the is_sac contract set" + ); + registry.set_sac_candidates(sac_contracts); + registry +} + #[derive(Debug, thiserror::Error)] pub enum SinkError { #[error("sink write failed: {0}")] @@ -104,6 +119,30 @@ impl ClickHouseSink { Ok(AssetRegistry::from_existing(existing)) } + /// The contracts BE flags `is_sac` (task 0242), for + /// [`AssetRegistry::set_sac_candidates`]. Separate from [`load_registry`]: + /// the local CLI has no BE tables, so `prices-cli` never calls it and still + /// mints an unproven SAC as a `Contract` identity. Local fixtures only. + /// + /// [`load_registry`]: ClickHouseSink::load_registry + pub async fn load_sac_contracts(&self) -> Result, SinkError> { + prices_ingest_core::load_sac_contracts(self.client(), prices_ingest_core::BE_DATABASE) + .await + .map_err(redact) + } + + /// The registry the live processor ingests with: [`load_registry`] armed + /// with BE's `is_sac` set by [`with_sac_candidates`], the two reads joined. + /// Either failing fails the cold start: with no candidates an unproven SAC + /// would be minted as a `Contract` identity (task 0242 D2, PC3). + /// + /// [`load_registry`]: ClickHouseSink::load_registry + pub async fn load_ingest_registry(&self) -> Result { + let (registry, sac_contracts) = + tokio::join!(self.load_registry(), self.load_sac_contracts()); + Ok(with_sac_candidates(registry?, sac_contracts?)) + } + /// Preload the discovered AMM pool registry from `prices.pool_registry` so the /// live processor resolves pools created before the cursor start (task 0078) /// — otherwise it only knows pools whose factory events appear in its own live diff --git a/packages/prices-ledger-processor/tests/sac_identity.rs b/packages/prices-ledger-processor/tests/sac_identity.rs new file mode 100644 index 00000000..2bb3e123 --- /dev/null +++ b/packages/prices-ledger-processor/tests/sac_identity.rs @@ -0,0 +1,395 @@ +//! Task 0242: a SAC never becomes a second asset identity on the live path. +//! +//! A SAC's own `transfer`/`mint`/`burn`/`clawback` names its asset (SEP-11) in +//! the last topic. `process_ledger` checks that claim against the emitter's +//! address and resolves the SAC to its classic identity, wherever the event sits +//! in the transaction. A contract BE flags `is_sac` that no proof resolves is +//! skipped and counted, never minted as a `Contract` identity. +//! +//! The ledgers are synthetic, like `pool_registry_persist.rs`, so these run +//! everywhere. + +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +use extractors_core::Venue; +use prices_ingest_core::{ + AssetIdentity, AssetRegistry, LedgerSoroban, OhlcvCandle, OracleSample, PoolRegistryRow, + Registries, process_ledger, +}; +use prices_ledger_processor::{ + cursor::{Cursor, StubFileCursor}, + galexie_key::ledger_s3_key, + object_fetcher::{FetchError, ObjectFetcher}, + reconcile::Reconciler, + sink::{CandleSink, SinkError, with_sac_candidates}, +}; +use stellar_xdr::{ + ContractEvent, ContractEventBody, ContractEventType, ContractEventV0, ContractId, Hash, + Int128Parts, LedgerCloseMeta, LedgerCloseMetaBatch, LedgerCloseMetaV2, Limits, OperationMetaV2, + ScAddress, ScString, ScSymbol, ScVal, TimePoint, TransactionMeta, TransactionMetaV4, + TransactionResultMetaV1, WriteXdr, +}; +use tempfile::tempdir; + +const FIRST: u64 = 70_000_000; +const BASE_MINUTE: u64 = 1_800_000_000 / 60 * 60; + +// Public on-chain identifiers: XCR, its SAC, and the XLM SAC. +const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; +const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; +const XLM_SAC: &str = "CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA"; + +const POOL: [u8; 32] = [0x55; 32]; +const IMPOSTOR: [u8; 32] = [0x77; 32]; +const GENUINE_TOKEN: [u8; 32] = [0x22; 32]; + +fn xcr() -> AssetIdentity { + AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + } +} + +fn xcr_sep11() -> String { + format!("XCR:{XCR_ISSUER}") +} + +fn strkey(bytes: [u8; 32]) -> String { + ContractId(Hash(bytes)).to_string() +} + +fn cid(strkey: &str) -> ContractId { + ContractId::from_str(strkey).unwrap() +} + +fn addr(strkey: &str) -> ScVal { + ScVal::Address(ScAddress::Contract(cid(strkey))) +} + +fn sym(s: &str) -> ScVal { + ScVal::Symbol(ScSymbol(s.try_into().unwrap())) +} + +fn i128v(lo: u64) -> ScVal { + ScVal::I128(Int128Parts { hi: 0, lo }) +} + +fn event(emitter: &str, topics: Vec, data: ScVal) -> ContractEvent { + ContractEvent { + ext: Default::default(), + contract_id: Some(cid(emitter)), + type_: ContractEventType::Contract, + body: ContractEventBody::V0(ContractEventV0 { + topics: topics.try_into().unwrap(), + data, + }), + } +} + +/// A SAC-shaped `transfer` from `emitter`, naming `sep11` in its last topic. +fn sac_transfer(emitter: &str, sep11: &str) -> ContractEvent { + event( + emitter, + vec![ + sym("transfer"), + addr(&strkey([0x66; 32])), + addr(&strkey(POOL)), + ScVal::String(ScString(sep11.try_into().unwrap())), + ], + i128v(10_000_000), + ) +} + +/// An Aquarius pool `trade` on [`POOL`]: `token_in` sold for `token_out`. +fn trade(token_in: &str, token_out: &str) -> ContractEvent { + event( + &strkey(POOL), + vec![ + sym("trade"), + addr(token_in), + addr(token_out), + addr(&strkey([0x66; 32])), + ], + ScVal::Vec(Some( + vec![i128v(10_000_000), i128v(20_000_000), i128v(3_000)] + .try_into() + .unwrap(), + )), + ) +} + +/// One ledger whose single transaction carries `events`, in order. +fn lcm_with(seq: u64, close_time: u64, events: Vec) -> LedgerCloseMeta { + let mut v2 = LedgerCloseMetaV2::default(); + v2.ledger_header.header.ledger_seq = seq as u32; + v2.ledger_header.header.scp_value.close_time = TimePoint(close_time); + if !events.is_empty() { + let op = OperationMetaV2 { + events: events.try_into().unwrap(), + ..Default::default() + }; + let meta = TransactionMetaV4 { + operations: vec![op].try_into().unwrap(), + ..Default::default() + }; + v2.tx_processing = vec![TransactionResultMetaV1 { + tx_apply_processing: TransactionMeta::V4(meta), + ..Default::default() + }] + .try_into() + .unwrap(); + } + LedgerCloseMeta::V2(v2) +} + +/// The same ledger, encoded as Galexie writes it. +fn ledger_with(seq: u64, close_time: u64, events: Vec) -> Vec { + let batch = LedgerCloseMetaBatch { + start_sequence: seq as u32, + end_sequence: seq as u32, + ledger_close_metas: vec![lcm_with(seq, close_time, events)].try_into().unwrap(), + }; + let xdr = batch.to_xdr(Limits::none()).unwrap(); + zstd::encode_all(&xdr[..], 0).unwrap() +} + +fn pool_registries() -> Registries { + let mut reg = Registries::new(); + reg.venue.insert(strkey(POOL), Venue::Aquarius); + reg +} + +/// Through the production step, so every case here runs the live arming. +fn with_candidates(assets: AssetRegistry) -> AssetRegistry { + with_sac_candidates(assets, HashSet::from([XCR_SAC.to_string()])) +} + +fn run(events: Vec, assets: &mut AssetRegistry) -> LedgerSoroban { + process_ledger( + &lcm_with(FIRST, BASE_MINUTE, events), + &mut pool_registries(), + assets, + ) +} + +fn pending(assets: &AssetRegistry) -> Vec { + assets.pending_new().cloned().collect() +} + +fn holds_contract(assets: &AssetRegistry, addr: &str) -> bool { + assets + .assets() + .any(|a| *a == AssetIdentity::Contract(addr.to_string())) +} + +/// `main.rs` needs the `lambda` feature and AWS, so no test runs it. Its cold +/// start must build the registry through `load_ingest_registry`: a plain +/// `load_registry` leaves the candidate set empty, and every unproven SAC would +/// mint a `Contract` identity again with all other tests green. +#[test] +fn the_lambda_cold_start_arms_the_sac_candidates() { + let main = include_str!("../src/main.rs"); + assert!(main.contains("sink.load_ingest_registry()")); + assert!( + !main.contains(".load_registry()"), + "a registry without candidates" + ); + let sink = include_str!("../src/sink/mod.rs"); + assert!(sink.contains("Ok(with_sac_candidates(registry?, sac_contracts?))")); +} + +/// The fixture constants are pinned by our own derivation, not trusted. +#[test] +fn the_fixture_sacs_are_the_derived_ones() { + let assets = AssetRegistry::from_existing(vec![]); + assert_eq!(assets.sac_address_of(&xcr()).as_deref(), Some(XCR_SAC)); + assert_eq!( + assets.sac_address_of(&AssetIdentity::Native).as_deref(), + Some(XLM_SAC) + ); +} + +/// T2 and T4: the proof resolves the SAC whether it comes before or after the +/// trade in the transaction, on an empty registry. +#[test] +fn a_sac_proven_in_its_transaction_trades_as_its_classic_in_either_order() { + let cases = [ + ( + "proof first", + vec![sac_transfer(XCR_SAC, &xcr_sep11()), trade(XLM_SAC, XCR_SAC)], + ), + ( + "proof after", + vec![trade(XLM_SAC, XCR_SAC), sac_transfer(XCR_SAC, &xcr_sep11())], + ), + ]; + for (name, events) in cases { + for candidates in [false, true] { + let mut assets = AssetRegistry::from_existing(vec![]); + if candidates { + assets = with_candidates(assets); + } + let out = run(events.clone(), &mut assets); + assert_eq!(out.amm_ticks.len(), 1, "{name}"); + let (source, tick) = &out.amm_ticks[0]; + assert_eq!(*source, "aquarius", "{name}"); + assert_eq!(tick.base, xcr(), "{name}"); + assert_eq!(tick.quote, AssetIdentity::Native, "{name}"); + assert!(out.sac_unproven.is_empty(), "{name}"); + assert!(pending(&assets).contains(&xcr()), "{name}"); + assert!(!holds_contract(&assets, XCR_SAC), "{name}: no Contract"); + } + } +} + +/// T3: one identity whichever path sees the asset first. +#[test] +fn classic_and_sac_meet_on_one_identity_in_either_order() { + // (a) The SDEX path interns the classic; the SAC then resolves through it, + // with no proof in the transaction. + let mut assets = with_candidates(AssetRegistry::from_existing(vec![])); + assert!(assets.intern(&xcr())); + let out = run(vec![trade(XLM_SAC, XCR_SAC)], &mut assets); + assert_eq!(out.amm_ticks.len(), 1); + assert_eq!(out.amm_ticks[0].1.base, xcr()); + assert!(out.sac_unproven.is_empty()); + assert_eq!(pending(&assets).iter().filter(|a| **a == xcr()).count(), 1); + assert!(!holds_contract(&assets, XCR_SAC)); + + // (b) The SAC trades first with its proof; the classic is then known. + let mut assets = with_candidates(AssetRegistry::from_existing(vec![])); + let out = run( + vec![sac_transfer(XCR_SAC, &xcr_sep11()), trade(XLM_SAC, XCR_SAC)], + &mut assets, + ); + assert_eq!(out.amm_ticks[0].1.base, xcr()); + assert!(!assets.intern(&xcr()), "already interned by the trade"); + assert_eq!(pending(&assets).iter().filter(|a| **a == xcr()).count(), 1); + assert!(!holds_contract(&assets, XCR_SAC)); +} + +/// T6-live and the impostor: an `is_sac` leg with no valid proof is skipped, +/// counted, and interns nothing. +#[test] +fn an_unproven_sac_trade_is_skipped_and_interns_nothing() { + let impostor = strkey(IMPOSTOR); + let cases = [ + ("no proof", vec![trade(XLM_SAC, XCR_SAC)]), + ( + "impostor proof", + vec![ + sac_transfer(&impostor, &xcr_sep11()), + trade(XLM_SAC, XCR_SAC), + ], + ), + ]; + for (name, events) in cases { + let mut assets = with_candidates(AssetRegistry::from_existing(vec![])); + let out = run(events, &mut assets); + assert!(out.amm_ticks.is_empty(), "{name}"); + assert_eq!(out.sac_unproven, vec![XCR_SAC.to_string()], "{name}"); + assert!(pending(&assets).is_empty(), "{name}: nothing interned"); + assert_eq!(assets.resolve_sac(XCR_SAC), None, "{name}"); + assert_eq!(assets.resolve_sac(&impostor), None, "{name}"); + } +} + +/// D2: a token BE does not flag `is_sac` keeps its `Contract` identity. +#[test] +fn a_genuine_soroban_token_keeps_its_contract_identity() { + let token = strkey(GENUINE_TOKEN); + let mut assets = with_candidates(AssetRegistry::from_existing(vec![])); + let out = run(vec![trade(XLM_SAC, &token)], &mut assets); + assert_eq!(out.amm_ticks.len(), 1); + assert_eq!( + out.amm_ticks[0].1.base, + AssetIdentity::Contract(token.clone()) + ); + assert!(out.sac_unproven.is_empty()); + assert!(holds_contract(&assets, &token)); +} + +#[derive(Clone, Default)] +struct MemoryFetcher { + objects: Arc>>>, +} + +impl MemoryFetcher { + fn publish(&self, seq: u64, bytes: Vec) { + self.objects + .lock() + .unwrap() + .insert(ledger_s3_key(seq as i64), bytes); + } +} + +impl ObjectFetcher for MemoryFetcher { + async fn fetch(&self, key: &str) -> Result>, FetchError> { + Ok(self.objects.lock().unwrap().get(key).cloned()) + } +} + +#[derive(Clone, Default)] +struct NullSink; + +impl CandleSink for NullSink { + async fn write_candles(&self, _: &[OhlcvCandle], _: &str) -> Result<(), SinkError> { + Ok(()) + } + + async fn write_oracle(&self, _: &[OracleSample]) -> Result<(), SinkError> { + Ok(()) + } + + async fn write_new_assets(&self, _: &AssetRegistry) -> Result<(), SinkError> { + Ok(()) + } + + async fn write_pool_rows(&self, _: &[PoolRegistryRow]) -> Result<(), SinkError> { + Ok(()) + } +} + +/// T6-live through the `Reconciler`: the count follows the WRITTEN minute, as +/// `UnregisteredPoolEvents` does, so a held-back ledger is not counted twice. +#[tokio::test] +async fn unproven_sac_trades_are_counted_once_when_their_minute_is_written() { + let dir = tempdir().unwrap(); + let cursor = StubFileCursor::new(dir.path().join("cursor.txt")); + cursor.write(FIRST - 1).await.unwrap(); + let fetcher = MemoryFetcher::default(); + let reconciler = Reconciler::new( + fetcher.clone(), + cursor, + NullSink, + with_candidates(AssetRegistry::from_existing(vec![])), + pool_registries(), + ); + let unproven = || vec![trade(XLM_SAC, XCR_SAC)]; + + for (seq, close) in [(FIRST, BASE_MINUTE), (FIRST + 1, BASE_MINUTE + 5)] { + fetcher.publish(seq, ledger_with(seq, close, unproven())); + let held = reconciler.run(32).await.unwrap(); + assert_eq!(held.end_cursor, FIRST - 1, "the minute is still open"); + assert_eq!(held.sac_unproven_skipped, 0); + } + + // The next minute's ledger is decoded but held back: 2, not 3. + fetcher.publish( + FIRST + 2, + ledger_with(FIRST + 2, BASE_MINUTE + 60, unproven()), + ); + let written = reconciler.run(32).await.unwrap(); + assert_eq!(written.end_cursor, FIRST + 1); + assert_eq!(written.sac_unproven_skipped, 2); + + fetcher.publish(FIRST + 3, ledger_with(FIRST + 3, BASE_MINUTE + 120, vec![])); + let next = reconciler.run(32).await.unwrap(); + assert_eq!(next.end_cursor, FIRST + 2); + assert_eq!( + next.sac_unproven_skipped, 1, + "only the minute this run wrote" + ); +} From 491d17a5984d767323093ee09dbc2dda500a5ea2 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:12:47 +0200 Subject: [PATCH 002/144] fix(lore-0242): prove SACs at events-backfill start and stop phase 3 on an unproven one events-backfill reads only pool events, so it never sees the SAC event that proves a leg's classic asset. One start-of-run query fetches a proof per is_sac contract assets.sac_address cannot resolve, and learn_sac verifies each before the first chunk. A swap on an is_sac leg that is still unproven is skipped and printed as `unproven sac swaps: N` (0 included); the 0286 orchestrator STOPs on N > 0 and passes a pre-0242 binary that prints no such line. --- packages/events-backfill/src/run.rs | 138 ++++++++- packages/prices-ingest-core/src/lib.rs | 5 +- packages/prices-ingest-core/src/sac_proof.rs | 147 +++++++++- packages/prices-ingest-core/src/soroban.rs | 4 + .../prices-ingest-core/tests/sac_proof_it.rs | 274 ++++++++++++++++++ tools/scripts/reingest_0286.py | 22 +- tools/scripts/reingest_0286_sac_test.py | 65 +++++ 7 files changed, 646 insertions(+), 9 deletions(-) create mode 100644 packages/prices-ingest-core/tests/sac_proof_it.rs create mode 100644 tools/scripts/reingest_0286_sac_test.py diff --git a/packages/events-backfill/src/run.rs b/packages/events-backfill/src/run.rs index 4791e915..9797b782 100644 --- a/packages/events-backfill/src/run.rs +++ b/packages/events-backfill/src/run.rs @@ -21,9 +21,9 @@ use serde_json::Value; use tracing::{info, warn}; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, DEFAULT_BACKOFF_MS, LedgerSoroban, OhlcvCandle, OhlcvWriter, - RawSorobanEvent, Registries, UnresolvedPool, UnresolvedPoolSwap, process_soroban_event_rows, - retry_with_backoff, + AssetRegistry, BE_DATABASE, CandleAccumulator, DEFAULT_BACKOFF_MS, LedgerSoroban, OhlcvCandle, + OhlcvWriter, PRICES_DATABASE, RawSorobanEvent, Registries, UnresolvedPool, UnresolvedPoolSwap, + preload_sac_resolver, process_soroban_event_rows, retry_with_backoff, }; use crate::cli::Cli; @@ -69,6 +69,7 @@ fn accumulate_ledger( ticks_by_source: &mut HashMap<&'static str, u64>, failed_by_source: &mut HashMap<&'static str, u64>, raw_unresolved: &mut Vec, + sac_unproven: &mut Vec, ) { let mut out = LedgerSoroban::default(); process_soroban_event_rows(ledger, closed_at, events, reg, assets, &mut out); @@ -80,6 +81,7 @@ fn accumulate_ledger( *failed_by_source.entry(source).or_default() += *swaps as u64; } raw_unresolved.extend(out.unresolved); + sac_unproven.extend(out.sac_unproven); // Minute-START timestamp of this ledger — the same key the accumulator buckets // on (`(closed_at/60)*60`), NOT the minute index; flush_older_than compares @@ -184,6 +186,25 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { // write only newly seen assets and resolve every seeded pool. let existing_assets = writer.load_assets().await?; let mut assets = AssetRegistry::from_existing(existing_assets); + // Task 0242 D1: BE's is_sac set, then one proof per SAC `assets` cannot + // resolve, both before the first chunk (this path reads no SAC events). + // Dry-run too, so its `unproven sac swaps` matches the write's. + // `execute_preloads_the_sac_resolver_before_the_first_chunk` pins this call. + let sac = + preload_sac_resolver(writer.client(), BE_DATABASE, PRICES_DATABASE, &mut assets).await?; + info!( + sac_contracts = sac.candidates, + sac_proofs = sac.proofs, + sacs_verified = sac.verified, + sacs_rejected = sac.rejected, + "preloaded SAC proofs (task 0242)" + ); + if sac.rejected > 0 { + warn!( + sacs_rejected = sac.rejected, + "SAC proofs that did not verify: their swaps are skipped, not minted (task 0242 D2)" + ); + } let mut reg = reprice_registry(writer.load_pool_registry().await?)?; // Every AMM pool has a `venue` entry (the registry superset); its strkeys are @@ -214,6 +235,8 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { let mut total_events: u64 = 0; let mut total_candles: u64 = 0; let mut raw_unresolved: Vec = Vec::new(); + // One entry per swap skipped on an is_sac leg with no proof (task 0242 D2). + let mut sac_unproven: Vec = Vec::new(); // Ledgers present in soroban_events but absent from default.ledgers (no close // time): their events are skipped but COUNTED, so the gap is visible instead // of a silent LEFT-JOIN drop. @@ -266,6 +289,7 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { &mut ticks_by_source, &mut failed_by_source, &mut raw_unresolved, + &mut sac_unproven, ); } cur_ledger = Some(r.ledger_sequence); @@ -359,6 +383,7 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { &mut ticks_by_source, &mut failed_by_source, &mut raw_unresolved, + &mut sac_unproven, ); } @@ -428,6 +453,18 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { ); } + if !sac_unproven.is_empty() { + let contracts: std::collections::BTreeSet<&str> = + sac_unproven.iter().map(String::as_str).collect(); + let sample: Vec<&str> = contracts.iter().copied().take(20).collect(); + warn!( + swaps = sac_unproven.len(), + contracts = contracts.len(), + ?sample, + "swaps skipped on SACs with no proof (task 0242 D2); the month is NOT complete" + ); + } + if ledgers_missing_close > 0 { warn!( ledgers_missing_close, @@ -456,6 +493,7 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { unresolved_genuine.len(), dropped_swaps, apply_order_fallbacks, + sac_unproven.len() as u64, ); info!( elapsed_s = run_start.elapsed().as_secs(), @@ -558,6 +596,9 @@ pub(crate) fn resolve_transaction_index(application_order: i16) -> (u16, bool) { (application_order as u16, false) } +/// Summary label the 0286 orchestrator reads (`tools/scripts/reingest_0286.py`). +pub(crate) const UNPROVEN_SAC_LABEL: &str = "unproven sac swaps:"; + #[allow(clippy::too_many_arguments)] fn print_summary( ticks_by_source: &HashMap<&'static str, u64>, @@ -568,6 +609,7 @@ fn print_summary( unresolved_contracts: usize, dropped_swaps: u64, apply_order_fallbacks: u64, + unproven_sac_swaps: u64, ) { println!(); println!("=== events-backfill complete ==="); @@ -596,6 +638,9 @@ fn print_summary( // event row, not a missing join row (task 0304). Non-zero means the range // is not repaired. println!("negative apply order: {apply_order_fallbacks}"); + // Always printed, 0 included. Non-zero means swaps were skipped on SACs with + // no proof, and the 0286 orchestrator STOPs on it (task 0242 D2). + println!("{UNPROVEN_SAC_LABEL:<27}{unproven_sac_swaps}"); } /// The registry the reprice reads events for: the LOADED `pool_registry` plus @@ -693,6 +738,7 @@ mod tests { ticks, &mut HashMap::new(), unresolved, + &mut Vec::new(), ); } @@ -782,6 +828,92 @@ mod tests { ); } + // ---- task 0242: a SAC leg is proven or skipped, never minted ---------- + + const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; + const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + + fn xcr() -> prices_ingest_core::AssetIdentity { + prices_ingest_core::AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + } + } + + /// One XLM-SAC / XCR-SAC Soroswap swap through `accumulate_ledger`: the + /// candles of its minute and the run's unproven list. + fn xcr_swap(assets: &mut AssetRegistry) -> (Vec, Vec) { + let mut reg = Registries::new(); + reg.venue.insert(POOL.to_string(), Venue::Soroswap); + reg.soroswap + .register(POOL.to_string(), T0.to_string(), XCR_SAC.to_string()); + let mut accs = HashMap::new(); + let mut unproven = Vec::new(); + accumulate_ledger( + 100, + 1_700_000_000, + &[soroswap_swap(100, 1_700_000_000)], + &mut reg, + assets, + &mut accs, + &mut HashMap::new(), + &mut HashMap::new(), + &mut HashMap::new(), + &mut Vec::new(), + &mut unproven, + ); + let candles = accs + .values_mut() + .flat_map(CandleAccumulator::flush_all) + .collect(); + (candles, unproven) + } + + #[test] + fn a_swap_on_an_unproven_sac_is_skipped_and_counted_and_a_proven_one_prices() { + let candidates = || std::collections::HashSet::from([XCR_SAC.to_string()]); + + let mut assets = AssetRegistry::from_existing(vec![]); + assert_eq!(assets.sac_address_of(&xcr()).as_deref(), Some(XCR_SAC)); + assets.set_sac_candidates(candidates()); + let (candles, unproven) = xcr_swap(&mut assets); + assert!(candles.is_empty(), "no tick"); + assert_eq!(unproven, [XCR_SAC]); + assert_eq!(assets.pending_new().count(), 0, "nothing interned"); + + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_sac_candidates(candidates()); + let proof = prices_ingest_core::SacProofRow { + contract_id: XCR_SAC.to_string(), + sep11: format!("XCR:{XCR_ISSUER}"), + }; + assert_eq!( + prices_ingest_core::apply_sac_proofs(&mut assets, &[proof]), + (1, 0) + ); + let (candles, unproven) = xcr_swap(&mut assets); + assert!(unproven.is_empty()); + assert_eq!(candles.len(), 1, "one tick, one candle"); + assert_eq!( + (&candles[0].base, &candles[0].quote), + (&xcr(), &prices_ingest_core::AssetIdentity::Native) + ); + } + + /// `execute` needs BE's and our tables, so no test runs it. It must arm the + /// registry before its first chunk: without the preload every swap on a SAC + /// whose classic is not in `assets` is skipped, and with the candidates gone + /// too it would be minted again (task 0242 D1/D2). `sac_proof_it` runs the + /// preload itself against real tables. + #[test] + fn execute_preloads_the_sac_resolver_before_the_first_chunk() { + let src = include_str!("run.rs"); + let execute = &src[src.find("pub async fn execute(").unwrap()..]; + let preload = execute.find("preload_sac_resolver(").unwrap(); + let first_chunk = execute.find("stream_chunk(").unwrap(); + assert!(preload < first_chunk, "the preload runs before any chunk"); + } + const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; /// Task 0300 D3a: the static pools never mask an unseeded `pool_registry` diff --git a/packages/prices-ingest-core/src/lib.rs b/packages/prices-ingest-core/src/lib.rs index 54370c1b..25116e25 100644 --- a/packages/prices-ingest-core/src/lib.rs +++ b/packages/prices-ingest-core/src/lib.rs @@ -54,7 +54,10 @@ pub use price::{ }; pub use registry_io::PoolRegistryRow; pub use retry::{DEFAULT_BACKOFF_MS, retry_with_backoff}; -pub use sac_proof::{BE_DATABASE, load_sac_contracts}; +pub use sac_proof::{ + BE_DATABASE, PRICES_DATABASE, SacPreload, SacProofRow, apply_sac_proofs, load_sac_contracts, + load_sac_proofs, preload_sac_resolver, +}; pub use safe_log::safe_response_token; pub use soroban::{ LedgerSoroban, RawSorobanEvent, Registries, UnresolvedPoolSwap, learn_factory_event, diff --git a/packages/prices-ingest-core/src/sac_proof.rs b/packages/prices-ingest-core/src/sac_proof.rs index d443e029..b4fa66f7 100644 --- a/packages/prices-ingest-core/src/sac_proof.rs +++ b/packages/prices-ingest-core/src/sac_proof.rs @@ -1,6 +1,7 @@ //! BE reads behind the SAC resolver (task 0242): which contracts BE flags as a -//! Stellar Asset Contract. A flagged contract no proof resolves is skipped, not -//! minted as a `Contract` identity (D2). +//! Stellar Asset Contract, and one proof event per flagged contract our +//! registry cannot resolve yet. A flagged contract no proof resolves is +//! skipped, not minted as a `Contract` identity (D2). //! //! No `SETTINGS` in any statement here: a read that crosses the client's GET //! length is sent as POST with `readonly=1`, which rejects per-query settings @@ -8,6 +9,7 @@ use std::collections::HashSet; +use crate::canonical::AssetRegistry; use crate::error::IngestError; /// BE's database, holding `soroban_contracts` and `soroban_events`. @@ -56,6 +58,91 @@ pub async fn load_sac_contracts( Ok(rows.into_iter().collect()) } +/// One SAC-signature event per `is_sac` contract whose address is no +/// `sac_address` in `.assets`: the contract and its event's last topic +/// (the SEP-11 asset). Without the "not resolvable" filter the scan times out +/// on production (0242 RESEARCH M16b). +pub(crate) fn sac_proof_preload_sql(be_db: &str, prices_db: &str) -> Result { + let be = identifier(be_db)?; + let prices = identifier(prices_db)?; + Ok(format!( + "SELECT sc.contract_id AS contract_id, \ + JSONExtractString(x.topics_xdr, JSONLength(x.topics_xdr), 'value') AS sep11 \ + FROM (SELECT contract_id, topics_xdr FROM {be}.soroban_events \ + WHERE contract_id IN (SELECT id FROM {be}.soroban_contracts \ + WHERE is_sac AND contract_id NOT IN \ + (SELECT sac_address FROM {prices}.assets FINAL WHERE sac_address != '')) \ + AND signature IN ('transfer', 'mint', 'burn', 'clawback') \ + LIMIT 1 BY contract_id) x \ + INNER JOIN (SELECT id, any(contract_id) AS contract_id FROM {be}.soroban_contracts \ + WHERE is_sac GROUP BY id) sc ON sc.id = x.contract_id" + )) +} + +/// A row of [`sac_proof_preload_sql`], fields in SELECT order. +#[derive(Debug, Clone, PartialEq, Eq, clickhouse::Row, serde::Deserialize)] +pub struct SacProofRow { + pub contract_id: String, + pub sep11: String, +} + +/// Load the proofs events-backfill feeds to `learn_sac` before its first chunk +/// (about 1,100 rows in 2.6 s on production). +pub async fn load_sac_proofs( + client: &clickhouse::Client, + be_db: &str, + prices_db: &str, +) -> Result, IngestError> { + Ok(client + .query(&sac_proof_preload_sql(be_db, prices_db)?) + .fetch_all::() + .await?) +} + +/// Feed every proof through `learn_sac`: (verified, rejected). +pub fn apply_sac_proofs(assets: &mut AssetRegistry, rows: &[SacProofRow]) -> (u64, u64) { + let verified = rows + .iter() + .filter(|r| assets.learn_sac(&r.contract_id, &r.sep11)) + .count() as u64; + (verified, rows.len() as u64 - verified) +} + +/// What [`preload_sac_resolver`] armed a registry with. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SacPreload { + /// BE's `is_sac` contracts, now the registry's candidates. + pub candidates: usize, + /// Proof rows read. + pub proofs: usize, + /// Proofs `learn_sac` accepted, and the ones it did not. + pub verified: u64, + pub rejected: u64, +} + +/// events-backfill's run-start step (task 0242 D1): BE's `is_sac` set as the +/// registry's candidates, then one proof per SAC `assets` cannot resolve, +/// through `learn_sac`. That path streams pool events only, so this is how it +/// resolves a SAC whose classic is not in `assets` yet. +pub async fn preload_sac_resolver( + client: &clickhouse::Client, + be_db: &str, + prices_db: &str, + assets: &mut AssetRegistry, +) -> Result { + let candidates = load_sac_contracts(client, be_db).await?; + let n = candidates.len(); + assets.set_sac_candidates(candidates); + let proofs = load_sac_proofs(client, be_db, prices_db).await?; + let (verified, rejected) = apply_sac_proofs(assets, &proofs); + Ok(SacPreload { + candidates: n, + proofs: proofs.len(), + verified, + rejected, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -78,5 +165,61 @@ mod tests { ); } assert!(sac_contracts_sql("it_0242_be").is_ok()); + for (be, prices) in [ + ("default; DROP", "prices"), + ("default", "pri ces"), + ("", ""), + ] { + assert!( + matches!( + sac_proof_preload_sql(be, prices), + Err(IngestError::Precondition(_)) + ), + "{be:?} {prices:?}" + ); + } + } + + #[test] + fn sac_proof_preload_sql_reads_one_proof_per_unresolvable_sac_without_settings() { + let sql = sac_proof_preload_sql("default", "prices").unwrap(); + for part in [ + "LIMIT 1 BY contract_id", + "signature IN ('transfer', 'mint', 'burn', 'clawback')", + "NOT IN (SELECT sac_address FROM prices.assets FINAL WHERE sac_address != '')", + "default.soroban_events", + "default.soroban_contracts", + "JSONLength(", + ] { + assert!(sql.contains(part), "{part} in {sql}"); + } + assert!(!sql.contains("SETTINGS"), "{sql}"); + } + + const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; + const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + + #[test] + fn apply_sac_proofs_learns_only_what_verifies() { + let xcr = crate::AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + }; + let mut assets = AssetRegistry::from_existing(vec![]); + assert_eq!(assets.sac_address_of(&xcr).as_deref(), Some(XCR_SAC)); + let impostor = stellar_strkey::Contract([7; 32]).to_string(); + let rows = [ + SacProofRow { + contract_id: XCR_SAC.to_string(), + sep11: format!("XCR:{XCR_ISSUER}"), + }, + SacProofRow { + contract_id: impostor.clone(), + sep11: format!("XCR:{XCR_ISSUER}"), + }, + ]; + assert_eq!(apply_sac_proofs(&mut assets, &rows), (1, 1)); + assert_eq!(assets.resolve_sac(XCR_SAC), Some(xcr)); + assert_eq!(assets.resolve_sac(&impostor), None); } } diff --git a/packages/prices-ingest-core/src/soroban.rs b/packages/prices-ingest-core/src/soroban.rs index 505bdfaf..111b400e 100644 --- a/packages/prices-ingest-core/src/soroban.rs +++ b/packages/prices-ingest-core/src/soroban.rs @@ -511,6 +511,10 @@ pub fn process_soroban_event_rows( .all(|e| e.transaction_index == transaction_index), "events of transaction {tx_id} disagree on transaction_index" ); + // Kept identical to `process_ledger`, but in production it learns + // nothing: events-backfill streams pool events only, so a SAC's own + // transfer/mint/burn/clawback never reaches this seam. There the proofs + // come from `preload_sac_resolver` at run start. learn_sacs( events[tx_start..tx_end] .iter() diff --git a/packages/prices-ingest-core/tests/sac_proof_it.rs b/packages/prices-ingest-core/tests/sac_proof_it.rs new file mode 100644 index 00000000..319eb8d9 --- /dev/null +++ b/packages/prices-ingest-core/tests/sac_proof_it.rs @@ -0,0 +1,274 @@ +//! Task 0242 — the two BE reads behind the SAC resolver: BE's `is_sac` set and +//! the events-backfill proof preload, run as rendered against real tables. +//! +//! tools/scripts/ignored-tests.sh # all of them: CI runs exactly this on every Rust PR +//! cargo test -p prices-ingest-core --test sac_proof_it -- --ignored --test-threads=1 +//! +//! Each test DROPs and CREATEs scratch databases `it_0242__be` (BE's +//! `soroban_events` / `soroban_contracts`, DDL as in coverage_sweep_it.rs) and +//! `it_0242__prices` (our `init.sql`). The real `default` and `prices` +//! are never touched. Never run against a shared cluster. + +use std::collections::HashSet; + +use clickhouse::Client; +use prices_clickhouse::USDC_ISSUER; +use prices_clickhouse::asset_id::fixture::{AssetFixture, assets_insert}; +use prices_ingest_core::{ + AssetIdentity, AssetRegistry, SacPreload, SacProofRow, apply_sac_proofs, load_sac_contracts, + load_sac_proofs, preload_sac_resolver, +}; + +const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; +const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + +fn ch_url() -> String { + std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".to_string()) +} + +fn admin() -> Client { + Client::default().with_url(ch_url()) +} + +async fn exec(sql: &str) { + admin().query(sql).execute().await.expect(sql); +} + +fn xcr() -> AssetIdentity { + AssetIdentity::Credit { + code: "XCR".to_string(), + issuer: XCR_ISSUER.to_string(), + } +} + +fn usdc() -> AssetIdentity { + AssetIdentity::Credit { + code: "USDC".to_string(), + issuer: USDC_ISSUER.to_string(), + } +} + +fn strkey(n: u8) -> String { + stellar_strkey::Contract([n; 32]).to_string() +} + +/// `[transfer, from, to, ]` in BE's typed-JSON shape. +fn topics(signature: &str, last: &str) -> String { + let from = stellar_strkey::ed25519::PublicKey([1; 32]).to_string(); + format!( + r#"[{{"type":"sym","value":"{signature}"}},{{"type":"address","value":"{from}"}},{{"type":"address","value":"{XCR_SAC}"}},{{"type":"string","value":"{last}"}}]"# + ) +} + +struct Fixture { + be: String, + prices: String, +} + +impl Fixture { + /// Fresh scratch databases, and the contracts every case shares: + /// - 1 XCR's SAC, is_sac, held as two RMT versions; classic absent from + /// `assets`; an `approve` and two `transfer`s. + /// - 2 USDC's SAC, is_sac; its classic is in `assets` with `sac_address`. + /// - 3 a non-SAC contract emitting `transfer` with XCR's SEP-11 topic. + /// - 4 an is_sac contract with no events. + async fn new(case: &str) -> Self { + let f = Fixture { + be: format!("it_0242_{case}_be"), + prices: format!("it_0242_{case}_prices"), + }; + for db in [&f.be, &f.prices] { + exec(&format!("DROP DATABASE IF EXISTS {db}")).await; + exec(&format!("CREATE DATABASE {db}")).await; + } + let init = prices_clickhouse::INIT_SQL + .replace("prices.", &format!("{}.", f.prices)) + .replace( + "IF NOT EXISTS prices", + &format!("IF NOT EXISTS {}", f.prices), + ); + prices_clickhouse::apply_sql(&admin(), &init) + .await + .expect("init schema"); + exec(&format!( + "CREATE TABLE {}.soroban_events ( + contract_id Int64, + ledger_sequence Int64, + transaction_index UInt32, + operation_index UInt16, + event_index UInt32, + application_order Int16, + event_type Int16, + signature LowCardinality(Nullable(String)), + topics_xdr String CODEC(ZSTD(3)), + data_xdr String CODEC(ZSTD(3)) + ) + ENGINE = ReplacingMergeTree + PARTITION BY intDiv(ledger_sequence, 500000) + ORDER BY (contract_id, ledger_sequence, transaction_index, operation_index, event_index)", + f.be + )) + .await; + exec(&format!( + "CREATE TABLE {}.soroban_contracts ( + id Int64, + contract_id String, + wasm_hash Nullable(FixedString(32)), + wasm_uploaded_at_ledger Int64 DEFAULT 0, + deployer_id Nullable(Int64), + deployed_at_ledger Nullable(Int64), + contract_type Nullable(Int16), + is_sac Bool + ) + ENGINE = ReplacingMergeTree(wasm_uploaded_at_ledger) + ORDER BY (contract_id)", + f.be + )) + .await; + + let usdc_sac = AssetRegistry::from_existing(vec![]) + .sac_address_of(&usdc()) + .expect("USDC derives a SAC"); + let (genuine, empty) = (strkey(3), strkey(4)); + // Two inserts, two parts: the RMT versions stay unmerged. + for (id, contract, version, is_sac) in [ + (1, XCR_SAC, 1, true), + (1, XCR_SAC, 2, true), + (2, usdc_sac.as_str(), 1, true), + (3, genuine.as_str(), 1, false), + (4, empty.as_str(), 1, true), + ] { + f.contract(id, contract, version, is_sac).await; + } + let xcr_sep11 = format!("XCR:{XCR_ISSUER}"); + f.event(1, 100, "approve", "NOT-A-PROOF").await; + f.event(1, 101, "transfer", &xcr_sep11).await; + f.event(1, 102, "transfer", &xcr_sep11).await; + f.event(2, 100, "transfer", &format!("USDC:{USDC_ISSUER}")) + .await; + f.event(3, 100, "transfer", &xcr_sep11).await; + + exec(&assets_insert( + &f.prices, + &[AssetFixture::new("USDC", "classic", USDC_ISSUER, "").with_sac(&usdc_sac)], + )) + .await; + f + } + + async fn contract(&self, id: i64, contract: &str, version: i64, is_sac: bool) { + exec(&format!( + "INSERT INTO {}.soroban_contracts (id, contract_id, wasm_uploaded_at_ledger, is_sac) \ + VALUES ({id}, '{contract}', {version}, {is_sac})", + self.be + )) + .await; + } + + async fn event(&self, id: i64, ledger: i64, signature: &str, last: &str) { + exec(&format!( + "INSERT INTO {}.soroban_events \ + (contract_id, ledger_sequence, transaction_index, operation_index, event_index, application_order, event_type, signature, topics_xdr, data_xdr) \ + VALUES ({id}, {ledger}, 0, 0, 0, 0, 1, '{signature}', '{}', '{{\"type\":\"i128\",\"value\":\"1\"}}')", + self.be, + topics(signature, last), + )) + .await; + } + + async fn drop(self) { + for db in [&self.be, &self.prices] { + exec(&format!("DROP DATABASE IF EXISTS {db}")).await; + } + } +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_is_sac_set_holds_each_flagged_contract_once() { + let f = Fixture::new("sacs").await; + let got = load_sac_contracts(&admin(), &f.be).await.unwrap(); + let usdc_sac = AssetRegistry::from_existing(vec![]) + .sac_address_of(&usdc()) + .unwrap(); + assert_eq!( + got, + HashSet::from([XCR_SAC.to_string(), usdc_sac, strkey(4)]), + "every is_sac contract once, the non-SAC one never" + ); + f.drop().await; +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_preload_returns_one_proof_per_sac_the_registry_cannot_resolve() { + let f = Fixture::new("preload").await; + let rows = load_sac_proofs(&admin(), &f.be, &f.prices).await.unwrap(); + assert_eq!( + rows, + [SacProofRow { + contract_id: XCR_SAC.to_string(), + sep11: format!("XCR:{XCR_ISSUER}"), + }], + "USDC is resolvable from assets, the non-SAC is no candidate, the eventless SAC has no proof" + ); + + let mut assets = AssetRegistry::from_existing(vec![]); + assert_eq!(assets.sac_address_of(&xcr()).as_deref(), Some(XCR_SAC)); + assert_eq!(apply_sac_proofs(&mut assets, &rows), (1, 0)); + assert_eq!(assets.resolve_sac(XCR_SAC), Some(xcr())); + f.drop().await; +} + +/// events-backfill's run-start step, end to end: the candidates are BE's +/// `is_sac` set, and the one provable SAC `assets` cannot resolve is learnt. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_run_start_preload_arms_candidates_and_learns_the_proofs() { + let f = Fixture::new("arm").await; + let mut assets = AssetRegistry::from_existing(vec![]); + let got = preload_sac_resolver(&admin(), &f.be, &f.prices, &mut assets) + .await + .unwrap(); + assert_eq!( + got, + SacPreload { + candidates: 3, + proofs: 1, + verified: 1, + rejected: 0, + } + ); + assert_eq!(assets.resolve_sac(XCR_SAC), Some(xcr())); + assert!(!assets.is_unproven_sac(XCR_SAC)); + assert!(assets.is_unproven_sac(&strkey(4)), "eventless SAC: skipped"); + assert!(!assets.is_unproven_sac(&strkey(3)), "not a SAC: a token"); + f.drop().await; +} + +/// IN-04: the `is_sac` read has no `FINAL`. A contract BE corrects to +/// `is_sac = false` stays a candidate (skipped, never minted) until BE's parts +/// merge, and leaves the set once they have. Merges are stopped so the two +/// versions cannot merge before the first read. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn an_unmerged_is_sac_correction_stays_a_candidate_until_merged() { + let f = Fixture::new("flip").await; + let table = format!("{}.soroban_contracts", f.be); + exec(&format!("SYSTEM STOP MERGES {table}")).await; + let flipped = strkey(5); + f.contract(5, &flipped, 1, true).await; + f.contract(5, &flipped, 1, false).await; + let got = load_sac_contracts(&admin(), &f.be).await.unwrap(); + assert!( + got.contains(&flipped), + "unmerged: either version may answer" + ); + + exec(&format!("SYSTEM START MERGES {table}")).await; + exec(&format!("OPTIMIZE TABLE {table} FINAL")).await; + let got = load_sac_contracts(&admin(), &f.be).await.unwrap(); + assert!(!got.contains(&flipped), "merged: the last insert wins"); + assert!(got.contains(XCR_SAC)); + f.drop().await; +} diff --git a/tools/scripts/reingest_0286.py b/tools/scripts/reingest_0286.py index c9610307..d2234114 100755 --- a/tools/scripts/reingest_0286.py +++ b/tools/scripts/reingest_0286.py @@ -674,7 +674,12 @@ def require_transport_flag(a): "(run it on the CH host as `default`) or --amm wait + amm-done.") -def amm_summary(text, logfile): +# events-backfill's task-0242 line (run.rs UNPROVEN_SAC_LABEL). Absent from a +# pre-0242 binary, which passes (D4: phase 3 is not held for the code). +UNPROVEN_SAC_LABEL = "unproven sac swaps:" + + +def amm_summary(text, logfile, *, dry_run): """Read events-backfill's closing summary, or Stop naming what is missing. Called after the --dry-run pass as well as the write, so a binary whose @@ -691,6 +696,17 @@ def amm_summary(text, logfile): f"see {logfile}. A binary without it predates task 0304: rebuild events-backfill " "from develop (on the CH host for --amm ssh)") found[key] = int(hit.group(1)) + hit = re.search(rf"^{re.escape(UNPROVEN_SAC_LABEL)}\s*(\d+)", text, re.M) + found["unproven_sacs"] = int(hit.group(1)) if hit else None + if found["unproven_sacs"]: + m = Path(logfile).parent.name + state = (f"This was the --dry-run pass: no AMM candle was written, but {m}'s 1m partition is " + "already dropped and holds SDEX-only candles. Fix it today and `run` again, or " + f"`rollback {m}`" if dry_run else + f"This was the write pass: {m}'s AMM candles are in without those swaps. `rollback {m}`") + raise Stop(f"events-backfill skipped {found['unproven_sacs']} swaps because their SAC could not be " + f"proven (task 0242 D2) — the contracts are in the WARN in {logfile}. {state}: " + "docs/runbooks/0242-sac-identity-heal.md §4c") return found @@ -915,7 +931,7 @@ def done(): code, text = stream(cmd, env, mdir / "events-backfill.log") if code != 0 or "=== events-backfill complete ===" not in text: raise Stop(f"events-backfill {' '.join(flag)} exit {code} — see {mdir}/events-backfill.log") - summary = amm_summary(text, mdir / "events-backfill.log") + summary = amm_summary(text, mdir / "events-backfill.log", dry_run=bool(flag)) record_amm_summary(ms, m, summary) elif a.amm == "ssh": for flag in (" --dry-run", ""): @@ -924,7 +940,7 @@ def done(): mdir / "events-backfill.log", stdin_text=pw) if code != 0 or "=== events-backfill complete ===" not in text: raise Stop(f"events-backfill{flag} exit {code} — see {mdir}/events-backfill.log") - summary = amm_summary(text, mdir / "events-backfill.log") + summary = amm_summary(text, mdir / "events-backfill.log", dry_run=bool(flag)) record_amm_summary(ms, m, summary) else: marker = mdir / "amm.done" diff --git a/tools/scripts/reingest_0286_sac_test.py b/tools/scripts/reingest_0286_sac_test.py new file mode 100644 index 00000000..720f8a90 --- /dev/null +++ b/tools/scripts/reingest_0286_sac_test.py @@ -0,0 +1,65 @@ +"""reingest_0286.py reads events-backfill's task-0242 `unproven sac swaps:` line. + +python3 -m unittest discover -s tools/scripts -p 'reingest_0286_sac_test.py' +""" + +import importlib.util +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +spec = importlib.util.spec_from_file_location("reingest_0286", HERE / "reingest_0286.py") +r = importlib.util.module_from_spec(spec) +spec.loader.exec_module(r) + +LOG = Path("/m/202609/events-backfill.log") + + +def summary(unproven=None, fallbacks=True): + """events-backfill's closing summary; `unproven=None` is a pre-0242 binary.""" + lines = ["=== events-backfill complete ===", + "swaps dropped (unresolved):0", + "swaps failed dispatch: 0"] + if fallbacks: + lines.append("negative apply order: 0") + if unproven is not None: + lines.append(f"unproven sac swaps: {unproven}") + return "\n".join(lines) + "\n" + + +class UnprovenSacLine(unittest.TestCase): + def test_the_line_is_read_and_only_non_zero_stops(self): + for unproven, expected in ((0, 0), (None, None)): + with self.subTest(unproven=unproven): + found = r.amm_summary(summary(unproven), LOG, dry_run=True) + self.assertEqual(found["unproven_sacs"], expected) + self.assertEqual((found["fallbacks"], found["dropped"]), (0, 0)) + + def test_a_non_zero_count_stops_naming_it_and_what_the_month_holds(self): + # The dry run comes after drop_1m and sdex: never "nothing was written". + for dry_run, says in ((True, ("--dry-run pass", "no AMM candle was written", + "202609's 1m partition is already dropped", "SDEX-only", + "Fix it today", "`rollback 202609`")), + (False, ("write pass", "AMM candles are in without those swaps", + "`rollback 202609`"))): + with self.subTest(dry_run=dry_run), self.assertRaises(r.Stop) as cm: + r.amm_summary(summary(3), LOG, dry_run=dry_run) + msg = str(cm.exception) + for part in ("3 swaps", "task 0242 D2", str(LOG), + "docs/runbooks/0242-sac-identity-heal.md §4c") + says: + self.assertIn(part, msg) + self.assertNotIn("nothing was written", msg) + + def test_a_missing_required_line_still_stops(self): + with self.assertRaises(r.Stop) as cm: + r.amm_summary(summary(0, fallbacks=False), LOG, dry_run=True) + self.assertIn("negative apply order:", str(cm.exception)) + + def test_the_label_is_the_one_events_backfill_prints(self): + run_rs = (REPO / "packages/events-backfill/src/run.rs").read_text() + self.assertIn(f'UNPROVEN_SAC_LABEL: &str = "{r.UNPROVEN_SAC_LABEL}"', run_rs) + + +if __name__ == "__main__": + unittest.main() From 4ace1ce8897c399c9d8c30e692790b3b41847fdf Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:19:24 +0200 Subject: [PATCH 003/144] fix(lore-0242): probe SAC contract identities and alarm on unproven SAC skips The resolver stops new SAC mints, but nothing watched for one. The rollup-freshness probe now publishes SacContractIdentities: contract rows of assets FINAL that are some row's sac_address or a BE is_sac contract, refused on an empty scan or an empty BE set. Production reads 36 until the heal, so its alarm ships with actions off behind opsAlarms.sacContractIdentitiesActionsEnabled. The live SacUnprovenSkipped counter gets its alarm, enabled, shaped like the 0291 one. --- infra/envs/production.json | 1 + infra/src/lib/mtls.ts | 4 +- infra/src/lib/stacks/observability-stack.ts | 77 ++++++++++ infra/src/lib/types.ts | 13 ++ .../src/asset_id_uniqueness.rs | 129 +++++++++++++++- packages/rollup-freshness-probe/src/main.rs | 42 ++++- .../tests/rollup_freshness_it.rs | 143 ++++++++++++++++++ 7 files changed, 401 insertions(+), 8 deletions(-) diff --git a/infra/envs/production.json b/infra/envs/production.json index 2263d683..f70088dd 100644 --- a/infra/envs/production.json +++ b/infra/envs/production.json @@ -56,6 +56,7 @@ "chDiskFreePercent": 20, "dlqEscalationDepths": [10, 50], "usdSanityEscalationCounts": [1, 100, 10000], + "sacContractIdentitiesActionsEnabled": false, "slack": { "workspaceIdSsmParam": "/prices/production/slack-workspace-id", "channelIdSsmParam": "/prices/production/slack-channel-id" diff --git a/infra/src/lib/mtls.ts b/infra/src/lib/mtls.ts index a5e709cb..3669d8d8 100644 --- a/infra/src/lib/mtls.ts +++ b/infra/src/lib/mtls.ts @@ -55,7 +55,9 @@ export function secretsManagerLayerArn(region: string): string { * `default.soroban_contracts` over this identity; those two SELECTs are * BE's to grant (request text: docs/runbooks/0100-coverage-sweep-triage.md * §4.1), and until they land the probe fails with Code 497. A dedicated - * read-only identity for it is task 0258's later clean-up. + * read-only identity for it is task 0258's later clean-up. Task 0242 also + * reads `default.soroban_contracts` over it, from the ledger processor's + * cold start and the rollup-freshness probe. * - `api` → CH user `prices_reader` (axum read handlers; * `SELECT ON prices.*`). */ diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index 19ed6799..7eb9df8d 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -339,6 +339,11 @@ export class ObservabilityStack extends cdk.Stack { public readonly ledgerProcessorForcedPartialFlushAlarm: cloudwatch.Alarm; /** Task 0291: live dropped trades from a pool missing from `pool_registry`. */ public readonly ledgerProcessorUnregisteredPoolAlarm: cloudwatch.Alarm; + /** + * Task 0242 D2: live skipped trades on a token BE flags `is_sac` that no SAC + * event proved, rather than minting it as a second identity. + */ + public readonly ledgerProcessorSacUnprovenAlarm: cloudwatch.Alarm; /** * Task 0100: the weekly coverage sweep found swap/trade emitters in neither * `pool_registry` nor the committed allow-list (layer 3). @@ -386,6 +391,12 @@ export class ObservabilityStack extends cdk.Stack { * 0139): a writer sent ids, not identities. Keyed by count. */ public readonly assetIdOrphanCandleAlarms: Record; + /** + * Contract identities in `assets FINAL` that are SACs (task 0242 D6). Actions + * behind `opsAlarms.sacContractIdentitiesActionsEnabled`: the baseline is 36 + * until the heal. + */ + public readonly sacContractIdentitiesAlarm: cloudwatch.Alarm; /** * A rollup MV that has lost `APPEND` (task 0204, gap 3) — history destroyed * on every refresh. Separate from {@link mvDriftAlarm} because this is the @@ -1468,6 +1479,38 @@ export class ObservabilityStack extends cdk.Stack { this.ledgerProcessorUnregisteredPoolAlarm.addAlarmAction(snsAction); this.ledgerProcessorUnregisteredPoolAlarm.addOkAction(snsAction); + // Task 0242 D2 — live skipped trades on an unproven SAC. A SAC leg resolves + // to its classic asset only when the same transaction carries the SAC's own + // transfer/mint/burn/clawback event; a token BE flags `is_sac` with no such + // proof is skipped rather than minted as a second identity. Same shape as + // the 0291 alarm above: `SacUnprovenSkipped` is published only when + // non-zero. + this.ledgerProcessorSacUnprovenAlarm = new cloudwatch.Alarm( + this, + 'LedgerProcessorSacUnprovenAlarm', + { + alarmName: `prices-${config.envName}-ledger-processor-sac-unproven`, + alarmDescription: + 'The ledger-processor skipped AMM trades with a leg that BE flags is_sac (default.soroban_contracts) but that no SAC transfer/mint/burn/clawback event in the same transaction proved (task 0242 D2). Those trades produce no candle; they are skipped rather than minted as a second, Contract identity of the asset. The WARN "skipped trades on is_sac contracts with no SAC proof (task 0242)" lists the contracts. Fix: docs/runbooks/0242-sac-identity-heal.md.', + metric: new cloudwatch.Metric({ + namespace: 'Prices/Ingest', + metricName: 'SacUnprovenSkipped', + dimensionsMap: { Environment: config.envName }, + statistic: 'Sum', + period: cdk.Duration.minutes(5), + }), + threshold: 1, + evaluationPeriods: 1, + datapointsToAlarm: 1, + comparisonOperator: + cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + // Emitted only when trades were skipped: "missing" is healthy. + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + }, + ); + this.ledgerProcessorSacUnprovenAlarm.addAlarmAction(snsAction); + this.ledgerProcessorSacUnprovenAlarm.addOkAction(snsAction); + // Task 0100 — layer 3 of the coverage model. Layer 2 (the alarm above) // sees only pools shaped like a venue we already index; the weekly coverage // sweep looks at EVERY contract emitting swap/trade-shaped events over a @@ -1750,6 +1793,40 @@ export class ObservabilityStack extends cdk.Stack { zeroLadder, ); + // SAC contract identities (task 0242 D6). A SAC is a facet of its classic + // asset; a contract row of assets FINAL that is a SAC is the 0242 defect. + // One alarm, not the ladder: usdSanityRungs takes no actionsEnabled, and the + // actions must start off because production reads 36 until the heal + // (13 split pairs + 23 SACs whose classic was never held, 2026-10-05). + // Actions off does not stop evaluation: the alarm sits in ALARM, red on the + // Row-0 strip, until the heal. It stays on the strip, which shows every + // alarm (SCF Tranche 3 AC 8); the description says the red is expected. + this.sacContractIdentitiesAlarm = new cloudwatch.Alarm( + this, + 'SacContractIdentitiesAlarm', + { + alarmName: `prices-${config.envName}-sac-contract-identities`, + alarmDescription: + "Contract rows of prices.assets FINAL whose address is a SAC: the sac_address of a classic row, or a contract BE flags is_sac in default.soroban_contracts (task 0242 D6). Baseline 36 until the 0242 heal, which is why actions start disabled (opsAlarms.sacContractIdentitiesActionsEnabled) and the state reads ALARM until then: that red tile on the dashboard's alarm strip is expected, not an incident. After the heal it reads 0, and a non-zero value means a SAC was minted as a Contract identity: a second asset_id for an asset we hold, splitting its candles and volume. Find it: SELECT asset_id, contract_address FROM prices.assets FINAL WHERE contract_address IN (SELECT sac_address FROM prices.assets WHERE sac_address != '') OR contract_address IN (SELECT contract_id FROM default.soroban_contracts WHERE is_sac). Runbook: docs/runbooks/0242-sac-identity-heal.md.", + metric: new cloudwatch.Metric({ + namespace: 'Prices/Rollup', + metricName: 'SacContractIdentities', + dimensionsMap: { Environment: config.envName }, + statistic: 'Maximum', + period: cdk.Duration.minutes(15), + }), + threshold: 1, + evaluationPeriods: 2, + datapointsToAlarm: 1, + comparisonOperator: + cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + actionsEnabled: config.opsAlarms.sacContractIdentitiesActionsEnabled, + }, + ); + this.sacContractIdentitiesAlarm.addAlarmAction(snsAction); + this.sacContractIdentitiesAlarm.addOkAction(snsAction); + // Materialized-view drift, on a schedule (task 0204, gap 3). Task 0142 built // `prices-clickhouse-drift` and NOTHING RAN IT — a check nobody runs is a // check that does not exist. It covers a condition no other alarm here can diff --git a/infra/src/lib/types.ts b/infra/src/lib/types.ts index 31e32902..0b664852 100644 --- a/infra/src/lib/types.ts +++ b/infra/src/lib/types.ts @@ -477,6 +477,14 @@ export interface EnvironmentConfig { * Must be strictly increasing positive integers. */ readonly usdSanityEscalationCounts: readonly number[]; + /** + * Actions of the `SacContractIdentities` alarm (task 0242 D6). Starts + * `false`: production holds 36 SAC contract identities until the 0242 heal, + * so an enabled alarm would latch from its first datum. The last step of + * docs/runbooks/0242-sac-identity-heal.md sets it `true`. A config key, not + * a console toggle, because the next CDK deploy would undo the toggle. + */ + readonly sacContractIdentitiesActionsEnabled: boolean; /** * Optional AWS Chatbot → Slack routing for the ops-alarms topic (task 0056). * When set, `ObservabilityStack` subscribes `prices-{env}-ops-alarms` to a @@ -1081,6 +1089,11 @@ export function validateConfig(config: EnvironmentConfig): void { `opsAlarms.currentPricesFreshnessSeconds must be an integer above the healthy peak of ${CURRENT_PRICES_HEALTHY_PEAK_SECONDS}s (60 s refresh interval + 40 s worst accepted refresh), or the alarm fires on a healthy table; got: ${ops.currentPricesFreshnessSeconds}`, ); } + if (typeof ops.sacContractIdentitiesActionsEnabled !== 'boolean') { + errors.push( + `opsAlarms.sacContractIdentitiesActionsEnabled must be a boolean, got: ${ops.sacContractIdentitiesActionsEnabled}`, + ); + } if ( !Number.isInteger(ops.ledgerProcessorLagSeconds) || ops.ledgerProcessorLagSeconds < 1 diff --git a/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs b/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs index e413611b..19c371f1 100644 --- a/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs +++ b/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs @@ -18,9 +18,13 @@ //! whose `asset_id` or `quote_asset_id` has no `assets` row. This is the //! post-window detector for a writer out of step with the registry, e.g. an //! old binary still writing counter ids after the swap (BRIEF §3.7). +//! 3. `SacContractIdentities` = contract rows of `assets FINAL` whose address +//! is a SAC: some row's `sac_address`, or a contract BE flags `is_sac` +//! (task 0242). A SAC is a facet of its classic asset, never an identity of +//! its own. Baseline 36 until the 0242 heal, then 0. //! -//! The two reads are independent: a failure in one must not suppress the -//! other's datum. A read that saw nothing is refused, never published as a +//! The reads are independent: a failure in one must not suppress another's +//! datum. A read that saw nothing is refused, never published as a //! healthy zero (same rule as [`crate::zero_invariants`]): the alarms are //! `NOT_BREACHING` on missing data, so an empty registry or an empty window //! must surface as a failed check. @@ -36,6 +40,12 @@ pub const ASSET_ID_COLLISIONS_METRIC: &str = "AssetIdCollisions"; /// CloudWatch metric: recent candles naming an id that has no `assets` row. pub const ASSET_ID_ORPHAN_CANDLES_METRIC: &str = "AssetIdOrphanCandles"; +/// CloudWatch metric: contract identities in `assets` that are SACs. +pub const SAC_CONTRACT_IDENTITIES_METRIC: &str = "SacContractIdentities"; + +/// BE's database, holding `soroban_contracts` (prices_writer has SELECT on it). +pub const BE_DATABASE: &str = "default"; + /// The tier every writer of candles writes; the coarse tiers derive from it. pub const ORPHAN_CANDLE_TABLE: &str = "price_ohlcv_1m"; @@ -84,6 +94,36 @@ pub fn orphan_candles_query() -> String { ) } +/// One reading of the contract rows of `assets FINAL` against the SAC sets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clickhouse::Row, serde::Deserialize)] +pub struct SacContractCounts { + /// Contract rows whose address is a SAC. + pub sac_rows: u64, + /// Contract rows examined. Zero means the scan measured nothing. + pub scanned: u64, + /// Contracts BE flags `is_sac`. Zero means the second test measured nothing. + pub be_sacs: u64, +} + +/// The SAC-identity read. A contract row is a SAC when its address is the +/// `sac_address` of a classic row, or when BE flags it `is_sac` (a SAC whose +/// classic we never held). `uniqExact`, because `soroban_contracts` is a +/// ReplacingMergeTree read without `FINAL`; `ifNull`, because a scalar +/// subquery is `Nullable` and RowBinary would misread it as a `u64`. +pub fn sac_contract_identities_query(be_db: &str) -> String { + format!( + "SELECT \ + countIf(contract_address IN (SELECT sac_address FROM assets WHERE sac_address != '') \ + OR contract_address IN (SELECT contract_id FROM {be_db}.soroban_contracts WHERE is_sac)) \ + AS sac_rows, \ + count() AS scanned, \ + ifNull((SELECT uniqExact(contract_id) FROM {be_db}.soroban_contracts WHERE is_sac), 0) \ + AS be_sacs \ + FROM assets FINAL \ + WHERE contract_address != ''" + ) +} + /// Why a reading was not published: the read measured nothing, so its zero /// would be a false healthy datum. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -92,6 +132,10 @@ pub enum UniquenessRefusal { EmptyRegistry, /// The candle window held no row. EmptyWindow, + /// `assets FINAL` held no contract row. + NoContractRows, + /// BE's `soroban_contracts` held no `is_sac` contract. + EmptyBeSacSet, } impl std::fmt::Display for UniquenessRefusal { @@ -110,6 +154,17 @@ impl std::fmt::Display for UniquenessRefusal { zero over a scan that measured nothing", ORPHAN_LOOKBACK_SECONDS, ORPHAN_CANDLE_TABLE ), + Self::NoContractRows => write!( + f, + "sac-contract-identities unreadable: assets FINAL holds no contract row, so a \ + count of 0 would measure nothing; the registry is empty or unreadable to this user" + ), + Self::EmptyBeSacSet => write!( + f, + "sac-contract-identities unreadable: {BE_DATABASE}.soroban_contracts holds no \ + is_sac contract, so SACs whose classic we never held go uncounted; BE's table \ + is empty or unreadable to this user" + ), } } } @@ -140,6 +195,22 @@ pub fn orphan_candles_metric( }) } +/// Shape a SAC-identity reading into its datum, or refuse it. +pub fn sac_contract_identities_metric( + counts: &SacContractCounts, +) -> Result { + if counts.scanned == 0 { + return Err(UniquenessRefusal::NoContractRows); + } + if counts.be_sacs == 0 { + return Err(UniquenessRefusal::EmptyBeSacSet); + } + Ok(SanityMetric { + name: SAC_CONTRACT_IDENTITIES_METRIC, + value: counts.sac_rows as f64, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -225,5 +296,59 @@ mod tests { assert!(window.contains("unreadable"), "{window}"); assert!(window.contains("price_ohlcv_1m"), "{window}"); assert!(window.contains("7200"), "{window}"); + let contracts = UniquenessRefusal::NoContractRows.to_string(); + assert!(contracts.contains("unreadable"), "{contracts}"); + assert!(contracts.contains("no contract row"), "{contracts}"); + let be = UniquenessRefusal::EmptyBeSacSet.to_string(); + assert!(be.contains("unreadable"), "{be}"); + assert!(be.contains("default.soroban_contracts"), "{be}"); + } + + #[test] + fn the_sac_query_counts_contract_rows_against_both_sac_sets() { + let sql = sac_contract_identities_query(BE_DATABASE); + for part in [ + "FROM assets FINAL", + "contract_address != ''", + "contract_address IN (SELECT sac_address FROM assets WHERE sac_address != '')", + "contract_address IN (SELECT contract_id FROM default.soroban_contracts WHERE is_sac)", + "ifNull((SELECT uniqExact(contract_id) FROM default.soroban_contracts WHERE is_sac), 0)", + "AS be_sacs", + "AS sac_rows", + "count() AS scanned", + ] { + assert!(sql.contains(part), "{part}: {sql}"); + } + assert!(!sql.contains("SETTINGS"), "{sql}"); + } + + #[test] + fn sac_contract_identities_are_published_or_refused() { + let cases = [ + ((0, 5, 10), Ok(0.0), "no SAC among the contracts"), + ((36, 59, 4_042), Ok(36.0), "production on 2026-10-05"), + ( + (0, 0, 10), + Err(UniquenessRefusal::NoContractRows), + "no contract row", + ), + ( + (0, 5, 0), + Err(UniquenessRefusal::EmptyBeSacSet), + "empty BE SAC set", + ), + ]; + for ((sac_rows, scanned, be_sacs), want, why) in cases { + let got = sac_contract_identities_metric(&SacContractCounts { + sac_rows, + scanned, + be_sacs, + }); + assert_eq!( + got.map(|m| (m.name, m.value)), + want.map(|v| (SAC_CONTRACT_IDENTITIES_METRIC, v)), + "{why}" + ); + } } } diff --git a/packages/rollup-freshness-probe/src/main.rs b/packages/rollup-freshness-probe/src/main.rs index 5b6b102c..a3102d3f 100644 --- a/packages/rollup-freshness-probe/src/main.rs +++ b/packages/rollup-freshness-probe/src/main.rs @@ -18,8 +18,9 @@ async fn main() -> Result<(), lambda_runtime::Error> { use lambda_runtime::{LambdaEvent, run, service_fn}; use rollup_freshness_probe::asset_id_uniqueness::{ - AssetIdCounts, OrphanCandleCounts, collisions_metric, collisions_query, - orphan_candles_metric, orphan_candles_query, + AssetIdCounts, BE_DATABASE, OrphanCandleCounts, SacContractCounts, collisions_metric, + collisions_query, orphan_candles_metric, orphan_candles_query, + sac_contract_identities_metric, sac_contract_identities_query, }; use rollup_freshness_probe::current_prices::{ CurrentPricesAge, current_prices_age_query, current_prices_metric, @@ -56,7 +57,8 @@ async fn main() -> Result<(), lambda_runtime::Error> { // secret/endpoint surfaces on the first query (the invocation fails and the // probe's error alarm fires) — no separate `SELECT 1` liveness probe is // needed. The `ingestion` mTLS identity (prices_writer) has SELECT on - // prices.*, which is all this probe needs. + // prices.*, and on `default.soroban_contracts`, which the SAC-identity + // read joins (task 0242). // // ⚠️ This comment used to claim the probe touches no `system.*` table. Since // task 0204 gap 3 it reads `system.tables`, and that is fine — `system.tables` @@ -74,6 +76,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { let zero_invariant_query = Arc::new(zero_invariant_query()); let collisions_query = Arc::new(collisions_query()); let orphan_candles_query = Arc::new(orphan_candles_query()); + let sac_contract_identities_query = Arc::new(sac_contract_identities_query(BE_DATABASE)); let refresh_waits_query = Arc::new(refresh_waits_query("prices")); // The mismatch reads are the only ones here whose cost grows with a week // of the child tier (the 15m read scans seven days of `_1m`: 0.7–4.0 s on @@ -103,6 +106,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { let zero_invariant_query = zero_invariant_query.clone(); let collisions_query = collisions_query.clone(); let orphan_candles_query = orphan_candles_query.clone(); + let sac_contract_identities_query = sac_contract_identities_query.clone(); let refresh_waits_query = refresh_waits_query.clone(); let mismatch_queries = mismatch_queries.clone(); let environment = environment.clone(); @@ -398,9 +402,9 @@ async fn main() -> Result<(), lambda_runtime::Error> { Err(e) => failures.push(format!("zero-invariants read: {e}")), } - // ---- 5b. Asset-id uniqueness (task 0139) ---------------------- + // ---- 5b. Asset-id uniqueness (tasks 0139, 0242) --------------- // - // Two independent reads, after the zero invariants for the same + // Three independent reads, after the zero invariants for the same // reason: they grow with the data (the registry, and the live tip // of `_1m` against it). A refusal is an empty read, recorded as a // failure rather than published as a healthy 0. @@ -446,6 +450,31 @@ async fn main() -> Result<(), lambda_runtime::Error> { Err(e) => failures.push(format!("asset-id-orphans read: {e}")), } + let mut sac_counts: Option = None; + match ch + .query(&sac_contract_identities_query) + .fetch_one::() + .await + { + Ok(counts) => { + sac_counts = Some(counts); + match sac_contract_identities_metric(&counts) { + Ok(metric) => { + if let Err(e) = + publish_sanity(&cw, &environment, std::slice::from_ref(&metric)) + .await + { + failures.push(format!("sac-contract-identities publish: {e}")); + } + } + Err(refusal) => { + failures.push(format!("sac-contract-identities: {refusal}")) + } + } + } + Err(e) => failures.push(format!("sac-contract-identities read: {e}")), + } + // ---- 6. Coarse buckets disagreeing with their source (0203) --- // // The new tail, for the reason the zero invariants were last: these @@ -592,6 +621,9 @@ async fn main() -> Result<(), lambda_runtime::Error> { "ids": id_counts.map(|c| c.ids), "orphan_candles": orphan_counts.map(|c| c.orphans), "orphan_scanned": orphan_counts.map(|c| c.scanned), + "sac_contract_rows": sac_counts.map(|c| c.sac_rows), + "sac_contract_scanned": sac_counts.map(|c| c.scanned), + "be_sac_contracts": sac_counts.map(|c| c.be_sacs), }, "mv_drift": { "critical": drift_critical, diff --git a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs index 27659200..97d72eb8 100644 --- a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs +++ b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs @@ -1929,3 +1929,146 @@ async fn an_empty_registry_or_window_is_refused_as_unreadable() { drop_scratch_db(db).await; } + +// ---- Task 0242: SAC contract identities ------------------------------------- +// +// A scratch prices database plus a scratch BE database holding only +// `soroban_contracts` (DDL as coverage_sweep_it.rs), read through the +// production query. + +const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; +const XCR: AssetFixture = AssetFixture::new( + "XCR", + "classic", + "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7", + "", +) +.with_sac(XCR_SAC); +const XCR_AS_CONTRACT: AssetFixture = AssetFixture::new("", "soroban", "", XCR_SAC); +/// An is_sac contract whose classic `assets` never held. +const ORPHAN_SAC: &str = "CORPHANSACXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"; +const ORPHAN_SAC_ROW: AssetFixture = AssetFixture::new("", "soroban", "", ORPHAN_SAC); +const TOKEN: &str = "CGENUINETOKENXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"; +const TOKEN_ROW: AssetFixture = AssetFixture::new("", "soroban", "", TOKEN); + +async fn scratch_be_db(db: &str) { + let admin = Client::default().with_url(ch_url()); + exec(&admin, &format!("DROP DATABASE IF EXISTS {db}")).await; + exec(&admin, &format!("CREATE DATABASE {db}")).await; + exec( + &admin, + &format!( + "CREATE TABLE {db}.soroban_contracts ( + id Int64, + contract_id String, + wasm_hash Nullable(FixedString(32)), + wasm_uploaded_at_ledger Int64 DEFAULT 0, + deployer_id Nullable(Int64), + deployed_at_ledger Nullable(Int64), + contract_type Nullable(Int16), + is_sac Bool + ) + ENGINE = ReplacingMergeTree(wasm_uploaded_at_ledger) + ORDER BY (contract_id)" + ), + ) + .await; +} + +async fn be_contract(be: &str, contract: &str, is_sac: bool) { + let admin = Client::default().with_url(ch_url()); + exec( + &admin, + &format!( + "INSERT INTO {be}.soroban_contracts (id, contract_id, is_sac) \ + VALUES (1, '{contract}', {is_sac})" + ), + ) + .await; +} + +async fn read_sac_contracts( + c: &Client, + be: &str, +) -> rollup_freshness_probe::asset_id_uniqueness::SacContractCounts { + c.query(&rollup_freshness_probe::asset_id_uniqueness::sac_contract_identities_query(be)) + .fetch_one() + .await + .expect("the SAC-identity query executes and deserializes") +} + +/// A contract row is counted through either disjunct: XCR's SAC through the +/// classic row's `sac_address`, a SAC with no classic through BE's `is_sac`. +/// A genuine token is scanned, not counted. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn sac_contract_identities_counts_both_disjuncts_and_not_a_token() { + use rollup_freshness_probe::asset_id_uniqueness::sac_contract_identities_metric; + + let (db, be) = ("it_probe_0242_sac", "it_probe_0242_sac_be"); + let c = scratch_db(db).await; + scratch_be_db(be).await; + // XCR_SAC is deliberately absent from BE: the first disjunct alone counts it. + // ORPHAN_SAC on two unmerged versions still counts once in be_sacs. + be_contract(be, ORPHAN_SAC, true).await; + be_contract(be, ORPHAN_SAC, true).await; + be_contract(be, TOKEN, false).await; + + let cases = [ + (XCR, (0, 0), "classic only: no contract row"), + (XCR_AS_CONTRACT, (1, 1), "XCR's SAC via sac_address"), + (ORPHAN_SAC_ROW, (2, 2), "a SAC with no classic via is_sac"), + (TOKEN_ROW, (2, 3), "a genuine token is not a SAC"), + ]; + for (row, (sac_rows, scanned), why) in cases { + exec(&c, &assets_insert(db, &[row])).await; + let counts = read_sac_contracts(&c, be).await; + assert_eq!( + (counts.sac_rows, counts.scanned), + (sac_rows, scanned), + "{why}" + ); + assert_eq!(counts.be_sacs, 1, "{why}"); + if scanned > 0 { + assert_eq!( + sac_contract_identities_metric(&counts).unwrap().value, + sac_rows as f64, + "{why}" + ); + } + } + + drop_scratch_db(db).await; + drop_scratch_db(be).await; +} + +/// No contract row, or no is_sac contract in BE, is unreadable: refused rather +/// than published as a healthy 0. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn sac_contract_identities_refuses_an_empty_scan_or_be_set() { + use rollup_freshness_probe::asset_id_uniqueness::{ + UniquenessRefusal, sac_contract_identities_metric, + }; + + let (db, be) = ("it_probe_0242_sac_empty", "it_probe_0242_sac_empty_be"); + let c = scratch_db(db).await; + scratch_be_db(be).await; + be_contract(be, ORPHAN_SAC, true).await; + exec(&c, &assets_insert(db, &[XCR])).await; + assert_eq!( + sac_contract_identities_metric(&read_sac_contracts(&c, be).await), + Err(UniquenessRefusal::NoContractRows) + ); + + scratch_be_db(be).await; + be_contract(be, TOKEN, false).await; + exec(&c, &assets_insert(db, &[TOKEN_ROW])).await; + assert_eq!( + sac_contract_identities_metric(&read_sac_contracts(&c, be).await), + Err(UniquenessRefusal::EmptyBeSacSet) + ); + + drop_scratch_db(db).await; + drop_scratch_db(be).await; +} From 3de111817ba8b6120df4f74a42f438c88dafdde5 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:28:01 +0200 Subject: [PATCH 004/144] fix(lore-0242): answer a SAC address as its classic asset A classic asset's SAC address answered 404, or, for a SAC we had minted as a second identity, a near-empty listing of the same token. Resolve a C... identifier against the stored assets.sac_address once at the handler boundary, so detail, price, ohlcv, oracles and batch answer the classic (XLM's SAC as native), and the classic wins over a pre-heal Contract row (D5). --- docs/prices-api-general-overview.md | 17 +- packages/prices-api/src/assets/handlers.rs | 34 +-- packages/prices-api/src/assets/queries_ch.rs | 209 +++++++++++++++++++ packages/prices-api/src/batch/handlers.rs | 24 ++- packages/prices-api/src/identity.rs | 3 +- packages/prices-api/src/oracles/handlers.rs | 8 +- packages/prices-api/tests/endpoints_it.rs | 187 ++++++++++++++++- web/portal/public/openapi.json | 10 +- 8 files changed, 461 insertions(+), 31 deletions(-) diff --git a/docs/prices-api-general-overview.md b/docs/prices-api-general-overview.md index e9d08204..7720f711 100644 --- a/docs/prices-api-general-overview.md +++ b/docs/prices-api-general-overview.md @@ -15,6 +15,7 @@ the API surface, or the cost / budget framing. | Date | Sections touched | Driver | Summary | | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-05 | §4.1, §4.3 | [Task 0242](../lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md) | **A SAC address is an alias of its classic asset.** A classic asset's Stellar Asset Contract address, XLM's included, used to answer `404` or, for a SAC we had minted as a second identity, a near-empty listing of the same token. It now answers as the classic asset on every identifier-taking endpoint, named by `{code}:{issuer}` or `native`, and the classic wins over a not-yet-healed second identity. In a batch the found entry carries the classic's name while `not_found` echoes what was asked. | | 2026-10-01 | §3.1, §3.2, §3.3, §3.4 | [Task 0139](../lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md) | **`asset_id` is derived by ClickHouse from the identity**, `xxh3` of `code:issuer:contract` as `UInt64`, instead of a `UInt32` counter in each writer. Two writers' counters had handed one id to two assets 3,315 times, so every join on `asset_id` fanned out. | | 2026-09-24 | §4 (intro, §4.1, §4.2, §4.3) | [Task 0306](../lore/1-tasks/active/0306_DOCS_published-docs-drift-from-production.md) · [Task 0178](../lore/1-tasks/archive/0178_BUG_current-prices-cannot-publish-the-quote-asset.md) | **§4 brought to what production answers**, next to the published API reference. The base URL is the production host, and a new note separates the two error bodies: the API's `{code, message}` and API Gateway's `{message}` for `403` and `429`, with the handler's own `401` not armed in production. The `/assets` and `/price` examples are the 2026-09-23 production snapshot (AQUA; XLM with its five venues) and carry `method`; `GET /assets/{asset_identifier}` and `POST /prices/batch` gain their response bodies. `volume_24h_usd` is defined as both sides of each trade (task 0178) against the base side in `sources` — the old example's venues summed exactly to the total, which production never shows. Recorded as not populated yet: `home_domain`, a classic asset's SAC in `contract_address`, and a Soroban token's `asset_code`. | | 2026-09-16 | §3.2, §4.2 (`GET /ohlcv`) | [Task 0286](../lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md) · [ADR 0287](../lore/2-adrs/0287_candle-prices-come-from-price-forming-fills-and-a-windowed-close.md) | **A candle's prices come only from the price-forming trades of its own bucket** — `open`/`close` the first and last of them, `high`/`low` their extremes, nothing carried from a neighbouring bucket. A fill's price is the ratio of two integer stroop amounts, so a few-stroop fill prints an exact small fraction that is arithmetically right and can sit hundreds of percent off the market; such fills no longer set a price. The task-0116 guidance to filter dust **on the client** is replaced by the rule and by three new `/ohlcv` fields — `pf_trade_count` (`0` means the bucket has no price and every price field is `null`), `pf_vwap` (the price-forming mean) and `close_divergent` (`close` more than 1% from it). Added the three `pf_*` columns to the §3.2 DDL and re-cut the rollup sketch to the shipped generator's pf-gated form, with the month rolled from the day. | @@ -634,7 +635,12 @@ Get single asset details. `asset_identifier` can be: - `{contract_address}` for Soroban tokens (e.g. `CABC...DEF`) - `native` for XLM -A classic asset's SAC address is not an identifier yet: it answers `404`. +A classic asset's Stellar Asset Contract address, XLM's included, is accepted as an alias of that +asset, and the answer names it by `{code}:{issuer}` or `native`. The same holds on `/price`, +`/ohlcv`, `GET /oracles/{asset_identifier}` and `POST /prices/batch`. For example, +`GET /assets/CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA` returns XLM with +`"asset": "native"`. Any other `C…` address is looked up as a contract: a Soroban token we hold +answers as itself, and an address we do not hold answers `404`. **Response:** `code` is the token's symbol for a Soroban token and `XLM` for `native`. @@ -863,9 +869,12 @@ Fetch current prices for multiple assets in one call. ``` **Response:** `prices` holds the `GET /assets/{asset_identifier}/price` object of every identifier -that has a current price, in the order of the request; `not_found` lists the rest, in canonical -form. A request names 1 to 100 identifiers, a repeated one is answered each time, and a malformed -body, an empty or over-long list or a single invalid identifier fails the whole request with `400`. +that has a current price, in the order of the request with the not-found ones left out; +`not_found` lists the rest as they were asked, in canonical form. Each entry's `asset` is the +canonical identifier of the asset found, so a classic asset's SAC address comes back as +`{code}:{issuer}` or `native` (§4.1), and a request naming both forms gets two identical entries. +A request names 1 to 100 identifiers, a repeated one is answered each time, and a malformed body, +an empty or over-long list or a single invalid identifier fails the whole request with `400`. ```json { diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index 336fc9c5..e94461e8 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -81,8 +81,8 @@ fn min_volume_error(v: Option) -> Option { ("asset_identifier" = String, Path, description = "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or \ the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic \ - asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract \ - addresses, which as a rule answer `404`"), + asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the \ + answer names the asset by `CODE:ISSUER` (or `native`)"), ("min_volume_usd" = Option, Query, minimum = 0, description = "Drop every venue whose trailing 24-hour USD volume is at or below this value, then \ recompute `vwap_24h` over the venues that remain. Applied exactly as given: it can \ @@ -114,6 +114,10 @@ pub async fn get_price( if let Some(resp) = min_volume_error(q.min_volume_usd) { return resp; } + let id = match queries_ch::resolve_sac_alias(state.ch(), id).await { + Ok(id) => id, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; match queries_ch::current_price(state.ch(), &id).await { Ok(Some(row)) => { @@ -143,8 +147,8 @@ pub async fn get_price( ("asset_identifier" = String, Path, description = "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or \ the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic \ - asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract \ - addresses, which as a rule answer `404`") + asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the \ + answer names the asset by `CODE:ISSUER` (or `native`)") ), responses( (status = 200, description = "Asset detail", body = AssetDetail), @@ -164,6 +168,10 @@ pub async fn get_asset( Ok(id) => id, Err(e) => return errors::bad_request(errors::INVALID_ID, e.to_string()), }; + let id = match queries_ch::resolve_sac_alias(state.ch(), id).await { + Ok(id) => id, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; match queries_ch::asset_detail(state.ch(), &id).await { Ok(Some(row)) => { @@ -403,8 +411,8 @@ pub struct OhlcvParams { params( ("asset_identifier" = String, Path, description = "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or \ the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic \ - asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract \ - addresses, which as a rule answer `404`"), + asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the \ + answer names the asset by `CODE:ISSUER` (or `native`)"), ("timeframe" = Option, Query, description = "Window ending now: `1h`, `24h` (default), `7d`, `30d`, `1y` or `all` (from Stellar \ genesis). `start` overrides its start"), @@ -533,6 +541,10 @@ pub async fn get_ohlcv( } // Resolve the base asset. + let id = match queries_ch::resolve_sac_alias(state.ch(), id).await { + Ok(id) => id, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; let asset_id = match queries_ch::resolve_asset_id(state.ch(), &id).await { Ok(Some(a)) => a, Ok(None) => return errors::not_found("unknown asset"), @@ -627,17 +639,13 @@ pub async fn get_ohlcv( limit: OHLCV_MAX_POINTS, }; // ADR 0011 §6: canonical USDC is only ever a quote leg, so a normal query - // for it matches zero rows and no amount of filter-dropping helps — its - // series is synthesized instead. Keyed on the requested identity, not on a - // resolved asset_id, because 0139 has ids serving more than one identity. - // ADR 0011 §6: canonical USDC is only ever a quote leg, so a normal query // for it matches zero rows in EITHER denomination and no amount of // filter-dropping helps — its series is synthesized instead. // // Matched on the natural identity (case-insensitively: `AssetIdentifier` - // preserves the code's case verbatim) OR on the resolved id. The id arm is - // what catches USDC addressed by its SAC contract address, which is a - // different identity for the same asset. + // preserves the code's case verbatim) OR on the resolved id. USDC's SAC + // address already arrives here as the classic identity (task 0242 alias); + // the id arm stays as a backstop. // ⚠️ [[0139]] means an `asset_id` can serve more than one identity, so the // id arm could in principle route a colliding asset here. Accepted: the // alternative is that a legitimate USDC identity silently falls back to the diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index 9ef713d0..5f13c7aa 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -506,6 +506,113 @@ fn resolve_asset_id_sql(where_sql: &str) -> String { ) } +// ---------------------------------------------------------------------------- +// SAC alias (task 0242, D5) +// ---------------------------------------------------------------------------- + +/// A classic row named by its SAC address. Positional: field order is the +/// [`sac_alias_sql`] SELECT order. +#[derive(Debug, clickhouse::Row, serde::Deserialize)] +struct SacAliasRow { + sac_address: String, + asset_code: String, + issuer_address: String, +} + +/// The classic rows whose stored `sac_address` is one of `n` bound addresses. +/// Read from the column only: every classic row carries it (XLM's included). +fn sac_alias_sql(n: usize) -> String { + let binds = vec!["?"; n].join(", "); + format!( + "SELECT a.sac_address, a.asset_code, a.issuer_address \ + FROM assets AS a FINAL \ + WHERE a.contract_address = '' AND a.sac_address IN ({binds})" + ) +} + +/// A classic row's identifier: XLM with no issuer is `native`. +fn alias_identifier(code: String, issuer: String) -> AssetIdentifier { + if code == "XLM" && issuer.is_empty() { + AssetIdentifier::Native + } else { + AssetIdentifier::Classic { code, issuer } + } +} + +/// The distinct contract addresses in `ids`, in first-seen order — the only +/// identifiers an alias can replace. Empty means no lookup is needed. +fn sac_alias_candidates(ids: &[AssetIdentifier]) -> Vec { + let mut out: Vec = Vec::new(); + for id in ids { + if let AssetIdentifier::Contract(c) = id + && !out.contains(c) + { + out.push(c.clone()); + } + } + out +} + +/// Replace each `Contract(c)` that `aliases` names with its classic; every +/// other identifier passes through. +fn apply_sac_aliases( + ids: Vec, + aliases: &std::collections::HashMap, +) -> Vec { + ids.into_iter() + .map(|id| match id { + AssetIdentifier::Contract(c) => aliases + .get(&c) + .cloned() + .unwrap_or(AssetIdentifier::Contract(c)), + other => other, + }) + .collect() +} + +/// Answer a classic asset's SAC address as that classic asset (task 0242, D5). +/// +/// Run once at the handler boundary, before any lookup. Together with +/// [`identity_where`]'s contract arm (`a.contract_address = ?`) a `C…` +/// identifier matches `contract_address = ? OR sac_address = ?`, and the +/// classic wins when both rows exist (a pre-heal SAC `Contract` row). The +/// response then echoes `CODE:ISSUER` / `native` through `to_canonical`. +/// No query when `ids` holds no contract. +pub async fn resolve_sac_aliases( + ch: &Client, + ids: Vec, +) -> Result, clickhouse::error::Error> { + let candidates = sac_alias_candidates(&ids); + if candidates.is_empty() { + return Ok(ids); + } + let sql = sac_alias_sql(candidates.len()); + let mut q = ch.query(&sql); + for c in candidates { + q = q.bind(c); + } + let aliases = q + .fetch_all::() + .await? + .into_iter() + .map(|r| { + ( + r.sac_address, + alias_identifier(r.asset_code, r.issuer_address), + ) + }) + .collect(); + Ok(apply_sac_aliases(ids, &aliases)) +} + +/// [`resolve_sac_aliases`] for one identifier. +pub async fn resolve_sac_alias( + ch: &Client, + id: AssetIdentifier, +) -> Result { + Ok(resolve_sac_aliases(ch, vec![id]).await?.remove(0)) +} + // ---------------------------------------------------------------------------- // OHLCV (overview §4.2) // ---------------------------------------------------------------------------- @@ -2335,6 +2442,108 @@ mod tests { ); } + const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; + const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + const XLM_SAC: &str = "CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA"; + + fn xcr() -> AssetIdentifier { + AssetIdentifier::Classic { + code: "XCR".into(), + issuer: XCR_ISSUER.into(), + } + } + + /// Task 0242: classic rows only, one bind per address, positional order + /// matching [`SacAliasRow`]. + #[test] + fn sac_alias_sql_reads_classic_rows_by_stored_sac_address() { + use clickhouse::Row; + + let sql = sac_alias_sql(2); + for needle in [ + "FROM assets AS a FINAL", + "a.contract_address = ''", + "a.sac_address IN (?, ?)", + ] { + assert!(sql.contains(needle), "missing {needle}: {sql}"); + } + let expected: Vec = SacAliasRow::COLUMN_NAMES + .iter() + .map(|s| (*s).to_string()) + .collect(); + assert_eq!(projected_columns(&sql), expected, "SQL: {sql}"); + } + + #[test] + fn alias_identifier_maps_xlm_to_native_and_the_rest_to_classic() { + for (code, issuer, expected) in [ + ("XLM", "", AssetIdentifier::Native), + ("XCR", XCR_ISSUER, xcr()), + ( + "XLM", + XCR_ISSUER, + AssetIdentifier::Classic { + code: "XLM".into(), + issuer: XCR_ISSUER.into(), + }, + ), + ] { + assert_eq!( + alias_identifier(code.into(), issuer.into()), + expected, + "{code}:{issuer}" + ); + } + } + + /// Only contracts need a lookup; none means no query at all. + #[test] + fn sac_alias_candidates_are_the_distinct_contracts() { + let c = |s: &str| AssetIdentifier::Contract(s.into()); + for (ids, expected) in [ + (vec![], vec![]), + (vec![AssetIdentifier::Native, xcr()], vec![]), + ( + vec![c(XCR_SAC), AssetIdentifier::Native, c(XLM_SAC), c(XCR_SAC)], + vec![XCR_SAC, XLM_SAC], + ), + ] { + assert_eq!(sac_alias_candidates(&ids), expected, "{ids:?}"); + } + } + + /// A matched SAC becomes its classic; an unmatched contract and every + /// non-contract pass through in order. + #[test] + fn apply_sac_aliases_replaces_only_matched_contracts() { + let c = |s: &str| AssetIdentifier::Contract(s.into()); + let other = stellar_strkey::Contract([9u8; 32]).to_string(); + let aliases = std::collections::HashMap::from([ + (XCR_SAC.to_string(), xcr()), + (XLM_SAC.to_string(), AssetIdentifier::Native), + ]); + let got = apply_sac_aliases( + vec![ + c(XCR_SAC), + c(&other), + xcr(), + c(XLM_SAC), + AssetIdentifier::Native, + ], + &aliases, + ); + assert_eq!( + got, + vec![ + xcr(), + c(&other), + xcr(), + AssetIdentifier::Native, + AssetIdentifier::Native + ] + ); + } + #[test] fn identity_where_native_is_literal_no_binds() { let (sql, binds) = identity_where(&AssetIdentifier::Native); diff --git a/packages/prices-api/src/batch/handlers.rs b/packages/prices-api/src/batch/handlers.rs index 46297edc..5cef874e 100644 --- a/packages/prices-api/src/batch/handlers.rs +++ b/packages/prices-api/src/batch/handlers.rs @@ -19,7 +19,9 @@ use crate::state::AppState; /// /// Validates the whole list first (400 if any identifier is malformed or the /// batch is empty / over `MAX_BATCH`). Assets with no current-price row are -/// returned in `not_found` rather than failing the request. Uncached. +/// returned in `not_found` rather than failing the request. A found entry is +/// named by the asset's canonical identifier, so a SAC address answers under +/// its classic name (task 0242 D5); `not_found` echoes what was asked. Uncached. #[utoipa::path( post, path = "/prices/batch", @@ -28,8 +30,12 @@ use crate::state::AppState; description = "The same snapshot `GET /assets/{asset_identifier}/price` returns, for up to 100 assets\nin \ one request. The whole list is validated first: a malformed identifier, an empty list\nor \ more than 100 entries fails the entire request with a 400. Assets without a current\nprice \ - are listed in `not_found` instead of failing the call. Results follow the order of\nthe \ - request; a duplicated identifier is answered each time. Not cached.", + are listed in `not_found` instead of failing the call. `prices` follows the order of the\nrequest \ + with the not-found identifiers left out, and a duplicated identifier is answered each\ntime. \ + Each entry's `asset` is the canonical identifier of the asset found: a classic asset's\nStellar \ + Asset Contract address is answered as `CODE:ISSUER` (or `native`), so a request naming\nboth \ + forms gets two identical entries. `not_found` echoes each identifier as it was asked,\nin \ + canonical form. Not cached.", request_body = BatchRequest, responses( (status = 200, description = "Current prices + not-found list", body = BatchResponse), @@ -67,6 +73,14 @@ pub async fn post_batch( } } + // A SAC address answers as its classic (task 0242); `not_found` still + // echoes what was asked. + let asked: Vec = ids.iter().map(AssetIdentifier::to_canonical).collect(); + let ids = match queries_ch::resolve_sac_aliases(state.ch(), ids).await { + Ok(ids) => ids, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; + // One query for the whole batch (vs. a per-asset N+1 loop), then map each // requested identifier back to its row by natural-identity key. let rows = match queries_ch::current_prices_batch(state.ch(), &ids).await { @@ -78,7 +92,7 @@ pub async fn post_batch( let mut prices = Vec::new(); let mut not_found = Vec::new(); - for id in &ids { + for (id, asked) in ids.iter().zip(asked) { match by_key.get(&queries_ch::IdentKey::of(id)) { Some(row) => prices.push(PriceResponse::from_row( id.to_canonical(), @@ -96,7 +110,7 @@ pub async fn post_batch( price_basis: row.price_basis.clone(), }, )), - None => not_found.push(id.to_canonical()), + None => not_found.push(asked), } } diff --git a/packages/prices-api/src/identity.rs b/packages/prices-api/src/identity.rs index ee01ae0c..01a29ec3 100644 --- a/packages/prices-api/src/identity.rs +++ b/packages/prices-api/src/identity.rs @@ -4,7 +4,8 @@ //! of three textual forms: //! - `native` → the native XLM asset //! - `{code}:{issuer}` → a classic credit asset (issuer is a G-strkey) -//! - `{contract_address}` → a Soroban token / SAC (a C-strkey) +//! - `{contract_address}` → a Soroban token / SAC (a C-strkey); a classic +//! asset's SAC is aliased to that classic (task 0242) //! //! This module only *parses + validates the shape* (real strkey CRC validation //! via `stellar-strkey`). Resolving a parsed identity to the internal diff --git a/packages/prices-api/src/oracles/handlers.rs b/packages/prices-api/src/oracles/handlers.rs index babd4e0c..a6a9b8db 100644 --- a/packages/prices-api/src/oracles/handlers.rs +++ b/packages/prices-api/src/oracles/handlers.rs @@ -29,8 +29,8 @@ use crate::state::AppState; ("asset_identifier" = String, Path, description = "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or \ the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic \ - asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract \ - addresses, which as a rule answer `404`") + asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the \ + answer names the asset by `CODE:ISSUER` (or `native`)") ), responses( (status = 200, description = "Oracle readings", body = OraclesResponse), @@ -50,6 +50,10 @@ pub async fn get_oracles( Ok(id) => id, Err(e) => return errors::bad_request(errors::INVALID_ID, e.to_string()), }; + let id = match assets_q::resolve_sac_alias(state.ch(), id).await { + Ok(id) => id, + Err(e) => return errors::db_error(&e, "asset lookup"), + }; let asset_id = match assets_q::resolve_asset_id(state.ch(), &id).await { Ok(Some(aid)) => aid, diff --git a/packages/prices-api/tests/endpoints_it.rs b/packages/prices-api/tests/endpoints_it.rs index b8acc0a4..ba9c8633 100644 --- a/packages/prices-api/tests/endpoints_it.rs +++ b/packages/prices-api/tests/endpoints_it.rs @@ -27,6 +27,11 @@ fn issuer() -> &'static str { prices_clickhouse::USDC_ISSUER } +/// XLM's SAC, stored on the XLM row as prod holds it (task 0242). +const XLM_SAC: &str = "CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA"; +const XCR_ISSUER: &str = "GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7"; +const XCR_SAC: &str = "CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI"; + /// Create + seed a scratch db with assets, current prices, oracle prices, and /// backfill progress; return a client scoped to it. async fn setup(db: &str) -> Client { @@ -46,7 +51,7 @@ async fn setup(db: &str) -> Client { .unwrap(); let assets = [ - AssetFixture::new("XLM", "native", "", ""), + AssetFixture::new("XLM", "native", "", "").with_sac(XLM_SAC), AssetFixture::new("USDC", "credit", issuer(), ""), ]; admin @@ -305,6 +310,186 @@ async fn asset_detail_unknown_is_404() { teardown(db).await; } +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn asset_detail_unknown_contract_is_404() { + let db = "it_ep_detail_unknown_c_0242"; + let client = setup(db).await; + let (status, _) = get(client, &format!("/v1/assets/{}", contract())).await; + assert_eq!(status, StatusCode::NOT_FOUND); + teardown(db).await; +} + +/// Seed XCR as a classic row carrying its SAC, priced at 0.25. `pre_heal` adds +/// the second identity task 0242 heals: a Contract row for the SAC, priced at +/// 9.0, so a lookup that reached it would show. +async fn seed_xcr(db: &str, pre_heal: bool) { + let admin = Client::default().with_url(ch_url()); + let classic = AssetFixture::new("XCR", "credit", XCR_ISSUER, "").with_sac(XCR_SAC); + let sac = AssetFixture::new("", "contract", "", XCR_SAC); + let usdc = AssetFixture::new("USDC", "credit", issuer(), "").id(); + let mut rows = vec![classic]; + let mut prices = vec![format!( + "({}, 0.25, 0.25, 10, '2026-02-10 12:00:30')", + classic.id() + )]; + // One XCR/USDC hour per identity, `close_usd = close` (par quote). + let candle = |id: String, px: f64, trades: u32| { + format!( + "('2026-02-10 12:00:00', {id}, {usdc}, 'soroswap', {px}, {px}, {px}, {px}, \ + 10, 10, {px}, {px}, {trades}, 1)" + ) + }; + let mut candles = vec![candle(classic.id(), 0.25, 7)]; + if pre_heal { + rows.push(sac); + prices.push(format!( + "({}, 9.0, 9.0, 1, '2026-02-10 12:00:30')", + sac.id() + )); + candles.push(candle(sac.id(), 9.0, 1)); + } + admin + .query(&format!( + "INSERT INTO {db}.price_ohlcv_1h \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote_usd, close_usd, vwap, trade_count, version) VALUES {}", + candles.join(", ") + )) + .execute() + .await + .unwrap(); + admin + .query(&assets_insert(db, &rows)) + .execute() + .await + .unwrap(); + admin + .query(&format!( + "INSERT INTO {db}.current_prices \ + (asset_id, price_usd, vwap_24h, volume_24h_usd, updated_at) VALUES {}", + prices.join(", ") + )) + .execute() + .await + .unwrap(); +} + +/// Task 0242 (D5): a classic asset's SAC address answers as the classic, and +/// the classic wins over a pre-heal Contract row for the same address. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn sac_address_answers_as_its_classic_asset() { + let xcr = format!("XCR:{XCR_ISSUER}"); + for (db, pre_heal) in [ + ("it_ep_sac_alias_0242", false), + ("it_ep_sac_alias_pre_heal_0242", true), + ] { + let client = setup(db).await; + seed_xcr(db, pre_heal).await; + + let (status, json) = get(client.clone(), &format!("/v1/assets/{XCR_SAC}")).await; + assert_eq!(status, StatusCode::OK, "{db}: body={json}"); + assert_eq!(json["asset"], xcr, "{db}"); + assert_eq!(json["asset_kind"], "credit", "{db}"); + assert_eq!(json["code"], "XCR", "{db}"); + assert_eq!(json["issuer"], XCR_ISSUER, "{db}"); + assert_eq!(json["contract"], "", "{db}"); + + let (status, json) = get(client.clone(), &format!("/v1/assets/{XCR_SAC}/price")).await; + assert_eq!(status, StatusCode::OK, "{db}: body={json}"); + assert_eq!(json["asset"], xcr, "{db}"); + approx(&json["price_usd"], 0.25); + + let (status, json) = get(client.clone(), &format!("/v1/oracles/{XCR_SAC}")).await; + assert_eq!(status, StatusCode::OK, "{db}: body={json}"); + assert_eq!(json["asset"], xcr, "{db}"); + + // The classic's series only: without the alias the clean case is 404 + // and the pre-heal case answers the SAC row's 9.0 candle. + let uri = format!( + "/v1/assets/{XCR_SAC}/ohlcv?granularity=1h\ + &start=2026-02-10T00:00:00Z&end=2026-02-11T00:00:00Z&base_currency=USD" + ); + let (status, json) = get(client, &uri).await; + assert_eq!(status, StatusCode::OK, "{db}: body={json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), 1, "{db}: body={json}"); + approx(&data[0]["close"], 0.25); + assert_eq!(data[0]["trade_count"], 7, "{db}"); + + teardown(db).await; + } +} + +/// XLM's SAC answers as `native`, off the stored `sac_address` alone. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn xlm_sac_address_answers_as_native() { + let db = "it_ep_xlm_sac_0242"; + let client = setup(db).await; + + let (status, json) = get(client.clone(), &format!("/v1/assets/{XLM_SAC}")).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + assert_eq!(json["asset"], "native"); + assert_eq!(json["asset_kind"], "native"); + assert_eq!(json["code"], "XLM"); + assert_eq!(json["is_active"], true); + + let (status, json) = get(client, &format!("/v1/assets/{XLM_SAC}/price")).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + assert_eq!(json["asset"], "native"); + approx(&json["price_usd"], 0.5); + + teardown(db).await; +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn batch_answers_sac_addresses_as_their_classic_assets() { + let db = "it_ep_batch_sac_0242"; + let client = setup(db).await; + seed_xcr(db, false).await; + + let body = json!({ "assets": [XCR_SAC, XLM_SAC] }); + let (status, json) = post(client, "/v1/prices/batch", body).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + let assets: Vec<&str> = json["prices"] + .as_array() + .unwrap() + .iter() + .map(|p| p["asset"].as_str().unwrap()) + .collect(); + assert_eq!(assets, [format!("XCR:{XCR_ISSUER}").as_str(), "native"]); + assert_eq!(json["not_found"], json!([])); + + teardown(db).await; +} + +/// Task 0242: an unpriced SAC is listed in `not_found` as the address the +/// client sent, not as its classic. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn batch_not_found_echoes_the_sac_address_asked() { + let db = "it_ep_batch_sac_nf_0242"; + let client = setup(db).await; + let classic = AssetFixture::new("XCR", "credit", XCR_ISSUER, "").with_sac(XCR_SAC); + Client::default() + .with_url(ch_url()) + .query(&assets_insert(db, &[classic])) + .execute() + .await + .unwrap(); + + let body = json!({ "assets": [XCR_SAC] }); + let (status, json) = post(client, "/v1/prices/batch", body).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + assert_eq!(json["prices"], json!([])); + assert_eq!(json["not_found"], json!([XCR_SAC])); + + teardown(db).await; +} + #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn batch_returns_found_and_not_found() { diff --git a/web/portal/public/openapi.json b/web/portal/public/openapi.json index 4ff31a01..a6db8b8f 100644 --- a/web/portal/public/openapi.json +++ b/web/portal/public/openapi.json @@ -234,7 +234,7 @@ { "name": "asset_identifier", "in": "path", - "description": "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract addresses, which as a rule answer `404`", + "description": "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the answer names the asset by `CODE:ISSUER` (or `native`)", "required": true, "schema": { "type": "string" @@ -325,7 +325,7 @@ { "name": "asset_identifier", "in": "path", - "description": "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract addresses, which as a rule answer `404`", + "description": "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the answer names the asset by `CODE:ISSUER` (or `native`)", "required": true, "schema": { "type": "string" @@ -471,7 +471,7 @@ { "name": "asset_identifier", "in": "path", - "description": "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract addresses, which as a rule answer `404`", + "description": "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the answer names the asset by `CODE:ISSUER` (or `native`)", "required": true, "schema": { "type": "string" @@ -633,7 +633,7 @@ { "name": "asset_identifier", "in": "path", - "description": "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic asset is named by `CODE:ISSUER` and XLM by `native`, not by their Stellar Asset Contract addresses, which as a rule answer `404`", + "description": "`native`, `CODE:ISSUER` (a classic asset's code and its issuer's `G…` public key) or the `C…` address of a Soroban contract. The code is case-sensitive (`yXLM` is not `YXLM`). A classic asset's Stellar Asset Contract address, XLM's included, is accepted as an alias, and the answer names the asset by `CODE:ISSUER` (or `native`)", "required": true, "schema": { "type": "string" @@ -718,7 +718,7 @@ "post": { "tags": ["prices"], "summary": "`POST /prices/batch` — current prices for many assets in one call.", - "description": "The same snapshot `GET /assets/{asset_identifier}/price` returns, for up to 100 assets\nin one request. The whole list is validated first: a malformed identifier, an empty list\nor more than 100 entries fails the entire request with a 400. Assets without a current\nprice are listed in `not_found` instead of failing the call. Results follow the order of\nthe request; a duplicated identifier is answered each time. Not cached.", + "description": "The same snapshot `GET /assets/{asset_identifier}/price` returns, for up to 100 assets\nin one request. The whole list is validated first: a malformed identifier, an empty list\nor more than 100 entries fails the entire request with a 400. Assets without a current\nprice are listed in `not_found` instead of failing the call. `prices` follows the order of the\nrequest with the not-found identifiers left out, and a duplicated identifier is answered each\ntime. Each entry's `asset` is the canonical identifier of the asset found: a classic asset's\nStellar Asset Contract address is answered as `CODE:ISSUER` (or `native`), so a request naming\nboth forms gets two identical entries. `not_found` echoes each identifier as it was asked,\nin canonical form. Not cached.", "operationId": "post_batch", "requestBody": { "content": { From d0026576d785593c3a19f0ab98267f8012e9b9cd Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:05:27 +0200 Subject: [PATCH 005/144] docs(lore-0242): runbook for healing SACs held as contract identities The code stops new mints; the 36 existing SAC contract identities need a data heal. The runbook orders it: 29-row seed after stage B, stage C to 202609, deploy, residual cleanup with an only-copy gate and rollback copies, then the alarm actions. It also covers a STOP on `unproven sac swaps:`, the --amm wait path, and keeps AC3 open for the five U3 assets. The 0286 runbook points to it. --- docs/runbooks/0242-sac-identity-heal.md | 849 ++++++++++++++++++++++++ docs/runbooks/0286-reingest-history.md | 14 + 2 files changed, 863 insertions(+) create mode 100644 docs/runbooks/0242-sac-identity-heal.md diff --git a/docs/runbooks/0242-sac-identity-heal.md b/docs/runbooks/0242-sac-identity-heal.md new file mode 100644 index 00000000..846dda71 --- /dev/null +++ b/docs/runbooks/0242-sac-identity-heal.md @@ -0,0 +1,849 @@ +# Runbook — healing SACs held as contract identities (task 0242) + +A Stellar Asset Contract (SAC) is a facet of its classic asset, never an asset of +its own. Production holds 36 SACs as `Contract` identities in `prices.assets`, +each with its own `asset_id`. This runbook removes them without a SQL rekey: +0286 phase 3 rebuilds the history under the classic ids, and a small cleanup +deletes what is left. + +**Applies to:** `prices.assets`, `prices.asset_symbol` and the seven +`prices.price_ohlcv_*` tiers on ch-prod-01; the Compute, EventBridge and +Observability stacks. + +**Read first:** [`0286-reingest-history.md`](0286-reingest-history.md) (phase 3, +its stages and its orchestrator) and lore task 0242 (decisions D1–D7, PC3, PC6, +PC9). + +--- + +## 0. What and why + +Measured 2026-10-05 (task 0242 §"Re-measured on prod, 2026-10-05"): + +| Group | Contract rows | What their candles are | +| ----- | ------------- | ----------------------------------------------------------------------------------------------------------- | +| A | 13 | Split pairs: the classic row exists. 145 of 151 SAC-keyed `1m` rows are byte-identical to a classic row | +| B | 23 | SACs whose classic was never in `assets`. Their candles are the **only copy** of those trades | +| — | 6 latent | Pool-leg SACs with no identity yet (SUSHI, HYPE, STELLA, TESTTTT, TEST77, TEST12). They mint on first trade | + +Group A's 13: POINTS, USDM1, SODA, VCHF, USDP, VEUR, WHLAQUA, USD, USDT0, XCR, +ESP, EUR, FrogST. + +The heal is the 0286 phase-3 re-ingest, not a SQL rekey (D3). A rekey would need +four kinds of row operation (delete the duplicates, move group B, merge three +mixed coarse buckets, re-enrich), plus a base/quote flip for Z/Q and XLD/SODA, +whose order changes when `Contract` becomes `Credit`. Phase 3 drops and rebuilds +every month anyway; with the classic identities in `assets` it writes them under +the classic ids by construction. + +The code of this task (one PR) stops new mints: a SAC leg resolves to its classic +only on a sha256-verified proof, and a SAC BE flags `is_sac` that nothing proves +is skipped and counted, never minted (D1, D2). The code protects the future; this +runbook heals the past. + +--- + +## 1. Order of events + +| # | Step | When | Section | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------- | +| 1 | Seed the 29 classic identities. Works with the `events-backfill` already on ch-prod-01 (D4) | After phase-3 stage B (202402–202404), before stage C | §3 | +| 2 | Re-plan phase 3 to 202609, run stage C with `--to-month 202609` (D4; the plan ended at 202608) | After the seed | §4a–§4d | +| 3 | Deploy the PR: Compute (live ledger processor + API), then EventBridge (probe), then Observability. Swap `events-backfill` on ch-prod-01 only between phase-3 stages, never while a month is at its `amm` step (D7) | Once merged. Independent of phase 3 (D4: phase 3 is not held for the code) | §4e | +| 4 | Residual cleanup (D3) | After stage D (`finish`) AND after the live slice is deployed with at least one cold start | §5 | +| 5 | Enable the `SacContractIdentities` alarm actions (D6) | After §5's post-checks are green | §6 | + +Step 4 waits for the live deploy because until then a warm container that predates +the seed, or a SAC the seed does not cover, can still mint a contract identity +and write candles under it. + +--- + +## 2. Operator identity and tooling + +- **CH shell:** `dev_shared` over mTLS, with the `chq` function, set up exactly + as in [`0139-asset-id-migration.md` → Who runs it, and where](0139-asset-id-migration.md#who-runs-it-and-where). + `echo "SELECT currentUser()" | chq` must print `dev_shared`. +- **Orchestrator shell:** `fishuser-hero`, the checkout `reingest_0286.py` runs + from, as in 0286 §3. +- **AWS shell:** production credentials, a checkout of the merged PR. +- **API:** `API=https://prices-api.sorobanscan.rumblefish.dev/v1` and an operator + `PRICES_API_KEY`. + +```bash +mkdir -p ~/heal-0242 +Q=~/Projects/stellar-prices-api/.planning/quick/261005-htu-0242-sac-minted-as-a-second-identity-res +XCR=XCR:GBLJBHWVORDFI4J7CLBDRPECMYT3XO5S6GERXGC74VXOJMZPLI6ZU3S7 +XCR_SAC=CDJQXBQO5ICVQUPHZHW7SHOM56K2UNNPPAIXUUSA3XACEI6Q4JQLXNVI +XLM_SAC=CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA +``` + +`$Q` is in Adam's checkout, under the gitignored `.planning/`. The seed and its +checks stay there, outside the repo. + +Every `chq` step stops at the first non-zero exit. Read the message before going on. + +--- + +## 3. Seed (after stage B, before stage C) + +`$Q/seed-0242.sql` inserts 29 classic identities: the 23 of group B plus the 6 +latent SACs. Every `sac_address` in it was re-derived from code + issuer by +sha256 (`$Q/measurements/sac_check.py`: ok 29, bad 0), and none of the 29 was in +`assets` on 2026-10-05. Adam runs it. + +**Why `sac_address` must be filled.** The ingest registry would resolve a seeded +classic's SAC even with the column empty, because `from_existing` derives it. +But `identity_by_contract`, the API alias (D5), the `events-backfill` preload +filter and the probe all read the stored column (0242 research, pitfall 8). + +**Pre-check.** + +- `reingest_0286.py status` shows 202404 done and no month in progress. +- ```bash + chq < $Q/seed-0242-check.sql | tee ~/heal-0242/seed-check-before.tsv + ``` + Query 1 prints `N N`, query 2 prints `N`. Record query 3 (`count() - uniqExact(asset_id)`) + as `COLLISIONS_BEFORE`. + - `N = 0` when the seed comes before the §4e deploy, the order of §1. + - `N > 0` when the 0242 build went live first: the live processor proved some of + the 29 in-band and interned their classics itself, with `sac_address`. Those + rows belong to live; the run below inserts only the `29 - N` others and keeps + only those for the seed rollback. + - The two numbers of query 1 differ: one of the 29 is held without + `sac_address`. Stop and bring it to Adam. + +**Run.** Stage the 29 rows, keep the ones `assets` does not hold yet, then +insert exactly those: + +```bash +echo "CREATE TABLE prices.bak_0242_seed_stage AS prices.assets" | chq +sed 's/^INSERT INTO prices\.assets /INSERT INTO prices.bak_0242_seed_stage /' $Q/seed-0242.sql | chq +chq <<'SQL' +CREATE TABLE prices.bak_0242_seed ENGINE = MergeTree ORDER BY asset_id AS +SELECT asset_id, asset_code, issuer_address, sac_address FROM prices.bak_0242_seed_stage +WHERE (asset_code, issuer_address) NOT IN (SELECT asset_code, issuer_address FROM prices.assets FINAL); +INSERT INTO prices.assets (asset_code, asset_type, issuer_address, contract_address, sac_address, is_active) +SELECT asset_code, asset_type, issuer_address, contract_address, sac_address, is_active +FROM prices.bak_0242_seed_stage +WHERE (asset_code, issuer_address) IN (SELECT asset_code, issuer_address FROM prices.bak_0242_seed); +SELECT count() FROM prices.bak_0242_seed_stage; +SELECT count() FROM prices.bak_0242_seed +SQL +``` + +The two counts print `29` and `29 - N`. `bak_0242_seed` holds the rows this seed +added and nothing live wrote; the seed rollback and §7 read it. + +**Post-check.** + +```bash +chq < $Q/seed-0242-check.sql | tee ~/heal-0242/seed-check-after.tsv +``` + +Query 1 prints `29 29` (29 rows, all with `sac_address`), whatever `N` was. +Query 2 prints `29` (each seeded SAC is a `credit` row of +`identity_by_contract`). Query 3 equals `COLLISIONS_BEFORE` (AC4). + +**When it takes effect.** Phase 3 reads `assets` at the start of every +`events-backfill` run, so stage C sees the seed at once. The live processor sees +it at its next cold start (`resumed cursor from ClickHouse` in +`/aws/lambda/prices-production-ledger-processor` after the seed time). Until +then a warm container can still mint one of the 29. + +**Seed rollback.** Only while no candle names a seeded id, i.e. before stage C +writes any: + +```bash +for T in 1m 15m 1h 4h 1d 1w 1M; do + echo "SELECT '$T', count() FROM prices.price_ohlcv_$T WHERE asset_id IN (SELECT asset_id FROM prices.bak_0242_seed) OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_seed)" | chq +done # every line 0 +echo "ALTER TABLE prices.assets DELETE WHERE (asset_code, issuer_address) IN (SELECT asset_code, issuer_address FROM prices.bak_0242_seed) SETTINGS mutations_sync = 2" | chq +echo "SELECT mutation_id, is_done, latest_fail_reason FROM system.mutations WHERE database = 'prices' AND table = 'assets' AND NOT is_done" | chq # no row +chq < $Q/seed-0242-check.sql # query 1 prints the `N N` of the pre-check again (`0 0` in the order of §1) +``` + +A mutation is fine on a ~211k-row table. `mutations_sync = 2` makes the +`ALTER` return only once the mutation is done on every replica, and fail if it +fails (`dev_shared` is not `readonly`, so SETTINGS are allowed). Do not re-seed +or start stage C before query 1 reads the pre-check's numbers again. Once stage C +has written candles under the seeded ids there is no seed rollback: those ids +are the right identities. + +--- + +## 4. Stage C and the deploy + +### 4a. Extend the plan to 202609 + +The phase-3 plan ends at 202608 (`plan` took the last complete month on the day +it ran). Stage C must reach 202609: it holds the only copies of the USDT0 and +SODA trades and part of 0139's hole (D4). + +Re-planning is safe: `cmd_plan` rewrites only the state's month map +(`st.d["plan"]`), and the progress of every month lives under a separate key +(`st.d["months"]`), which `plan` does not touch (`tools/scripts/reingest_0286.py` +`cmd_plan`). + +**Pre-check** (orchestrator shell, between stages, no `run` holding the lock): + +```bash +cp -p ~/reingest-0286/state.json ~/reingest-0286/state.json.pre0242 +python3 -c "import json,os; p=json.load(open(os.path.expanduser('~/reingest-0286/state.json')))['plan']; print(min(p), max(p), len(p))" +``` + +It prints ` 202608 `. Pass `` back explicitly, so the re-plan +cannot start from a different first month. + +**Run.** + +```bash +python3 tools/scripts/reingest_0286.py plan --from-month --to-month 202609 +``` + +**Post-check.** The last line reads ` months, ..202609, archive tip …`. +`reingest_0286.py status` still shows every finished month as done. + +**Rollback.** `cp -p ~/reingest-0286/state.json.pre0242 ~/reingest-0286/state.json`. + +### 4b. Run stage C + +**Pre-check: the seed is in** (§3 post-check: query 1 prints `29 29`). It is +mandatory, not just ordered first. The `events-backfill` on ch-prod-01 until the +§4e swap resolves a SAC only when its classic is in `assets`, so stage C without +the seed rebuilds group B's history, BLTA/BLTB/BLTC/PPRIME/LumenJoule (U3, §8) +included, under the SAC ids again: the second identities this task removes. + +The flags are stage B's (task 0286, "Stage B command"), plus `--ack-0285` (202607 +on) and `--ack-0139-binaries` (0286 §10a). Use the restart wrapper of 0286 §3: + +```bash +python3 tools/scripts/reingest_0286.py run --to-month 202609 --ack-0285 --ack-0139-binaries --yes +``` + +**Which `events-backfill` is on the host.** A one-ledger dry run tells (CH host, +as `default`): + +```bash +read -rs CH_PW +CLICKHOUSE_PASSWORD="$CH_PW" ~/events-backfill --start 51500460 --end 51500460 \ + --clickhouse-url http://localhost:8123 --dry-run | grep 'unproven sac swaps:' +``` + +A 0242 binary prints `unproven sac swaps:` followed by a count; a binary built +before 0242 prints nothing. Both are fine for stage C once the seed is in (PC6). + +**Post-check: U3's candles are back under the classic ids.** Once 202609 is done +(CH shell): + +```bash +chq <<'SQL' | tee ~/heal-0242/u3-post-stage-c.tsv +WITH ['CCUYL75XNUTAHFE3RZAN7NEFQWZ7OIRJZNN4KUXXPA34RAHVUGLTBGT5', -- BLTA + 'CAANCS7TKC7IFWLRACKL4T4HMNMDUWSWNOL5EV244TVQCU3O7BM7N4VD', -- BLTB + 'CB2J5FHCJJBQE5UPQ5FPZAWIEQD5KKP4GWLZZ3QT5EFVIT3FW5CI2OGK', -- BLTC + 'CBVWPBYEDJ7GYIUHL2HITMEEWM75WAMFINIQCR4ZAFZ62ISDFBVERQCX', -- LumenJoule + 'CBI2NXIZQ33L3K5RMQW53OGV52HDHZU2AUISCUFXDYTDR345VKPHAQEP' -- PPRIME + ] AS sacs, + (SELECT groupArray(asset_id) FROM prices.assets FINAL + WHERE contract_address = '' AND has(sacs, sac_address)) AS classic, + (SELECT groupArray(asset_id) FROM prices.assets FINAL + WHERE has(sacs, contract_address)) AS contract +SELECT toYYYYMM(timestamp) AS month, length(classic) AS classics, + countIf(has(classic, asset_id) OR has(classic, quote_asset_id)) AS classic_rows, + sumIf(trade_count, has(classic, asset_id) OR has(classic, quote_asset_id)) AS classic_trades, + sumIf(pf_trade_count, has(classic, asset_id) OR has(classic, quote_asset_id)) AS classic_pf_trades, + countIf(has(contract, asset_id) OR has(contract, quote_asset_id)) AS sac_id_rows +FROM prices.price_ohlcv_1h FINAL +WHERE source = 'soroswap' AND timestamp >= '2026-03-01 00:00:00' AND timestamp < '2026-10-01 00:00:00' + AND (has(classic, asset_id) OR has(classic, quote_asset_id) + OR has(contract, asset_id) OR has(contract, quote_asset_id)) +GROUP BY month ORDER BY month +FORMAT TSVWithNames +SQL +``` + +`classics` = 5 and `sac_id_rows` = 0 on every line. `classic_trades` equals BE's +swap count on the four registry pools of these SACs (two BLT pools, +LumenJoule/USDC, PPRIME/USDC; measured 2026-10-05, `$Q/U3-root-cause.md`): + +| month | 202603 | 202604 | 202605 | 202606 | 202607 | 202608 | 202609 | total | +| ------------------- | ------ | ------ | ------ | ------ | ------ | ------ | ------ | ------ | +| `classic_trades` | 24 | 7,037 | 3,301 | 23,422 | 10,189 | 6,958 | 24,649 | 75,580 | +| `classic_pf_trades` | 24 | 0 | 0 | 6 | 0 | 0 | 0 | 30 | + +The query counts candle rows, not assets, so a BLTB/BLTA trade, which names two +of the five, counts once. `classic_pf_trades` is LumenJoule's 24 and PPRIME's 6: every BLT fill is +dust (about 0.0001 units each), so BLTA, BLTB and BLTC come back with volume and +`trade_count` but `pf_trade_count = 0` and no price (ADR 0287 §1). That is the +rule working, not a gap. Any `sac_id_rows` above 0, a month short of the table, +or `classics` below 5: stop before §4e and bring the output to Adam. + +### 4c. A STOP on `unproven sac swaps:` + +A 0242 binary prints, after `negative apply order:`, + +``` +unproven sac swaps: 0 +``` + +Anything but `0` means swaps were skipped because a leg is a contract BE flags +`is_sac` and no proof resolved it (D2). The orchestrator STOPs on it +(`amm_summary`), after either pass, and the STOP names the pass. It reads the +line after the `--dry-run` pass first, so a STOP there comes after `drop_1m` and +`sdex` and before anything AMM is written: the month's `1m` holds SDEX-only +candles, its coarse tiers are untouched, and the month stays at its `amm` step. +A STOP on the write pass is step 4's last case. + +1. **Find the contracts.** In `~/reingest-0286//events-backfill.log`: + - the WARN `swaps skipped on SACs with no proof (task 0242 D2); the month is NOT complete`, + whose `sample` lists up to 20 contracts; + - the start-of-run INFO `preloaded SAC proofs (task 0242)` (`sacs_verified`, + `sacs_rejected`) and, when `sacs_rejected > 0`, the WARN + `SAC proofs that did not verify`. +2. **Check each contract** (CH shell; `C1,C2` = the contracts, quoted): + + ```bash + chq <<'SQL' + SELECT contract_id, groupUniqArray(is_sac) AS is_sac_versions FROM default.soroban_contracts + WHERE contract_id IN ('C1', 'C2') GROUP BY contract_id; + SELECT sc.addr AS contract, x.signature, x.ledger_sequence, + JSONExtractString(x.topics_xdr, JSONLength(x.topics_xdr), 'value') AS sep11 + FROM (SELECT contract_id, signature, ledger_sequence, topics_xdr FROM default.soroban_events + WHERE contract_id IN (SELECT id FROM default.soroban_contracts WHERE contract_id IN ('C1', 'C2')) + AND signature IN ('transfer', 'mint', 'burn', 'clawback') + LIMIT 3 BY contract_id) AS x + INNER JOIN (SELECT id, any(contract_id) AS addr FROM default.soroban_contracts + WHERE contract_id IN ('C1', 'C2') GROUP BY id) AS sc ON sc.id = x.contract_id + SQL + ``` + + Then derive the SAC of each `sep11` (`CODE:ISSUER`, or `native`) with + `$Q/measurements/sac_check.py` and compare it with the contract. + +3. **Fix, then `run` again.** + - **The SEP-11 derives to the contract:** the SAC is real and its proof did not + reach the run. Seed its classic as §3 seeded the 29: one row + `(asset_code, 'classic', issuer, '', sac_address = the contract, 1)`, with + the §3 checks narrowed to it. Then `run` with the same flags. It resumes at + the `amm` step, and its `--dry-run` pass is the proof: the binary derives + the SAC itself, so a wrong seed STOPs again with the same count and writes + nothing. Then remove the wrong row (§3's seed rollback, narrowed to it) + before anything else: the API alias would answer the contract as it. + - **BE has corrected `is_sac` to false:** the next run treats it as a genuine + token only once no version of its row says `true`. The code reads BE's + `soroban_contracts` (a ReplacingMergeTree) without `FINAL`, on purpose, so an + unmerged old version keeps the contract a candidate and its swaps skipped. + `run` again only when `is_sac_versions` above is `[false]`. While it holds + both values, BE's parts have not merged: wait for them (never `OPTIMIZE` + BE's table), or take step 4. +4. **`rollback` the month instead** (`reingest_0286.py rollback `, which + restores `1m` and the four fine tiers from the month's snapshot and forgets + the month, so the next `run` starts it at `gates`): + - when no SAC event exists for the contract, or its SEP-11 does not derive to + it. No data fix can prove it. Stop phase 3 there and bring it to Adam; + - when the fix cannot land the same day: the month serves SDEX-only `1m` + candles while it waits; + - when the non-zero line came from the **write** pass (the log shows that + attempt's dry-run summary reading `0`, then the write's non-zero): the + month's AMM candles are in without those swaps. + +### 4d. `--amm wait` + +`amm-done` records only `--fallbacks`. It does not read `unproven sac swaps:` +(nor `swaps dropped (unresolved):`). On this path the operator is the gate: + +1. Run the host command the orchestrator prints with `--dry-run` first, and read + `unproven sac swaps:`. Non-zero: do not run the write, do not `amm-done`; + follow §4c. +2. Run the write. Read the line again. Non-zero: do not `amm-done`; `rollback` + the month (§4c step 4). +3. Only on `0` (or no such line, a pre-0242 binary): `amm-done --fallbacks `. + +### 4e. Deploy the PR + +**Pre-check before the live deploy: `prices_writer` can read BE's SAC set.** The +live Lambda loads `default.soroban_contracts` at cold start as `prices_writer` +and fails Init when it cannot (PC3): live ingest would stop. + +```bash +chq <<'SQL' +SELECT user_name, role_name, access_type, database, table FROM system.grants +WHERE (user_name = 'prices_writer' + OR role_name IN (SELECT granted_role_name FROM system.role_grants WHERE user_name = 'prices_writer')) + AND access_type IN ('SELECT', 'ALL') + AND (database = 'default' OR database IS NULL) + AND (table = 'soroban_contracts' OR table IS NULL) +SQL +``` + +At least one row (on 2026-10-05: `prices_writer SELECT default soroban_contracts`). +Then the read itself, as that identity (fishuser-hero, the orchestrator's writer +bundle): + +```bash +curl -sS --fail-with-body --cert ~/prices-mtls/prices_writer.crt --key ~/prices-mtls/prices_writer.key \ + --cacert ~/prices-mtls/ca.crt https://ch.sorobanscan.rumblefish.dev/ \ + --data-binary "SELECT count() FROM (SELECT DISTINCT contract_id FROM default.soroban_contracts WHERE is_sac)" +``` + +About 4,042 (M13b). An error or `0`: **do not deploy Compute**; fix the grant first. + +**Record what runs now** (AWS shell), for the rollback: + +```bash +for f in ledger-processor api-handler rollup-freshness-probe; do + aws lambda get-function-configuration --function-name "prices-production-$f" \ + --query '[FunctionName,CodeSha256,LastModified]' --output text +done | tee ~/heal-0242/deployed-before.tsv +``` + +**Deploy** (AWS shell, `cd infra`): + +| Order | Target | Ships | +| ----- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 1 | `make deploy-production-compute` | the live ledger processor (proof pre-pass, skip, `SacUnprovenSkipped`) and the API alias | +| 2 | `make deploy-production-eventbridge` | `rollup-freshness-probe` with `SacContractIdentities` | +| 3 | `make deploy-production-observability` | `prices-production-ledger-processor-sac-unproven` (actions on) and `prices-production-sac-contract-identities` (actions off) | + +**Post-checks.** + +- The live Lambda cold-started on the new build: `/aws/lambda/prices-production-ledger-processor` + shows `loaded the is_sac contract set` with `sac_contracts` ≈ 4,042, then + `resumed cursor from ClickHouse`. No Init error. +- The probe: `aws lambda invoke --function-name prices-production-rollup-freshness-probe ~/heal-0242/probe.json`, + then `jq '.asset_id_uniqueness | {sac_contract_rows, sac_contract_scanned, be_sac_contracts}' ~/heal-0242/probe.json`. + `sac_contract_rows` = 36 (the baseline until §5), `be_sac_contracts` ≈ 4,042. + If the file holds an `errorMessage`, read the metric instead: + `aws cloudwatch get-metric-statistics --namespace Prices/Rollup --metric-name SacContractIdentities --dimensions Name=Environment,Value=production --start-time "$(date -u -d '-30 min' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" --period 900 --statistics Maximum`. +- `aws cloudwatch describe-alarms --alarm-names prices-production-sac-contract-identities --query 'MetricAlarms[0].[ActionsEnabled,StateValue]'` + → `false`, `"ALARM"` (or `"INSUFFICIENT_DATA"` before the first probe run). + **`ALARM` is expected until §6**: actions off stops notifications, not + evaluation, and the metric reads 36 until §5. The alarm shows red on the + dashboard's Row-0 alarm strip for the whole window (stage C, stage D, §5); its + description says so. Tell whoever reports from that strip (tasks 0294, 0296) + before the deploy. +- `curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XLM_SAC" | jq '{asset, code}'` → `"native"`, `"XLM"`. + +**`events-backfill` on ch-prod-01.** Build it from the merge commit as in 0286 +§10a, keep the current one as `~/events-backfill.pre0242` (never overwrite it), +and swap only between phase-3 stages, never while a month is at its `amm` step +(D7). Confirm with the one-ledger dry run of §4b. + +**Rollback.** Redeploy the previous commit with the same `make` targets (the +CodeSha256 in `deployed-before.tsv` is what must come back). On the host, +`cp -p ~/events-backfill.pre0242 ~/events-backfill`. Nothing in the PR writes a +schema change. + +--- + +## 5. Residual cleanup (after stage D and the live cold start) + +What phase 3 does not rewrite stays under the SAC ids: 202404 (stage B ran before +the seed, so Z/Q was re-minted under its SAC ids, U2), any month after 202609, +and the 1w/1M rows `finish` rolled from those. Then 72 metadata rows: 36 in +`assets`, 36 in `asset_symbol`. + +**Precondition.** `reingest_0286.py finish` done (stage D); §4e post-checks green +(the live slice has cold-started). + +### 5a. Capture the SAC ids + +The probe's predicate, with each contract's classic beside it: + +```bash +chq <<'SQL' +CREATE TABLE prices.bak_0242_sac_ids +ENGINE = MergeTree ORDER BY contract_address AS +SELECT c.contract_address AS contract_address, + c.asset_id AS asset_id, + cl.asset_id AS classic_id, + cl.asset_code AS classic_code, + cl.issuer_address AS classic_issuer, + now() AS captured_at +FROM (SELECT contract_address, asset_id FROM prices.assets FINAL + WHERE contract_address != '' + AND (contract_address IN (SELECT sac_address FROM prices.assets WHERE sac_address != '') + OR contract_address IN (SELECT contract_id FROM default.soroban_contracts WHERE is_sac))) AS c +LEFT JOIN (SELECT sac_address, asset_id, asset_code, issuer_address FROM prices.assets FINAL + WHERE sac_address != '') AS cl ON cl.sac_address = c.contract_address; +SELECT count(), countIf(classic_id = 0) FROM prices.bak_0242_sac_ids; +SELECT count() AS identities, uniqExact(asset_id) AS ids, identities - ids AS collisions FROM prices.assets FINAL +SQL +echo "SELECT * FROM prices.bak_0242_sac_ids ORDER BY classic_code FORMAT TSVWithNames" | chq > ~/heal-0242/sac_ids.tsv +``` + +**Check.** 36 rows, 0 without a classic (the seed gave group B theirs). Group A +is the 13 codes of §0. More than 36: the probe has seen new mints. They are in +the capture and go through the same gates; name them on the task. A row with +`classic_id = 0` has no classic to re-key to: every candle under it is an only +copy (5b). Record `identities`, `ids` and `collisions` as the AC4 "before". + +### 5b. Count the residual, and gate it + +**The window after the last phase-3 month.** Measured 2026-10-05: 0 rows since +2026-10-01 on any tier keyed on the 36 SAC ids. The window closes only once the +seed is in AND live has cold-started since, or the 0242 build is deployed; until +then live can still write under a SAC id. Re-count it now, for every month after +202609 (`1w`/`1M` roll from `1d`, so `1d` decides): + +```bash +for T in 1m 15m 1h 4h 1d; do + chq <= '2026-10-01 00:00:00' + AND (asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids) + OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids)) +GROUP BY month ORDER BY month +SQL +done | tee ~/heal-0242/window.tsv +``` + +- Empty → go on. +- Any row → it is one of the residual rows below, and the PC9 gate decides it + like any other. A month after 202609 can only be re-run once it is complete + (`plan` refuses the current month). + +**Every residual row is an only copy until proven otherwise.** A residual row +exists only where phase 3 did not rebuild its bucket under the classic id after +the seed, so it may hold trades that no classic row holds. A trade count cannot +tell: a classic row with more trades can hold different ones (for example a SAC +hour written before the live cold start beside a classic hour written after it), +and by §5 the `1m`/`15m` retention has removed the finer rows that would show it. + +The only proof that releases a row is a **byte-level duplicate**: a classic row +at the same pair after re-keying (both orientations, because Z/Q and XLD/SODA +flip), the same source and bucket, the same `trade_count` and the same volume on +each leg. That is the double count of group A (145 of 151 SAC-keyed `1m` rows on +2026-10-05). Every other row is an _only copy_, whatever the classic row holds. +Per tier and partition: + +```bash +for T in 1m 15m 1h 4h 1d 1w 1M; do + chq < t.1, sc) AS s, arrayMap(t -> t.2, sc) AS c +SELECT '$T' AS tier, x.p AS partition, count() AS residual_rows, + countIf(y.hit) AS byte_identical, residual_rows - byte_identical AS only_copy, + sumIf(x.trade_count, NOT y.hit) AS only_copy_trades, + groupUniqArrayIf(x.sac, NOT y.hit) AS only_copy_sacs +FROM (SELECT _partition_id AS p, if(has(s, asset_id), asset_id, quote_asset_id) AS sac, + transform(asset_id, s, c, asset_id) AS a, transform(quote_asset_id, s, c, quote_asset_id) AS q, + least(a, q) AS lo, greatest(a, q) AS hi, source, timestamp, trade_count, + if(a = lo, volume_base, volume_quote) AS vol_lo, if(a = lo, volume_quote, volume_base) AS vol_hi + FROM prices.price_ohlcv_$T FINAL + WHERE has(s, asset_id) OR has(s, quote_asset_id)) AS x +ANY LEFT JOIN (SELECT least(asset_id, quote_asset_id) AS lo, greatest(asset_id, quote_asset_id) AS hi, + source, timestamp, trade_count, + if(asset_id = lo, volume_base, volume_quote) AS vol_lo, + if(asset_id = lo, volume_quote, volume_base) AS vol_hi, true AS hit + FROM prices.price_ohlcv_$T FINAL + WHERE has(c, asset_id) OR has(c, quote_asset_id)) AS y + USING (lo, hi, source, timestamp, trade_count, vol_lo, vol_hi) +GROUP BY partition ORDER BY partition +SQL +done | tee ~/heal-0242/residual.tsv +``` + +`s` and `c` come from one read of pairs, so `s[i]` and `c[i]` are always the same +capture row. Two separate `groupArray` reads need not return the same order once +`bak_0242_sac_ids` has more than one part, and would re-key a SAC to the wrong +classic. + +`vol_lo`/`vol_hi` are each leg's volume, whichever side it is stored on, so a +flipped pair compares leg by leg. A row with `classic_id = 0` re-keys to `0`, +matches nothing and counts as an only copy. On a scratch fixture a SAC hour of 1 +trade beside a classic hour of 2 different trades reads `only_copy = 1`. + +`quote_asset_id` is not a key prefix, so this scans each tier: run it off-peak. + +**The PC9 gate. It STOPs §5 on any partition with `only_copy > 0`**, in any tier, +`1w` and `1M` included. A coarse bucket or a month phase 3 did not re-ingest after +the seed is no exception: only the byte-level match above releases a row. An +only-copy row is never deleted silently. For each such partition Adam picks one +of the two, and it is recorded on task 0242 before 5c: + +- **Re-run the month through phase 3.** In a fresh state dir, as the 0139 second + pass does (0286 §10d), so a finished month can run again: + + ```bash + mkdir -p ~/reingest-0286-0242 && cp -r ~/reingest-0286/ledger-cache ~/reingest-0286-0242/ + python3 tools/scripts/reingest_0286.py plan --state-dir ~/reingest-0286-0242 --from-month --to-month + python3 tools/scripts/reingest_0286.py run --state-dir ~/reingest-0286-0242 --from-month --to-month --yes + ``` + + That rewrites `1m` and the four fine tiers. The month's `1w`/`1M` rows are only + rebuilt by `finish`, which TRUNCATEs both tables whole (hours, then the + whole-history re-enrichment). Until it runs again they still read as only + copies: run `finish` again, or record them as below. Then repeat 5b; the gate + passes only when every partition reads `only_copy = 0` or carries a recorded + loss. + +- **Record the loss** on task 0242, with Adam's ack: tier, partition, + `only_copy_trades`, `only_copy_sacs`. For `1w`/`1M` rows of a re-run month, say + that the classic's `1w`/`1M` lack those trades until the next `finish`. + +Either way the rows are in the 5c copy before 5d deletes them. **Expected here:** +202404's Z/Q rows, in every tier stage B wrote and in the `1w`/`1M` rows `finish` +rolled from them (U2). + +### 5c. Rollback copies, before any delete + +Every residual candle row and the 72 metadata rows. The column lists come from +`system.columns` without MATERIALIZED, ALIAS and EPHEMERAL columns: +`assets.asset_id` is MATERIALIZED (re-derived on re-insert), and the candles' +`base_*`/`quote_*` identity columns are EPHEMERAL (their ids are stored). + +**Pre-check.** `echo "SHOW TABLES FROM prices LIKE 'bak_0242_%'" | chq` lists only +`bak_0242_sac_ids`, `bak_0242_seed` and `bak_0242_seed_stage`. `CREATE TABLE` below has no `IF NOT EXISTS`, +so a second run fails instead of overwriting the first copy. + +```bash +chq <<'SQL' > ~/heal-0242/bak.sql +SELECT concat( + 'CREATE TABLE prices.bak_0242_', table, ' AS prices.', table, ';\n', + 'INSERT INTO prices.bak_0242_', table, ' (', cols, ') SELECT ', cols, ' FROM prices.', table, ' FINAL WHERE ', + if(table IN ('assets', 'asset_symbol'), + 'contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)', + 'asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids) OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids)'), + ';') +FROM (SELECT table, arrayStringConcat(groupArray(name), ', ') AS cols + FROM system.columns + WHERE database = 'prices' + AND table IN ('assets', 'asset_symbol', 'price_ohlcv_1m', 'price_ohlcv_15m', 'price_ohlcv_1h', + 'price_ohlcv_4h', 'price_ohlcv_1d', 'price_ohlcv_1w', 'price_ohlcv_1M') + AND default_kind NOT IN ('MATERIALIZED', 'ALIAS', 'EPHEMERAL') + GROUP BY table) +ORDER BY table +FORMAT TSVRaw +SQL +cat ~/heal-0242/bak.sql && chq < ~/heal-0242/bak.sql +``` + +This creates `prices.bak_0242_assets`, `prices.bak_0242_asset_symbol` and +`prices.bak_0242_price_ohlcv_` for the seven tiers. + +**Post-check.** + +```bash +for t in assets asset_symbol price_ohlcv_1m price_ohlcv_15m price_ohlcv_1h price_ohlcv_4h \ + price_ohlcv_1d price_ohlcv_1w price_ohlcv_1M; do + echo "SELECT '$t', count() FROM prices.bak_0242_$t" | chq +done | tee ~/heal-0242/bak-counts.tsv +``` + +`assets` and `asset_symbol` equal the 5a count (36). Each tier equals the sum of +its `residual_rows` in `residual.tsv`. + +### 5d. Delete the residual candles, fine tiers first + +One lightweight DELETE per tier and partition, over the partitions 5c copied. +Fine before coarse, so a re-roll of a coarse tier between two statements reads an +already cleaned finer tier. + +```bash +for T in 1m 15m 1h 4h 1d 1w 1M; do + for P in $(echo "SELECT DISTINCT _partition_id FROM prices.bak_0242_price_ohlcv_$T ORDER BY 1 FORMAT TSVRaw" | chq); do + echo "DELETE FROM prices.price_ohlcv_$T IN PARTITION ID '$P' WHERE asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids) OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids);" + done +done | tee ~/heal-0242/delete-candles.sql +cat ~/heal-0242/delete-candles.sql && chq < ~/heal-0242/delete-candles.sql +``` + +**Post-check (M4).** Every line `0`: + +```bash +for T in 1m 15m 1h 4h 1d 1w 1M; do + echo "SELECT '$T', count() FROM prices.price_ohlcv_$T FINAL WHERE asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids) OR quote_asset_id IN (SELECT asset_id FROM prices.bak_0242_sac_ids)" | chq +done +``` + +### 5e. Delete the 36 `assets` rows + +**Pre-check.** `echo "SELECT count() FROM prices.assets FINAL WHERE contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)" | chq` +equals the 5a count. + +```bash +echo "DELETE FROM prices.assets WHERE contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)" | chq +``` + +**Post-check.** The same count reads `0`. + +### 5f. Delete the 36 `asset_symbol` rows + +After `assets`, because the symbol queue is driven by the `assets` contract rows +(`asset-discovery` `symbols.rs`): deleted first, they would be fetched again. + +**Pre-check.** `echo "SELECT count() FROM prices.asset_symbol FINAL WHERE contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)" | chq` +equals the 5a count. + +```bash +echo "DELETE FROM prices.asset_symbol WHERE contract_address IN (SELECT contract_address FROM prices.bak_0242_sac_ids)" | chq +``` + +**Post-check.** The same count reads `0`. + +### 5g. Post-checks + +```bash +chq <<'SQL' +SELECT count() FROM prices.assets FINAL +WHERE contract_address != '' + AND (contract_address IN (SELECT sac_address FROM prices.assets WHERE sac_address != '') + OR contract_address IN (SELECT contract_id FROM default.soroban_contracts WHERE is_sac)); +SELECT count() FROM prices.assets FINAL WHERE contract_address != ''; +SELECT uniqExact(contract), countIf(asset_kind = 'contract'), max(n) +FROM (SELECT contract, any(asset_kind) AS asset_kind, count() AS n FROM prices.identity_by_contract + WHERE contract IN (SELECT contract_address FROM prices.bak_0242_sac_ids) GROUP BY contract); +SELECT count() AS identities, uniqExact(asset_id) AS ids, identities - ids AS collisions FROM prices.assets FINAL +SQL +``` + +| Check | Expected | +| ----------------------------------- | ------------------------------------------------------------------------------ | +| SAC contract rows (M1) | `0` | +| Contract rows left | the genuine tokens only: 23 on 2026-10-05, plus any minted since | +| `identity_by_contract` | the 5a count, `0` contract rows, `max(n)` = `1`: one classic row per SAC | +| identities / ids / collisions (AC4) | identities and ids each down by exactly the 5a count; collisions equal to 5a's | +| SAC-keyed candles (M4) | `0` in every tier (5d post-check) | + +The probe and the alarm metric: + +```bash +aws lambda invoke --function-name prices-production-rollup-freshness-probe ~/heal-0242/probe.json +jq '.asset_id_uniqueness | {sac_contract_rows, collisions: (.identities - .ids)}' ~/heal-0242/probe.json +``` + +`sac_contract_rows` = `0`, and `SacContractIdentities` reads `0` in CloudWatch +(§4e's `get-metric-statistics`). `AssetIdCollisions` unchanged (AC4). + +The API (D5, AC2, AC3): + +```bash +curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR_SAC" | jq '{asset, asset_kind, contract}' +curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR_SAC/price" | jq -S '{asset, price_usd, volume_24h_usd}' +curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR/price" | jq -S '{asset, price_usd, volume_24h_usd}' +W="granularity=1d&start=2026-09-01T00:00:00Z&end=2026-10-01T00:00:00Z" +diff <(curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR_SAC/ohlcv?$W" | jq -S .data) \ + <(curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XCR/ohlcv?$W" | jq -S .data) && echo SAME +curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XLM_SAC" | jq '{asset, asset_kind, code}' +``` + +- XCR by its SAC: `asset` = `XCR:GBLJ…`, `asset_kind` = `credit`, `contract` = `""`. +- Both `/price` calls print the same object: one asset, one volume. `SAME`. +- The XLM SAC answers 200 as XLM: `asset` = `native`, `code` = `XLM`. A 404 + means prod's XLM row lost its `sac_address`: read it first + (`SELECT sac_address FROM prices.assets FINAL WHERE asset_code = 'XLM' AND issuer_address = ''`). + On 2026-10-05 it held `CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA`, + and no classic row lacked one. + +### 5h. Rollback + +Re-insert from the copies with the same column lists, metadata first: + +```bash +chq <<'SQL' > ~/heal-0242/restore.sql +SELECT concat('INSERT INTO prices.', table, ' (', cols, ') SELECT ', cols, ' FROM prices.bak_0242_', table, ';') +FROM (SELECT table, arrayStringConcat(groupArray(name), ', ') AS cols + FROM system.columns + WHERE database = 'prices' + AND table IN ('assets', 'asset_symbol', 'price_ohlcv_1m', 'price_ohlcv_15m', 'price_ohlcv_1h', + 'price_ohlcv_4h', 'price_ohlcv_1d', 'price_ohlcv_1w', 'price_ohlcv_1M') + AND default_kind NOT IN ('MATERIALIZED', 'ALIAS', 'EPHEMERAL') + GROUP BY table) +ORDER BY table IN ('assets', 'asset_symbol') DESC, table +FORMAT TSVRaw +SQL +cat ~/heal-0242/restore.sql && chq < ~/heal-0242/restore.sql +``` + +Check: §5a's capture predicate counts the 5a count again, and each tier's +SAC-keyed count equals `bak-counts.tsv`. `asset_id` re-derives to the same value +from the same identity. Keep the `bak_0242_*` tables until 0242 is closed; then +`DROP TABLE prices.bak_0242_ SYNC`, one by one. + +--- + +## 6. Enable the alarm (D6) + +The `SacContractIdentities` alarm was deployed with its actions off, because +production read 36 until §5. + +**Pre-check.** `SacContractIdentities` reads `0` for at least two periods (30 min), +and + +```bash +aws cloudwatch describe-alarms --alarm-names prices-production-sac-contract-identities \ + --query 'MetricAlarms[0].StateValue' +``` + +prints `"OK"`. CloudWatch notifies only on a transition: enabled while in ALARM, +the alarm would say nothing about the next mint. + +**Run.** Set `opsAlarms.sacContractIdentitiesActionsEnabled` to `true` in +`infra/envs/production.json` through a PR; once merged: + +```bash +cd infra && make deploy-production-observability +``` + +**Post-check.** The same `describe-alarms` with `--query 'MetricAlarms[0].ActionsEnabled'` +prints `true`. + +**Rollback.** The same key back to `false`, the same deploy. + +--- + +## 7. Acceptance evidence + +| AC | Evidence | +| -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| AC1 a SAC no longer mints a second identity, both orders | `packages/prices-ledger-processor/tests/sac_identity.rs` (T2 SAC first, T3 both orders, T4 the SAC event after the trade), through the real `process_ledger`; `events-backfill` `a_swap_on_an_unproven_sac_is_skipped_and_counted_and_a_proven_one_prices` | +| AC2 the existing pairs are resolved | §5g: 0 SAC contract rows, one `identity_by_contract` row per SAC, 0 SAC-keyed candles. Group A by phase 3 plus 5d; group B by the seed plus phase 3 | +| AC3 one asset, one volume (XCR) | §5g: XCR by SAC and by `CODE:ISSUER` print the same price, volume and candles. For BLTA/BLTB/BLTC/PPRIME/LumenJoule, §4b's U3 post-check: their volume under the classic ids; BLTA/BLTB/BLTC carry no price by design (dust) | +| AC4 no new duplicate `asset_id` | §3 query 3 and §5a/§5g: `count() - uniqExact(asset_id)` unchanged at every step; `AssetIdCollisions` unchanged | + +Paste the outputs of §3, §5a, §5b, §5g and §6 into task 0242, then tell BE once +the heal lands: they read `current_price_usd` and `identity_by_contract`, and +each SAC now resolves to one classic row. + +--- + +## 8. Open items + +- **U3, settled 2026-10-05** (`$Q/U3-root-cause.md`). The 75,580 Soroswap swaps + on BLTA, BLTB, BLTC, PPRIME and LumenJoule had candles until 2026-10-02. The + task-0139 re-key left them in `price_ohlcv_*__pre0139`: each of the five SAC + contract identities shared its old id with an unrelated classic asset, so all + five were `colliding`, and `fill` copies only `mapped`/`sentinel` ids. It is a + sub-case of 0139's hole (colliding ids have no candles 2024-02..2026-09), not + an ingest or dust-rule bug, and needs no code. Stage C after the seed rebuilds + them from BE's events under the classic ids; §4b's post-check proves it, and + BLTA/BLTB/BLTC come back with volume and no price (dust). Without the seed + stage C would rebuild them under the SAC ids, which is why §4b requires it. +- **U5.** USDT0's 17:26 SAC trade came after its classic's 17:00-hour SDEX + candle: consistent with the warm-container race of 0139, not traced to a + container. +- **Known residual risk.** The live processor reads BE's `is_sac` set once per + cold start. A SAC deployed after it, traded with no SAC event in the same + transaction, is not a candidate yet and still mints a `Contract` identity; so + does a SAC BE does not flag. The in-band proof missed 0 of 338,601 sampled + legs, so this should stay rare. `SacContractIdentities` counts it, §9. + +--- + +## 9. When an alarm fires + +**`prices-production-ledger-processor-sac-unproven`** (live, actions on). The +live processor skipped trades on a contract BE flags `is_sac` that no SAC event +in the same transaction proved. Those trades have no candle. The WARN +`skipped trades on is_sac contracts with no SAC proof (task 0242)` in +`/aws/lambda/prices-production-ledger-processor` names the contracts. Check each +as in §4c step 2. A real SAC: seed its classic as §3 did; the next cold start +resolves it. The skipped minutes stay without those trades until a re-ingest of +their month. + +**`prices-production-sac-contract-identities`** (after §6). A contract row of +`assets` is a SAC again. The alarm's description holds the query that names it. +Treat it as a one-row §5: capture, gate its candles, copy, delete. Find which +path minted it first (a pre-0242 binary somewhere, or a SAC BE does not flag). diff --git a/docs/runbooks/0286-reingest-history.md b/docs/runbooks/0286-reingest-history.md index e7757d01..cab257cf 100644 --- a/docs/runbooks/0286-reingest-history.md +++ b/docs/runbooks/0286-reingest-history.md @@ -160,6 +160,10 @@ Four of those steps exist because of one fact and nothing else: wrapper exists to prevent. Watch `restarts.log` and answer the prompt, or run those stages without the wrapper. + Before stage C: the task-0242 seed, then `plan` and `run` with + `--to-month 202609` ([`0242-sac-identity-heal.md`](0242-sac-identity-heal.md) + §1, §3, §4). + - `events-backfill` reads `default.*` AND writes `prices.*`, so it runs **on the CH host as the `default` user** against `localhost:8123` — the prices mTLS user cannot read `default.*` @@ -457,6 +461,16 @@ negative apply order: 0 repaired** — do not record it as done. The run also emits one WARN, once, naming the first ledger and `application_order` it saw. +Since task 0242 the summary also prints `unproven sac swaps: N`: swaps skipped +because a leg is a contract BE flags `is_sac` and no proof resolved it. Anything +but `0` means the month is NOT complete, and the orchestrator STOPs on it after +either pass. After the `--dry-run` pass no AMM candle is written, but the month's +`1m` is already dropped and holds SDEX-only candles; after the write pass its AMM +candles lack those swaps. The STOP names the pass. A binary built before 0242 prints +no such line and is not checked. What to do: +[`0242-sac-identity-heal.md`](0242-sac-identity-heal.md) §4c, and §4d for +`--amm wait`, where `amm-done` does not read the line. + --- ## 7. After the last month: 1w, 1M, and the re-enrichment From 7080070a126b6576808182760d690b1746513c9b Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:39:03 +0200 Subject: [PATCH 006/144] docs(lore-0294): shorten the form answers for submission --- docs/scf/milestone-3-form-answers.md | 218 ++++++++------------------- 1 file changed, 66 insertions(+), 152 deletions(-) diff --git a/docs/scf/milestone-3-form-answers.md b/docs/scf/milestone-3-form-answers.md index 0cbadd34..4a3c99a1 100644 --- a/docs/scf/milestone-3-form-answers.md +++ b/docs/scf/milestone-3-form-answers.md @@ -1,11 +1,9 @@ # SCF Milestone 3 — Form Answers (stellar-prices-api) -> ⚠️ **DRAFT — first full draft 2026-10-02.** Mirrors the Milestone 2 form -> ([`milestone-2-form-answers.md`](milestone-2-form-answers.md)). Every figure is -> taken from [`milestone-3-evidence.md`](milestone-3-evidence.md); none may -> outrun it. Anything in `` waits for the release run on `master`, -> the recording, or the asset-id migration (task 0139); anything in -> `` is a link to place on submission day. +> Mirrors the Milestone 2 form +> ([`milestone-2-form-answers.md`](milestone-2-form-answers.md)), shortened +> before submission. Every figure is taken from +> [`milestone-3-evidence.md`](milestone-3-evidence.md); none may outrun it. > > Copy the text inside each blockquote into the matching field of the Stellar > Community Fund **Deliverable Verification** form. @@ -14,168 +12,84 @@ ## Field 1 — Tranche Deliverables -> **Deliverable 3 — Production Launch & Validation** (as originally approved). -> -> Milestone 3 opens the Stellar Prices API to developers. The service went -> public on **2026-09-23 at 09:40 CEST**, when the staging password came off -> the onboarding portal; the API itself had been serving on its custom domain -> since Milestone 2. Everything below is the production deployment. -> -> What is live and verifiable today: -> -> 1. **A self-service onboarding portal.** At -> `https://sorobanscan.rumblefish.dev/prices-api/` (at `/api/` until -> 2026-10-02; every old link answers 301 to the same page). Sign-in is -> through Discord, and eligibility is membership in the official Stellar -> Developers server. A key is issued at once on the free plan, one request a -> second and a hundred thousand a month, and the dashboard shows it with its -> plan, limits and usage. The quick start matches the published contract, -> and the portal carries its own privacy policy. The video records the walk -> on production on 2026-10-05: sign-in at 13:00:40 UTC, a `/v1` call with the -> key at 13:01:28 (200), the key revoked at 13:06:01 and refused with a 403 -> from 13:07. -> 2. **A published contract.** The API serves its own OpenAPI 3.1.0 document -> at `/api-docs-json`, generated from the handler code, with 74 -> production-shaped examples. CI lints it on every Rust change with -> Redocly's strict ruleset, and the portal renders it as an API reference. -> The release run on `master` (2026-10-05) lints it with no errors or -> warnings: `https://github.com/rumblefishdev/stellar-prices-api/actions/runs/37298469682` -> 3. **Integration tests in CI against a real database.** Since 2026-09-22, -> CI starts ClickHouse, applies the schema, starts the mTLS proxy and runs -> the integration suite against it, covering each of the seven `/v1` -> routes. On the first run on `master` after the release (2026-10-05) -> 1,407 unit and 306 ClickHouse integration tests pass, 0 failed: -> `https://github.com/rumblefishdev/stellar-prices-api/actions/runs/37298469682` -> 4. **Load-tested at the approved target.** 100 requests per second for five -> minutes: **p95 49.0 ms** against a 100 ms bar, **0 errors in 30,001 -> requests**, on the purpose-built plan the criterion asks us to name. The -> report also carries the miss-only row and the 500 and 1000 req/s rows. -> 5. **A security review.** ClickHouse is reachable only over mutual TLS; the -> client certificates live in Secrets Manager, not in environment variables; -> every invalid input gets a 400. No IAM statement grants every action or a -> whole service, and every statement with a wildcard resource is listed -> with what limits it: 22 in the release's templates, each for an action -> that has no resource ARN. -> 6. **Observability.** X-Ray tracing is active on the API handler, the ledger -> processor, the oracle and the enrichment worker, and on the gateway stage. -> The `prices-production-overview` dashboard covers API latency and errors, -> ingestion lag, ClickHouse write latency, the workers and an alarm strip with -> every `prices-production-*` alarm; all 83 were OK on 2026-10-05. -> 7. **Seven days of post-launch monitoring.** From 2026-09-23 09:40 to -> 2026-09-30 09:40 CEST: **uptime 100.000 %, 0 × 5XX in 65,806 requests, -> gateway p95 145.2 ms.** The report recomputes from a raw CloudWatch export -> kept in the repository, so a reviewer can rerun it without AWS access. -> 8. **A public repository.** Public since 2026-09-16, with a root README -> leading to the architecture documents, the operator runbooks and a -> fresh-account deployment runbook. -> -> The tranche's backfill target, SDEX history back to January 2018, was -> reached early, in Milestone 2: the archive completed on 2026-07-27 and -> reaches back to **2015-11-18**. -> -> **Declared deviations.** Four places where the delivery departs from the -> wording, each with its reason in the deviations document: the backfill that -> finished early (so liveness is graded on the ingestion signals), the two -> monitoring-report metrics it made flat, Redocly and the portal's own renderer -> in place of `openapi-validator` and Swagger UI, and the scope of the -> load-test figure. -> -> **Known issues and limitations are listed, each with the task that owns it:** -> the history recomputation that finishes after delivery, the history of the -> assets that shared an id until the asset-id fix (task 0139), still being -> re-ingested, a slow path we measured but did not instrument, and the ceiling -> of the database server we share with the Soroban Block Explorer. -> -> **Full evidence — acceptance-criteria mapping with runnable commands, the -> deviation rationale, known issues and limitations:** -> `https://drive.google.com/file/d/1AktIqEMocAW_DtaHiISdZTfptMu58Xg2/view?usp=sharing` +> Deliverable 3 — Production Launch & Validation (as originally approved). +> +> The Stellar Prices API went public on 2026-09-23 at 09:40 CEST. Everything +> below is the production deployment: +> +> 1. Self-service onboarding portal: https://sorobanscan.rumblefish.dev/prices-api/ +> — Discord sign-in for members of the Stellar Developers server, a +> free-plan key issued at once, usage on the dashboard. The full key flow was +> recorded on production on 2026-10-05. +> 2. Published contract: an OpenAPI 3.1.0 document generated from the handler +> code, linted clean in CI with Redocly and rendered in the portal. +> 3. Integration tests in CI against a real ClickHouse: 1,407 unit and 306 +> integration tests, 0 failed, on the release run on master (2026-10-05). +> 4. Load test: 100 req/s for five minutes, p95 49.0 ms against a 100 ms bar, +> 0 errors in 30,001 requests. +> 5. Security: ClickHouse over mutual TLS only, secrets in Secrets Manager, +> every input validated; the 22 IAM statements with a wildcard resource are +> each listed with what limits them. +> 6. Observability: X-Ray tracing end to end, the prices-production-overview +> dashboard, all 83 alarms OK on 2026-10-05. +> 7. Seven days of post-launch monitoring (2026-09-23 to 09-30): uptime +> 100.000 %, 0 × 5XX in 65,806 requests, gateway p95 145.2 ms. +> 8. A public repository with a fresh-account deployment runbook. +> +> SDEX history reaches back to 2015-11-18, past the January 2018 target. Four +> deviations from the wording are declared with their reasons; known issues and +> limitations are listed, each with the task that owns it. +> +> Full evidence: https://drive.google.com/file/d/1AktIqEMocAW_DtaHiISdZTfptMu58Xg2/view?usp=sharing --- ## Field 2 — Deliverable Verification - Video -> `https://drive.google.com/file/d/1KTFv9A58sU1FLtnqa4xBYYE8oaCI3DgW/view?usp=sharing` +> https://drive.google.com/file/d/1KTFv9A58sU1FLtnqa4xBYYE8oaCI3DgW/view?usp=sharing > -> About eight minutes, all on production: (1) scope; (2) the portal, end to -> end — sign-in, a key, a `/v1` call with it, revocation; (3) the OpenAPI -> document, the rendered reference and a 404 in the error envelope; (4) the -> integration tests in CI and the load test; (5) the security checklist; -> (6) operations — backfill status against Horizon, the dashboard and its -> alarms, the monitoring report; (7) the public repository and the -> fresh-account runbook; (8) the deviations and limitations. +> About eight minutes, all on production: the portal and the key flow, the +> OpenAPI reference, CI and the load test, security, operations (backfill, +> dashboard, alarms, monitoring report), the repository and the deviations. --- ## Field 3 — Additional Deliverable Verification -> **Evidence package (Google Drive):** `https://drive.google.com/drive/folders/1XnHTsg1hI7e7XR9z5G1MZ2zuyjVmO2K0?usp=sharing` -> -> - Evidence PDF: `https://drive.google.com/file/d/1AktIqEMocAW_DtaHiISdZTfptMu58Xg2/view?usp=sharing` -> - Demo video: `https://drive.google.com/file/d/1KTFv9A58sU1FLtnqa4xBYYE8oaCI3DgW/view?usp=sharing` -> -> The nine acceptance criteria, in order. Deviations are set out once, in -> `.../blob/master/docs/scf/milestone-3-rfp-deviations.md`, and referred to -> here by number. -> -> 1. **Backfill status: depth met by two years.** `earliest_data_available` -> is 2015-11-18 against a 2018-01-01 bar; liveness is graded on the -> ingestion alarms and `realtime_tip_ledger` (deviation 1). -> `GET https://prices-api.sorobanscan.rumblefish.dev/v1/backfill/status` -> 2. **OpenAPI lints clean; reference deployed.** Met: the release run on -> `master` (2026-10-05) lints it with no errors or warnings. Reference: -> `https://sorobanscan.rumblefish.dev/prices-api/docs` (deviation 3). -> Reproduce: `npm run openapi:lint` -> 3. **Portal accessible; self-service key flow works.** -> `https://sorobanscan.rumblefish.dev/prices-api/`, anonymous since -> 2026-09-23; Discord sign-in for keys. Met: recorded on 2026-10-05, -> sign-in 13:00:40 UTC, `/v1` call 13:01:28 (200), revoked 13:06:01, 403 -> from 13:07 (evidence AC 3). -> 4. **Integration suite passes on CI.** Met: first run on `master` after the -> release, `https://github.com/rumblefishdev/stellar-prices-api/actions/runs/37298469682` -> (2026-10-05): 1,407 unit and 306 ClickHouse integration tests, 0 failed. -> Reproduce locally: -> `tools/scripts/ignored-tests.sh` -> 5. **Load test: met.** p95 49.0 ms at 100 req/s for five minutes, 0 errors -> in 30,001 requests, plan `prices-production-loadtest-plan`; scope in -> deviation 4. Report: -> `.../blob/master/docs/prices-api-load-test-100rps.md` -> 6. **Security checklist: met.** mTLS-only ClickHouse, secrets in Secrets -> Manager, validated inputs, no `Action: "*"` or `service:*`; every -> `Resource: "*"` statement inventoried in evidence §5, AC 6. -> 7. **Repository public; deployable from the README: met on the -> fresh-account runbook.** `https://github.com/rumblefishdev/stellar-prices-api`, -> `README.md` → `infra/README.md` §"Fresh-account deployment". Not run in an -> empty AWS account; the claim rests on the runbook, a credential-free synth -> in CI and a name-by-name check of the runbook against the code. -> 8. **Dashboard and alarms: met.** -> `prices-production-overview`, `eu-central-1`; all 83 alarms OK on -> 2026-10-05. Read-only access for a named reviewer on request -> (see Field 4). -> 9. **7-day post-launch report: met.** Uptime 100.000 %, 0 × 5XX in 65,806 -> requests, gateway p95 145.2 ms; two named metrics replaced by live -> signals (deviation 2). Reproduce offline: -> `python3 docs/scf/milestone-3-monitoring/report.py` -> -> **Live & anonymous:** -> -> - OpenAPI document: `https://prices-api.sorobanscan.rumblefish.dev/api-docs-json` -> - Rendered API reference: `https://sorobanscan.rumblefish.dev/prices-api/docs` -> - Health probe: `https://prices-api.sorobanscan.rumblefish.dev/health` -> -> **Key-gated** (`x-api-key`; the Milestone 2 reviewer key works, or issue -> one in the portal): the seven `/v1` route groups listed in evidence §9. +> Evidence package: https://drive.google.com/drive/folders/1XnHTsg1hI7e7XR9z5G1MZ2zuyjVmO2K0?usp=sharing +> +> Deviations: https://github.com/rumblefishdev/stellar-prices-api/blob/master/docs/scf/milestone-3-rfp-deviations.md +> +> 1. Backfill status — depth met: earliest_data_available is 2015-11-18 against +> 2018-01-01; liveness graded on the ingestion signals (deviation 1). +> GET https://prices-api.sorobanscan.rumblefish.dev/v1/backfill/status +> 2. OpenAPI lints clean, reference deployed — met on the release run on master +> (2026-10-05): https://sorobanscan.rumblefish.dev/prices-api/docs +> (deviation 3) +> 3. Portal and self-service keys — met, recorded on 2026-10-05: +> https://sorobanscan.rumblefish.dev/prices-api/ +> 4. Integration suite on CI — met: +> https://github.com/rumblefishdev/stellar-prices-api/actions/runs/37298469682 +> 5. Load test — met: p95 49.0 ms at 100 req/s (deviation 4) +> 6. Security checklist — met: evidence §5, AC 6 +> 7. Public repository, deployable from the README — met on the fresh-account +> runbook, not run in an empty AWS account: +> https://github.com/rumblefishdev/stellar-prices-api +> 8. Dashboard and alarms — met: all 83 alarms OK on 2026-10-05; read-only +> access for a named reviewer on request +> 9. 7-day post-launch report — met: uptime 100.000 %, 0 × 5XX in 65,806 +> requests (deviation 2) +> +> No key needed: https://prices-api.sorobanscan.rumblefish.dev/api-docs-json and +> https://prices-api.sorobanscan.rumblefish.dev/health. The seven /v1 route +> groups need an x-api-key: the Milestone 2 reviewer key works, or issue one in +> the portal. --- ## Field 4 — Support Needed -> — No support is needed to complete or verify this tranche: nothing in it -> depends on the Stellar side. SDF settled the Discord integration by asking -> the project to run it itself. -> -> To open the CloudWatch dashboard and the alarms, ask us with your name, -> surname, e-mail, purpose and the end date of access; a read-only IAM Identity -> Center user is issued for the review window. +> None. --- From b10007dab0f2401734c649cded704985cb732ffe Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:58:49 +0200 Subject: [PATCH 007/144] docs(lore-0242): settle U3 and record the October window U3 was the 0139 re-key hole, not dust or an ingest bug: the five SAC ids collided with classic ids and their candles stayed in __pre0139. Stage C with the 29-row seed rebuilds them; U3 stays inside 0242. --- ...ac-minted-as-a-second-identity-splits-volume.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md index 5dbaab82..0e00885c 100644 --- a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md +++ b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md @@ -210,6 +210,20 @@ Plan choices confirmed by Adam on 2026-10-05 (GSD plan `261005-htu`): - **PC9 Residual rows of a SAC with no classic.** Such a row is the only copy of its trades, so it is never deleted silently. The operator either re-runs that month through phase 3 or records the loss on this task. The rows are copied beside the 72 metadata rows for rollback. The 202404 Z/Q row, from stage B before the seed, is the expected case. Open: **U3**. About 75.6k Soroswap swaps on BLTA/BLTB/BLTC/PPRIME/LumenJoule produce no candle under any id, and the cause is not established. It must be settled before AC3 promises numbers for those assets. +Stays in scope of 0242 (Adam, 2026-10-05): this task closes everything that concerns SACs. + +**U3 settled (2026-10-05).** Neither the dust rule nor an ingest bug: those swaps were missing from the candle tables because of the 0139 re-key. +- Each of the five SAC contract identities shared its old UInt32 id with an unrelated classic asset: 123376 LumenJoule/BTC, 123377 BLTB/ETH, 123378 BLTA/AST, 123379 BLTC/ASC1148, 123380 PPRIME/KWD. +- `asset_id_map_0139` therefore marks all five `colliding`. The re-key copies only `mapped` and `sentinel` (`rekey.rs` `COPYABLE`), so their candles stayed in `price_ohlcv_*__pre0139`. +- Before 0139, the 1h counts matched BE's swaps exactly for 202603–202606. +- Jul–Sep were already short before 0139: the 07-09 Soroswap gap (0101) and burst minutes before the 0282 fix. +- Locally, five real transactions from 202603–202609 produce candles under the classic identity on the 0242 branch. + +No code fix is needed. Phase 3 stage C, run with the 29-row seed, rebuilds them under the classic ids. AC3 for these five then rests on stage C's post-check; the BLT assets will have volume but no price, because their fills are dust. + +The six latent SACs have other causes. SUSHI and HYPE traded before SushiSwap was indexed, and phase 3 covers that under 0290. STELLA, TESTTTT, TEST77 and TEST12 have pools with no swaps. + +**October window (measured 2026-10-05, `dev_read`).** Since 2026-10-01, no candle on any tier is keyed on the 36 SAC ids, and no contract row has been minted since BURNMKR. No decision is needed now. The runbook re-checks months after 202609 before the cleanup, and any non-zero result goes through PC9. ## Acceptance Criteria From 77e6ef8d943a7a228dbc1fdefbf40870e0f62228 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:10:23 +0200 Subject: [PATCH 008/144] docs(lore-0294): tick the pre-submission checklist Every item re-checked on 2026-10-06, with the observable beside each: the reviewer key's plan read from API Gateway, the release run's jobs, the alarm states and the /api/ redirects. --- docs/scf/milestone-3-form-answers.md | 30 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/scf/milestone-3-form-answers.md b/docs/scf/milestone-3-form-answers.md index 4a3c99a1..3d0e7db9 100644 --- a/docs/scf/milestone-3-form-answers.md +++ b/docs/scf/milestone-3-form-answers.md @@ -95,20 +95,28 @@ ## Pre-submission checklist -- [ ] Every figure in Fields 1 and 3 appears, with the same value and date, in +Checked 2026-10-06. + +- [x] Every figure in Fields 1 and 3 appears, with the same value and date, in the evidence package -- [ ] Every deviation the package declares is mentioned in Field 3 by number -- [ ] Every `` and `` placeholder replaced -- [ ] The reviewer key still works and is on the free plan -- [ ] `develop` released to `master`: the repository's default branch shows +- [x] Every deviation the package declares is mentioned in Field 3 by number + → deviations 1–4, all four the deviations document declares +- [x] Every `` and `` placeholder replaced +- [x] The reviewer key still works and is on the free plan + → `prices-production-scf-reviewer-key-20260909T120021Z`: enabled, on + `pricing-api-free-production` +- [x] `develop` released to `master`: the repository's default branch shows the root `README.md` and the fresh-account runbook (AC 7), and every - `.../blob/master/...` link above resolves. On 2026-09-28 `master` still - had neither -- [ ] The first CI run on `master` after the release is green in every job, + `.../blob/master/...` link above resolves + → release #381 (2026-10-05), the package in #391 +- [x] The first CI run on `master` after the release is green in every job, `XDR protocol lag` included (fixed by task 0325, PR #383) -- [ ] The 0139 window is closed: alarms back to OK, the alarm count updated + → run 37298469682, all four jobs green +- [x] The 0139 window is closed: alarms back to OK, the alarm count updated wherever "65" appears (evidence, video script, Field 1 item 6) -- [ ] Portal links use `/prices-api/`, and an old `/api/` link still answers + → 83 of 83 `prices-production-*` alarms OK; no alarm count of 65 left +- [x] Portal links use `/prices-api/`, and an old `/api/` link still answers `301` on submission day -- [ ] The video was recorded against the public deployment, no secrets on + → `/api/` and `/api/dashboard` answer `301` to `/prices-api/…` +- [x] The video was recorded against the public deployment, no secrets on screen (see the scenario's ⚠️ section) From 35c2b62dc33ab85538cf87f06c6ee6b89775ea23 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:10:46 +0200 Subject: [PATCH 009/144] chore(lore-0149): activate task --- .../0149_BUG_sweep-mv-version-ownership-close-usd.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename lore/1-tasks/{backlog => active}/0149_BUG_sweep-mv-version-ownership-close-usd.md (96%) diff --git a/lore/1-tasks/backlog/0149_BUG_sweep-mv-version-ownership-close-usd.md b/lore/1-tasks/active/0149_BUG_sweep-mv-version-ownership-close-usd.md similarity index 96% rename from lore/1-tasks/backlog/0149_BUG_sweep-mv-version-ownership-close-usd.md rename to lore/1-tasks/active/0149_BUG_sweep-mv-version-ownership-close-usd.md index 3962a3c5..e8224720 100644 --- a/lore/1-tasks/backlog/0149_BUG_sweep-mv-version-ownership-close-usd.md +++ b/lore/1-tasks/active/0149_BUG_sweep-mv-version-ownership-close-usd.md @@ -2,7 +2,8 @@ id: "0149" title: "Two writers with incompatible version arithmetic own close_usd — the 0114 sweep's repair is overwritten inside the MV window" type: BUG -status: backlog +status: active +assignee: akot related_adr: [] related_tasks: ["0144", "0146", "0148", "0114", "0095", "0143"] tags: @@ -12,6 +13,10 @@ links: - "../../../packages/enrichment-worker/src/repair.rs" - "../../../packages/prices-clickhouse/schema/rollups.sql" history: + - date: "2026-10-06" + status: active + who: akot + note: "Activated and assigned to akot to work through the options." - date: 2026-08-05 status: backlog who: okarcz From 8ae35d1a4f48baf9aea8930a33734cd9aa33cd98 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:18:50 +0200 Subject: [PATCH 010/144] docs(lore-0294): complete and archive the task --- ...CS_scf-milestone-3-verification-package.md | 140 +++++++++++++++--- 1 file changed, 122 insertions(+), 18 deletions(-) rename lore/1-tasks/{active => archive}/0294_DOCS_scf-milestone-3-verification-package.md (68%) diff --git a/lore/1-tasks/active/0294_DOCS_scf-milestone-3-verification-package.md b/lore/1-tasks/archive/0294_DOCS_scf-milestone-3-verification-package.md similarity index 68% rename from lore/1-tasks/active/0294_DOCS_scf-milestone-3-verification-package.md rename to lore/1-tasks/archive/0294_DOCS_scf-milestone-3-verification-package.md index 93db0bca..7d896406 100644 --- a/lore/1-tasks/active/0294_DOCS_scf-milestone-3-verification-package.md +++ b/lore/1-tasks/archive/0294_DOCS_scf-milestone-3-verification-package.md @@ -2,17 +2,33 @@ id: "0294" title: "SCF Milestone 3 verification package — evidence doc, form answers, video scenario, deviations" type: DOCS -status: active +status: completed related_adr: [] -related_tasks: ["0102", "0128", "0293", "0260", "0275", "0164", "0249", "0179", "0233", "0239", "0194", "0047", "0295", "0296", "0297", "0311"] +related_tasks: ["0102", "0128", "0293", "0260", "0275", "0164", "0249", "0179", "0233", "0239", "0194", "0047", "0295", "0296", "0297", "0311", "0101", "0139", "0274"] tags: [layer-docs, priority-high, effort-medium, milestone-M3, scf, submission, evidence] milestone: 3 links: + - "../../../docs/scf/milestone-3-evidence.md" + - "../../../docs/scf/milestone-3-rfp-deviations.md" - "../../../docs/scf/milestone-2-evidence.md" - "../../../docs/scf/milestone-2-rfp-deviations.md" - "../../../docs/prices-api-general-overview.md" - "../../../docs/prices-api-load-test-100rps.md" history: + - date: 2026-10-06 + status: completed + who: stkrolikiewicz + note: > + Submitted: Aga sent the SCF Deliverable Verification form on 2026-10-06 + with the shortened answers of #392. The package — + milestone-3-evidence.md (510 lines, 16-page PDF), four declared + deviations, the form answers, the video scenario and a 7:39 film — + merged in #354 (2026-10-05 17:27 UTC) and reached master in #391 three + minutes later; every repo link the PDF and the form make answered 200 + on master. The PDF, the film and their folder are on Google Drive with + link sharing. The form's G- and C-address fields do not apply: the API + owns no Stellar account and no contract. Closing decisions and the gaps + found on the way out are under Design Decisions and Issues Encountered. - date: 2026-09-29 status: active who: claude @@ -206,23 +222,111 @@ issue with its task. Kept here so the package is not written from memory. ## Acceptance Criteria -- [ ] `milestone-3-evidence.md` covers all 9 Tranche 3 criteria, each with the - observable it is graded against and a reproducible source -- [ ] Every §9 Tranche 3 Work bullet is addressed, including the two without a - numbered criterion -- [ ] `milestone-3-rfp-deviations.md` declares every known deviation, including - AC 1, the AC 5 companion numbers and the endpoint-group count -- [ ] Every criterion that is open on 2026-09-18 is closed, declared as a +- [x] `milestone-3-evidence.md` covers all 9 Tranche 3 criteria, each with the + observable it is graded against and a reproducible source — **§5**, each + with a command, a live URL or a CI run, and the date it was read. +- [x] Every §9 Tranche 3 Work bullet is addressed, including the two without a + numbered criterion — **§6**: X-Ray tracing end to end and the CloudWatch + dashboard. +- [x] `milestone-3-rfp-deviations.md` declares every known deviation, including + AC 1, the AC 5 companion numbers and the endpoint-group count — **four**: + AC 1 (the backfill finished early), AC 9 (two metrics flat since), AC 2 + (Redocly and the portal's renderer in place of `openapi-validator` and + Swagger UI), AC 5 (scope and companion numbers). The endpoint-group count + turned out not to be a deviation (Design Decision 7). +- [x] Every criterion that is open on 2026-09-18 is closed, declared as a deviation, or listed under "deliberately not claimed" with an owner — - none is silently absent -- [ ] "What is deliberately not claimed" section present, each row with a - destination -- [ ] Live endpoints + access table current, including how the Stellar team - reaches the dashboard -- [ ] `milestone-3-form-answers.md` and `milestone-3-video-scenario.md` written -- [ ] All cited figures re-run within days of submission, with the date beside - each -- [ ] No claim in the package lacks a task, query or URL behind it + none is silently absent. AC 3, 4, 8 and 9 closed; AC 7 met on the + fresh-account runbook, not run in an empty account ([[0297]] reopens on a + reviewer's request, §8); AC 6's 22 wildcard-resource statements each + named with what limits them. +- [x] "What is deliberately not claimed" section present, each row with a + destination — as **§7 Known issues** and **§8 Limitations**, each row + naming its task (Design Decision 6). +- [x] Live endpoints + access table current, including how the Stellar team + reaches the dashboard — **§9**: read-only access per named reviewer + through IAM Identity Center ([[0295]]'s runbook). +- [x] `milestone-3-form-answers.md` and `milestone-3-video-scenario.md` written + — the form went out with the shortened answers of #392. +- [x] All cited figures re-run within days of submission, with the date beside + each — AC 2, 3, 4, 6 and 8 on 2026-10-05, AC 9's window 09-23 → 09-30. + **Two exceptions, both dated in the package:** AC 5 (2026-09-18, no + re-run by the decision of 09-28) and AC 1 (2026-10-01, Design Decision 3). +- [x] No claim in the package lacks a task, query or URL behind it + +## Implementation Notes + +- `docs/scf/`: `milestone-3-evidence.md` (510 lines, §1–§10), + `milestone-3-rfp-deviations.md`, `milestone-3-form-answers.md`, + `milestone-3-video-scenario.md`, `screenshots/m3-ac2-api-reference.png`, + and `repo-links.lua`, which `build-pdf.sh` now runs. Side edits: the + load-test report, the 0295 dashboard-access runbook, the monitoring report + and the general overview. +- PRs: #354 (the package), #391 (release to master), #392 (shortened answers). + The AC 2 lint and the AC 4 test counts come from the master run of the + earlier release #381 (37298469682). +- Google Drive, link-shared: the evidence PDF, the film (15 clips stitched, + 7:39) and the folder holding both. + +## Issues Encountered + +- **Dead links in the PDF.** Typst renders a relative Markdown link as an + in-document reference, so every link to a repo document went nowhere. + Reported on 2026-10-05; fixed by `repo-links.lua` (Design Decision 4). +- **Two sentences written before the release reached Drive**: AC 7's + "on `master` from the next release" and the access table's "request by + e-mail". Fixed (b943f811, a5816d8e); each PDF went up as a new version under + the same Drive link, checked by its size each time. +- **The film.** QuickTime's variable-frame-rate clips froze their picture, and + one clip's narration ran 12 s late; the stitch padded the frames and cut + 12 s of silence to resync. The ffmpeg scripts were not kept in the repo. +- **Not in the package:** [[0101]]'s AMM history holes (Soroswap + 2026-07-06 → 07-11, Phoenix ~2 % short; blocked) are named in neither the + Milestone 2 nor the Milestone 3 known issues. Found at closing. + +## Design Decisions + +### From Plan + +1. **The M2 layout and the same four documents**: evidence per criterion with + its observable, source and date; deviations in their own document; form + answers and video scenario mirrored from M2. +2. **Launch at 2026-09-23 09:40 CEST** for the AC 9 window and the narrative + ([[0296]]). + +### Emerged + +3. **AC 1 stays at its 2026-10-01 reading** (decided 2026-10-05). A re-read + that day found that the explorer's Galexie export stalled from 2026-10-01 + 19:08 to 10-02 09:07 CEST: no ledger reached the pipeline for 14 h, + `ledger-processor-no-invocations` and `rollup-freshness-1m` fired within + 25 minutes, and `ledger-processor-lag` stayed in ALARM 09:17–13:27 while + the backlog drained. The [[0139]] window then held the writers + 14:08–14:57. Both postdate the reading; neither is in the package. +4. **Repo links in the PDF point at `blob/master`** (`repo-links.lua`): a PDF + has no repository beside it. `build-pdf.sh` stops if the filter no longer + rewrites. +5. **No published address for dashboard access** (decided 2026-10-05): the + evidence says "on request", the submitted Field 4 is "None." and the access + line sits in AC 8. +6. **"What is deliberately not claimed" became §7 Known issues and §8 + Limitations.** M3 is the last tranche, so no row can point at a later + milestone; each names its task instead. +7. **The endpoint-group count is not a deviation**: the evidence counts seven + `/v1` routes as the M2 package did, and the design document's §4 files the + same routes under five headings (evidence §5, AC 4). +8. **Answers shortened before submission** (#392): about 3,700 characters + instead of 8,800, every figure and date unchanged, detail left to the PDF. +9. **`ch-demo-queries.sql` not refreshed**: M3's evidence runs on the API, + CloudWatch and CI, and adds no ClickHouse query. +10. **#391 released #373 ([[0274]], `price_basis`) to master before it was + deployed** — declared in #391's description. The package does not + describe that field. + +## Future Work + +None spawned. Post-delivery work stays with the tasks the evidence names in §7 +and §8. ## Notes From 3bf9ebd6a261106247e8c7ff03ba54c5cc93cc68 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:58:43 +0200 Subject: [PATCH 011/144] docs(lore-0328): open the board page assignee bug --- ...oard-page-ignores-the-computed-assignee.md | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 lore/1-tasks/active/0328_BUG_board-page-ignores-the-computed-assignee.md diff --git a/lore/1-tasks/active/0328_BUG_board-page-ignores-the-computed-assignee.md b/lore/1-tasks/active/0328_BUG_board-page-ignores-the-computed-assignee.md new file mode 100644 index 00000000..031118b0 --- /dev/null +++ b/lore/1-tasks/active/0328_BUG_board-page-ignores-the-computed-assignee.md @@ -0,0 +1,52 @@ +--- +id: "0328" +title: "The board page recomputes the assignee from the last history entry and ignores the one board.json carries" +type: BUG +status: active +assignee: akot +related_adr: [] +related_tasks: ["0327", "0149", "0041"] +tags: ["lore-board", "tooling", "effort-small", "priority-medium"] +links: + - "../../board.html" + - "../../../tools/scripts/generate-lore-board.mjs" +history: + - date: "2026-10-06" + status: active + who: akot + note: > + Found on 0149: board.json says akot, the board page shows okarcz. + [[0327]] fixed the generator only; the page has its own rule. +--- + +# The board page ignores the computed assignee + +## Summary + +[[0327]] made `getAssignee` in `generate-lore-board.mjs` honour `assignee:`, +then the newest-dated history entry, skipping `who: claude`. `board.json` has +carried the right value since. The page never reads it. +`getTaskAssignee` in `lore/board.html:934` still takes the `who` of the +**last** history entry. On a newest-first history that entry is the author. + +## Evidence (2026-10-06) + +- `board.json` on GitHub Pages: 0149 → `"assignee": "akot"`. +- The board page shows 0149 as `@okarcz`, from its 2026-08-05 spawn entry. +- 0327 checked its fix against the `board.json` diff (25 of 318 tasks moved), + not against the page, so the page kept the old rule. + +## Fix + +- `getTaskAssignee(t)` returns `t.assignee`. The generator already applies the + status rule (active and archive only) and the history fallback, so the page + keeps no rule of its own. +- The same function feeds the filter counts, the card tag, the table and the + modal (`board.html:883, 951, 999, 1052, 1086`), so all of them follow. + +## Acceptance Criteria + +- [ ] `board.html` reads `assignee` from `board.json` and has no assignee + logic of its own. +- [ ] On the deployed board, 0149, 0242 and 0139 show `@akot`. +- [ ] The assignee filter counts match `board.json`'s `assignee` values. From 0df1e1c4d4174b15d785d00661597597e06d0e3c Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:02:41 +0200 Subject: [PATCH 012/144] fix(lore-0328): read the assignee from board.json on the page 0327 fixed getAssignee in the generator, but board.html kept its own rule: the who of the last history entry. On a newest-first history that is the author, so 0149 showed okarcz while board.json said akot. --- lore/board.html | 7 +++---- tools/scripts/generate-lore-board.test.mjs | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lore/board.html b/lore/board.html index 3c13b8d4..f1243693 100644 --- a/lore/board.html +++ b/lore/board.html @@ -931,11 +931,10 @@

Stellar Prices API Backlog Board

`; } + // The generator owns the rule (getAssignee, task 0327); the page only + // reads its result (task 0328). function getTaskAssignee(t) { - if (t.status !== 'active' && t.status !== 'archive') return null; - const history = Array.isArray(t.history) ? t.history : []; - const lastEntry = history[history.length - 1]; - return lastEntry?.who || null; + return t.assignee || null; } function filtered() { diff --git a/tools/scripts/generate-lore-board.test.mjs b/tools/scripts/generate-lore-board.test.mjs index 30edd9a9..e2f772a8 100644 --- a/tools/scripts/generate-lore-board.test.mjs +++ b/tools/scripts/generate-lore-board.test.mjs @@ -1,9 +1,11 @@ -// Tests for getAssignee in generate-lore-board.mjs (task 0327). +// Tests for getAssignee in generate-lore-board.mjs (task 0327) and its use in +// board.html (task 0328). // // Run: node --test tools/scripts/generate-lore-board.test.mjs import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; import { getAssignee } from './generate-lore-board.mjs'; @@ -95,3 +97,14 @@ for (const { name, dir = 'active', task, want } of CASES) { assert.equal(getAssignee({ _dir: dir, ...task }), want); }); } + +// The page must show what board.json carries, not re-derive it (task 0328). +test('board.html reads the assignee from board.json', () => { + const html = readFileSync( + new URL('../../lore/board.html', import.meta.url), + 'utf-8', + ); + const fn = html.match(/function getTaskAssignee\(t\) \{([\s\S]*?)\n\s*\}/); + assert.ok(fn, 'getTaskAssignee not found in board.html'); + assert.equal(fn[1].trim(), 'return t.assignee || null;'); +}); From 1e4d63e9a4d98b9653e78d914ab6f067dc767ee5 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:04:16 +0200 Subject: [PATCH 013/144] docs(lore-0149): close as superseded by 0286 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-measured on prod 2026-10-06: no coarse row on any of the six tiers holds a USD value its children do not support. 0286's rate-form close_usd (ADR 0287 §5) removed TEST E's zero; the version arithmetic is unchanged and documented in ADR 0317. No follow-up tasks. --- ...UG_sweep-mv-version-ownership-close-usd.md | 99 ---------- ...UG_sweep-mv-version-ownership-close-usd.md | 181 ++++++++++++++++++ 2 files changed, 181 insertions(+), 99 deletions(-) delete mode 100644 lore/1-tasks/active/0149_BUG_sweep-mv-version-ownership-close-usd.md create mode 100644 lore/1-tasks/archive/0149_BUG_sweep-mv-version-ownership-close-usd.md diff --git a/lore/1-tasks/active/0149_BUG_sweep-mv-version-ownership-close-usd.md b/lore/1-tasks/active/0149_BUG_sweep-mv-version-ownership-close-usd.md deleted file mode 100644 index e8224720..00000000 --- a/lore/1-tasks/active/0149_BUG_sweep-mv-version-ownership-close-usd.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -id: "0149" -title: "Two writers with incompatible version arithmetic own close_usd — the 0114 sweep's repair is overwritten inside the MV window" -type: BUG -status: active -assignee: akot -related_adr: [] -related_tasks: ["0144", "0146", "0148", "0114", "0095", "0143"] -tags: - ["priority-medium", "effort-medium", "clickhouse", "data-correctness", "materialized-view", "milestone-M2"] -milestone: 2 -links: - - "../../../packages/enrichment-worker/src/repair.rs" - - "../../../packages/prices-clickhouse/schema/rollups.sql" -history: - - date: "2026-10-06" - status: active - who: akot - note: "Activated and assigned to akot to work through the options." - - date: 2026-08-05 - status: backlog - who: okarcz - note: > - Spawned from [[0144]] future work (phase 7) — finding 3ii-b, confirmed on - the prod CH pin (TEST E). Deliberately ordered after [[0146]], which - removes most of its consequence. ---- - -# Sweep vs MV: incompatible version arithmetic on `close_usd` - -## Summary - -Two writers update `close_usd` on the same coarse rows and claim the row by -different rules: - -- The [[0114]] coarse sweep wins by **`version + 1`** (`repair.rs:20-22`). -- The rollup MVs write at **`version = sum(version)`** over their sub-rows — - and every enrichment event underneath adds 1 to that sum. - -Two enrichment events and the MV overtakes the sweep. Worse, the MV gets far -more attempts: `mv_ohlcv_15m_to_1h` re-appends the same hour **every 15 minutes -for 8 hours** (`REFRESH EVERY 15 MINUTE`, window `now() - INTERVAL 8 HOUR`). -The sweep gets one. - -Confirmed on CH **26.3.10.60** ([[0144]] `repro/04_sweep_durability.sql`, -TEST E): - -``` -after the sweep repairs it close_usd 0.171 version 401 -after the next MV refresh close_usd 0 version 402 -what the view publishes now (empty) -``` - -So the repair path we already built is defeated inside the re-aggregation -window — the bucket vanishes from `price_usd_series*` again, which is BE's -observation exactly. - -## Why this is ordered late - -**[[0146]] removes most of the consequence.** Once the MVs use `argMaxIf`, the -re-append writes a *correct* value rather than a zero — so being overtaken by -the MV stops being harmful inside the window. And outside the window the MV -does not write at all, so [[0148]]'s historical sweep is unaffected either way. - -That leaves this as a latent trap rather than a live defect: **two writers with -incompatible version arithmetic on one column**, with no mechanism preventing -the next such writer from colliding the same way. Worth closing on its own -terms, not urgent. - -Fixing only this and not [[0146]] would leave the zero-propagation completely -intact — the sweep would win a race to publish a value the MV is about to -recompute as zero anyway. - -## Implementation - -Options to weigh, none obviously correct yet: - -- Give the sweep a version domain the MV cannot reach (e.g. a large offset), at - the cost of making `version` no longer a plain event count. -- Have the sweep write through the same path as the MV so there is one writer. -- Make the sweep skip rows still inside an MV's re-aggregation window entirely, - and rely on [[0146]] for those — smallest change, but encodes the window - boundaries in two places. -- Reconsider whether `close_usd` should be MV-carried at all rather than always - derived by the sweep. - -Related: [[0143]] records that there is no `DEPENDS ON` anywhere in the cascade, -so ordering between tiers is already unsynchronised; any fix here should not -assume refresh ordering it does not control. - -## Acceptance Criteria - -- [ ] A single documented owner for `close_usd` on coarse rows, or a version - scheme under which the sweep's repair provably survives an MV refresh. -- [ ] Regression test on CH 26.3.10.60 reproducing TEST E and showing the - repair now survives. -- [ ] The `version` column's semantics restated in the schema header if they - change. -- [ ] Confirmed not to reintroduce the [[0095]] replace-mode invariants problem. diff --git a/lore/1-tasks/archive/0149_BUG_sweep-mv-version-ownership-close-usd.md b/lore/1-tasks/archive/0149_BUG_sweep-mv-version-ownership-close-usd.md new file mode 100644 index 00000000..ee84f16e --- /dev/null +++ b/lore/1-tasks/archive/0149_BUG_sweep-mv-version-ownership-close-usd.md @@ -0,0 +1,181 @@ +--- +id: "0149" +title: "Two writers with incompatible version arithmetic own close_usd — the 0114 sweep's repair is overwritten inside the MV window" +type: BUG +status: completed +assignee: akot +related_adr: ["0287", "0317", "0292"] +related_tasks: ["0144", "0146", "0148", "0114", "0095", "0143", "0286", "0203", "0151", "0228"] +tags: + ["priority-medium", "effort-medium", "clickhouse", "data-correctness", "materialized-view", "milestone-M2"] +milestone: 2 +links: + - "../../../packages/enrichment-worker/src/repair.rs" + - "../../../packages/prices-clickhouse/schema/rollups.sql" +history: + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed as superseded by [[0286]] (ADR 0287 §5), like [[0146]]. + Re-measured on prod: 0 coarse rows on any of the six tiers hold a USD + value their children do not support, so TEST E's case cannot recur. The + version arithmetic is unchanged and documented in ADR 0317. Further + paths are hypothetical; no follow-up tasks, by Adam's decision. No code + change, no regression test. + - date: "2026-10-06" + status: active + who: akot + note: "Activated and assigned to akot to work through the options." + - date: "2026-08-05" + status: backlog + who: okarcz + note: > + Spawned from [[0144]] future work (phase 7) — finding 3ii-b, confirmed on + the prod CH pin (TEST E). Deliberately ordered after [[0146]], which + removes most of its consequence. +--- + +# Sweep vs MV: incompatible version arithmetic on `close_usd` + +## Summary + +Two writers update `close_usd` on the same coarse rows and claim the row by +different rules: + +- The [[0114]] coarse sweep wins by **`version + 1`** (`repair.rs:20-22`). +- The rollup MVs write at **`version = sum(version)`** over their sub-rows — + and every enrichment event underneath adds 1 to that sum. + +Two enrichment events and the MV overtakes the sweep. Worse, the MV gets far +more attempts: `mv_ohlcv_15m_to_1h` re-appends the same hour **every 15 minutes +for 8 hours** (`REFRESH EVERY 15 MINUTE`, window `now() - INTERVAL 8 HOUR`). +The sweep gets one. + +Confirmed on CH **26.3.10.60** ([[0144]] `repro/04_sweep_durability.sql`, +TEST E): + +``` +after the sweep repairs it close_usd 0.171 version 401 +after the next MV refresh close_usd 0 version 402 +what the view publishes now (empty) +``` + +So the repair path we already built is defeated inside the re-aggregation +window — the bucket vanishes from `price_usd_series*` again, which is BE's +observation exactly. + +## Why this is ordered late + +**[[0146]] removes most of the consequence.** Once the MVs use `argMaxIf`, the +re-append writes a *correct* value rather than a zero — so being overtaken by +the MV stops being harmful inside the window. And outside the window the MV +does not write at all, so [[0148]]'s historical sweep is unaffected either way. + +That leaves this as a latent trap rather than a live defect: **two writers with +incompatible version arithmetic on one column**, with no mechanism preventing +the next such writer from colliding the same way. Worth closing on its own +terms, not urgent. + +Fixing only this and not [[0146]] would leave the zero-propagation completely +intact — the sweep would win a race to publish a value the MV is about to +recompute as zero anyway. + +## Implementation + +Options to weigh, none obviously correct yet: + +- Give the sweep a version domain the MV cannot reach (e.g. a large offset), at + the cost of making `version` no longer a plain event count. +- Have the sweep write through the same path as the MV so there is one writer. +- Make the sweep skip rows still inside an MV's re-aggregation window entirely, + and rely on [[0146]] for those — smallest change, but encodes the window + boundaries in two places. +- Reconsider whether `close_usd` should be MV-carried at all rather than always + derived by the sweep. + +Related: [[0143]] records that there is no `DEPENDS ON` anywhere in the cascade, +so ordering between tiers is already unsynchronised; any fix here should not +assume refresh ordering it does not control. + +## Re-measured on prod, 2026-10-06 + +Measured as `dev_read` (read-only), 08:23–08:40 UTC. The queries are kept +outside the repo. + +**What changed since this task was written.** + +- The version arithmetic is unchanged. The sweep still writes `version + 1` + (`repair.rs:22`, `ch_enrich.rs:2446/2490/2717`). Every MV still writes + `sum(version)` (`rollups.sql:151-163`), and 0286 kept it. +- [[0146]] was closed as superseded by [[0286]]. Since 2026-09-22 a coarse + `close_usd` is `close × the latest priced child's rate` (ADR 0287 §5, + `rollups.sql:65-75`). The MV writes 0 only when no child is priced, not + whenever the last child is unpriced. TEST E's step 4 now yields the same value + the sweep wrote. +- [[0143]]/[[0203]] (live 2026-10-05) added a third writer: six reconcile MVs + with a 7-day window, hourly. They write only buckets whose `trade_count` or + `volume_base` disagree, so they do not touch a sweep repair by themselves. +- ADR 0317 "Version analysis" already describes the arithmetic (sweep `+1`, + `coarse-repair` `+2`, ties go to the last insert) and accepts that a + re-derived `close_usd` "can briefly go to 0 … until the next sweep pass". +- The one design that would resolve this by construction, a USD rate table with + `close_usd` derived ([[0151]]), was rejected in ADR 0292. +- The fast windows are not all short. 1w re-rolls 60 days and 1M re-rolls + **400 days**, daily. The recurring sweep covers 2 months + (`COARSE_SWEEP_LOOKBACK_MONTHS`). + +**Exposure.** A row is at risk when it holds a USD value that its children do +not support, because the next MV write that wins would replace it. + +| tier (fast window) | rows | priced | last written by sweep | priced, no priced child | +|---|---|---|---|---| +| 15m (2 h) | 10,716 | 2,236 | 0 | 0 | +| 1h (8 h) | 25,444 | 5,948 | 11 | 0 | +| 4h (1 d) | 46,901 | 11,220 | 35 | 0 | +| 1d (7 d) | 117,586 | 29,977 | 241 | 0 | +| 1w (60 d) | 251,015 | 88,003 | 905 | 0 | +| 1M (400 d) | 564,712 | 275,966 | 448 | 0 | + +- "Last written by sweep" is `version > sum(children)`. Each of these rows holds + the USD value its children already give, so the sweep only got there first. + The same holds for `volume_quote_usd`: 0 rows priced over unpriced children, + on every tier. +- Residue at the 1e-12 floor, not this defect: + - 1d: one row priced at 1.0e-12 over children below the floor, and 15 rows + whose `close_usd` is ~1e-13. + - 1M: 1,302 rows at `close_usd = 0` whose 1d children are all below the floor + (max 9.9e-13), all last written by the MV. That is the floor working as + `rollups.sql:70-73` intends. + +**Verdict.** TEST E's case cannot recur. Its zero came from the MV carrying the +last child's `close_usd` when that child was unpriced. Since 0286 the MV +re-prices the bucket from the latest priced child, so it writes the same value +the sweep wrote. The race still happens, because the version arithmetic is +unchanged, but losing it costs nothing. + +## Closing decision (2026-10-06, Adam) + +Closed as superseded by [[0286]] (ADR 0287 §5), like [[0146]]. The measurement +above is the evidence. Other ways the race could matter again are hypothetical, +and we deliberately create no tasks for them: + +- a coarse-only price correction inside an MV window; +- a coarse tier pricing where its children cannot; +- a regression of the rate form; +- a new writer with its own version arithmetic. + +## Acceptance Criteria + +- [x] A single documented owner for `close_usd` on coarse rows, or a version + scheme under which the sweep's repair provably survives an MV refresh. + Met differently: the repair is still overwritten, but with the same + value. The arithmetic and the accepted brief zero are documented in ADR + 0317 "Version analysis"; the rate form in ADR 0287 §5. +- [ ] Regression test on CH 26.3.10.60 reproducing TEST E and showing the + repair now survives. Not written, by decision; the prod measurement of + 2026-10-06 stands in for it. +- [x] The `version` column's semantics restated in the schema header if they + change. They did not change. +- [x] Confirmed not to reintroduce the [[0095]] replace-mode invariants problem. + No schema or MV change was made; every MV is still `APPEND`. From 7cffc7abe3df4ec07c9a165772730aa50ea9d377 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:26:44 +0200 Subject: [PATCH 014/144] fix(lore-0236): count stored candles outside the OHLC band on all seven tiers Since 0229 the /ohlcv clamp repairs a crossed candle on the way out, so nothing noticed a corrupt stored row. The probe now reads every price_ohlcv tier FINAL over a per-tier bucket window (2 days widened by one bucket) and publishes CandleBandViolations: priced rows outside the band or with any price <= 0, zero tolerance. Seven independent reads so one failing tier cannot hide the others; a zero is refused unless all seven were read and the 1m window was non-empty. --- packages/rollup-freshness-probe/src/lib.rs | 6 + packages/rollup-freshness-probe/src/main.rs | 70 ++- .../rollup-freshness-probe/src/ohlc_band.rs | 444 ++++++++++++++++++ .../tests/rollup_freshness_it.rs | 150 ++++++ 4 files changed, 669 insertions(+), 1 deletion(-) create mode 100644 packages/rollup-freshness-probe/src/ohlc_band.rs diff --git a/packages/rollup-freshness-probe/src/lib.rs b/packages/rollup-freshness-probe/src/lib.rs index de13b29a..5101189b 100644 --- a/packages/rollup-freshness-probe/src/lib.rs +++ b/packages/rollup-freshness-probe/src/lib.rs @@ -75,6 +75,12 @@ pub mod zero_invariants; /// recent candles naming an id `assets` does not hold. See /// [`asset_id_uniqueness`] for why each refuses an empty read. pub mod asset_id_uniqueness; +/// Stored candles that break the OHLC band or carry a price `<= 0`, on all seven +/// tiers (task 0236). Asserted at the source, with zero tolerance, because task +/// 0229's read-path clamp repairs a crossed candle on its way out of `/ohlcv` — +/// so the API can no longer show one. See [`ohlc_band`] for the per-tier window +/// and the refusal rule. +pub mod ohlc_band; /// `current_prices` writer liveness (task 0243). Rides in the same invocation /// and publishes under the same [`METRIC_NAME`] with `Table = current_prices` — diff --git a/packages/rollup-freshness-probe/src/main.rs b/packages/rollup-freshness-probe/src/main.rs index 5b6b102c..118eb634 100644 --- a/packages/rollup-freshness-probe/src/main.rs +++ b/packages/rollup-freshness-probe/src/main.rs @@ -29,6 +29,9 @@ async fn main() -> Result<(), lambda_runtime::Error> { MV_DRIFT_CRITICAL_METRIC, MV_DRIFT_METRIC, describe, drift_metrics, publish_drift, visible_objects_query, }; + use rollup_freshness_probe::ohlc_band::{ + OHLC_BAND_TIERS, OhlcBandCounts, ohlc_band_detail, ohlc_band_metric, ohlc_band_queries, + }; use rollup_freshness_probe::reconcile_mismatch::{ MISMATCH_MIN_READ_SECS, MISMATCH_RESERVE, MismatchCount, mismatch_metric, mismatch_queries, mismatch_read_bound, publish_mismatch, @@ -86,6 +89,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { // deadline, and a read that would get under 2 s is skipped and recorded as // a failure — see `reconcile_mismatch` (review WR-04). let mismatch_queries = Arc::new(mismatch_queries()); + let ohlc_band_queries = Arc::new(ohlc_band_queries()); let aws_cfg = aws_config::defaults(aws_config::BehaviorVersion::latest()) .load() @@ -105,6 +109,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { let orphan_candles_query = orphan_candles_query.clone(); let refresh_waits_query = refresh_waits_query.clone(); let mismatch_queries = mismatch_queries.clone(); + let ohlc_band_queries = ohlc_band_queries.clone(); let environment = environment.clone(); let deadline = event.context.deadline(); async move { @@ -361,7 +366,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { // ---- 5. The zero sentinel's stored-data invariants (ADR 0292) --- // - // Late on purpose (only 5b and the mismatch reads, 6, come after it). It is the one unscoped read here: `timestamp` is the + // Late on purpose (only 5b, 5c and the mismatch reads, 6, come after it). It is the one unscoped read here: `timestamp` is the // fourth sort-key column, so the 48 h window prunes only to the monthly // partition, which is then merged `FINAL` across every pair. Measured on // production 2026-09-18 it is cheap (0.04 s, 650k rows read) — but it @@ -446,6 +451,51 @@ async fn main() -> Result<(), lambda_runtime::Error> { Err(e) => failures.push(format!("asset-id-orphans read: {e}")), } + // ---- 5c. Stored-candle OHLC band + positive prices, all seven tiers (task 0236) --- + // + // Late, for the reason block 5 gives: these reads grow with the + // tables. Before the mismatch reads (6), which stay last because + // their budget is what is left of the invocation. Measured on + // production 2026-10-06 the seven reads together are cheap: 0.29 s + // of server time, 3.7 M rows read. + // + // Seven INDEPENDENT reads, not one `UNION ALL`: a statement fails as a + // unit, so one dropped tier or lost grant would hide the other six + // counts, and the alarm is `NOT_BREACHING` on missing data. A failed + // tier is recorded and the loop moves on. + // + // Independent of block 5, although a priced `_1m` row with + // `close = 0` fires both: that overlap is expected (see `ohlc_band`). + let mut band_readings: Vec<(&'static str, OhlcBandCounts)> = Vec::new(); + for (table, sql) in ohlc_band_queries.iter() { + match ch.query(sql).fetch_one::().await { + Ok(counts) => band_readings.push((*table, counts)), + Err(e) => failures.push(format!("ohlc-band read {table}: {e}")), + } + } + match ohlc_band_metric(&band_readings) { + Ok(metric) => { + if let Err(e) = + publish_sanity(&cw, &environment, std::slice::from_ref(&metric)).await + { + failures.push(format!("ohlc-band publish: {e}")); + } + } + Err(refusal) => failures.push(format!("ohlc-band: {refusal}")), + } + // Totals only when every tier was read: a partial sum logged as a + // number would read as the whole chain. The detail names each tier. + let band_totals = (band_readings.len() == OHLC_BAND_TIERS.len()).then(|| { + band_readings.iter().fold((0u64, 0u64, 0u64), |(v, b, n), (_, c)| { + (v + c.violations(), b + c.band, n + c.nonpositive) + }) + }); + let band_scanned_1m = band_readings + .iter() + .find(|(t, _)| *t == OHLC_BAND_TIERS[0].0) + .map(|(_, c)| c.scanned); + let band_detail = ohlc_band_detail(&band_readings); + // ---- 6. Coarse buckets disagreeing with their source (0203) --- // // The new tail, for the reason the zero invariants were last: these @@ -548,6 +598,12 @@ async fn main() -> Result<(), lambda_runtime::Error> { mv_drift_critical = %reading(drift_critical, drift_missing), mv_drift = %reading(drift_count, drift_missing), mv_visible_objects = %reading(visible_objects, FAILED), + ohlc_band_violations = %reading(band_totals.map(|t| t.0), FAILED), + ohlc_band_band = %reading(band_totals.map(|t| t.1), FAILED), + ohlc_band_nonpositive = %reading(band_totals.map(|t| t.2), FAILED), + ohlc_band_scanned_1m = %reading(band_scanned_1m, FAILED), + ohlc_band_tiers_read = band_readings.len(), + ohlc_band_detail = %band_detail, mv_detail = %drift_detail, mv_refresh_waiting = %reading(refresh_waiting, refresh_missing), mv_refresh_disabled = %reading(refresh_disabled, refresh_missing), @@ -593,6 +649,18 @@ async fn main() -> Result<(), lambda_runtime::Error> { "orphan_candles": orphan_counts.map(|c| c.orphans), "orphan_scanned": orphan_counts.map(|c| c.scanned), }, + "ohlc_band": { + "violations": band_totals.map(|t| t.0), + "tiers": band_readings + .iter() + .map(|(t, c)| serde_json::json!({ + "tier": t, + "band": c.band, + "nonpositive": c.nonpositive, + "scanned": c.scanned, + })) + .collect::>(), + }, "mv_drift": { "critical": drift_critical, "drifted": drift_count, diff --git a/packages/rollup-freshness-probe/src/ohlc_band.rs b/packages/rollup-freshness-probe/src/ohlc_band.rs new file mode 100644 index 00000000..b6dde0ec --- /dev/null +++ b/packages/rollup-freshness-probe/src/ohlc_band.rs @@ -0,0 +1,444 @@ +//! Stored candles that are internally inconsistent, on all seven tiers (task 0236). +//! +//! A candle's four prices obey two shapes: `low` and `high` bracket the open and +//! the close, and every price is positive. Since task 0229 the `/ohlcv` read path +//! CLAMPS a crossed candle into shape on its way out (ADR-0011 §3), which is the +//! right thing for a read API to do — and it removed the only place anyone would +//! have noticed a corrupt stored row. This check moves that detector to the +//! SOURCE: it reads the stored rows, not what the API serves. +//! +//! # The predicate — exact, with zero tolerance +//! +//! A priced row (`pf_trade_count > 0`) is a violation when +//! +//! ```text +//! low > least(open, close) OR high < greatest(open, close) OR low > high +//! OR open <= 0 OR high <= 0 OR low <= 0 OR close <= 0 +//! ``` +//! +//! There is no epsilon. The stored decimals are produced by `min`/`max`/ +//! `argMin`/`argMax` over the same fills (and, up the chain, over the tier +//! below), which COPY a value bit for bit — nothing is computed, so nothing can +//! round. Any crossing in storage is a real defect, and the comparisons are plain +//! `Decimal` against integer literals: no `toFloat`, no float path at all. +//! +//! The predicate is split into two **disjoint** classes, so their sum counts ROWS: +//! +//! - `band` — the three band arms, over rows whose four prices are ALL `> 0`; +//! - `nonpositive` — any of the four prices `<= 0`. +//! +//! Their union is exactly the predicate above. Without the positive gate a row +//! like `open = 0, low = 3` would fire both `low > least(open, close)` and +//! `open <= 0` and count twice. `low > high` follows logically from the other two +//! band arms (if `low > high` then either `low > least(o, c)`, or +//! `least(o, c) >= low > high`, and then `greatest(o, c) > high`); it is kept as +//! the literal invariant the reader expects to find. +//! +//! # The window is per tier, on bucket START +//! +//! `timestamp >= now() - 2 days - ` (`_1m` plain 2 days). A +//! bucket of length `L` starting at `s` overlaps the last two days iff +//! `s > now - 2d - L`. A flat 2-day window would NEVER see a `_1w` or `_1M` +//! bucket for most of the week or month — its start is older than two days. The +//! widening is by one bucket, and since weeks are Monday-aligned and month +//! subtraction clamps to the month's end, `now - 1 MONTH` is always at or before +//! the current month's start: the window always reaches the current bucket. +//! +//! ⚠️ **Bucket time, not write time** — as with [`crate::zero_invariants`], the +//! tables carry no insert-time column. A backfill, pre-roll or operator +//! `INSERT … SELECT` that writes buckets older than the window is NOT covered; +//! a historical rewrite needs its own assertion over the partitions it touched. +//! +//! ⚠️ **A scheduled assertion, not a ClickHouse `CHECK` constraint** — for the +//! reason ADR 0292 §6 gives: a violated CHECK fails the whole insert, so one bad +//! row would stall a refreshable MV's tier and turn a data defect into a +//! freshness incident. +//! +//! # Overlap and multiplication — both expected +//! +//! - A priced `_1m` row with `close = 0` is also [`crate::zero_invariants`] +//! invariant 3, so both alarms fire. `low = 0` beside a positive close is +//! caught ONLY here — the shape the prod baseline was mostly made of. +//! - The metric sums TIER-ROWS. The rollups carry a bad minute into every coarse +//! bucket above it, so one defect can count up to seven times. Read the per-tier +//! detail: a violation on a coarse tier with a clean `_1m` is a ROLLUP defect, +//! not an ingest one. +//! +//! # Seven reads, not one `UNION ALL` +//! +//! One statement fails as a unit: a dropped or renamed tier, or a lost grant, +//! would take every other tier's count down with it, and the alarm is +//! `NOT_BREACHING` on missing data. Seven independent reads keep the six that +//! work. +//! +//! # When a reading is refused +//! +//! A positive sum is always published — it can never be a false OK. A ZERO is +//! published only when all seven tiers were read AND the `_1m` window was +//! non-empty; otherwise [`OhlcBandRefusal`] says why, and the probe's `-errors` +//! alarm carries it. Coarse tiers with nothing in their window are fine. +//! +//! # Baseline +//! +//! Measured on production 2026-09-28: band violations 0 on every tier, every +//! source, since 2015. Priced rows with a price `<= 0` are all exact zeros and +//! all older than the task 0286 rollout (2026-09-22 12:00 UTC); 0 after it. That +//! legacy residue is outside this window by construction and is re-ingested by +//! 0286 phase 3. A healthy reading is therefore exactly **0**. + +use crate::usd_sanity::SanityMetric; + +/// CloudWatch metric: stored candles in the per-tier windows that break the band +/// or carry a price `<= 0`, summed over all seven tiers. +pub const OHLC_BAND_METRIC: &str = "CandleBandViolations"; + +/// Two days, the same base window as +/// [`crate::zero_invariants::ZERO_INVARIANT_LOOKBACK_SECONDS`]; the coarse tiers +/// widen it by one bucket (see [`OHLC_BAND_TIERS`]). +pub const OHLC_BAND_LOOKBACK_SECONDS: i64 = 2 * 86_400; + +/// `(table, widening)` per tier, in [`crate::ROLLUP_TIERS`] order (a unit test +/// pins the two together). The widening is one bucket length, as a ClickHouse +/// `INTERVAL` literal; `_1m` is not widened. +pub const OHLC_BAND_TIERS: [(&str, Option<&str>); 7] = [ + ("price_ohlcv_1m", None), + ("price_ohlcv_15m", Some("15 MINUTE")), + ("price_ohlcv_1h", Some("1 HOUR")), + ("price_ohlcv_4h", Some("4 HOUR")), + ("price_ohlcv_1d", Some("1 DAY")), + ("price_ohlcv_1w", Some("1 WEEK")), + ("price_ohlcv_1M", Some("1 MONTH")), +]; + +/// One tier's reading. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clickhouse::Row, serde::Deserialize)] +pub struct OhlcBandCounts { + /// Priced rows with four positive prices that break the band. + pub band: u64, + /// Priced rows with any of the four prices `<= 0`. Disjoint from `band`. + pub nonpositive: u64, + /// Rows examined. Zero on `_1m` means the scan measured nothing. + pub scanned: u64, +} + +impl OhlcBandCounts { + /// Rows breaking either shape. The classes are disjoint, so this is a row + /// count, not a count of arms. + pub fn violations(&self) -> u64 { + self.band + self.nonpositive + } +} + +/// The seven assertion queries, `(table, sql)` in [`OHLC_BAND_TIERS`] order. +/// +/// `FINAL`, because a corrected row is re-inserted at a higher `version` and the +/// alarm must stop once the data is fixed. Takes no argument: the SQL is built +/// only from `'static` constants, so no caller-supplied string reaches `format!`. +pub fn ohlc_band_queries() -> Vec<(&'static str, String)> { + OHLC_BAND_TIERS + .iter() + .map(|&(table, widen)| { + let widening = widen + .map(|w| format!(" - INTERVAL {w}")) + .unwrap_or_default(); + let sql = format!( + "SELECT \ + countIf(pf_trade_count > 0 AND open > 0 AND high > 0 AND low > 0 AND close > 0 \ + AND (low > least(open, close) OR high < greatest(open, close) OR low > high)) \ + AS band, \ + countIf(pf_trade_count > 0 AND (open <= 0 OR high <= 0 OR low <= 0 OR close <= 0)) \ + AS nonpositive, \ + count() AS scanned \ + FROM {table} FINAL \ + WHERE timestamp >= now() - INTERVAL {lookback} SECOND{widening}", + lookback = OHLC_BAND_LOOKBACK_SECONDS, + ); + (table, sql) + }) + .collect() +} + +/// Why a zero was not published. +/// +/// Its own type, NOT [`crate::usd_sanity::SanityRefusal`], for the reason +/// [`crate::zero_invariants::ZeroInvariantRefusal`] gives: that one blames the +/// USDT identity, and this check resolves no quote leg. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OhlcBandRefusal { + /// The `_1m` window held no candle at all. + EmptyScan, + /// At least one tier's read failed in this run. + Incomplete, +} + +impl std::fmt::Display for OhlcBandRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::EmptyScan => write!( + f, + "no candle of any pair was found in the last {} s of {} — ingestion has \ + stopped or the table is empty (this check is not scoped to a quote leg); \ + refusing to publish a healthy zero over a scan that measured nothing", + OHLC_BAND_LOOKBACK_SECONDS, OHLC_BAND_TIERS[0].0 + ), + Self::Incomplete => write!( + f, + "fewer than {} tiers were read in this run (each failed tier read is \ + reported alongside) — a zero would claim tiers nobody measured", + OHLC_BAND_TIERS.len() + ), + } + } +} + +impl std::error::Error for OhlcBandRefusal {} + +/// Shape the readings into the one datum to publish, or refuse. +/// +/// In order: a positive sum is always published (never a false OK, even with a +/// tier missing); a zero with any tier unread is [`OhlcBandRefusal::Incomplete`]; +/// a zero over an empty `_1m` window is [`OhlcBandRefusal::EmptyScan`]; otherwise +/// a healthy zero. +pub fn ohlc_band_metric( + readings: &[(&'static str, OhlcBandCounts)], +) -> Result { + let total: u64 = readings.iter().map(|(_, c)| c.violations()).sum(); + if total > 0 { + return Ok(SanityMetric { + name: OHLC_BAND_METRIC, + value: total as f64, + }); + } + let read = |table: &str| readings.iter().find(|(t, _)| *t == table).map(|(_, c)| c); + if OHLC_BAND_TIERS.iter().any(|(t, _)| read(t).is_none()) { + return Err(OhlcBandRefusal::Incomplete); + } + if read(OHLC_BAND_TIERS[0].0).is_some_and(|c| c.scanned == 0) { + return Err(OhlcBandRefusal::EmptyScan); + } + Ok(SanityMetric { + name: OHLC_BAND_METRIC, + value: 0.0, + }) +} + +/// One ` band= nonpositive= scanned=` segment per reading, +/// joined by `; ` — the per-tier breakdown the alarm text points at. +pub fn ohlc_band_detail(readings: &[(&'static str, OhlcBandCounts)]) -> String { + readings + .iter() + .map(|(t, c)| { + format!( + "{t} band={} nonpositive={} scanned={}", + c.band, c.nonpositive, c.scanned + ) + }) + .collect::>() + .join("; ") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ROLLUP_TIERS; + + fn counts(band: u64, nonpositive: u64, scanned: u64) -> OhlcBandCounts { + OhlcBandCounts { + band, + nonpositive, + scanned, + } + } + + /// Seven readings, one per tier, all the given counts. + fn all_tiers(c: OhlcBandCounts) -> Vec<(&'static str, OhlcBandCounts)> { + OHLC_BAND_TIERS.iter().map(|(t, _)| (*t, c)).collect() + } + + #[test] + fn every_tier_query_asserts_every_predicate_arm() { + let queries = ohlc_band_queries(); + assert_eq!(queries.len(), 7); + for (table, sql) in &queries { + for arm in [ + "low > least(open, close)", + "high < greatest(open, close)", + "low > high", + "open <= 0 OR high <= 0 OR low <= 0 OR close <= 0", + ] { + assert!(sql.contains(arm), "{table} is missing `{arm}`: {sql}"); + } + } + } + + /// The two classes are disjoint: `band` only over four positive prices, so + /// a row breaking both shapes (open = 0 with a positive low) counts once, + /// and both only over priced rows. + #[test] + fn the_classes_are_disjoint_and_both_gated_on_a_priced_row() { + for (table, sql) in ohlc_band_queries() { + assert!( + sql.contains( + "countIf(pf_trade_count > 0 AND open > 0 AND high > 0 AND low > 0 AND close > 0 \ + AND (low > least(open, close) OR high < greatest(open, close) OR low > high)) \ + AS band" + ), + "{table}: {sql}" + ); + assert!( + sql.contains( + "countIf(pf_trade_count > 0 AND (open <= 0 OR high <= 0 OR low <= 0 OR close <= 0)) \ + AS nonpositive" + ), + "{table}: {sql}" + ); + assert!(sql.contains("count() AS scanned"), "{table}: {sql}"); + assert!( + !sql.contains("toFloat"), + "exact Decimal, no float path: {sql}" + ); + } + } + + /// BRIEF point 4: the window is on bucket START, widened by one bucket per + /// tier. One assertion per tier, each pinned to the END of the statement. + #[test] + fn each_tier_reads_its_table_final_with_its_own_bucket_window() { + let expected = [ + ( + "price_ohlcv_1m", + "timestamp >= now() - INTERVAL 172800 SECOND", + ), + ( + "price_ohlcv_15m", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 15 MINUTE", + ), + ( + "price_ohlcv_1h", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 1 HOUR", + ), + ( + "price_ohlcv_4h", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 4 HOUR", + ), + ( + "price_ohlcv_1d", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 1 DAY", + ), + ( + "price_ohlcv_1w", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 1 WEEK", + ), + ( + "price_ohlcv_1M", + "timestamp >= now() - INTERVAL 172800 SECOND - INTERVAL 1 MONTH", + ), + ]; + let queries = ohlc_band_queries(); + assert_eq!(queries.len(), expected.len()); + for ((table, sql), (want_table, want_window)) in queries.iter().zip(expected) { + assert_eq!(*table, want_table); + assert!( + sql.contains(&format!("FROM {want_table} FINAL WHERE")), + "{table} must read FINAL: {sql}" + ); + assert!( + sql.ends_with(&format!("WHERE {want_window}")), + "{table} window: {sql}" + ); + } + } + + /// An eighth tier added to `ROLLUP_TIERS` must not go unwatched here. + #[test] + fn the_tiers_are_exactly_the_rollup_tiers_in_order() { + let ours: Vec<&str> = OHLC_BAND_TIERS.iter().map(|(t, _)| *t).collect(); + let rollup: Vec<&str> = ROLLUP_TIERS.iter().map(|t| t.table).collect(); + assert_eq!(ours, rollup); + } + + #[test] + fn the_metric_is_the_sum_of_both_classes_over_every_tier() { + let readings: Vec<(&'static str, OhlcBandCounts)> = OHLC_BAND_TIERS + .iter() + .enumerate() + .map(|(i, (t, _))| (*t, counts(i as u64, 2 * i as u64, 100))) + .collect(); + // band 0+1+..+6 = 21, nonpositive 2*21 = 42 + let metric = ohlc_band_metric(&readings).unwrap(); + assert_eq!(metric.name, OHLC_BAND_METRIC); + assert_eq!(metric.name, "CandleBandViolations"); + assert_eq!(metric.value, 63.0); + assert_eq!(counts(2, 3, 9).violations(), 5); + } + + #[test] + fn an_empty_1m_scan_is_refused_not_published_as_healthy() { + let refusal = ohlc_band_metric(&all_tiers(counts(0, 0, 0))).unwrap_err(); + assert_eq!(refusal, OhlcBandRefusal::EmptyScan); + } + + #[test] + fn a_zero_over_a_missing_tier_is_refused_as_incomplete() { + let mut readings = all_tiers(counts(0, 0, 10)); + readings.remove(3); + assert_eq!( + ohlc_band_metric(&readings).unwrap_err(), + OhlcBandRefusal::Incomplete + ); + } + + /// A positive value is never a false OK, so a failed tier elsewhere does + /// not hide the violations the other six found. + #[test] + fn a_positive_sum_is_published_even_with_a_tier_missing() { + let mut readings = all_tiers(counts(0, 0, 10)); + readings.remove(6); + readings[2].1 = counts(1, 2, 10); + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 3.0); + } + + #[test] + fn empty_coarse_tiers_beside_a_scanned_1m_publish_a_healthy_zero() { + let mut readings = all_tiers(counts(0, 0, 0)); + readings[0].1 = counts(0, 0, 500); + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 0.0); + } + + #[test] + fn a_coarse_violation_is_published_even_when_1m_scanned_nothing() { + let mut readings = all_tiers(counts(0, 0, 0)); + readings[5].1 = counts(2, 0, 4); + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 2.0); + } + + #[test] + fn the_empty_scan_refusal_names_the_table_and_not_a_quote_leg() { + let message = OhlcBandRefusal::EmptyScan.to_string(); + assert!(message.contains("price_ohlcv_1m"), "{message}"); + assert!(message.contains("ingestion"), "{message}"); + assert!(!message.contains("USDT"), "{message}"); + assert!(!message.contains("0139"), "{message}"); + } + + #[test] + fn the_incomplete_refusal_says_a_tier_read_failed_in_this_run() { + let message = OhlcBandRefusal::Incomplete.to_string(); + assert!(message.contains("tier"), "{message}"); + assert!(message.contains("read"), "{message}"); + assert!(message.contains("this run"), "{message}"); + } + + #[test] + fn the_detail_names_every_reading() { + let detail = ohlc_band_detail(&[ + ("price_ohlcv_1m", counts(1, 2, 6)), + ("price_ohlcv_1w", counts(0, 0, 0)), + ]); + assert_eq!( + detail, + "price_ohlcv_1m band=1 nonpositive=2 scanned=6; \ + price_ohlcv_1w band=0 nonpositive=0 scanned=0" + ); + } +} diff --git a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs index 27659200..bdaf99e8 100644 --- a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs +++ b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs @@ -1929,3 +1929,153 @@ async fn an_empty_registry_or_window_is_refused_as_unreadable() { drop_scratch_db(db).await; } + +// ---- task 0236: stored candles outside the OHLC band, on all seven tiers ---- + +/// Clear all seven tiers, `_1m` FIRST and then the coarse tiers finest to +/// coarsest. The rollup MVs run in this database: clearing a source before its +/// target stops a refresh landing between the two from re-feeding the tier just +/// cleared. Its own helper — [`reset_sanity_tables`] is shared and clears only +/// two tiers. +async fn reset_ohlc_band_tables(c: &Client) { + for table in [ + "price_ohlcv_1m", + "price_ohlcv_15m", + "price_ohlcv_1h", + "price_ohlcv_4h", + "price_ohlcv_1d", + "price_ohlcv_1w", + "price_ohlcv_1M", + ] { + exec(c, &format!("TRUNCATE TABLE prices.{table}")).await; + } +} + +/// One candle on any tier with its four prices spelled out, each as an exact +/// `Decimal` literal — no Float64 path, so a fixture one ulp off the band is +/// stored as written. Every other column as in [`insert_invariant_row`]: +/// `pf_trade_count` explicit, never left to its DEFAULT. +/// +/// `ts` is a unix timestamp fixed by the caller, so a repair at a higher +/// `version` lands on the same key as the row it supersedes. +async fn insert_ohlc_row( + c: &Client, + table: &str, + ts: u32, + asset_id: u32, + (open, high, low, close): (&str, &str, &str, &str), + pf: u32, + version: u32, +) { + exec( + c, + &format!( + "INSERT INTO prices.{table} \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, \ + version, pf_trade_count, pf_volume, pf_price_volume) \ + SELECT toDateTime({ts}), {asset_id}, 2, 'sdex', \ + toDecimal128('{open}', 14), toDecimal128('{high}', 14), \ + toDecimal128('{low}', 14), toDecimal128('{close}', 14), \ + 1, 1, 0, 0, 1, 3, {version}, {pf}, {pf}, {pf}" + ), + ) + .await; +} + +/// Run every production query — `ohlc_band_queries()` itself, never a copy. +async fn read_ohlc_band( + c: &Client, +) -> Vec<( + &'static str, + rollup_freshness_probe::ohlc_band::OhlcBandCounts, +)> { + let mut readings = Vec::new(); + for (table, sql) in rollup_freshness_probe::ohlc_band::ohlc_band_queries() { + let counts = c + .query(&sql) + .fetch_one() + .await + .unwrap_or_else(|e| panic!("the ohlc-band query for {table} executes: {e}")); + readings.push((table, counts)); + } + readings +} + +/// A server-side unix timestamp for `now() - `, computed ONCE so every +/// fixture of a test shares it. +async fn ts_ago(c: &Client, interval: &str) -> u32 { + c.query(&format!("SELECT toUnixTimestamp(now() - {interval})")) + .fetch_one() + .await + .unwrap() +} + +fn reading( + readings: &[( + &'static str, + rollup_freshness_probe::ohlc_band::OhlcBandCounts, + )], + table: &str, +) -> rollup_freshness_probe::ohlc_band::OhlcBandCounts { + readings + .iter() + .find(|(t, _)| *t == table) + .unwrap_or_else(|| panic!("{table} was read")) + .1 +} + +/// The seven queries must **execute and deserialize** on the production build, +/// and count exactly the rows outside the band — none of the healthy shapes a +/// careless predicate flags: a single-trade candle (O=H=L=C), a dust-only +/// candle (all four 0 with `pf_trade_count = 0`, correct per ADR 0287), and a +/// priced candle still pending enrichment (`close_usd = 0`). The row breaking +/// BOTH shapes (`open = 0` with a positive low) counts ONCE — RED if the band +/// class loses its four-positive-prices gate. +/// +/// `_1m` fixtures sit at `now() - 3 HOUR`: inside the probe's 2-day window, +/// outside the 1m→15m MV's 2-hour window, so no refresh copies them upward +/// mid-test and every coarse tier reads exactly zero. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_counts_only_rows_outside_the_band_on_every_tier() { + use rollup_freshness_probe::ohlc_band::{OHLC_BAND_TIERS, OhlcBandCounts, ohlc_band_metric}; + + let c = client().await; + reset_ohlc_band_tables(&c).await; + let ts = ts_ago(&c, "INTERVAL 3 HOUR").await; + let m = "price_ohlcv_1m"; + + insert_ohlc_row(&c, m, ts, 20, ("5", "5", "5", "5"), 1, 1).await; // single trade + insert_ohlc_row(&c, m, ts, 21, ("0", "0", "0", "0"), 0, 1).await; // dust only + insert_ohlc_row(&c, m, ts, 22, ("4", "6", "3", "5"), 3, 1).await; // pending enrichment + insert_ohlc_row(&c, m, ts, 23, ("5", "4.5", "3", "5"), 2, 1).await; // ⛔ high < close + insert_ohlc_row(&c, m, ts, 24, ("4", "6", "0", "5"), 2, 1).await; // ⛔ low = 0 + insert_ohlc_row(&c, m, ts, 25, ("0", "6", "3", "5"), 2, 1).await; // ⛔ open = 0, low > open + + let readings = read_ohlc_band(&c).await; + assert_eq!(readings.len(), OHLC_BAND_TIERS.len()); + assert_eq!( + reading(&readings, m), + OhlcBandCounts { + band: 1, + nonpositive: 2, + scanned: 6 + }, + "_1m: one band violation, two non-positive rows (the double-shaped one counted once)" + ); + for (table, counts) in &readings[1..] { + assert_eq!( + *counts, + OhlcBandCounts { + band: 0, + nonpositive: 0, + scanned: 0 + }, + "{table} holds nothing" + ); + } + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 3.0); + + reset_ohlc_band_tables(&c).await; +} From 8344bdeebb25d296ec4cc962f469edb9196053f8 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:49:26 +0200 Subject: [PATCH 015/144] fix(lore-0236): pin each band arm, the per-tier window and FINAL against ClickHouse Seven ITs on the production queries: high below close, low above open, low above high (counted once), a zero low beside a positive close, _1w and _1M violations older than 2 days inside their widened windows, violations outside the window, and a bad row superseded by a good higher version. Each was shown red with its clause removed. --- .../tests/rollup_freshness_it.rs | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs index bdaf99e8..9913c91c 100644 --- a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs +++ b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs @@ -2079,3 +2079,219 @@ async fn the_ohlc_band_scan_counts_only_rows_outside_the_band_on_every_tier() { reset_ohlc_band_tables(&c).await; } + +/// Seed the one healthy `_1m` row every arm test needs so the scan is not +/// refused as empty, and return the shared `now() - 3 HOUR` timestamp. +async fn seed_healthy_1m(c: &Client) -> u32 { + let ts = ts_ago(c, "INTERVAL 3 HOUR").await; + insert_ohlc_row(c, "price_ohlcv_1m", ts, 30, ("5", "5", "5", "5"), 1, 1).await; + ts +} + +fn band_only( + band: u64, + nonpositive: u64, + scanned: u64, +) -> rollup_freshness_probe::ohlc_band::OhlcBandCounts { + rollup_freshness_probe::ohlc_band::OhlcBandCounts { + band, + nonpositive, + scanned, + } +} + +/// The arm `high < greatest(open, close)`, and ONLY that arm: `o4 h4.5 l3 c5` +/// has `low (3) <= least (4)` and `low <= high`. RED without the arm. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_catches_a_high_below_the_close() { + let c = client(); + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + insert_ohlc_row(&c, "price_ohlcv_1m", ts, 31, ("4", "4.5", "3", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + band_only(1, 0, 2), + "high 4.5 below close 5 is outside the band" + ); + + reset_ohlc_band_tables(&c).await; +} + +/// The arm `low > least(open, close)`, and ONLY that arm: `o3 h6 l4 c5` has +/// `high (6) >= greatest (5)` and `low <= high`. RED without the arm. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_catches_a_low_above_the_open() { + let c = client(); + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + insert_ohlc_row(&c, "price_ohlcv_1m", ts, 32, ("3", "6", "4", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + band_only(1, 0, 2), + "low 4 above open 3 is outside the band" + ); + + reset_ohlc_band_tables(&c).await; +} + +/// `low > high` — `o5 h4 l6 c5` — counted ONCE although it fires all three band +/// arms (countIf counts rows). The arm is implied by the other two (if +/// `low > high`, either `low > least(o, c)`, or `least(o, c) >= low > high` and +/// then `greatest(o, c) > high`), so this goes RED only with all three band arms +/// removed; it pins that the literal invariant is caught and counted once. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_catches_a_low_above_the_high_once() { + let c = client(); + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + insert_ohlc_row(&c, "price_ohlcv_1m", ts, 33, ("5", "4", "6", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + band_only(1, 0, 2), + "a low above the high is one bad row" + ); + + reset_ohlc_band_tables(&c).await; +} + +/// `low = 0` beside a positive close — the shape most of the prod baseline was +/// made of, and the one `zero_invariants` cannot see (its invariant 3 reads +/// `close = 0`). Non-positive, not band: the classes are disjoint. RED without +/// `low <= 0`. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_catches_a_zero_low_beside_a_positive_close() { + let c = client(); + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + insert_ohlc_row(&c, "price_ohlcv_1m", ts, 34, ("4", "6", "0", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + band_only(0, 1, 2), + "a priced low of 0 is a non-positive price" + ); + + reset_ohlc_band_tables(&c).await; +} + +/// BRIEF point 4, the regression it exists for: a `_1w` bucket starting 5 days +/// ago and a `_1M` bucket starting 12 days ago both overlap the last two days +/// once each is widened by its own length, and a FLAT 2-day window never sees +/// them. RED with the widening removed. Only the terminal sinks `_1w`/`_1M` +/// are used (fed daily from `_1d`, which is empty here), and both sit days away +/// from either window edge. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_reads_each_coarse_tier_over_its_own_bucket_window() { + use rollup_freshness_probe::ohlc_band::ohlc_band_metric; + + let c = client(); + reset_ohlc_band_tables(&c).await; + seed_healthy_1m(&c).await; + let week = ts_ago(&c, "INTERVAL 5 DAY").await; + let month = ts_ago(&c, "INTERVAL 12 DAY").await; + insert_ohlc_row(&c, "price_ohlcv_1w", week, 35, ("5", "4.5", "3", "5"), 2, 1).await; + insert_ohlc_row(&c, "price_ohlcv_1M", month, 36, ("3", "6", "4", "5"), 2, 1).await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1w"), + band_only(1, 0, 1), + "a week bucket older than 2 days but inside its widened window" + ); + assert_eq!( + reading(&readings, "price_ohlcv_1M"), + band_only(1, 0, 1), + "a month bucket older than 2 days but inside its widened window" + ); + assert_eq!(reading(&readings, "price_ohlcv_1m"), band_only(0, 0, 1)); + assert_eq!(ohlc_band_metric(&readings).unwrap().value, 2.0); + + reset_ohlc_band_tables(&c).await; +} + +/// The window has an edge. A `_1m` violation an hour beyond 2 days and a `_1w` +/// violation an hour beyond `2 days + 1 week` are NOT counted — nor scanned. +/// RED with the WHERE removed (the legacy pre-0286 residue would page forever). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn the_ohlc_band_scan_ignores_violations_outside_the_window() { + let c = client(); + reset_ohlc_band_tables(&c).await; + seed_healthy_1m(&c).await; + let old_minute = ts_ago(&c, "INTERVAL 2 DAY - INTERVAL 1 HOUR").await; + let old_week = ts_ago(&c, "INTERVAL 2 DAY - INTERVAL 1 WEEK - INTERVAL 1 HOUR").await; + insert_ohlc_row( + &c, + "price_ohlcv_1m", + old_minute, + 37, + ("5", "4.5", "3", "5"), + 2, + 1, + ) + .await; + insert_ohlc_row( + &c, + "price_ohlcv_1w", + old_week, + 38, + ("4", "6", "0", "5"), + 2, + 1, + ) + .await; + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + band_only(0, 0, 1), + "only the healthy seed is inside the 1m window" + ); + assert_eq!( + reading(&readings, "price_ohlcv_1w"), + band_only(0, 0, 0), + "a week bucket starting before now - 2d - 1w is outside" + ); + for (table, counts) in &readings { + assert_eq!(counts.violations(), 0, "{table}"); + } + + reset_ohlc_band_tables(&c).await; +} + +/// The alarm must stop once the data is fixed: a bad row re-inserted good at a +/// higher `version` on the same key is superseded, so `FINAL` neither counts +/// nor scans the bad version. The unrepaired neighbour still counts. RED +/// without `FINAL` (the superseded version is read as a second row). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn a_repaired_candle_stops_counting_in_the_ohlc_band_scan() { + let c = client(); + reset_ohlc_band_tables(&c).await; + let ts = seed_healthy_1m(&c).await; + let m = "price_ohlcv_1m"; + insert_ohlc_row(&c, m, ts, 40, ("5", "4.5", "3", "5"), 2, 1).await; // bad + insert_ohlc_row(&c, m, ts, 40, ("4", "6", "3", "5"), 2, 2).await; // its repair + insert_ohlc_row(&c, m, ts, 41, ("5", "4.5", "3", "5"), 2, 1).await; // left bad + + let readings = read_ohlc_band(&c).await; + assert_eq!( + reading(&readings, m), + band_only(1, 0, 3), + "the superseded version is neither counted nor scanned" + ); + + reset_ohlc_band_tables(&c).await; +} From c0e983e3a678bbb4faae90279ca2a4e738a30925 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:51:17 +0200 Subject: [PATCH 016/144] fix(lore-0236): page on CandleBandViolations with an ohlc-band ladder pinned at 1 Alarms prices-{env}-ohlc-band-{1,100,10000} on the probe's new metric, through the shared usdSanityRungs helper with the first rung fixed at 1 as zeroInvariantAlarms does. The escalation-counts doc now names the fourth ladder those rungs drive. --- infra/src/lib/stacks/observability-stack.ts | 24 +++++++++++++++++++++ infra/src/lib/types.ts | 8 +++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index 19ed6799..88ac3915 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -386,6 +386,14 @@ export class ObservabilityStack extends cdk.Stack { * 0139): a writer sent ids, not identities. Keyed by count. */ public readonly assetIdOrphanCandleAlarms: Record; + /** + * Stored candles on all seven `price_ohlcv_*` tiers that break the OHLC band + * or carry a price <= 0 (task 0236), keyed by count. Asserted by the probe at + * the source with zero tolerance, because task 0229's read-path clamp repairs + * a crossed candle on its way out of `/ohlcv` and so hides it from the API. + * The healthy reading is exactly 0. + */ + public readonly ohlcBandAlarms: Record; /** * A rollup MV that has lost `APPEND` (task 0204, gap 3) — history destroyed * on every refresh. Separate from {@link mvDriftAlarm} because this is the @@ -1749,6 +1757,22 @@ export class ObservabilityStack extends cdk.Stack { `${count} or more price_ohlcv_1m candles of the last 2 h carry an asset_id or quote_asset_id that prices.assets does not hold. Since task 0139 a writer sends the identity and ClickHouse derives both ids; an orphan means a writer sent ids itself (a stale pre-0139 binary or Lambda version: RowBinary is width-exact, so an old u32 writer misaligns rather than failing) or wrote candles before their assets rows. Stop that writer first, then find it from the rows' source column. Runbook: docs/runbooks/0139-asset-id-migration.md. First rung is fixed at 1.`, zeroLadder, ); + // Stored candles outside the OHLC band, or with a price <= 0, on all seven + // tiers (task 0236). Same ladder, same reasoning as the zero invariants + // above: the healthy reading is exactly 0, so the first rung carries the + // meaning and the rest say how fast it is growing. Since task 0229 the + // /ohlcv clamp repairs a crossed candle on the way out, so this is the only + // place a corrupt stored row is seen. See + // packages/rollup-freshness-probe/src/ohlc_band.rs and ADR-0011 section 3. + this.ohlcBandAlarms = usdSanityRungs( + 'CandleBandViolations', + 'OhlcBandAlarmCount', + 'ohlc-band', + (count) => + `${count} or more stored candles on price_ohlcv_{1m,15m,1h,4h,1d,1w,1M} (FINAL) are internally inconsistent: priced (pf_trade_count > 0) yet low > least(open, close), high < greatest(open, close) or low > high, or any price <= 0. Healthy is exactly 0 (prod baseline 2026-09-28). A priced _1m row with close = 0 also fires zero-invariant (expected overlap); low = 0 beside a positive close fires only this. The window is on BUCKET time, 48 h widened by one bucket per tier: a backfill into older buckets is NOT covered. The value sums tier-rows, so one bad minute can count up to seven times; a violation on a coarse tier with a clean _1m means a rollup defect. Find the writer before repairing - /ohlcv clamps since task 0229, so the API will not show it. Per-tier counts: probe log field ohlc_band_detail. See packages/rollup-freshness-probe/src/ohlc_band.rs and ADR-0011 s3. First rung is fixed at 1.`, + // The zero ladder above: the first rung is pinned at 1. + zeroLadder, + ); // Materialized-view drift, on a schedule (task 0204, gap 3). Task 0142 built // `prices-clickhouse-drift` and NOTHING RAN IT — a check nobody runs is a diff --git a/infra/src/lib/types.ts b/infra/src/lib/types.ts index 31e32902..7c191e2f 100644 --- a/infra/src/lib/types.ts +++ b/infra/src/lib/types.ts @@ -451,8 +451,12 @@ export interface EnvironmentConfig { * and the rationale below for `100` / `10000` is about the USDT metrics, not * about it. That ladder therefore pins its first rung at `1` in the stack * and borrows only the rungs above it from here, so raising the first rung - * to quiet a USDT ladder cannot hide a violation there. **Changing the - * higher rungs still re-tunes all three ladders.** + * to quiet a USDT ladder cannot hide a violation there. Since task 0236 + * they drive a FOURTH ladder, `CandleBandViolations`: all seven + * `price_ohlcv_*` tiers, `FINAL`, over a 48 h window widened by one bucket + * per tier. Its healthy value is also exactly 0, and it pins its first rung + * at `1` the same way. **Changing the higher rungs still re-tunes all four + * ladders.** * * ⚠️ **Why a ladder and not a single `>= 1`.** A wrong `close_usd` is a * **standing condition** — it stays wrong until a person repairs it — so it From 0bebfad2ad1f0140b9fee99d96581d96f77c6f46 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:51:17 +0200 Subject: [PATCH 017/144] fix(lore-0236): confirm the read-path clamp in ADR-0011 s3 now the source is asserted The 0229 clamp stays: stored-row consistency is now asserted at the source with zero tolerance by rollup-freshness-probe. Records that the as-stored QuoteLeg arm does not clamp O/H/L, the 2026-09-28 prod baseline, and why there is no API clamp metric. --- ...ncy-is-a-denomination-not-a-pair-filter.md | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/lore/2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md b/lore/2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md index 17cd84ad..314cd05e 100644 --- a/lore/2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md +++ b/lore/2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md @@ -3,7 +3,7 @@ id: "0011" title: "`base_currency` on the read surfaces is a DENOMINATION, not a quote-leg pair filter" status: accepted deciders: [okarcz, stkrolikiewicz] -related_tasks: ["0170", "0178", "0120", "0127", "0128", "0211", "0165", "0114", "0201", "0116", "0212"] +related_tasks: ["0170", "0178", "0120", "0127", "0128", "0211", "0165", "0114", "0201", "0116", "0212", "0236"] related_adrs: ["0003", "0004", "0008"] tags: [api, read-surface, contract, usd, denomination, provenance, milestone-M2] links: @@ -240,6 +240,68 @@ as the extremes, and scaling by one positive factor is monotonic, so `low <= open <= high` holds within a row and survives `min`/`max` across rows. Only the exact/derived boundary was ever at risk. +#### ✅ AMENDED 2026-09-28 — the clamp is CONFIRMED; stored-row consistency is asserted at the source + +Task [[0236]]. **The 0229 clamp stays: it is the right behaviour for the read +path.** A read API's job is to return well-formed candles, and a consumer cannot +act on `high < close` except by breaking. The alternative the 0229 review floated +— bounding the clamp so only ulp-sized crossings are repaired — would buy a +detection signal by **serving a malformed candle to every consumer** whenever a +source row is genuinely corrupt. That is paying in the wrong currency. + +🔑 **It is safe because stored-row consistency is now asserted at the source, +with ZERO tolerance.** Before 0229 a corrupt stored row leaked into `/ohlcv`, +where [[0120]]'s conformance suite could stumble over it; the clamp removed that +accidental detector, so it was moved to where it belongs. +`rollup-freshness-probe` reads every tier, on a schedule — +[`packages/rollup-freshness-probe/src/ohlc_band.rs`](../../packages/rollup-freshness-probe/src/ohlc_band.rs): + +- **predicate**, per priced row (`pf_trade_count > 0`): `low > least(open, close) + OR high < greatest(open, close) OR low > high OR` any of the four prices `<= 0`. + Exact `Decimal` comparisons, no epsilon — `min`/`max`/`argMin`/`argMax` copy + stored values bit for bit, so any crossing in storage is a real defect; +- **scope**: all seven tiers `price_ohlcv_{1m,15m,1h,4h,1d,1w,1M}`, `FINAL`, a + 48 h window on bucket start widened by one bucket length per tier (a flat 48 h + would never see a `_1w`/`_1M` bucket); +- **signal**: metric `Prices/Rollup` `CandleBandViolations`, alarms + `prices-{env}-ohlc-band-{1,100,10000}`, first rung pinned at 1. + +⚠️ **The as-stored `Denomination::QuoteLeg` arm does NOT clamp O/H/L** — only +`vwap` (`packages/prices-api/src/assets/queries_ch.rs`, the `QuoteLeg` arm, ~L1714). +It applies no rate, so the four prices are the stored decimals and cannot cross +unless the stored row itself is corrupt — which is exactly what the detector +above sees. On that path a corrupt row still reaches the consumer as-is. + +**Baseline, measured on prod 2026-09-28** (read-only, without `FINAL`, every +row of every tier): + +- **Band violations: 0** on every tier, every source, since 2015 — before and + after [[0286]]. +- **Priced rows with a price `<= 0`**: every one an exact zero (none negative), + mostly `low = 0` beside a positive `close`, and **all written before the 0286 + rollout (2026-09-22 12:00 UTC); 0 after it**: + + | tier | before 2026-09-22 12:00 UTC | after | + |---|---:|---:| + | `_1m` | 7,651 | 0 | + | `_15m` | 3,041 | 0 | + | `_1h` | 15,003 | 0 | + | `_4h` | 12,326 | 0 | + | `_1d` | 9,382 | 0 | + | `_1w` | 5,010 | — | + | `_1M` | 4,350 | — | + + These are **upper bounds** — read without `FINAL`, so superseded versions are + included (`_1m` 2026-09 reads 143 without `FINAL`, 52 with it). They are + pre-0286 residue that 0286 phase 3 re-ingests, and they sit outside the + detector's window by construction. The healthy reading is therefore exactly 0. + +⚠️ **No API clamp metric, on purpose.** The only crossing the clamp repairs in +normal operation is the structural ulp between the exact `close_usd` and the +rate-derived extremes — the 1.343e-11 BTC 1h measurement above. Counting it would +page on arithmetic, not on data, and the read path has no per-request metric +path worth adding for it. A corrupt stored row is caught at the source instead. + ### 4. Provenance reuses [[0165]]'s vocabulary `method` is propagated from the underlying `close_usd`, using the values 0165 From c2eeddaff1d8c9dbc77c3834468b2f37c011cbdd Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:27:16 +0200 Subject: [PATCH 018/144] test(lore-0236): await the shared client that stops the reconcile MVs On develop client() became async: it now stops the shared prices reconcile MVs before a test seeds the tiers (0203 review WR-05). --- .../tests/rollup_freshness_it.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs index 9913c91c..64d97999 100644 --- a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs +++ b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs @@ -2105,7 +2105,7 @@ fn band_only( #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn the_ohlc_band_scan_catches_a_high_below_the_close() { - let c = client(); + let c = client().await; reset_ohlc_band_tables(&c).await; let ts = seed_healthy_1m(&c).await; insert_ohlc_row(&c, "price_ohlcv_1m", ts, 31, ("4", "4.5", "3", "5"), 2, 1).await; @@ -2125,7 +2125,7 @@ async fn the_ohlc_band_scan_catches_a_high_below_the_close() { #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn the_ohlc_band_scan_catches_a_low_above_the_open() { - let c = client(); + let c = client().await; reset_ohlc_band_tables(&c).await; let ts = seed_healthy_1m(&c).await; insert_ohlc_row(&c, "price_ohlcv_1m", ts, 32, ("3", "6", "4", "5"), 2, 1).await; @@ -2148,7 +2148,7 @@ async fn the_ohlc_band_scan_catches_a_low_above_the_open() { #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn the_ohlc_band_scan_catches_a_low_above_the_high_once() { - let c = client(); + let c = client().await; reset_ohlc_band_tables(&c).await; let ts = seed_healthy_1m(&c).await; insert_ohlc_row(&c, "price_ohlcv_1m", ts, 33, ("5", "4", "6", "5"), 2, 1).await; @@ -2170,7 +2170,7 @@ async fn the_ohlc_band_scan_catches_a_low_above_the_high_once() { #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn the_ohlc_band_scan_catches_a_zero_low_beside_a_positive_close() { - let c = client(); + let c = client().await; reset_ohlc_band_tables(&c).await; let ts = seed_healthy_1m(&c).await; insert_ohlc_row(&c, "price_ohlcv_1m", ts, 34, ("4", "6", "0", "5"), 2, 1).await; @@ -2196,7 +2196,7 @@ async fn the_ohlc_band_scan_catches_a_zero_low_beside_a_positive_close() { async fn the_ohlc_band_scan_reads_each_coarse_tier_over_its_own_bucket_window() { use rollup_freshness_probe::ohlc_band::ohlc_band_metric; - let c = client(); + let c = client().await; reset_ohlc_band_tables(&c).await; seed_healthy_1m(&c).await; let week = ts_ago(&c, "INTERVAL 5 DAY").await; @@ -2227,7 +2227,7 @@ async fn the_ohlc_band_scan_reads_each_coarse_tier_over_its_own_bucket_window() #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn the_ohlc_band_scan_ignores_violations_outside_the_window() { - let c = client(); + let c = client().await; reset_ohlc_band_tables(&c).await; seed_healthy_1m(&c).await; let old_minute = ts_ago(&c, "INTERVAL 2 DAY - INTERVAL 1 HOUR").await; @@ -2278,7 +2278,7 @@ async fn the_ohlc_band_scan_ignores_violations_outside_the_window() { #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn a_repaired_candle_stops_counting_in_the_ohlc_band_scan() { - let c = client(); + let c = client().await; reset_ohlc_band_tables(&c).await; let ts = seed_healthy_1m(&c).await; let m = "price_ohlcv_1m"; From 76ac5584fd276f75808198fdb28b3db5fbe72aa9 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:48:58 +0200 Subject: [PATCH 019/144] docs(lore-0242): deploy Observability after the cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sac-contract-identities alarm reads 36 until the heal, so deployed with the PR it would sit red on the Row-0 alarm strip, which is the SCF evidence of 0294. Observability now ships in §6 with its actions on (D8); until then the operator reads SacUnprovenSkipped by hand. --- docs/runbooks/0242-sac-identity-heal.md | 93 ++++++++++++++++--------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/docs/runbooks/0242-sac-identity-heal.md b/docs/runbooks/0242-sac-identity-heal.md index 846dda71..95078bcf 100644 --- a/docs/runbooks/0242-sac-identity-heal.md +++ b/docs/runbooks/0242-sac-identity-heal.md @@ -11,7 +11,7 @@ deletes what is left. Observability stacks. **Read first:** [`0286-reingest-history.md`](0286-reingest-history.md) (phase 3, -its stages and its orchestrator) and lore task 0242 (decisions D1–D7, PC3, PC6, +its stages and its orchestrator) and lore task 0242 (decisions D1–D8, PC3, PC6, PC9). --- @@ -45,13 +45,13 @@ runbook heals the past. ## 1. Order of events -| # | Step | When | Section | -| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------- | -| 1 | Seed the 29 classic identities. Works with the `events-backfill` already on ch-prod-01 (D4) | After phase-3 stage B (202402–202404), before stage C | §3 | -| 2 | Re-plan phase 3 to 202609, run stage C with `--to-month 202609` (D4; the plan ended at 202608) | After the seed | §4a–§4d | -| 3 | Deploy the PR: Compute (live ledger processor + API), then EventBridge (probe), then Observability. Swap `events-backfill` on ch-prod-01 only between phase-3 stages, never while a month is at its `amm` step (D7) | Once merged. Independent of phase 3 (D4: phase 3 is not held for the code) | §4e | -| 4 | Residual cleanup (D3) | After stage D (`finish`) AND after the live slice is deployed with at least one cold start | §5 | -| 5 | Enable the `SacContractIdentities` alarm actions (D6) | After §5's post-checks are green | §6 | +| # | Step | When | Section | +| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------- | +| 1 | Seed the 29 classic identities. Works with the `events-backfill` already on ch-prod-01 (D4) | After phase-3 stage B (202402–202404), before stage C | §3 | +| 2 | Re-plan phase 3 to 202609, run stage C with `--to-month 202609` (D4; the plan ended at 202608) | After the seed | §4a–§4d | +| 3 | Deploy the PR: Compute (live ledger processor + API), then EventBridge (probe). Not Observability (D8). Swap `events-backfill` on ch-prod-01 only between phase-3 stages, never while a month is at its `amm` step (D7) | Once merged. Independent of phase 3 (D4: phase 3 is not held for the code) | §4e | +| 4 | Residual cleanup (D3) | After stage D (`finish`) AND after the live slice is deployed with at least one cold start | §5 | +| 5 | Deploy Observability with the `SacContractIdentities` alarm actions on (D6, D8) | After §5's post-checks are green | §6 | Step 4 waits for the live deploy because until then a warm container that predates the seed, or a SAC the seed does not cover, can still mint a contract identity @@ -399,11 +399,31 @@ done | tee ~/heal-0242/deployed-before.tsv **Deploy** (AWS shell, `cd infra`): -| Order | Target | Ships | -| ----- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| 1 | `make deploy-production-compute` | the live ledger processor (proof pre-pass, skip, `SacUnprovenSkipped`) and the API alias | -| 2 | `make deploy-production-eventbridge` | `rollup-freshness-probe` with `SacContractIdentities` | -| 3 | `make deploy-production-observability` | `prices-production-ledger-processor-sac-unproven` (actions on) and `prices-production-sac-contract-identities` (actions off) | +| Order | Target | Ships | +| ----- | ------------------------------------ | ---------------------------------------------------------------------------------------- | +| 1 | `make deploy-production-compute` | the live ledger processor (proof pre-pass, skip, `SacUnprovenSkipped`) and the API alias | +| 2 | `make deploy-production-eventbridge` | `rollup-freshness-probe` with `SacContractIdentities` | + +**Observability waits for §6 (D8).** Deployed now, `sac-contract-identities` +would sit in ALARM, red on the dashboard's Row-0 alarm strip, through stage C, +stage D and §5. That strip is the SCF evidence of task 0294 (all alarms OK). +Its two alarms come in §6 instead. Every Observability deploy from a commit +that holds this PR creates them, so tell the team: no +`make deploy-production-observability` from `develop` until §6. A deploy that +cannot wait shows the red tile; its description says it is expected. + +Until §6 no alarm watches `SacUnprovenSkipped`. Read it by hand after the cold +start and at least daily until §6: + +```bash +aws cloudwatch get-metric-statistics --namespace Prices/Ingest --metric-name SacUnprovenSkipped \ + --dimensions Name=Environment,Value=production \ + --start-time "$(date -u -d '-1 day' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" \ + --period 3600 --statistics Sum +``` + +No datapoints is healthy: the metric is published only when trades were +skipped. Any datapoint: §9. **Post-checks.** @@ -415,13 +435,6 @@ done | tee ~/heal-0242/deployed-before.tsv `sac_contract_rows` = 36 (the baseline until §5), `be_sac_contracts` ≈ 4,042. If the file holds an `errorMessage`, read the metric instead: `aws cloudwatch get-metric-statistics --namespace Prices/Rollup --metric-name SacContractIdentities --dimensions Name=Environment,Value=production --start-time "$(date -u -d '-30 min' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" --period 900 --statistics Maximum`. -- `aws cloudwatch describe-alarms --alarm-names prices-production-sac-contract-identities --query 'MetricAlarms[0].[ActionsEnabled,StateValue]'` - → `false`, `"ALARM"` (or `"INSUFFICIENT_DATA"` before the first probe run). - **`ALARM` is expected until §6**: actions off stops notifications, not - evaluation, and the metric reads 36 until §5. The alarm shows red on the - dashboard's Row-0 alarm strip for the whole window (stage C, stage D, §5); its - description says so. Tell whoever reports from that strip (tasks 0294, 0296) - before the deploy. - `curl -sS -H "x-api-key: $PRICES_API_KEY" "$API/assets/$XLM_SAC" | jq '{asset, code}'` → `"native"`, `"XLM"`. **`events-backfill` on ch-prod-01.** Build it from the merge commit as in 0286 @@ -429,7 +442,7 @@ done | tee ~/heal-0242/deployed-before.tsv and swap only between phase-3 stages, never while a month is at its `amm` step (D7). Confirm with the one-ledger dry run of §4b. -**Rollback.** Redeploy the previous commit with the same `make` targets (the +**Rollback.** Redeploy the previous commit with the same two `make` targets (the CodeSha256 in `deployed-before.tsv` is what must come back). On the host, `cp -p ~/events-backfill.pre0242 ~/events-backfill`. Nothing in the PR writes a schema change. @@ -764,21 +777,24 @@ from the same identity. Keep the `bak_0242_*` tables until 0242 is closed; then --- -## 6. Enable the alarm (D6) +## 6. Deploy Observability with the alarm on (D6, D8) -The `SacContractIdentities` alarm was deployed with its actions off, because -production read 36 until §5. +Observability was held back in §4e (D8), so this deploy creates both 0242 +alarms, `prices-production-ledger-processor-sac-unproven` and +`prices-production-sac-contract-identities`, the second with its actions on. -**Pre-check.** `SacContractIdentities` reads `0` for at least two periods (30 min), -and +**Pre-check.** §5g is green and `SacContractIdentities` reads `0` for at least +two periods (30 min): ```bash -aws cloudwatch describe-alarms --alarm-names prices-production-sac-contract-identities \ - --query 'MetricAlarms[0].StateValue' +aws cloudwatch get-metric-statistics --namespace Prices/Rollup --metric-name SacContractIdentities \ + --dimensions Name=Environment,Value=production \ + --start-time "$(date -u -d '-45 min' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" \ + --period 900 --statistics Maximum ``` -prints `"OK"`. CloudWatch notifies only on a transition: enabled while in ALARM, -the alarm would say nothing about the next mint. +Every `Maximum` is `0`. Created with its actions on while the metric read 36, +the alarm would fire at once. **Run.** Set `opsAlarms.sacContractIdentitiesActionsEnabled` to `true` in `infra/envs/production.json` through a PR; once merged: @@ -787,10 +803,21 @@ the alarm would say nothing about the next mint. cd infra && make deploy-production-observability ``` -**Post-check.** The same `describe-alarms` with `--query 'MetricAlarms[0].ActionsEnabled'` -prints `true`. +**Post-check.** + +```bash +aws cloudwatch describe-alarms \ + --alarm-names prices-production-sac-contract-identities prices-production-ledger-processor-sac-unproven \ + --query 'MetricAlarms[].[AlarmName,ActionsEnabled,StateValue]' --output text +``` + +Both alarms print `True`, then `OK` (`INSUFFICIENT_DATA` until the first +evaluation). The `INSUFFICIENT_DATA` → `OK` transition sends one OK +notification: expected. -**Rollback.** The same key back to `false`, the same deploy. +**Rollback.** The same key back to `false` and the same deploy turns the +actions off. Redeploying Observability from a commit without this PR removes +both alarms. --- From cebecc92f9b9572042204ed418025eae0a995bdf Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:49:16 +0200 Subject: [PATCH 020/144] docs(lore-0242): record D8 and the seed Observability ships after the cleanup so the red baseline alarm stays off the SCF alarm strip. The seed ran on 2026-10-06; bak_0242_seed was rebuilt from its rows because the staged variant was not used. --- ...UG_sac-minted-as-a-second-identity-splits-volume.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md index 0e00885c..86a90871 100644 --- a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md +++ b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md @@ -202,6 +202,10 @@ Prior art: soroban-block-explorer ADR 0051 (a SAC is a facet of the classic asse - **One PR** with ordered commit slices: resolver + live, then `events-backfill`, then probe + alarm, then API alias. This overrides the ~400-line norm for this task. - The heal is a runbook section, with no window. - The new `events-backfill` binary is swapped on ch-prod-01 only between phase-3 stages, never during an `amm` step. +- **D8 Observability after the cleanup (Adam, 2026-10-06).** The PR deploys Compute and EventBridge only. Observability, with both 0242 alarms, is deployed in runbook §6, after §5, with the `SacContractIdentities` actions already on. + - Why: deployed earlier, that alarm would sit in ALARM on the dashboard's Row-0 alarm strip through stage C, stage D and the cleanup. That strip is [[0294]]'s SCF evidence (all alarms OK). + - Cost: until §6 no alarm watches `SacUnprovenSkipped`. The runbook has the operator read the metric by hand after the cold start and at least daily. + - Constraint on the team: no `make deploy-production-observability` from `develop` between the merge and §6. Such a deploy creates the alarms early, and the red tile's description says it is expected. Plan choices confirmed by Adam on 2026-10-05 (GSD plan `261005-htu`): @@ -225,6 +229,12 @@ The six latent SACs have other causes. SUSHI and HYPE traded before SushiSwap wa **October window (measured 2026-10-05, `dev_read`).** Since 2026-10-01, no candle on any tier is keyed on the 36 SAC ids, and no contract row has been minted since BURNMKR. No decision is needed now. The runbook re-checks months after 202609 before the cleanup, and any non-zero result goes through PC9. +## Heal log + +- **2026-10-06, seed (runbook §3).** Stage B finished 202404. The seed ran at 10:29:11 UTC as `dev_shared`, straight from `seed-0242.sql` rather than through the staged variant of §3. There is therefore no `bak_0242_seed_stage`, and `bak_0242_seed` was created afterwards from the 29 rows with that `created_at`. + - Check: `29 29 / 29 / 0`, with collisions at 0. No candle under any seeded id on any tier. + - Comet's `pool_registry` row is present (1). + ## Acceptance Criteria - [ ] A SAC whose classic asset is in the registry no longer mints a second From df792d9b83e177663a9b83a395770aa6a2cd9a9f Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:00:49 +0200 Subject: [PATCH 021/144] refactor(lore-0329): move soroban rpc simulate into ingest-core The ingest needs decimals() from the same simulateTransaction call the asset-discovery symbol stage makes. Move the envelope builder and the Absent/Transient classification into prices_ingest_core::soroban_rpc so both readers share one boundary. symbols.rs re-exports the old names. --- Cargo.lock | 3 +- packages/asset-discovery/Cargo.toml | 6 +- packages/asset-discovery/src/symbols.rs | 291 ++-------------- packages/prices-ingest-core/Cargo.toml | 8 + packages/prices-ingest-core/src/lib.rs | 3 + .../prices-ingest-core/src/soroban_rpc.rs | 313 ++++++++++++++++++ 6 files changed, 353 insertions(+), 271 deletions(-) create mode 100644 packages/prices-ingest-core/src/soroban_rpc.rs diff --git a/Cargo.lock b/Cargo.lock index f04e0152..2f6a9512 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,7 +158,6 @@ dependencies = [ name = "asset-discovery" version = "0.1.0" dependencies = [ - "base64", "clickhouse", "lambda_runtime 0.13.0", "prices-clickhouse", @@ -2918,6 +2917,7 @@ name = "prices-ingest-core" version = "0.1.0" dependencies = [ "aquarius-extractor", + "base64", "clickhouse", "comet-extractor", "extractors-core", @@ -2925,6 +2925,7 @@ dependencies = [ "ledger-processor", "phoenix-extractor", "prices-clickhouse", + "reqwest", "rust_decimal", "serde", "serde_json", diff --git a/packages/asset-discovery/Cargo.toml b/packages/asset-discovery/Cargo.toml index 81f6604c..4cbb4e63 100644 --- a/packages/asset-discovery/Cargo.toml +++ b/packages/asset-discovery/Cargo.toml @@ -31,10 +31,10 @@ prices-clickhouse = { path = "../prices-clickhouse" } stellar-xdr = { workspace = true } # Task 0210 symbol stage: Soroban RPC `simulateTransaction` against a token -# contract's `symbol()`. Same set (and the same reqwest feature list) -# oracle-worker uses for its Reflector calls. +# contract's `symbol()`. The envelope and response classification live in +# prices-ingest-core::soroban_rpc since task 0329; reqwest stays for the client +# type in this crate's signatures. stellar-strkey = { workspace = true } -base64 = "0.22" reqwest = { version = "0.12", default-features = false, features = [ "rustls-tls-webpki-roots", "json", diff --git a/packages/asset-discovery/src/symbols.rs b/packages/asset-discovery/src/symbols.rs index 55a7f4eb..2da7ac77 100644 --- a/packages/asset-discovery/src/symbols.rs +++ b/packages/asset-discovery/src/symbols.rs @@ -19,24 +19,16 @@ //! The three-way outcome policy *is* borrowed from `supply-worker::run_supply`, //! because it is what keeps the queue from starving: see [`Outcome`]. -use base64::Engine; use clickhouse::Client; -use serde::Serialize; -use stellar_xdr::{ - ContractId, Hash, HostFunction, InvokeContractArgs, InvokeHostFunctionOp, Limits, Memo, - MuxedAccount, Operation, OperationBody, Preconditions, ReadXdr, ScAddress, ScString, ScSymbol, - ScVal, SequenceNumber, Transaction, TransactionEnvelope, TransactionExt, TransactionV1Envelope, - Uint256, VecM, WriteXdr, +use prices_ingest_core::soroban_rpc::{Simulated, simulate}; +// Re-exported so `main.rs` and the RPC integration test keep their paths; the +// envelope and the Absent/Transient boundary moved to `prices-ingest-core` +// (task 0329), which the ingest's `decimals()` lookup shares. +pub use prices_ingest_core::soroban_rpc::{ + DEFAULT_SOROBAN_RPC, RPC_TIMEOUT_SECS, build_simulate_envelope, http_client, }; - -/// Default public Soroban RPC endpoint (overridable via `SOROBAN_RPC_URL`). -/// Needs no auth — same endpoint and same reasoning as `oracle-worker`. -pub const DEFAULT_SOROBAN_RPC: &str = "https://mainnet.sorobanrpc.com"; - -/// Per-request RPC timeout. Together with [`MAX_CONTRACTS_PER_RUN`] this is the -/// stage's entire time bound: worst case is the product of the two, which must -/// leave the ledger scan its headroom inside the Lambda's 5-minute limit. -pub const RPC_TIMEOUT_SECS: u64 = 5; +use serde::Serialize; +use stellar_xdr::{ScString, ScSymbol, ScVal}; /// How many unresolved contracts one run will attempt. /// @@ -99,92 +91,6 @@ pub struct SymbolStats { pub skipped: usize, } -/// An HTTP client bounded by [`RPC_TIMEOUT_SECS`]. -/// -/// Built here rather than at the call site so the stage's time bound stays next -/// to the constant that states it. -/// -/// # Panics -/// -/// If the client cannot be built — in practice a TLS backend that failed to -/// initialise, which is an environment fault at cold start, identical on every -/// invocation. Deliberately not a silent fallback to `Client::default()`: that -/// client carries **no timeout**, so the stage's whole `MAX_CONTRACTS_PER_RUN × -/// RPC_TIMEOUT_SECS` bound would quietly become unbounded, and 25 hung requests -/// would eat the Lambda's five minutes ahead of the ledger scan that runs after -/// it. A loud failure at startup beats an hourly discovery outage with no -/// obvious cause. -pub fn http_client() -> reqwest::Client { - reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(RPC_TIMEOUT_SECS)) - .build() - .expect("build an HTTP client with a timeout") -} - -/// Base64 `TransactionEnvelope` invoking the zero-argument `func()` on -/// `contract`, for a read-only `simulateTransaction` (source account / fee / seq -/// are not checked by simulation). -/// -/// `None` when `contract` is not a well-formed C-strkey — a data-integrity -/// problem in `prices.assets`, permanent for that row, so the caller sentinels -/// it rather than failing the run. -pub fn build_simulate_envelope(contract: &str, func: &str) -> Option { - let contract_hash = stellar_strkey::Contract::from_string(contract).ok()?.0; - let invoke = InvokeContractArgs { - contract_address: ScAddress::Contract(ContractId(Hash(contract_hash))), - function_name: ScSymbol(func.try_into().ok()?), - args: VecM::default(), - }; - let op = Operation { - source_account: None, - body: OperationBody::InvokeHostFunction(InvokeHostFunctionOp { - host_function: HostFunction::InvokeContract(invoke), - auth: VecM::default(), - }), - }; - let tx = Transaction { - source_account: MuxedAccount::Ed25519(Uint256([0u8; 32])), - fee: 0, - seq_num: SequenceNumber(0), - cond: Preconditions::None, - memo: Memo::None, - operations: vec![op].try_into().ok()?, - ext: TransactionExt::V0, - }; - let envelope = TransactionEnvelope::Tx(TransactionV1Envelope { - tx, - signatures: VecM::default(), - }); - let xdr = envelope.to_xdr(Limits::none()).ok()?; - Some(base64::engine::general_purpose::STANDARD.encode(xdr)) -} - -#[derive(serde::Deserialize)] -struct RpcResponse { - result: Option, - /// JSON-RPC 2.0 reports failure *here*, with HTTP 200 — which is how public - /// endpoints signal quota exhaustion, a disabled method, and a node that is - /// not synced. Without this field such a body deserialises to - /// `result: None`, and every contract in the run sentinels permanently. - #[serde(default)] - error: Option, -} -#[derive(serde::Deserialize)] -struct RpcResult { - #[serde(default)] - results: Vec, - #[serde(default)] - error: Option, - /// Set when the contract's instance or Wasm is archived but restorable — - /// a live token, not one that lacks a symbol. - #[serde(default, rename = "restorePreamble")] - restore_preamble: Option, -} -#[derive(serde::Deserialize)] -struct RpcInvokeResult { - xdr: String, -} - /// Extract a UTF-8 string from a `symbol()` return value. /// /// Both arms are needed: the SEP-41 interface declares `Symbol`, but deployed @@ -227,99 +133,19 @@ pub fn sanitize_symbol(raw: &str) -> Option { /// /// Never returns an error: every failure mode is either Absent (permanent for /// this contract) or Transient (retry next run), which is exactly the decision -/// the caller has to make. +/// the caller has to make. The line between the two is +/// [`prices_ingest_core::soroban_rpc::simulate`]'s. pub async fn resolve_symbol(http: &reqwest::Client, rpc_url: &str, contract: &str) -> Outcome { - let Some(envelope) = build_simulate_envelope(contract, "symbol") else { - tracing::warn!(contract, "not a well-formed contract address; sentinelling"); - return Outcome::Absent; - }; - let body = serde_json::json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "simulateTransaction", - "params": { "transaction": envelope }, - }); - - // A transport-level failure — connection, timeout, non-2xx (a 429 or 5xx - // from the public endpoint lands here), unparseable body — is systemic, not - // a fact about this contract. `error_for_status` is what keeps a - // rate-limited run from sentinelling every contract it touches. - let resp = match http - .post(rpc_url) - .json(&body) - .send() - .await - .and_then(|r| r.error_for_status()) - { - Ok(r) => r, - Err(err) => { - tracing::warn!(contract, error = %err, "symbol() rpc failed; retrying next run"); - return Outcome::Transient; - } - }; - let resp: RpcResponse = match resp.json().await { - Ok(r) => r, - Err(err) => { - tracing::warn!(contract, error = %err, "symbol() rpc body unparseable; retrying next run"); - return Outcome::Transient; - } - }; - - classify(resp, contract) -} - -/// Decide what a parsed simulate response means for this contract. -/// -/// The line between the two failure arms is **whether the simulation actually -/// evaluated the contract**. If it did, its outcome is a fact that will repeat -/// on every future run, so it sentinels. If we never got that far — a JSON-RPC -/// error, archived state, an empty body — we learned nothing about the -/// contract, so it must retry. Getting this backwards is unrecoverable in one -/// direction only: a wrong `Transient` costs one RPC call per run, a wrong -/// `Absent` publishes a permanent empty symbol that nothing re-polls. -fn classify(resp: RpcResponse, contract: &str) -> Outcome { - // Arrives with HTTP 200, so `error_for_status` above cannot see it. - if let Some(err) = resp.error { - tracing::warn!(contract, error = %err, "symbol() rpc returned a json-rpc error; retrying next run"); - return Outcome::Transient; + match simulate(http, rpc_url, contract, "symbol").await { + Simulated::Value(v) => symbol_outcome(&v, contract), + Simulated::Absent => Outcome::Absent, + Simulated::Transient => Outcome::Transient, } - let Some(result) = resp.result else { - tracing::warn!( - contract, - "symbol() rpc returned neither result nor error; retrying next run" - ); - return Outcome::Transient; - }; - if result.restore_preamble.is_some() { - tracing::warn!( - contract, - "symbol() needs a state restore; retrying next run" - ); - return Outcome::Transient; - } - // The simulation ran and the contract itself errored — deterministic, so a - // fact. This is the one arm that stays permanent. - if let Some(err) = result.error { - tracing::debug!(contract, error = %err, "symbol() simulate returned error; sentinelling"); - return Outcome::Absent; - } - if result.results.is_empty() { - tracing::warn!( - contract, - "symbol() simulated without a result or an error; retrying next run" - ); - return Outcome::Transient; - } - - let raw = result - .results - .into_iter() - .next() - .and_then(|r| base64::engine::general_purpose::STANDARD.decode(r.xdr).ok()) - .and_then(|bytes| ScVal::from_xdr(&bytes, Limits::none()).ok()) - .as_ref() - .and_then(scval_to_string); +} +/// A `symbol()` return value as an [`Outcome`]. +fn symbol_outcome(v: &ScVal, contract: &str) -> Outcome { + let raw = scval_to_string(v); match raw.as_deref().and_then(sanitize_symbol) { Some(s) => Outcome::Symbol(s), // It answered, and the answer is unusable. A fact — but a permanent and @@ -529,50 +355,14 @@ mod tests { assert!(sanitize_symbol("v1.2_beta").is_some()); } - fn classify_json(body: &str) -> Outcome { - classify( - serde_json::from_str(body).expect("test body parses"), - "C_TEST", - ) - } - - #[test] - fn a_json_rpc_error_retries_instead_of_sentinelling() { - // JSON-RPC reports this with HTTP 200, so `error_for_status` cannot see - // it. Sentinelling here would name every contract in the run `""` - // permanently, with nothing that ever re-polls them. - let out = classify_json( - r#"{"jsonrpc":"2.0","id":1,"error":{"code":-32601,"message":"method not enabled"}}"#, - ); - assert_eq!(out, Outcome::Transient); - } - #[test] - fn an_empty_or_archived_response_retries() { - // Neither result nor error: we learned nothing about the contract. - assert_eq!( - classify_json(r#"{"jsonrpc":"2.0","id":1}"#), - Outcome::Transient - ); - // Archived-but-restorable state is a live token, not a symbol-less one. + fn an_unusable_answer_is_absent() { + // Answered, but not a string: a fact about the contract, not a retry. + assert_eq!(symbol_outcome(&ScVal::U32(7), "C_TEST"), Outcome::Absent); + let sym = ScVal::Symbol(ScSymbol("XAUM".try_into().unwrap())); assert_eq!( - classify_json(r#"{"result":{"results":[],"restorePreamble":{"minResourceFee":"1"}}}"#), - Outcome::Transient, - ); - // Simulated clean, returned nothing at all — not a fact either. - assert_eq!( - classify_json(r#"{"result":{"results":[]}}"#), - Outcome::Transient - ); - } - - #[test] - fn a_contract_error_is_the_one_permanent_arm() { - // The simulation evaluated the contract and it errored. That repeats on - // every run, so it must leave the queue or it starves the tail. - assert_eq!( - classify_json(r#"{"result":{"results":[],"error":"HostError: missing symbol"}}"#), - Outcome::Absent, + symbol_outcome(&sym, "C_TEST"), + Outcome::Symbol("XAUM".to_string()) ); } @@ -595,37 +385,4 @@ mod tests { assert_eq!(scval_to_string(&ScVal::U32(7)), None); } - - #[test] - fn builds_an_envelope_that_decodes_back_to_symbol_on_the_contract() { - // A locally-derived valid C-strkey: self-contained, no network, and it - // cannot rot the way a pasted mainnet address can. - let contract = stellar_strkey::Contract([7u8; 32]).to_string(); - let b64 = build_simulate_envelope(&contract, "symbol").expect("valid contract address"); - let raw = base64::engine::general_purpose::STANDARD - .decode(&b64) - .expect("valid base64"); - let env = TransactionEnvelope::from_xdr(&raw, Limits::none()).expect("valid xdr"); - let TransactionEnvelope::Tx(v1) = env else { - panic!("expected a v1 envelope") - }; - let OperationBody::InvokeHostFunction(op) = &v1.tx.operations[0].body else { - panic!("expected InvokeHostFunction") - }; - let HostFunction::InvokeContract(args) = &op.host_function else { - panic!("expected InvokeContract") - }; - assert_eq!(args.function_name.0.to_utf8_string().unwrap(), "symbol"); - assert_eq!(args.args.len(), 0, "symbol() takes no arguments"); - assert_eq!( - args.contract_address, - ScAddress::Contract(ContractId(Hash([7u8; 32]))) - ); - } - - #[test] - fn rejects_a_malformed_contract_address() { - assert_eq!(build_simulate_envelope("not-a-contract", "symbol"), None); - assert_eq!(build_simulate_envelope("", "symbol"), None); - } } diff --git a/packages/prices-ingest-core/Cargo.toml b/packages/prices-ingest-core/Cargo.toml index d898ffa5..547ea808 100644 --- a/packages/prices-ingest-core/Cargo.toml +++ b/packages/prices-ingest-core/Cargo.toml @@ -11,6 +11,14 @@ path = "src/lib.rs" [dependencies] stellar-xdr = { workspace = true } stellar-strkey = { workspace = true } +# Task 0329: `decimals()` over Soroban RPC `simulateTransaction` (moved here +# from asset-discovery's symbol stage, which now shares it). Same feature list +# as asset-discovery and oracle-worker. +base64 = "0.22" +reqwest = { version = "0.12", default-features = false, features = [ + "rustls-tls-webpki-roots", + "json", +] } sha2 = { workspace = true } xdr-parser = { workspace = true } clickhouse = { workspace = true } diff --git a/packages/prices-ingest-core/src/lib.rs b/packages/prices-ingest-core/src/lib.rs index 5b659e4f..079ed092 100644 --- a/packages/prices-ingest-core/src/lib.rs +++ b/packages/prices-ingest-core/src/lib.rs @@ -13,6 +13,8 @@ //! Layers, in pipeline order: //! - [`filter`] — classic SDEX trades from `LedgerCloseMeta` operation results. //! - [`soroban`] — Soroban AMM trades + oracle samples from contract events. +//! - [`soroban_rpc`] — read-only token calls (`decimals()`, `symbol()`) over +//! RPC `simulateTransaction`. //! - [`static_pools`] — the committed list of factory-less pools //! ([`STATIC_POOLS`], task 0300) merged into the AMM registries. //! - [`canonical`] — asset identity, the [`AssetRegistry`] of known and newly @@ -34,6 +36,7 @@ pub mod registry_io; pub mod retry; pub mod safe_log; pub mod soroban; +pub mod soroban_rpc; pub mod static_pools; pub mod tick; pub mod writer; diff --git a/packages/prices-ingest-core/src/soroban_rpc.rs b/packages/prices-ingest-core/src/soroban_rpc.rs new file mode 100644 index 00000000..ae51c274 --- /dev/null +++ b/packages/prices-ingest-core/src/soroban_rpc.rs @@ -0,0 +1,313 @@ +//! Read-only calls to a Soroban token contract over RPC `simulateTransaction`. +//! +//! Two readers share this: asset-discovery's `symbol()` stage (task 0210) and +//! the ingest's `decimals()` lookup (task 0329). Both need the same envelope and +//! the same answer to one question — did the simulation actually evaluate the +//! contract? — so that boundary lives here once rather than in each caller. + +use base64::Engine; +use stellar_xdr::{ + ContractId, Hash, HostFunction, InvokeContractArgs, InvokeHostFunctionOp, Limits, Memo, + MuxedAccount, Operation, OperationBody, Preconditions, ReadXdr, ScAddress, ScSymbol, ScVal, + SequenceNumber, Transaction, TransactionEnvelope, TransactionExt, TransactionV1Envelope, + Uint256, VecM, WriteXdr, +}; + +/// Default public Soroban RPC endpoint (overridable via `SOROBAN_RPC_URL`). +/// Needs no auth — same endpoint and same reasoning as `oracle-worker`. +pub const DEFAULT_SOROBAN_RPC: &str = "https://mainnet.sorobanrpc.com"; + +/// Per-request RPC timeout. Each caller's time bound is this times the number +/// of calls it allows itself per run. +pub const RPC_TIMEOUT_SECS: u64 = 5; + +/// `SOROBAN_RPC_URL`, or [`DEFAULT_SOROBAN_RPC`]. +pub fn rpc_url_from_env() -> String { + std::env::var("SOROBAN_RPC_URL").unwrap_or_else(|_| DEFAULT_SOROBAN_RPC.to_string()) +} + +/// An HTTP client bounded by [`RPC_TIMEOUT_SECS`]. +/// +/// # Panics +/// +/// If the client cannot be built — in practice a TLS backend that failed to +/// initialise, which is an environment fault at cold start, identical on every +/// invocation. Deliberately not a silent fallback to `Client::default()`: that +/// client carries **no timeout**, so every caller's time bound would quietly +/// become unbounded. A loud failure at startup beats an outage with no obvious +/// cause. +pub fn http_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(RPC_TIMEOUT_SECS)) + .build() + .expect("build an HTTP client with a timeout") +} + +/// Base64 `TransactionEnvelope` invoking the zero-argument `func()` on +/// `contract`, for a read-only `simulateTransaction` (source account / fee / seq +/// are not checked by simulation). +/// +/// `None` when `contract` is not a well-formed C-strkey. +pub fn build_simulate_envelope(contract: &str, func: &str) -> Option { + let contract_hash = stellar_strkey::Contract::from_string(contract).ok()?.0; + let invoke = InvokeContractArgs { + contract_address: ScAddress::Contract(ContractId(Hash(contract_hash))), + function_name: ScSymbol(func.try_into().ok()?), + args: VecM::default(), + }; + let op = Operation { + source_account: None, + body: OperationBody::InvokeHostFunction(InvokeHostFunctionOp { + host_function: HostFunction::InvokeContract(invoke), + auth: VecM::default(), + }), + }; + let tx = Transaction { + source_account: MuxedAccount::Ed25519(Uint256([0u8; 32])), + fee: 0, + seq_num: SequenceNumber(0), + cond: Preconditions::None, + memo: Memo::None, + operations: vec![op].try_into().ok()?, + ext: TransactionExt::V0, + }; + let envelope = TransactionEnvelope::Tx(TransactionV1Envelope { + tx, + signatures: VecM::default(), + }); + let xdr = envelope.to_xdr(Limits::none()).ok()?; + Some(base64::engine::general_purpose::STANDARD.encode(xdr)) +} + +/// What one simulated call produced. +/// +/// The Absent/Transient split is load-bearing for every caller: Absent is a +/// fact about the contract that will repeat, Transient means we learned +/// nothing and must ask again. +#[derive(Debug, PartialEq)] +pub enum Simulated { + /// The contract evaluated the call and returned this value. + Value(ScVal), + /// The contract evaluated the call and errored, or returned undecodable + /// bytes, or the address is not a contract at all. Repeats on every call. + Absent, + /// No answer about the contract: transport failure, JSON-RPC error, + /// archived state, an empty body. Ask again later. + Transient, +} + +#[derive(serde::Deserialize)] +struct RpcResponse { + result: Option, + /// JSON-RPC 2.0 reports failure *here*, with HTTP 200 — which is how public + /// endpoints signal quota exhaustion, a disabled method, and a node that is + /// not synced. Without this field such a body deserialises to + /// `result: None` and would read as a fact about the contract. + #[serde(default)] + error: Option, +} +#[derive(serde::Deserialize)] +struct RpcResult { + #[serde(default)] + results: Vec, + #[serde(default)] + error: Option, + /// Set when the contract's instance or Wasm is archived but restorable — + /// a live contract, not one that lacks the function. + #[serde(default, rename = "restorePreamble")] + restore_preamble: Option, +} +#[derive(serde::Deserialize)] +struct RpcInvokeResult { + xdr: String, +} + +/// Call the zero-argument `func()` on `contract` by simulation and classify +/// the result. Never returns an error: every failure is Absent or Transient, +/// which is exactly the decision a caller has to make. +pub async fn simulate( + http: &reqwest::Client, + rpc_url: &str, + contract: &str, + func: &str, +) -> Simulated { + let Some(envelope) = build_simulate_envelope(contract, func) else { + tracing::warn!(contract, func, "not a well-formed contract address"); + return Simulated::Absent; + }; + let body = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "simulateTransaction", + "params": { "transaction": envelope }, + }); + + // A transport-level failure — connection, timeout, non-2xx (a 429 or 5xx + // from the public endpoint lands here), unparseable body — is systemic, not + // a fact about this contract. `error_for_status` is what keeps a + // rate-limited run from recording every contract it touches as Absent. + let resp = match http + .post(rpc_url) + .json(&body) + .send() + .await + .and_then(|r| r.error_for_status()) + { + Ok(r) => r, + Err(err) => { + tracing::warn!(contract, func, error = %err, "simulate rpc failed"); + return Simulated::Transient; + } + }; + match resp.json::().await { + Ok(r) => classify(r, contract, func), + Err(err) => { + tracing::warn!(contract, func, error = %err, "simulate rpc body unparseable"); + Simulated::Transient + } + } +} + +/// Decide what a parsed simulate response means for this contract. +/// +/// The line between the two failure arms is **whether the simulation actually +/// evaluated the contract**. If it did, its outcome is a fact that will repeat, +/// so it is Absent. If we never got that far — a JSON-RPC error, archived +/// state, an empty body — we learned nothing, so it is Transient. A wrong +/// Transient costs a repeated call; a wrong Absent can make a caller give up +/// on a contract for good. +fn classify(resp: RpcResponse, contract: &str, func: &str) -> Simulated { + // Arrives with HTTP 200, so `error_for_status` cannot see it. + if let Some(err) = resp.error { + tracing::warn!(contract, func, error = %err, "simulate returned a json-rpc error"); + return Simulated::Transient; + } + let Some(result) = resp.result else { + tracing::warn!(contract, func, "simulate returned neither result nor error"); + return Simulated::Transient; + }; + if result.restore_preamble.is_some() { + tracing::warn!(contract, func, "simulate needs a state restore"); + return Simulated::Transient; + } + // The simulation ran and the contract itself errored — deterministic, so a + // fact. This is the one arm that stays permanent. + if let Some(err) = result.error { + tracing::debug!(contract, func, error = %err, "contract errored in simulation"); + return Simulated::Absent; + } + let Some(first) = result.results.into_iter().next() else { + tracing::warn!(contract, func, "simulated without a result or an error"); + return Simulated::Transient; + }; + match base64::engine::general_purpose::STANDARD + .decode(first.xdr) + .ok() + .and_then(|bytes| ScVal::from_xdr(&bytes, Limits::none()).ok()) + { + Some(v) => Simulated::Value(v), + // It answered, and the answer does not decode. A fact, and otherwise an + // invisible one, so name the contract. + None => { + tracing::warn!(contract, func, "simulate result is not a decodable ScVal"); + Simulated::Absent + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn classify_json(body: &str) -> Simulated { + classify( + serde_json::from_str(body).expect("test body parses"), + "C_TEST", + "symbol", + ) + } + + #[test] + fn a_json_rpc_error_retries_instead_of_being_absent() { + // JSON-RPC reports this with HTTP 200, so `error_for_status` cannot see + // it. Reading it as Absent would turn an RPC outage into a fact about + // every contract the run touched. + let out = classify_json( + r#"{"jsonrpc":"2.0","id":1,"error":{"code":-32601,"message":"method not enabled"}}"#, + ); + assert_eq!(out, Simulated::Transient); + } + + #[test] + fn an_empty_or_archived_response_retries() { + // Neither result nor error: we learned nothing about the contract. + assert_eq!( + classify_json(r#"{"jsonrpc":"2.0","id":1}"#), + Simulated::Transient + ); + // Archived-but-restorable state is a live contract. + assert_eq!( + classify_json(r#"{"result":{"results":[],"restorePreamble":{"minResourceFee":"1"}}}"#), + Simulated::Transient, + ); + // Simulated clean, returned nothing at all — not a fact either. + assert_eq!( + classify_json(r#"{"result":{"results":[]}}"#), + Simulated::Transient + ); + } + + #[test] + fn a_contract_error_is_the_one_permanent_arm() { + assert_eq!( + classify_json(r#"{"result":{"results":[],"error":"HostError: missing symbol"}}"#), + Simulated::Absent, + ); + } + + #[test] + fn a_returned_value_is_decoded() { + let xdr = base64::engine::general_purpose::STANDARD + .encode(ScVal::U32(8).to_xdr(Limits::none()).unwrap()); + let body = format!(r#"{{"result":{{"results":[{{"xdr":"{xdr}"}}]}}}}"#); + assert_eq!(classify_json(&body), Simulated::Value(ScVal::U32(8))); + // Bytes that are not an ScVal: it answered, with garbage. + assert_eq!( + classify_json(r#"{"result":{"results":[{"xdr":"AAAA//8="}]}}"#), + Simulated::Absent + ); + } + + #[test] + fn builds_an_envelope_that_decodes_back_to_the_call() { + // A locally-derived valid C-strkey: self-contained, no network, and it + // cannot rot the way a pasted mainnet address can. + let contract = stellar_strkey::Contract([7u8; 32]).to_string(); + let b64 = build_simulate_envelope(&contract, "decimals").expect("valid contract address"); + let raw = base64::engine::general_purpose::STANDARD + .decode(&b64) + .expect("valid base64"); + let env = TransactionEnvelope::from_xdr(&raw, Limits::none()).expect("valid xdr"); + let TransactionEnvelope::Tx(v1) = env else { + panic!("expected a v1 envelope") + }; + let OperationBody::InvokeHostFunction(op) = &v1.tx.operations[0].body else { + panic!("expected InvokeHostFunction") + }; + let HostFunction::InvokeContract(args) = &op.host_function else { + panic!("expected InvokeContract") + }; + assert_eq!(args.function_name.0.to_utf8_string().unwrap(), "decimals"); + assert_eq!(args.args.len(), 0, "the call takes no arguments"); + assert_eq!( + args.contract_address, + ScAddress::Contract(ContractId(Hash([7u8; 32]))) + ); + } + + #[test] + fn rejects_a_malformed_contract_address() { + assert_eq!(build_simulate_envelope("not-a-contract", "symbol"), None); + assert_eq!(build_simulate_envelope("", "symbol"), None); + } +} From f9530d862eed85525959ea81e4418ab8d2807007 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:03:41 +0200 Subject: [PATCH 022/144] fix(lore-0329): scale each amm leg by its token's decimals amm_trade_to_tick scaled both legs at a fixed 7, so a contract token with d decimals priced at 10^(7 - d) of market. Classic identities and their SACs stay at 7; a pure Soroban token uses its decimals() from the asset registry, and a token whose decimals are unknown yields no tick and is reported in LedgerSoroban::missing_decimals instead of being guessed. --- packages/prices-ingest-core/src/bucket.rs | 2 +- packages/prices-ingest-core/src/canonical.rs | 25 ++ packages/prices-ingest-core/src/price.rs | 4 +- packages/prices-ingest-core/src/soroban.rs | 240 ++++++++++++++++-- .../src/soroban/comet_tests.rs | 1 + 5 files changed, 246 insertions(+), 26 deletions(-) diff --git a/packages/prices-ingest-core/src/bucket.rs b/packages/prices-ingest-core/src/bucket.rs index c0ad6a2e..0d5657ad 100644 --- a/packages/prices-ingest-core/src/bucket.rs +++ b/packages/prices-ingest-core/src/bucket.rs @@ -843,7 +843,7 @@ mod tests { /// The fixture is the shape that actually reaches this: a Soroban AMM fill /// whose raw i128 amounts PASS the rounding bound (both above 1000, product /// of the shifted factors far above 10^6) yet whose quotient is enormous — - /// 1001 units in against 7.9e28 out. After `AMM_AMOUNT_SCALE` the price is + /// 1001 units in against 7.9e28 out. Scaled at 7 decimals the price is /// ~7.9e25, well past the 10^24 an integer part may reach, and so is the /// vwap derived from the same two volumes. Clamping the volume sums alone /// would leave `vwap` to be saturated by `decimal_to_i128` at `i128::MAX` diff --git a/packages/prices-ingest-core/src/canonical.rs b/packages/prices-ingest-core/src/canonical.rs index 02eae991..4b9a35a1 100644 --- a/packages/prices-ingest-core/src/canonical.rs +++ b/packages/prices-ingest-core/src/canonical.rs @@ -80,6 +80,10 @@ pub(crate) use prices_clickhouse::{USDC_ISSUER, USDT_ISSUER}; /// liquidity when someone debugs it. To support a non-mainnet backfill, thread the /// network passphrase through config and into [`AssetRegistry::from_existing`] /// (the single bake-in point, below) rather than reading this const. +/// Decimals of every classic asset, and so of every SAC: fixed by the protocol +/// (stroops). A pure Soroban token has its own, read from `decimals()`. +pub const CLASSIC_DECIMALS: u32 = 7; + const MAINNET_PASSPHRASE: &str = "Public Global Stellar Network ; September 2015"; fn mainnet_network_id() -> [u8; 32] { @@ -168,6 +172,10 @@ pub struct AssetRegistry { /// intern order. Replaces the id watermark (task 0139): a set of /// identities says "new" without assuming ids are handed out in order. pending: Vec, + /// `decimals()` of each pure Soroban token resolved so far, by contract + /// address (`prices.asset_decimals`, task 0329). Classic identities need + /// no entry — see [`CLASSIC_DECIMALS`]. + decimals: HashMap, } impl AssetRegistry { @@ -180,6 +188,7 @@ impl AssetRegistry { network_id: mainnet_network_id(), sac_index: HashMap::new(), pending: Vec::new(), + decimals: HashMap::new(), }; // Pre-seed the canonical quote SACs so an AMM-via-SAC USDC/USDT/XLM // collapses even before that asset's first classic (SDEX) sighting in the @@ -242,6 +251,22 @@ impl AssetRegistry { identity_to_asset(identity).and_then(|asset| sac_address(&asset, &self.network_id)) } + /// The decimals an AMM leg in `identity` is scaled by, or `None` for a pure + /// Soroban token whose `decimals()` is not known yet. Never a guess: the + /// fixed 7 this replaced priced every non-7-decimal token off by + /// 10^(7 − decimals) (task 0329). + pub fn decimals_of(&self, identity: &AssetIdentity) -> Option { + match identity { + AssetIdentity::Contract(addr) => self.decimals.get(addr).copied(), + AssetIdentity::Native | AssetIdentity::Credit { .. } => Some(CLASSIC_DECIMALS), + } + } + + /// Record a Soroban token's resolved `decimals()`. + pub fn set_decimals(&mut self, contract: String, decimals: u32) { + self.decimals.insert(contract, decimals); + } + pub fn assets(&self) -> impl Iterator { self.known.iter() } diff --git a/packages/prices-ingest-core/src/price.rs b/packages/prices-ingest-core/src/price.rs index 81000e97..a3138e56 100644 --- a/packages/prices-ingest-core/src/price.rs +++ b/packages/prices-ingest-core/src/price.rs @@ -164,7 +164,7 @@ pub fn price_forming_i64(amount_sold: i64, amount_bought: i64) -> bool { /// [`rounding_bound_holds`] for a Soroban AMM fill's RAW i128 amounts. /// /// Must be called on the amounts as the event carries them, in each token's own -/// decimals — BEFORE `soroban::AMM_AMOUNT_SCALE` converts them to `Decimal`. +/// decimals — BEFORE they are scaled by those decimals into `Decimal`. /// After that conversion the integer unit the bound reasons about is gone, and /// every fill looks equally precise. pub fn price_forming_i128(amount_in: i128, amount_out: i128) -> bool { @@ -340,7 +340,7 @@ mod tests { } /// Soroban amounts are raw i128 in each token's own decimals — classified - /// BEFORE `AMM_AMOUNT_SCALE`, because after scaling the integer information + /// BEFORE scaling, because after scaling the integer information /// the bound reads is gone. #[test] fn the_i128_wrapper_widens_without_losing_the_verdict() { diff --git a/packages/prices-ingest-core/src/soroban.rs b/packages/prices-ingest-core/src/soroban.rs index 7181884c..7f40ff31 100644 --- a/packages/prices-ingest-core/src/soroban.rs +++ b/packages/prices-ingest-core/src/soroban.rs @@ -32,10 +32,6 @@ use crate::price::{price_forming_i128, price_survives_column_scale}; use crate::tick::TradeTick; use crate::writer::OracleSample; -/// AMM token amounts are treated as 7-decimal (Stellar SAC convention). Token -/// decimals vary; this is a documented sizing-measurement approximation. -const AMM_AMOUNT_SCALE: u32 = 7; - /// Per-run registries, grown incrementally from in-window factory events. /// /// Oracle assets are NOT kept here: they are resolved through the same @@ -195,6 +191,12 @@ pub struct LedgerSoroban { /// re-read ledger and in callers that never read the count, so it only /// logs at debug level. pub unregistered_pool_contracts: Vec, + /// The Soroban token behind every AMM trade dropped because its + /// `decimals()` is not known yet (task 0329): one entry per unknown leg of + /// each dropped trade. A caller that can reach RPC resolves these and + /// decodes the ledger again; whatever is still here after that is lost + /// volume. + pub missing_decimals: Vec, } fn collect_tx_metas(lcm: &LedgerCloseMeta) -> Vec<&TransactionMeta> { @@ -565,8 +567,13 @@ fn classify_amm_groups( match dispatch(&rows, ®.venue, ®.phoenix, reg.pair_registries()) { Ok(trades) => { for t in trades { - if let Some(tick) = amm_trade_to_tick(&t, transaction_index, closed_at, assets) - { + if let Some(tick) = amm_trade_to_tick( + &t, + transaction_index, + closed_at, + assets, + &mut out.missing_decimals, + ) { out.amm_ticks.push((source, tick)); } } @@ -826,11 +833,15 @@ fn resolve_amm_token(contract_addr: &str, assets: &AssetRegistry) -> AssetIdenti } /// Convert a venue `TradeRow` into a `TradeTick` for the candle accumulator. +/// +/// `None` also when a leg's decimals are unknown; that leg's contract is then +/// pushed onto `missing_decimals`. fn amm_trade_to_tick( trade: &extractors_core::TradeRow, transaction_index: u16, closed_at: i64, assets: &mut AssetRegistry, + missing_decimals: &mut Vec, ) -> Option { // Collapse a SAC token onto its underlying classic identity (§12.4) so // AMM-via-SAC and SDEX-classic share one asset_id; a pure Soroban token keeps @@ -848,16 +859,34 @@ fn amm_trade_to_tick( return None; } + // Task 0329: each leg is scaled by its own token's decimals. A pure Soroban + // token whose `decimals()` is not known yet gets no tick at all — never the + // fixed 7 this replaced, which put SolvBTC (8) at a tenth of its price and + // every 18-decimal token 10^11 too low. Checked before `canonicalise`, so a + // dropped trade interns nothing. + let (Some(decimals_in), Some(decimals_out)) = + (assets.decimals_of(&sold), assets.decimals_of(&bought)) + else { + for leg in [&sold, &bought] { + if let AssetIdentity::Contract(addr) = leg + && assets.decimals_of(leg).is_none() + { + missing_decimals.push(addr.clone()); + } + } + return None; + }; + let pair = canonicalise(&sold, &bought, assets); // Classified on the RAW i128 amounts, in each token's own decimals, BEFORE - // the scaling below (task 0286, ADR 0287 §1): `AMM_AMOUNT_SCALE` turns them - // into a Decimal, and after that the integer unit the rounding bound reasons - // about is gone — every fill would look equally precise. + // the scaling below (task 0286, ADR 0287 §1): scaling turns them into a + // Decimal, and after that the integer unit the rounding bound reasons about + // is gone — every fill would look equally precise. let bound_holds = price_forming_i128(trade.amount_in, trade.amount_out); - let amount_in = Decimal::try_from_i128_with_scale(trade.amount_in, AMM_AMOUNT_SCALE).ok()?; - let amount_out = Decimal::try_from_i128_with_scale(trade.amount_out, AMM_AMOUNT_SCALE).ok()?; + let amount_in = Decimal::try_from_i128_with_scale(trade.amount_in, decimals_in).ok()?; + let amount_out = Decimal::try_from_i128_with_scale(trade.amount_out, decimals_out).ok()?; if amount_in.is_zero() || amount_out.is_zero() { return None; } @@ -1628,6 +1657,7 @@ mod tests { .register(POOL.to_string(), T0.to_string(), T1.to_string()); let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(T1.to_string(), 7); let mut out = LedgerSoroban::default(); classify_amm_groups(groups, 0, ®, &mut assets, SEQ, CLOSED_AT, &mut out); @@ -1825,6 +1855,7 @@ mod tests { .register(POOL.to_string(), T0.to_string(), T1.to_string()); let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(T1.to_string(), 7); let mut out = LedgerSoroban::default(); process_soroban_event_rows(SEQ, CLOSED_AT, &[swap], &mut reg, &mut assets, &mut out); @@ -2032,6 +2063,8 @@ mod tests { ); let mut assets = AssetRegistry::from_existing(vec![]); + // Pinned at 7 so the fixture keeps the scale it was written for. + assets.set_decimals(T1.to_string(), 7); let tick = amm_trade_to_tick( &extractors_core::TradeRow { venue: Venue::Soroswap, @@ -2049,6 +2082,7 @@ mod tests { 0, 1_700_000_000, &mut assets, + &mut Vec::new(), ) .expect("the swap still produces a tick"); // T0 is XLM's SAC: the tick carries the collapsed identity (D6). @@ -2104,16 +2138,30 @@ mod tests { } let mut assets = AssetRegistry::from_existing(vec![]); + // Pinned at 7 so the fixture keeps the scale it was written for. + assets.set_decimals(T1.to_string(), 7); assert!(price_forming_i128(2_000, AMOUNT_OUT)); - let at_floor = amm_trade_to_tick(&trade(2_000, AMOUNT_OUT), 0, 1_700_000_000, &mut assets) - .expect("the swap prices"); + let at_floor = amm_trade_to_tick( + &trade(2_000, AMOUNT_OUT), + 0, + 1_700_000_000, + &mut assets, + &mut Vec::new(), + ) + .expect("the swap prices"); assert_eq!(at_floor.price, Decimal::new(1, 12), "the fixture is 1e-12"); assert!(at_floor.price_forming, "the floor itself is a price"); assert!(price_forming_i128(1_980, AMOUNT_OUT)); - let under = amm_trade_to_tick(&trade(1_980, AMOUNT_OUT), 0, 1_700_000_000, &mut assets) - .expect("the swap still produces a tick"); + let under = amm_trade_to_tick( + &trade(1_980, AMOUNT_OUT), + 0, + 1_700_000_000, + &mut assets, + &mut Vec::new(), + ) + .expect("the swap still produces a tick"); assert_eq!(under.price, Decimal::new(99, 14), "the fixture is 9.9e-13"); assert!( !under.price_forming, @@ -2127,7 +2175,7 @@ mod tests { /// Task 0286 / ADR 0287 §1 — an AMM fill is classified on the RAW i128 /// amounts the swap event carries, in each token's own decimals, BEFORE - /// `AMM_AMOUNT_SCALE` turns them into `Decimal`s. Once scaled they are + /// scaling turns them into `Decimal`s. Once scaled they are /// fractions and every fill looks equally precise, so the bound has to be /// evaluated on the way in or not at all. #[test] @@ -2156,15 +2204,23 @@ mod tests { } let mut assets = AssetRegistry::from_existing(vec![]); + // Pinned at 7 so the fixture keeps the scale it was written for. + assets.set_decimals(T1.to_string(), 7); - let ordinary = amm_trade_to_tick(&trade(1_000_000, 914_145), 4, CLOSED_AT, &mut assets) - .expect("an ordinary swap prices"); + let ordinary = amm_trade_to_tick( + &trade(1_000_000, 914_145), + 4, + CLOSED_AT, + &mut assets, + &mut Vec::new(), + ) + .expect("an ordinary swap prices"); assert!( ordinary.price_forming, "a million units against ~914k clears the bound by orders of magnitude" ); - let dust = amm_trade_to_tick(&trade(34, 5), 4, CLOSED_AT, &mut assets) + let dust = amm_trade_to_tick(&trade(34, 5), 4, CLOSED_AT, &mut assets, &mut Vec::new()) .expect("a dust swap still produces a tick"); assert!( !dust.price_forming, @@ -2172,7 +2228,7 @@ mod tests { ); assert_eq!( dust.volume_base + dust.volume_quote, - Decimal::try_from_i128_with_scale(39, AMM_AMOUNT_SCALE).unwrap(), + Decimal::try_from_i128_with_scale(39, 7).unwrap(), "the dust fill keeps its volumes: it is a real trade" ); @@ -2227,12 +2283,19 @@ mod tests { reg } + /// SEAM_T0 is XLM's SAC; SEAM_T1 is a pure Soroban token, pinned at 7. + fn seam_assets() -> AssetRegistry { + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(SEAM_T1.to_string(), 7); + assets + } + /// The events-backfill seam carries BE's apply order onto the tick, so a /// repriced candle orders its fills exactly like the live path does. #[test] fn the_seam_carries_the_groups_apply_order_onto_its_ticks() { let mut reg = seam_registries(); - let mut assets = AssetRegistry::from_existing(vec![]); + let mut assets = seam_assets(); let mut out = LedgerSoroban::default(); process_soroban_event_rows( SEAM_SEQ, @@ -2257,7 +2320,7 @@ mod tests { #[should_panic(expected = "disagree on transaction_index")] fn a_group_disagreeing_on_its_apply_order_is_refused_in_debug() { let mut reg = seam_registries(); - let mut assets = AssetRegistry::from_existing(vec![]); + let mut assets = seam_assets(); let mut out = LedgerSoroban::default(); process_soroban_event_rows( SEAM_SEQ, @@ -2294,8 +2357,139 @@ mod tests { let mut assets = AssetRegistry::from_existing(vec![]); let before = assets.assets().count(); - assert!(amm_trade_to_tick(&trade, 0, 1_700_000_000, &mut assets).is_none()); + assert!( + amm_trade_to_tick(&trade, 0, 1_700_000_000, &mut assets, &mut Vec::new()).is_none() + ); + assert_eq!(assets.assets().count(), before, "nothing interned"); + } + + // ---- task 0329: each leg in its own token's decimals -------------------- + + /// SolvBTC, the token SBE reported at a tenth of its price. + const SOLVBTC: &str = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN"; + + fn usdc() -> AssetIdentity { + AssetIdentity::Credit { + code: "USDC".to_string(), + issuer: USDC_ISSUER.to_string(), + } + } + + /// One whole `token` sold for `usdc_raw` stroops of USDC, through its SAC. + fn sell_for_usdc( + assets: &AssetRegistry, + token: &str, + token_raw: i128, + usdc_raw: i128, + ) -> extractors_core::TradeRow { + extractors_core::TradeRow { + venue: Venue::Aquarius, + contract_id: "CDE57N6XTUPBKYYDGQMXX7E7SLNOLFY3JEQB4MULSMR2AKTSAENGX2HC".to_string(), + transaction_id: "tx".to_string(), + ledger_sequence: 100, + first_event_index: 0, + token_in: token.to_string(), + token_out: assets.sac_address_of(&usdc()).expect("USDC has a SAC"), + amount_in: token_raw, + amount_out: usdc_raw, + fee: None, + trader: None, + } + } + + /// The defect SBE reported: amounts were scaled at a fixed 7, so a token + /// with d decimals priced at 10^(7 − d) of market. Each case sells exactly + /// one whole token for a USDC (7-decimal) amount and must read back that + /// amount as the price and one token as the base volume. + #[test] + fn each_leg_is_scaled_by_its_own_tokens_decimals() { + for (decimals, usdc_per_token) in [(8u32, 86_055i128), (18, 1), (6, 2), (7, 41)] { + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(SOLVBTC.to_string(), decimals); + let one_token = 10i128.pow(decimals); + let trade = sell_for_usdc(&assets, SOLVBTC, one_token, usdc_per_token * 10_000_000); + + let tick = amm_trade_to_tick(&trade, 0, 1_700_000_000, &mut assets, &mut Vec::new()) + .unwrap_or_else(|| panic!("{decimals}-decimal leg prices")); + + assert_eq!( + (&tick.base, &tick.quote), + (&AssetIdentity::Contract(SOLVBTC.to_string()), &usdc()), + "the token is the base, USDC the quote" + ); + assert_eq!( + tick.price, + Decimal::from(usdc_per_token), + "{decimals} decimals" + ); + assert_eq!(tick.volume_base, Decimal::ONE, "{decimals} decimals"); + assert_eq!( + tick.volume_quote, + Decimal::from(usdc_per_token), + "{decimals} decimals" + ); + } + } + + /// A pure Soroban token whose decimals are unknown is never guessed at 7: + /// the trade yields no tick, interns nothing, and every unknown leg is + /// reported so the caller can resolve it. + #[test] + fn a_leg_with_unknown_decimals_prices_nothing_and_is_reported() { + let mut assets = AssetRegistry::from_existing(vec![]); + let before = assets.assets().count(); + let mut missing = Vec::new(); + let trade = sell_for_usdc(&assets, SOLVBTC, 100_000_000, 860_550_000_000); + + assert!(amm_trade_to_tick(&trade, 0, 1_700_000_000, &mut assets, &mut missing).is_none()); + assert_eq!(missing, vec![SOLVBTC.to_string()], "only the unknown leg"); assert_eq!(assets.assets().count(), before, "nothing interned"); + + // Two unknown legs: both reported, so one resolve pass covers the trade. + let other = "CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO"; + let both = extractors_core::TradeRow { + token_out: other.to_string(), + ..trade + }; + missing.clear(); + assert!(amm_trade_to_tick(&both, 0, 1_700_000_000, &mut assets, &mut missing).is_none()); + assert_eq!(missing, vec![SOLVBTC.to_string(), other.to_string()]); + } + + /// The contract the callers rely on: a ledger decoded before a token's + /// decimals are known reports the token and prices nothing; decoded again + /// once they are set, it prices. + #[test] + fn a_ledger_decoded_again_after_its_decimals_resolve_prices() { + let mut reg = seam_registries(); + let mut assets = AssetRegistry::from_existing(vec![]); + let mut out = LedgerSoroban::default(); + let events = [seam_swap("tx-a", 0, 0)]; + + process_soroban_event_rows( + SEAM_SEQ, + SEAM_CLOSED_AT, + &events, + &mut reg, + &mut assets, + &mut out, + ); + assert!(out.amm_ticks.is_empty()); + assert_eq!(out.missing_decimals, vec![SEAM_T1.to_string()]); + assert!(out.unresolved.is_empty(), "a known pool is not unresolved"); + + assets.set_decimals(SEAM_T1.to_string(), 7); + let mut out = LedgerSoroban::default(); + process_soroban_event_rows( + SEAM_SEQ, + SEAM_CLOSED_AT, + &events, + &mut reg, + &mut assets, + &mut out, + ); + assert_eq!(out.amm_ticks.len(), 1); + assert!(out.missing_decimals.is_empty()); } } diff --git a/packages/prices-ingest-core/src/soroban/comet_tests.rs b/packages/prices-ingest-core/src/soroban/comet_tests.rs index cc4b2630..814e587d 100644 --- a/packages/prices-ingest-core/src/soroban/comet_tests.rs +++ b/packages/prices-ingest-core/src/soroban/comet_tests.rs @@ -218,6 +218,7 @@ fn an_aquarius_self_trade_yields_no_tick_and_no_error() { let mut reg = Registries::new(); reg.venue.insert(AQUA.to_string(), Venue::Aquarius); let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(TOKEN.to_string(), 7); let out = run( 62_078_348, std::slice::from_ref(&trade), From d99c600e4a6be843f0d023cf4469c6d1c1f6cb57 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:13:38 +0200 Subject: [PATCH 023/144] feat(lore-0329): persist soroban token decimals resolved over rpc Add prices.asset_decimals and a DecimalsResolver that asks a token's SEP-41 decimals() by simulation, rejects anything above rust_decimal's scale, and leaves a failed contract alone for ten minutes. The writer loads the table into the asset registry and writes new answers. --- packages/prices-clickhouse/schema/init.sql | 25 +++ packages/prices-clickhouse/src/lib.rs | 3 +- packages/prices-ingest-core/src/decimals.rs | 159 ++++++++++++++++++ packages/prices-ingest-core/src/lib.rs | 4 + packages/prices-ingest-core/src/writer.rs | 32 ++++ .../prices-ingest-core/tests/decimals_it.rs | 59 +++++++ .../tests/decimals_rpc_it.rs | 47 ++++++ 7 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 packages/prices-ingest-core/src/decimals.rs create mode 100644 packages/prices-ingest-core/tests/decimals_it.rs create mode 100644 packages/prices-ingest-core/tests/decimals_rpc_it.rs diff --git a/packages/prices-clickhouse/schema/init.sql b/packages/prices-clickhouse/schema/init.sql index 74399235..94c73066 100644 --- a/packages/prices-clickhouse/schema/init.sql +++ b/packages/prices-clickhouse/schema/init.sql @@ -413,6 +413,31 @@ ENGINE = ReplacingMergeTree(fetched_at) ORDER BY (contract_address) SETTINGS index_granularity = 8192; +---------------------------------------------------------------------- +-- `decimals()` of each pure Soroban token an AMM swap has touched (task 0329). +-- The ingest scales each swap leg's raw i128 amount by its token's decimals. +-- Before this table it assumed 7 for every token, the classic convention, and +-- priced every other token off by 10^(7 - decimals). Classic assets and their +-- SACs are 7 by protocol and have no row. +-- +-- Written by the ingest itself (live processor, events-backfill, sdex-backfill) +-- the first time a swap names a token it has no row for: one Soroban RPC +-- `decimals()` call, then this row. Never refreshed: decimals are fixed at +-- deploy, the same reasoning as `asset_symbol`. Only answers are stored; a +-- token whose call failed has no row and is asked again later. +-- +-- Read in full at cold start. The live processor's init FAILS if the table is +-- absent, so apply this before deploying an ingest that reads it. +---------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS prices.asset_decimals ( + contract_address String, + decimals UInt8, + fetched_at DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(fetched_at) +ORDER BY (contract_address) +SETTINGS index_granularity = 8192; + ---------------------------------------------------------------------- -- Oracle reference prices (§3.4). ReplacingMergeTree, monthly partitions. -- Written by the Oracle Fetcher Lambda in production; the backfill writes diff --git a/packages/prices-clickhouse/src/lib.rs b/packages/prices-clickhouse/src/lib.rs index c6fe2b1b..ae0f3881 100644 --- a/packages/prices-clickhouse/src/lib.rs +++ b/packages/prices-clickhouse/src/lib.rs @@ -512,8 +512,9 @@ mod tests { // (−1 = 42: task 0256 removed `discovery_state` with the ledger scan // that was its only reader and writer.) // (+1 = 43: task 0274's `current_prices.price_basis` ALTER, same shape.) + // (+1 = 44: task 0329's `asset_decimals` table.) let stmts = split_statements(INIT_SQL); - assert_eq!(stmts.len(), 43, "got {}", stmts.len()); + assert_eq!(stmts.len(), 44, "got {}", stmts.len()); } /// The single `CREATE TABLE … IF NOT EXISTS
(` statement of `sql`. diff --git a/packages/prices-ingest-core/src/decimals.rs b/packages/prices-ingest-core/src/decimals.rs new file mode 100644 index 00000000..acc364bd --- /dev/null +++ b/packages/prices-ingest-core/src/decimals.rs @@ -0,0 +1,159 @@ +//! Soroban token decimals for the AMM path (task 0329). +//! +//! A swap event carries each leg as a raw i128 in its token's own decimals. +//! Classic assets and their SACs are 7 by protocol; a pure Soroban token's +//! decimals are whatever its SEP-41 `decimals()` returns. Decoding is +//! synchronous and never guesses, so a trade whose token is unknown is dropped +//! and reported ([`crate::LedgerSoroban::missing_decimals`]). The caller then +//! asks [`DecimalsResolver`], persists what it learned to +//! `prices.asset_decimals`, records it in the [`AssetRegistry`], and decodes +//! the ledger again. +//! +//! [`AssetRegistry`]: crate::AssetRegistry + +use std::collections::{HashMap, HashSet}; +use std::time::{Duration, Instant}; + +use serde::Serialize; +use stellar_xdr::ScVal; + +use crate::canonical::AssetRegistry; +use crate::soroban_rpc::{Simulated, http_client, rpc_url_from_env, simulate}; + +/// The largest scale `rust_decimal` holds. A token claiming more cannot be +/// represented, so it is treated as unresolved rather than truncated. +pub const MAX_DECIMALS: u32 = 28; + +/// How long a contract whose `decimals()` gave no usable answer is left alone +/// before it is asked again. Bounds the RPC cost of a token that trades in +/// every ledger and never answers, without parking a transient failure for a +/// warm container's whole lifetime. +pub const RETRY_AFTER: Duration = Duration::from_secs(600); + +/// One `prices.asset_decimals` row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, clickhouse::Row)] +pub struct DecimalsRow { + pub contract_address: String, + pub decimals: u8, +} + +impl DecimalsRow { + /// Record this row in `assets`, after it is durably written. + pub fn record(&self, assets: &mut AssetRegistry) { + assets.set_decimals(self.contract_address.clone(), self.decimals.into()); + } +} + +/// Asks Soroban RPC for the decimals a decode reported missing. +pub struct DecimalsResolver { + http: reqwest::Client, + rpc_url: String, + /// When each contract last failed to resolve. + failed: HashMap, +} + +impl DecimalsResolver { + pub fn new(rpc_url: String) -> Self { + Self { + http: http_client(), + rpc_url, + failed: HashMap::new(), + } + } + + /// Against `SOROBAN_RPC_URL`, or the public default. + pub fn from_env() -> Self { + Self::new(rpc_url_from_env()) + } + + /// Resolve each distinct contract in `missing` that has not failed within + /// [`RETRY_AFTER`]. Returns the answers, which the caller persists and then + /// [`record`](DecimalsRow::record)s. Empty means nothing new was learned, + /// so decoding again would price nothing more. + pub async fn resolve(&mut self, missing: &[String]) -> Vec { + let mut rows = Vec::new(); + let mut asked = HashSet::new(); + for contract in missing { + if !asked.insert(contract) + || self + .failed + .get(contract) + .is_some_and(|at| at.elapsed() < RETRY_AFTER) + { + continue; + } + match decimals_of(&self.http, &self.rpc_url, contract).await { + Some(decimals) => { + tracing::info!(contract, decimals, "resolved soroban token decimals"); + self.failed.remove(contract); + rows.push(DecimalsRow { + contract_address: contract.clone(), + decimals, + }); + } + None => { + tracing::warn!( + contract, + retry_after_secs = RETRY_AFTER.as_secs(), + "decimals() unresolved — this token's AMM trades are dropped until it resolves" + ); + self.failed.insert(contract.clone(), Instant::now()); + } + } + } + rows + } +} + +/// `decimals()` of `contract`, or `None` when there is no usable answer. +pub async fn decimals_of(http: &reqwest::Client, rpc_url: &str, contract: &str) -> Option { + match simulate(http, rpc_url, contract, "decimals").await { + Simulated::Value(v) => usable_decimals(&v, contract), + Simulated::Absent | Simulated::Transient => None, + } +} + +/// SEP-41 declares `decimals() -> u32`; anything else, or more than +/// [`MAX_DECIMALS`], is not a scale we can apply. +fn usable_decimals(v: &ScVal, contract: &str) -> Option { + match v { + ScVal::U32(d) if *d <= MAX_DECIMALS => Some(*d as u8), + other => { + tracing::warn!(contract, value = ?other, "decimals() returned no usable scale"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_a_u32_within_the_decimal_scale_is_usable() { + assert_eq!(usable_decimals(&ScVal::U32(8), "C"), Some(8)); + assert_eq!(usable_decimals(&ScVal::U32(0), "C"), Some(0)); + assert_eq!(usable_decimals(&ScVal::U32(28), "C"), Some(28)); + assert_eq!(usable_decimals(&ScVal::U32(29), "C"), None); + assert_eq!(usable_decimals(&ScVal::I32(7), "C"), None); + assert_eq!(usable_decimals(&ScVal::Void, "C"), None); + } + + #[tokio::test] + async fn a_failed_contract_is_not_asked_again_inside_the_retry_window() { + // Port 1 refuses at once, so each attempt is a fast Transient. + let mut r = DecimalsResolver::new("http://127.0.0.1:1/".to_string()); + let c = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN".to_string(); + assert!(r.resolve(&[c.clone(), c.clone()]).await.is_empty()); + let first = r.failed[&c]; + assert!(r.resolve(std::slice::from_ref(&c)).await.is_empty()); + assert_eq!(r.failed[&c], first, "not asked again inside RETRY_AFTER"); + + r.failed.insert(c.clone(), Instant::now() - RETRY_AFTER); + assert!(r.resolve(std::slice::from_ref(&c)).await.is_empty()); + assert!( + r.failed[&c] > first, + "asked again once the window has passed" + ); + } +} diff --git a/packages/prices-ingest-core/src/lib.rs b/packages/prices-ingest-core/src/lib.rs index 079ed092..f0181ee4 100644 --- a/packages/prices-ingest-core/src/lib.rs +++ b/packages/prices-ingest-core/src/lib.rs @@ -19,6 +19,8 @@ //! ([`STATIC_POOLS`], task 0300) merged into the AMM registries. //! - [`canonical`] — asset identity, the [`AssetRegistry`] of known and newly //! seen identities, and `(base, quote)` canonicalisation. +//! - [`decimals`] — Soroban token decimals: the RPC resolver the AMM path's +//! callers use when a decode reports a token it cannot scale. //! - [`price`] / [`tick`] — per-trade price + the [`TradeTick`] the bucketer eats. //! - [`bucket`] — 1-minute OHLCV accumulation ([`CandleAccumulator`]). //! - [`writer`] — the transport-agnostic ClickHouse [`OhlcvWriter`] (works with a @@ -28,6 +30,7 @@ pub mod bucket; pub mod canonical; +pub mod decimals; pub mod decode; pub mod error; pub mod filter; @@ -43,6 +46,7 @@ pub mod writer; pub use bucket::{CandleAccumulator, OhlcvCandle}; pub use canonical::{AssetIdentity, AssetRegistry, CanonicalPair, canonicalise}; +pub use decimals::{DecimalsResolver, DecimalsRow}; pub use decode::{decode_object, ledger_close_time, ledger_sequence}; pub use error::IngestError; pub use filter::{ diff --git a/packages/prices-ingest-core/src/writer.rs b/packages/prices-ingest-core/src/writer.rs index 42a8bd56..2f66f933 100644 --- a/packages/prices-ingest-core/src/writer.rs +++ b/packages/prices-ingest-core/src/writer.rs @@ -18,6 +18,7 @@ use tracing::info; use crate::bucket::OhlcvCandle; use crate::canonical::{AssetIdentity, AssetRegistry}; +use crate::decimals::DecimalsRow; use crate::error::IngestError; use crate::price::CANDLE_PRICE_SCALE; use crate::registry_io::PoolRegistryRow; @@ -105,6 +106,37 @@ impl OhlcvWriter { Ok(assets) } + /// Load every resolved Soroban token decimals from `prices.asset_decimals` + /// into `registry` (task 0329), so a cold start scales known tokens + /// without an RPC call. The table must exist: an ingest that cannot read + /// it fails init rather than dropping every pure Soroban token's trades. + pub async fn load_decimals(&self, registry: &mut AssetRegistry) -> Result<(), IngestError> { + let rows = self + .client + .query("SELECT contract_address, decimals FROM prices.asset_decimals FINAL") + .fetch_all::<(String, u8)>() + .await?; + info!(tokens = rows.len(), "loaded soroban token decimals"); + for (contract, decimals) in rows { + registry.set_decimals(contract, decimals.into()); + } + Ok(()) + } + + /// Persist decimals a run resolved over RPC to `prices.asset_decimals` + /// (task 0329). A no-op on an empty slice. + pub async fn write_decimals(&self, rows: &[DecimalsRow]) -> Result<(), IngestError> { + if rows.is_empty() { + return Ok(()); + } + let mut insert = self.client.insert("prices.asset_decimals")?; + for row in rows { + insert.write(row).await?; + } + insert.end().await?; + Ok(()) + } + /// Rehydrate the discovered AMM pool [`Registries`] from `prices.pool_registry` /// so a consumer resolves pools created before its processing window instead of /// re-deriving from Soroban activation (task 0053 decision #4). Returns empty diff --git a/packages/prices-ingest-core/tests/decimals_it.rs b/packages/prices-ingest-core/tests/decimals_it.rs new file mode 100644 index 00000000..0e055aa6 --- /dev/null +++ b/packages/prices-ingest-core/tests/decimals_it.rs @@ -0,0 +1,59 @@ +//! Task 0329 — `prices.asset_decimals` round-trips through the writer. +//! +//! tools/scripts/ignored-tests.sh # all of them: CI runs exactly this on every Rust PR +//! cargo test -p prices-ingest-core --test decimals_it -- --ignored --test-threads=1 +//! +//! The writer hardcodes `prices.asset_decimals`, so this resets the real +//! `prices` database's table. Never run against a shared cluster. + +use clickhouse::Client; +use prices_ingest_core::{AssetIdentity, AssetRegistry, DecimalsRow, OhlcvWriter}; + +fn ch_url() -> String { + std::env::var("CLICKHOUSE_URL").unwrap_or_else(|_| "http://localhost:8123".to_string()) +} + +const SOLVBTC: &str = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN"; +const BNUSD: &str = "CCT4ZYIYZ3TUO2AWQFEOFGBZ6HQP3GW5TA37CK7CRZVFRDXYTHTYX7KP"; + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn written_decimals_load_back_into_the_registry() { + let c = Client::default().with_url(ch_url()); + prices_clickhouse::apply_sql(&c, prices_clickhouse::INIT_SQL) + .await + .expect("apply init schema"); + c.query("TRUNCATE TABLE prices.asset_decimals") + .execute() + .await + .unwrap(); + + let writer = OhlcvWriter::new(c.clone()); + let row = |contract: &str, decimals| DecimalsRow { + contract_address: contract.to_string(), + decimals, + }; + writer + .write_decimals(&[row(SOLVBTC, 8), row(BNUSD, 18)]) + .await + .unwrap(); + // A re-resolve writes the same row again; FINAL collapses it. + writer.write_decimals(&[row(SOLVBTC, 8)]).await.unwrap(); + writer.write_decimals(&[]).await.unwrap(); + + let mut assets = AssetRegistry::from_existing(vec![]); + writer.load_decimals(&mut assets).await.unwrap(); + let of = |c: &str| assets.decimals_of(&AssetIdentity::Contract(c.to_string())); + assert_eq!(of(SOLVBTC), Some(8)); + assert_eq!(of(BNUSD), Some(18)); + assert_eq!( + of("CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO"), + None, + "a token with no row stays unknown" + ); + + c.query("TRUNCATE TABLE prices.asset_decimals") + .execute() + .await + .unwrap(); +} diff --git a/packages/prices-ingest-core/tests/decimals_rpc_it.rs b/packages/prices-ingest-core/tests/decimals_rpc_it.rs new file mode 100644 index 00000000..5ad4900b --- /dev/null +++ b/packages/prices-ingest-core/tests/decimals_rpc_it.rs @@ -0,0 +1,47 @@ +//! Live Soroban RPC check for the task-0329 `decimals()` lookup. Gated +//! `#[ignore]` because it talks to the public mainnet endpoint: +//! +//! cargo test -p prices-ingest-core --test decimals_rpc_it -- --ignored +//! +//! Recorded in the `#[ignore]` inventory (tools/scripts/ignored-tests.sh) as a +//! public-network test and never run by CI: third-party uptime must not gate a +//! PR. The parser is pinned offline in `decimals.rs` and `soroban_rpc.rs`; this +//! proves the two ends meet against real tokens. + +use prices_ingest_core::decimals::decimals_of; +use prices_ingest_core::soroban_rpc::{http_client, rpc_url_from_env}; + +#[tokio::test] +#[ignore = "requires public network — third-party uptime; never gates a PR"] +async fn reads_the_decimals_sbe_checked_on_chain() { + // The tokens SBE reported mispriced (task 0329), with the decimals they + // read by calling decimals() themselves. + for (name, contract, want) in [ + ( + "SolvBTC", + "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN", + 8, + ), + ( + "XAUM", + "CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO", + 9, + ), + ( + "XRP", + "CB7OOP3VSAWBZOOTOG2YEFANVU45GVWYUUM5HI32DKLHVKUDOFVQ37XP", + 6, + ), + ] { + let got = decimals_of(&http_client(), &rpc_url_from_env(), contract).await; + assert_eq!(got, Some(want), "{name} ({contract})"); + } +} + +#[tokio::test] +#[ignore = "requires public network — third-party uptime; never gates a PR"] +async fn a_contract_that_was_never_deployed_has_no_decimals() { + let never_deployed = stellar_strkey::Contract([3u8; 32]).to_string(); + let got = decimals_of(&http_client(), &rpc_url_from_env(), &never_deployed).await; + assert_eq!(got, None); +} From 232975559a0981377d06ba4a7a012f68c304269d Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:13:41 +0200 Subject: [PATCH 024/144] fix(lore-0329): resolve unknown decimals and decode the ledger again The live processor, events-backfill and sdex-backfill (combined mode) now resolve the tokens a decode reported missing, persist them, and decode that ledger again. Cold starts load prices.asset_decimals first. events-backfill prints the trades it still could not reprice. --- packages/events-backfill/src/run.rs | 103 +++++++++++++++--- .../prices-ledger-processor/src/reconcile.rs | 27 ++++- .../prices-ledger-processor/src/sink/mod.rs | 34 +++++- packages/sdex-backfill/src/ingest.rs | 21 +++- packages/sdex-backfill/src/run.rs | 11 +- packages/sdex-backfill/src/sink.rs | 14 ++- 6 files changed, 180 insertions(+), 30 deletions(-) diff --git a/packages/events-backfill/src/run.rs b/packages/events-backfill/src/run.rs index 4791e915..6a756239 100644 --- a/packages/events-backfill/src/run.rs +++ b/packages/events-backfill/src/run.rs @@ -21,9 +21,9 @@ use serde_json::Value; use tracing::{info, warn}; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, DEFAULT_BACKOFF_MS, LedgerSoroban, OhlcvCandle, OhlcvWriter, - RawSorobanEvent, Registries, UnresolvedPool, UnresolvedPoolSwap, process_soroban_event_rows, - retry_with_backoff, + AssetRegistry, CandleAccumulator, DEFAULT_BACKOFF_MS, DecimalsResolver, LedgerSoroban, + OhlcvCandle, OhlcvWriter, RawSorobanEvent, Registries, UnresolvedPool, UnresolvedPoolSwap, + process_soroban_event_rows, retry_with_backoff, }; use crate::cli::Cli; @@ -47,11 +47,47 @@ where .map_err(EventsBackfillError::from) } -/// Run the shared seam over one ledger's events, merge the resulting ticks into -/// the **run-level** per-source accumulators, then move every candle for a minute -/// strictly older than this ledger's minute out of the accumulators and into the -/// per-source write buffers. Synchronous (no I/O): buffered candles are written -/// later, in batches, at chunk end. +/// Run the shared seam over one ledger's events. +/// +/// A trade naming a Soroban token whose decimals are not known yet is dropped by +/// the decode, never guessed (task 0329). Those tokens are resolved over RPC, +/// persisted (not in a dry-run), and the ledger is decoded again — decoding is +/// deterministic and the first result is discarded whole. Whatever is still in +/// `missing_decimals` afterwards is volume this run did not reprice. +#[allow(clippy::too_many_arguments)] +async fn decode_ledger( + ledger: u32, + closed_at: i64, + events: &[RawSorobanEvent], + reg: &mut Registries, + assets: &mut AssetRegistry, + decimals: &mut DecimalsResolver, + writer: &OhlcvWriter, + dry_run: bool, +) -> Result { + let mut out = LedgerSoroban::default(); + process_soroban_event_rows(ledger, closed_at, events, reg, assets, &mut out); + if !out.missing_decimals.is_empty() { + let rows = decimals.resolve(&out.missing_decimals).await; + if !rows.is_empty() { + if !dry_run { + retry_write(|| async { writer.write_decimals(&rows).await }).await?; + } + for row in &rows { + row.record(assets); + } + out = LedgerSoroban::default(); + process_soroban_event_rows(ledger, closed_at, events, reg, assets, &mut out); + } + } + Ok(out) +} + +/// Merge one decoded ledger's ticks into the **run-level** per-source +/// accumulators, then move every candle for a minute strictly older than this +/// ledger's minute out of the accumulators and into the per-source write +/// buffers. Synchronous (no I/O): buffered candles are written later, in +/// batches, at chunk end. /// /// Because events arrive in ledger (hence minute) order and the accumulators /// persist across chunks, only the current minute stays open — so a minute split @@ -59,19 +95,16 @@ where /// RMT-colliding partials. #[allow(clippy::too_many_arguments)] fn accumulate_ledger( - ledger: u32, + out: LedgerSoroban, closed_at: i64, - events: &[RawSorobanEvent], - reg: &mut Registries, - assets: &mut AssetRegistry, accumulators: &mut HashMap<&'static str, CandleAccumulator>, buffers: &mut HashMap<&'static str, Vec>, ticks_by_source: &mut HashMap<&'static str, u64>, failed_by_source: &mut HashMap<&'static str, u64>, raw_unresolved: &mut Vec, + missing_decimals: &mut u64, ) { - let mut out = LedgerSoroban::default(); - process_soroban_event_rows(ledger, closed_at, events, reg, assets, &mut out); + *missing_decimals += out.missing_decimals.len() as u64; for (source, tick) in out.amm_ticks { accumulators.entry(source).or_default().merge(tick); *ticks_by_source.entry(source).or_default() += 1; @@ -184,6 +217,8 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { // write only newly seen assets and resolve every seeded pool. let existing_assets = writer.load_assets().await?; let mut assets = AssetRegistry::from_existing(existing_assets); + writer.load_decimals(&mut assets).await?; + let mut decimals = DecimalsResolver::from_env(); let mut reg = reprice_registry(writer.load_pool_registry().await?)?; // Every AMM pool has a `venue` entry (the registry superset); its strkeys are @@ -225,6 +260,9 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { // not cover this ledger" — it can only be a corrupt or unexpected row, and // any non-zero count means those fills ARE in the wrong order. let mut apply_order_fallbacks: u64 = 0; + // Unknown-decimals legs of AMM trades the run could not reprice because + // the token's `decimals()` did not resolve (task 0329). + let mut missing_decimals: u64 = 0; // Run-level state (persists across chunks): one accumulator per source and // one write buffer per source. `flush_older_than` keeps the current minute @@ -255,17 +293,26 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { if let Some(l) = cur_ledger && !cur_missing { - accumulate_ledger( + let out = decode_ledger( l, cur_closed_at, &cur_events, &mut reg, &mut assets, + &mut decimals, + &writer, + cli.dry_run, + ) + .await?; + accumulate_ledger( + out, + cur_closed_at, &mut accumulators, &mut buffers, &mut ticks_by_source, &mut failed_by_source, &mut raw_unresolved, + &mut missing_decimals, ); } cur_ledger = Some(r.ledger_sequence); @@ -348,17 +395,26 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { if let Some(l) = cur_ledger && !cur_missing { - accumulate_ledger( + let out = decode_ledger( l, cur_closed_at, &cur_events, &mut reg, &mut assets, + &mut decimals, + &writer, + cli.dry_run, + ) + .await?; + accumulate_ledger( + out, + cur_closed_at, &mut accumulators, &mut buffers, &mut ticks_by_source, &mut failed_by_source, &mut raw_unresolved, + &mut missing_decimals, ); } @@ -456,6 +512,7 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { unresolved_genuine.len(), dropped_swaps, apply_order_fallbacks, + missing_decimals, ); info!( elapsed_s = run_start.elapsed().as_secs(), @@ -568,6 +625,7 @@ fn print_summary( unresolved_contracts: usize, dropped_swaps: u64, apply_order_fallbacks: u64, + missing_decimals: u64, ) { println!(); println!("=== events-backfill complete ==="); @@ -596,6 +654,9 @@ fn print_summary( // event row, not a missing join row (task 0304). Non-zero means the range // is not repaired. println!("negative apply order: {apply_order_fallbacks}"); + // Always printed, 0 included: non-zero means a token's `decimals()` did not + // resolve and its trades are NOT in this range's candles (task 0329). + println!("trades dropped (decimals): {missing_decimals}"); } /// The registry the reprice reads events for: the LOADED `pool_registry` plus @@ -664,6 +725,7 @@ mod tests { reg.soroswap .register(POOL.to_string(), T0.to_string(), T1.to_string()); let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(T1.to_string(), 7); let mut accs: HashMap<&'static str, CandleAccumulator> = HashMap::new(); let mut buffers: HashMap<&'static str, Vec> = HashMap::new(); let mut ticks: HashMap<&'static str, u64> = HashMap::new(); @@ -682,17 +744,24 @@ mod tests { ticks: &mut HashMap<&'static str, u64>, unresolved: &mut Vec, ) { - accumulate_ledger( + let mut out = LedgerSoroban::default(); + process_soroban_event_rows( ledger, closed_at, &[soroswap_swap(ledger, closed_at)], reg, assets, + &mut out, + ); + accumulate_ledger( + out, + closed_at, accs, buffers, ticks, &mut HashMap::new(), unresolved, + &mut 0, ); } diff --git a/packages/prices-ledger-processor/src/reconcile.rs b/packages/prices-ledger-processor/src/reconcile.rs index 6d57c99c..65a07878 100644 --- a/packages/prices-ledger-processor/src/reconcile.rs +++ b/packages/prices-ledger-processor/src/reconcile.rs @@ -63,9 +63,9 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::Instant; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, OfferLookupCounts, OracleSample, PoolRegistryRow, Registries, - decode_object, extract_trades_with_counts, ledger_close_time, ledger_sequence, process_ledger, - raw_trade_to_tick, + AssetRegistry, CandleAccumulator, DecimalsResolver, OfferLookupCounts, OracleSample, + PoolRegistryRow, Registries, decode_object, extract_trades_with_counts, ledger_close_time, + ledger_sequence, process_ledger, raw_trade_to_tick, }; use tokio::sync::Mutex; use tracing::info; @@ -137,6 +137,10 @@ pub struct ProcessingState { /// here only *after* the write succeeds, so a failed write is retried next /// run. Init: everything loaded at cold start is already in the table. pub persisted_pools: HashMap, + /// Resolves Soroban token decimals a decode reports missing (task 0329). + /// Warm across invocations, like the registries, so a token that will not + /// resolve is not re-asked on every ledger. + pub decimals: DecimalsResolver, } /// The cold-start [`ProcessingState`] built from what was loaded. @@ -160,6 +164,7 @@ fn initial_state(assets: AssetRegistry, mut registries: Registries) -> Processin assets, registries, persisted_pools, + decimals: DecimalsResolver::from_env(), } } @@ -279,7 +284,21 @@ where sdex.merge(raw_trade_to_tick(&trade, &mut state.assets)); } // Soroban AMM trades + oracle samples. - let sob = process_ledger(lcm, &mut state.registries, &mut state.assets); + let mut sob = process_ledger(lcm, &mut state.registries, &mut state.assets); + // Task 0329: a trade naming a token whose decimals are not known + // yet was dropped, not guessed. Resolve, persist, and decode the + // ledger again — decoding is deterministic, and the first + // result is discarded whole, so nothing is counted twice. + if !sob.missing_decimals.is_empty() { + let rows = state.decimals.resolve(&sob.missing_decimals).await; + if !rows.is_empty() { + self.sink.write_decimals(&rows).await?; + for row in &rows { + row.record(&mut state.assets); + } + sob = process_ledger(lcm, &mut state.registries, &mut state.assets); + } + } for (source, tick) in sob.amm_ticks { amm.entry(source).or_default().merge(tick); } diff --git a/packages/prices-ledger-processor/src/sink/mod.rs b/packages/prices-ledger-processor/src/sink/mod.rs index c435da3d..53ce4f7c 100644 --- a/packages/prices-ledger-processor/src/sink/mod.rs +++ b/packages/prices-ledger-processor/src/sink/mod.rs @@ -11,8 +11,8 @@ use std::future::Future; use prices_ingest_core::{ - AssetRegistry, DEFAULT_BACKOFF_MS, OhlcvCandle, OhlcvWriter, OracleSample, PoolRegistryRow, - Registries, retry_with_backoff, + AssetRegistry, DEFAULT_BACKOFF_MS, DecimalsRow, OhlcvCandle, OhlcvWriter, OracleSample, + PoolRegistryRow, Registries, retry_with_backoff, }; #[derive(Debug, thiserror::Error)] @@ -52,6 +52,16 @@ pub trait CandleSink { &self, rows: &[PoolRegistryRow], ) -> impl Future> + Send; + + /// Persist Soroban token decimals a run resolved over RPC (task 0329). A + /// no-op by default: a sink that drops them only costs an RPC call per + /// token at the next cold start, because the registry keeps them warm. + fn write_decimals( + &self, + _rows: &[DecimalsRow], + ) -> impl Future> + Send { + async { Ok(()) } + } } /// ClickHouse sink backed by the shared [`OhlcvWriter`]. Works against either a @@ -101,7 +111,14 @@ impl ClickHouseSink { /// identity (task 0139), so live and backfill ids agree by construction. pub async fn load_registry(&self) -> Result { let existing = self.writer.load_assets().await.map_err(redact)?; - Ok(AssetRegistry::from_existing(existing)) + let mut registry = AssetRegistry::from_existing(existing); + // Task 0329: the AMM path scales a pure Soroban token's legs by its + // `decimals()`; loading the resolved ones here saves an RPC call each. + self.writer + .load_decimals(&mut registry) + .await + .map_err(redact)?; + Ok(registry) } /// Preload the discovered AMM pool registry from `prices.pool_registry` so the @@ -162,6 +179,17 @@ impl CandleSink for ClickHouseSink { .await .map(|_| ()) } + + async fn write_decimals(&self, rows: &[DecimalsRow]) -> Result<(), SinkError> { + // Idempotent (RMT on contract_address) → retried like the other writes. + retry_with_backoff( + &DEFAULT_BACKOFF_MS, + |_| true, + || async { self.writer.write_decimals(rows).await.map_err(redact) }, + ) + .await + .map(|_| ()) + } } /// Map an ingest error into a sink error. `IngestError`'s `Display` is already diff --git a/packages/sdex-backfill/src/ingest.rs b/packages/sdex-backfill/src/ingest.rs index 3ce7c2f9..1f5d5a6f 100644 --- a/packages/sdex-backfill/src/ingest.rs +++ b/packages/sdex-backfill/src/ingest.rs @@ -5,8 +5,9 @@ use std::time::{Duration, Instant}; use tracing::{info, warn}; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, OhlcvCandle, Registries, TradeTick, UnresolvedPoolSwap, - extract_trades, ledger_sequence, offer_lookup_counts, process_ledger, raw_trade_to_tick, + AssetRegistry, CandleAccumulator, DecimalsResolver, OhlcvCandle, Registries, TradeTick, + UnresolvedPoolSwap, extract_trades, ledger_sequence, offer_lookup_counts, process_ledger, + raw_trade_to_tick, }; use crate::error::BackfillError; @@ -333,6 +334,7 @@ pub async fn index_partition( completed: &HashSet, registry: &mut AssetRegistry, reg: &mut Registries, + decimals: &mut DecimalsResolver, mode: ExtractMode, accs: &mut RunAccumulators, end: PartitionEnd, @@ -401,7 +403,20 @@ pub async fn index_partition( // Only in Combined mode — pre-Soroban ledgers carry no Soroban // events, so SdexOnly skips the decode entirely. if mode == ExtractMode::Combined { - let sob = process_ledger(lcm, reg, registry); + let mut sob = process_ledger(lcm, reg, registry); + // Task 0329: a trade naming a token whose decimals are unknown + // was dropped, not guessed. Resolve, persist, decode again — + // the same step as the live processor. + if !sob.missing_decimals.is_empty() { + let rows = decimals.resolve(&sob.missing_decimals).await; + if !rows.is_empty() { + sink.write_decimals(&rows).await?; + for row in &rows { + row.record(registry); + } + sob = process_ledger(lcm, reg, registry); + } + } for (source, tick) in sob.amm_ticks { accs.merge_amm(source, tick); stats.amm_ticks += 1; diff --git a/packages/sdex-backfill/src/run.rs b/packages/sdex-backfill/src/run.rs index 6aef8925..69203f93 100644 --- a/packages/sdex-backfill/src/run.rs +++ b/packages/sdex-backfill/src/run.rs @@ -5,7 +5,9 @@ use tokio::process::Command; use tokio::task::JoinHandle; use tracing::{info, warn}; -use prices_ingest_core::{AssetRegistry, Registries, UnresolvedPool, UnresolvedPoolSwap}; +use prices_ingest_core::{ + AssetRegistry, DecimalsResolver, Registries, UnresolvedPool, UnresolvedPoolSwap, +}; use crate::error::BackfillError; use crate::ingest::{ @@ -146,6 +148,12 @@ pub async fn execute( let existing_assets = sink.load_assets().await?; let mut registry = AssetRegistry::from_existing(existing_assets); + // Task 0329: only the AMM path scales by token decimals, so an SDEX-only + // run neither reads `prices.asset_decimals` nor needs it to exist. + if mode == ExtractMode::Combined { + sink.load_decimals(&mut registry).await?; + } + let mut decimals = DecimalsResolver::from_env(); // Venue / pool registries. Preloaded from the persisted `pool_registry` // artifact (decision #4) so a window starting after activation still // resolves earlier-created pools; empty on a fresh full run. Then grown @@ -193,6 +201,7 @@ pub async fn execute( &completed, &mut registry, &mut reg, + &mut decimals, mode, &mut accs, partition_end, diff --git a/packages/sdex-backfill/src/sink.rs b/packages/sdex-backfill/src/sink.rs index 0c2cf1bf..f9bd099a 100644 --- a/packages/sdex-backfill/src/sink.rs +++ b/packages/sdex-backfill/src/sink.rs @@ -13,8 +13,8 @@ use std::collections::HashSet; use clickhouse::Row; use prices_ingest_core::canonical::AssetIdentity; use prices_ingest_core::{ - AssetRegistry, DEFAULT_BACKOFF_MS, OhlcvCandle, OhlcvWriter, Registries, UnresolvedPool, - retry_with_backoff, + AssetRegistry, DEFAULT_BACKOFF_MS, DecimalsRow, OhlcvCandle, OhlcvWriter, Registries, + UnresolvedPool, retry_with_backoff, }; use serde::{Deserialize, Serialize}; use tracing::info; @@ -72,6 +72,11 @@ impl Sink { Ok(self.writer.load_assets().await?) } + /// Resolved Soroban token decimals into `registry` (task 0329). + pub async fn load_decimals(&self, registry: &mut AssetRegistry) -> Result<(), BackfillError> { + Ok(self.writer.load_decimals(registry).await?) + } + // All `prices.*` writes below are idempotent (ReplacingMergeTree keyed by // `version`), so a retried INSERT can only replace, never duplicate. That // lets the sink retry every failure as transient (`|_| true`) — a bounded @@ -116,6 +121,11 @@ impl Sink { .await } + pub async fn write_decimals(&self, rows: &[DecimalsRow]) -> Result<(), BackfillError> { + self.retry_write(|| async { self.writer.write_decimals(rows).await }) + .await + } + pub async fn write_unresolved_pools( &self, pools: &[UnresolvedPool], From ad54c29c954fc5b97250fd8bdc4e40136fe784a9 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:15:07 +0200 Subject: [PATCH 025/144] docs(lore-0329): document asset_decimals and the decimals summary line Schema overview gains section 3.10b, amm-trades-schema closes its open decimals question, and the 0286 re-ingest runbook explains the new trades dropped (decimals) line events-backfill prints. --- docs/database-schema/amm-trades-schema.md | 11 ++- .../database-schema-overview.md | 68 ++++++++++++++++--- docs/runbooks/0286-reingest-history.md | 15 +++- docs/runbooks/running-ingestion-components.md | 20 +++--- 4 files changed, 86 insertions(+), 28 deletions(-) diff --git a/docs/database-schema/amm-trades-schema.md b/docs/database-schema/amm-trades-schema.md index 822fa346..f1aa4e09 100644 --- a/docs/database-schema/amm-trades-schema.md +++ b/docs/database-schema/amm-trades-schema.md @@ -466,12 +466,11 @@ references. The remaining item is still pending BE input. `Symbol("swap")` emitters outside the three venues are dropped via a strict allowlist (no `venue: unknown` bucket) — see lore 0005. -3. **Decimals normalisation.** `amount_in` and `amount_out` are stored - as `NUMERIC(28,14)`. Confirm whether the BE Ledger Processor will - divide raw on-chain amounts by each token's `decimals()` before - insertion, or write the raw integer scalar. The Prices API expects - the **decimal-normalised** value (matching how it stores quantities - throughout `price_ohlcv` and `current_prices`). _Still open._ +3. **~~Decimals normalisation.~~** **Resolved** by lore 0329. The Prices + ingest normalises raw on-chain amounts itself: classic assets and their + SACs at 7, a pure Soroban token at its `decimals()`, read once over RPC + and kept in `prices.asset_decimals`. Until then it assumed 7 for every + token, which priced non-7-decimal tokens off by 10^(7 − decimals). --- diff --git a/docs/database-schema/database-schema-overview.md b/docs/database-schema/database-schema-overview.md index d5738acd..eac690f9 100644 --- a/docs/database-schema/database-schema-overview.md +++ b/docs/database-schema/database-schema-overview.md @@ -11,6 +11,7 @@ | Date | Sections | Driver | Summary | | ---------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-10-06 | §2, §3.0 note, §3.10b (new), §5, §13, App. A | [Task 0329](../../lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md) | **Added §3.10b `asset_decimals`**: each pure Soroban token's `decimals()`, by which the ingest now scales that token's AMM swap legs. Before it, every leg was scaled at a fixed 7 and every non-7-decimal token priced off by 10^(7 − decimals). | | 2026-10-01 | §3.1, §3.2, §3.3, §3.4, §3.9, §3.10, App. A | [Task 0139](../../lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md) | **`asset_id` is derived by ClickHouse from the identity** — `xxh3(concat(asset_code, ':', issuer_address, ':', contract_address))`, `UInt64`, rendered once by `prices_clickhouse::asset_id`. `assets.asset_id` is `MATERIALIZED` (a writer naming it is refused); candle tiers and `oracle_prices` take the identity in `EPHEMERAL` columns and derive the ids as `DEFAULT` (the rollup MVs and enrichment still copy ids with `INSERT … SELECT`); `current_prices`, `asset_supply` and `asset_metadata` hold plain `UInt64`. A `CHECK` refuses 0 and the id of a blank identity; on `oracle_prices` a blank identity is the REDSTONE sentinel 0. Two identities can no longer share an id, which closes the fan-out of every `asset_id` join. | | 2026-09-21 | §3.0 note, §3.6, §3.8 (removed), §13, App. A | [Task 0256](../../lore/1-tasks/active/0256_BUG_asset-discovery-ledger-scan-never-runs.md) | **`prices.discovery_state` removed** together with asset-discovery's ledger scan, which never ran on production. §3.8 is kept as a numbered tombstone so §3.9 onwards do not shift; the table leaves the storage-engine list (§3.0), the §13 at-a-glance table, and App. A's index and ER diagram. `init.sql` no longer creates it — a database created earlier keeps the (always empty) table until an operator drops it. The Asset Discovery Lambda is no longer listed as a `pool_registry` writer (§3.6, §13): the live Ledger Processor maintains that table (task 0291). | | 2026-09-17 | §3.6 (`pool_registry`), App. A | [Task 0291](../../lore/1-tasks/active/0291_BUG_pool-registry-is-stale-since-the-backfill-ended.md) | **`pool_registry` has two new writers.** The live Ledger Processor persists pools it learns from factory events (new rows only), and `events-backfill --discover-pools` fills a range's missing pools. Corrected the Asset Discovery claim: its scan has never run on production (task 0256). | @@ -149,16 +150,16 @@ are pushed to the Hetzner cluster via separate post-backfill tools. ## 2. Database Tech Stack -| Component | Technology | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Database engine | **ClickHouse** on BE's shared Hetzner cluster (separate `prices` database, ADR 0007) | -| Storage engines | `ReplacingMergeTree(version)` for OHLCV and `unresolved_pools`; `ReplacingMergeTree(updated_at)` for `current_prices` / `assets` / `asset_metadata` / `backfill_progress` / `pool_registry`; `ReplacingMergeTree(fetched_at)` for `asset_supply` / `asset_symbol`; `ReplacingMergeTree(ledger)` for `ingest_cursor` (highest-ledger-wins, §3.12); bare `ReplacingMergeTree` for `oracle_prices` / `backfill_sdex_ledgers`; `ReplacingMergeTree(version)` for `usd_rate` | -| Rollups | Chain of CH materialised views: `price_ohlcv_1m → _15m → _1h → _4h → _1d → _1w → _1M` (replaces the OHLCV Rollup Lambda) | -| Partitioning | `PARTITION BY toYYYYMM(timestamp)` on every OHLCV/oracle table; cleanup via `ALTER TABLE … DROP PARTITION` | -| Database client (Rust) | [`clickhouse`](https://crates.io/crates/clickhouse) — async, native protocol over HTTPS-mTLS | -| Schema tooling | Plain SQL DDL applied by the prices-api schema applier on first deploy; prices-api owns `prices.*` migrations unilaterally (ADR 0007 §3.7) | -| Hosting | BE-managed Hetzner box behind Caddy:443; cross-cloud (AWS → Hetzner) hop, ~80–130 ms RTT mitigated by warm connection reuse and batched per-ledger writes | -| Credentials | AWS Secrets Manager — per-env client `{cert,key,ca}` as a single JSON bundle secret per identity (one secret per identity per env, named by `MTLS_SECRET_NAME`; ADR 0007 / task 0063) | +| Component | Technology | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Database engine | **ClickHouse** on BE's shared Hetzner cluster (separate `prices` database, ADR 0007) | +| Storage engines | `ReplacingMergeTree(version)` for OHLCV and `unresolved_pools`; `ReplacingMergeTree(updated_at)` for `current_prices` / `assets` / `asset_metadata` / `backfill_progress` / `pool_registry`; `ReplacingMergeTree(fetched_at)` for `asset_supply` / `asset_symbol` / `asset_decimals`; `ReplacingMergeTree(ledger)` for `ingest_cursor` (highest-ledger-wins, §3.12); bare `ReplacingMergeTree` for `oracle_prices` / `backfill_sdex_ledgers`; `ReplacingMergeTree(version)` for `usd_rate` | +| Rollups | Chain of CH materialised views: `price_ohlcv_1m → _15m → _1h → _4h → _1d → _1w → _1M` (replaces the OHLCV Rollup Lambda) | +| Partitioning | `PARTITION BY toYYYYMM(timestamp)` on every OHLCV/oracle table; cleanup via `ALTER TABLE … DROP PARTITION` | +| Database client (Rust) | [`clickhouse`](https://crates.io/crates/clickhouse) — async, native protocol over HTTPS-mTLS | +| Schema tooling | Plain SQL DDL applied by the prices-api schema applier on first deploy; prices-api owns `prices.*` migrations unilaterally (ADR 0007 §3.7) | +| Hosting | BE-managed Hetzner box behind Caddy:443; cross-cloud (AWS → Hetzner) hop, ~80–130 ms RTT mitigated by warm connection reuse and batched per-ledger writes | +| Credentials | AWS Secrets Manager — per-env client `{cert,key,ca}` as a single JSON bundle secret per identity (one secret per identity per env, named by `MTLS_SECRET_NAME`; ADR 0007 / task 0063) | **Why ClickHouse on a BE-shared cluster (ADR 0007):** @@ -313,7 +314,7 @@ erDiagram > The registry, bookkeeping, and enrichment side tables — `pool_registry` > (§3.6), `unresolved_pools` (§3.7), `asset_metadata` > (§3.9), `asset_supply` (§3.10), `asset_symbol` (§3.10a), -> `backfill_sdex_ledgers` (§3.11), and +> `asset_decimals` (§3.10b), `backfill_sdex_ledgers` (§3.11), and > `ingest_cursor` (§3.12) — are omitted here to keep the price path legible. > **Appendix A** carries every `prices.*` table. @@ -1481,6 +1482,40 @@ each of which rules out an easier home: --- +### 3.10b `prices.asset_decimals` — Soroban token decimals (task 0329) + +A swap event carries each leg as a raw i128 in its token's own decimals. Classic +assets and their SACs are 7-decimal by protocol and have no row here; a pure +Soroban token's decimals are whatever its SEP-41 `decimals()` returns. + +```sql +CREATE TABLE prices.asset_decimals ( + contract_address String, + decimals UInt8, + fetched_at DateTime DEFAULT now() -- RMT version +) +ENGINE = ReplacingMergeTree(fetched_at) +ORDER BY (contract_address) +SETTINGS index_granularity = 8192; +``` + +**Written by:** the ingest itself: the live ledger processor, `events-backfill`, +and `sdex-backfill` in `combined` mode. The first time a swap names a token with +no row, the decode drops that trade rather than guessing a scale. The caller +asks `decimals()` over Soroban RPC, writes the row, and decodes the ledger +again. **Read by:** the same three, in full, at startup. + +> **Only answers are stored.** A token whose call failed has no row and is asked +> again after ten minutes. Until then its trades are not priced, and +> `events-backfill` counts them in its `trades dropped (decimals)` summary line. +> Rows are never refreshed, because decimals are fixed at deploy. + +> ⚠️ **Deploy order.** The live processor's cold start reads this table and +> fails init if it is absent, so apply `init.sql` before deploying an ingest +> built with task 0329. + +--- + ### 3.11 `prices.backfill_sdex_ledgers` — Per-ledger done-marks One row per processed ledger sequence — the resume source for the SDEX stream. @@ -1658,6 +1693,7 @@ data by orders of magnitude without the per-row cost of B-tree indexes. | `prices.asset_metadata` | `(asset_id)` | — (small table, no partitioning) | One row per asset; enrichment `LEFT JOIN` target | | `prices.asset_supply` | `(asset_id)` | — (small table, no partitioning) | One row per asset; supply `LEFT JOIN` for `market_cap_usd` | | `prices.asset_symbol` | `(contract_address)` | — (small table, no partitioning) | One row per Soroban contract; symbol `LEFT JOIN` for the displayed `asset_code` | +| `prices.asset_decimals` | `(contract_address)` | — (small table, no partitioning) | One row per Soroban token; read in full at ingest startup | | `prices.backfill_sdex_ledgers` | `(sequence)` | — | Set-membership probe: "is this ledger already done?" | | `prices.ingest_cursor` | `(id)` | — | One row per consumer; live resume point | @@ -2362,6 +2398,7 @@ criteria from the delivery plan, restated against the canonical | `prices.asset_metadata` | `ReplacingMergeTree(updated_at)` | none | `(asset_id)` | Discovery/enrichment worker (sole writer) | `GET /assets`, `GET /assets/{id}` — API queries `LEFT JOIN` it on `asset_id` (no view reads it) | | `prices.asset_supply` | `ReplacingMergeTree(fetched_at)` | none | `(asset_id)` | Supply worker (sole writer, hourly) | `current_prices` path, via `LEFT JOIN` for `market_cap_usd` | | `prices.asset_symbol` | `ReplacingMergeTree(fetched_at)` | none | `(contract_address)` | Asset Discovery worker, symbol stage (sole writer, hourly) | `GET /assets`, `GET /assets/{id}` — composed into `asset_code` at read time (§3.10a) | +| `prices.asset_decimals` | `ReplacingMergeTree(fetched_at)` | none | `(contract_address)` | the ingest (live processor, `events-backfill`, `sdex-backfill` combined) on first sight of a token | the same writers, at startup — scales AMM legs (§3.10b) | | `prices.backfill_sdex_ledgers` | `ReplacingMergeTree()` | none | `(sequence)` | SDEX backfill stream — one row per processed ledger | SDEX backfill at startup (skip already-done ledgers) | | `prices.ingest_cursor` | `ReplacingMergeTree(ledger)` | none | `(id)` | Ledger Processor reconcile loop (written last each run) | Ledger Processor at cold start | | `prices.price_usd_series` / `_1h`, `usd_reference` / `_1h`, `identity_by_contract` | `VIEW` (plain, derived) | none (read-through) | n/a (defined over `price_ohlcv_1d` / `_1h`) | n/a — derived at read time from `close_usd` / `close` on the OHLCV tables (task 0061) | BE historical USD close series (BE task 0199); `price_usd_at` endpoint (task 0040) | @@ -2399,6 +2436,7 @@ erDiagram assets ||--o| asset_metadata : "asset_id (logical, 1:1 enrichment)" assets ||--o| asset_supply : "asset_id (logical, 1:1 supply)" assets ||--o| asset_symbol : "contract_address (logical, 1:1 soroban symbol)" + assets ||--o| asset_decimals : "contract_address (logical, 1:1 soroban decimals)" price_ohlcv_1m ||--o{ price_ohlcv_15m : "MV: 1m → 15m" price_ohlcv_15m ||--o{ price_ohlcv_1h : "MV: 15m → 1h" price_ohlcv_1h ||--o{ price_ohlcv_4h : "MV: 1h → 4h" @@ -2571,6 +2609,14 @@ erDiagram ORDER_BY sort_key "(contract_address)" } + asset_decimals { + String contract_address PK "logical FK to assets.contract_address" + UInt8 decimals "SEP-41 decimals(); only answers are stored" + DateTime fetched_at "DEFAULT now() — RMT version column" + ENGINE engine "ReplacingMergeTree(fetched_at)" + ORDER_BY sort_key "(contract_address)" + } + backfill_sdex_ledgers { UInt32 sequence PK "processed ledger; set-membership marker" ENGINE engine "ReplacingMergeTree()" diff --git a/docs/runbooks/0286-reingest-history.md b/docs/runbooks/0286-reingest-history.md index e7757d01..f7a556f6 100644 --- a/docs/runbooks/0286-reingest-history.md +++ b/docs/runbooks/0286-reingest-history.md @@ -447,7 +447,7 @@ month cannot be silently ordered the old way any more. What survives is the value check. `application_order` is `Int16`, and a negative value is not a position — `resolve_transaction_index` degrades it to 0 and counts -it. The run's last summary line reports it: +it. The run's summary reports it: ``` negative apply order: 0 @@ -457,6 +457,19 @@ negative apply order: 0 repaired** — do not record it as done. The run also emits one WARN, once, naming the first ledger and `application_order` it saw. +Since task 0329 the summary ends with one more line: + +``` +trades dropped (decimals): 0 +``` + +A pure Soroban token's legs are scaled by its `decimals()`, read over Soroban +RPC the first time the run meets it and kept in `prices.asset_decimals`. +**Anything but `0` means a token did not resolve and its trades are NOT in the +month's candles.** The run WARNs naming the contract. Re-run the month once it +resolves. The binary needs outbound HTTPS to `SOROBAN_RPC_URL` (default +`https://mainnet.sorobanrpc.com`), and `prices.asset_decimals` must exist. + --- ## 7. After the last month: 1w, 1M, and the re-enrichment diff --git a/docs/runbooks/running-ingestion-components.md b/docs/runbooks/running-ingestion-components.md index 0081977c..d6180154 100644 --- a/docs/runbooks/running-ingestion-components.md +++ b/docs/runbooks/running-ingestion-components.md @@ -45,16 +45,16 @@ seam section. ## Component summary -| Component | Kind | Writes | How it runs | Built? | -| ------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | ------------ | -| `sdex-backfill` | operator CLI | `price_ohlcv_1m` (`sdex` + AMM sources), `assets`, `oracle_prices`, `backfill_progress`, `pool_registry`, `unresolved_pools`, `backfill_sdex_ledgers` | manual, `--mode`/`--start`/`--end` | ✅ | -| `prices-ledger-processor` | Lambda / fixture CLI | `price_ohlcv_1m` (all sources), `oracle_prices`, `assets` | SQS doorbell (prod) or `--cursor` (local) | ✅ | -| `enrichment-worker` | Lambda / CLI | `price_ohlcv_1m` (`_usd` cols) | scheduled (prod) or CLI (local) | ✅ prototype | -| `asset-discovery` | scheduled Lambda | `assets`, `asset_symbol` | EventBridge `rate(1h)` | ✅ | -| `oracle-worker` | scheduled Lambda | `oracle_prices` | EventBridge schedule | ✅ | -| `supply-worker` | scheduled Lambda | `asset_supply` | EventBridge schedule | ✅ | -| `cleanup-worker` | scheduled Lambda | TTL/partition maintenance | EventBridge schedule | ✅ | -| `prices-api` | HTTP Lambda | reads only | API Gateway (prod) or `serve` bin (local) | ✅ | +| Component | Kind | Writes | How it runs | Built? | +| ------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | ------------ | +| `sdex-backfill` | operator CLI | `price_ohlcv_1m` (`sdex` + AMM sources), `assets`, `asset_decimals`, `oracle_prices`, `backfill_progress`, `pool_registry`, `unresolved_pools`, `backfill_sdex_ledgers` | manual, `--mode`/`--start`/`--end` | ✅ | +| `prices-ledger-processor` | Lambda / fixture CLI | `price_ohlcv_1m` (all sources), `oracle_prices`, `assets`, `asset_decimals` | SQS doorbell (prod) or `--cursor` (local) | ✅ | +| `enrichment-worker` | Lambda / CLI | `price_ohlcv_1m` (`_usd` cols) | scheduled (prod) or CLI (local) | ✅ prototype | +| `asset-discovery` | scheduled Lambda | `assets`, `asset_symbol` | EventBridge `rate(1h)` | ✅ | +| `oracle-worker` | scheduled Lambda | `oracle_prices` | EventBridge schedule | ✅ | +| `supply-worker` | scheduled Lambda | `asset_supply` | EventBridge schedule | ✅ | +| `cleanup-worker` | scheduled Lambda | TTL/partition maintenance | EventBridge schedule | ✅ | +| `prices-api` | HTTP Lambda | reads only | API Gateway (prod) or `serve` bin (local) | ✅ | --- From 3ce7b0cd8157544e559e881dcef9c72a441ea9db Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:17:36 +0200 Subject: [PATCH 026/144] docs(lore-0329): open and activate the 7-decimal soroban amount bug Reported by SBE: contract-token prices are off by 10^(7 - decimals). All seven contract tokens the API lists have decimals != 7. Design: SEP-41 decimals() over RPC, resolved by the ingest on a cache miss. Was committed as 0328, which another task took while this sat unpushed. --- ...ban-token-amounts-assume-seven-decimals.md | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md diff --git a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md new file mode 100644 index 00000000..157f0084 --- /dev/null +++ b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md @@ -0,0 +1,154 @@ +--- +id: "0329" +title: "Soroban AMM amounts are scaled as if every token had 7 decimals — every contract token's price is off by 10^(7 − decimals)" +type: BUG +status: active +assignee: stkrolikiewicz +related_adr: [] +related_tasks: ["0286", "0242", "0210", "0097", "0060", "0048", "0018"] +tags: [layer-backend, priority-high, effort-medium, ingest, soroban, data-correctness, defect] +links: + - "../../../packages/prices-ingest-core/src/soroban.rs" + - "../../../packages/asset-discovery/src/symbols.rs" + - "../../../docs/database-schema/amm-trades-schema.md" +history: + - date: "2026-10-06" + status: active + who: stkrolikiewicz + note: > + Renumbered from 0328. This task was committed as 0328 but its push did + not land, and another 0328 (board page assignee) was pushed and merged + meanwhile. An unpushed task reserves no ID. + - date: "2026-10-06" + status: active + who: stkrolikiewicz + note: > + Activated for implementation with the proposed design accepted: + SEP-41 decimals() over RPC, resolved by the ingest on a cache miss, + persisted in prices.asset_decimals, never defaulting to 7. + - date: "2026-10-06" + status: backlog + who: stkrolikiewicz + note: > + Reported by Karol (SBE) on 2026-10-05: SolvBTC, XAUM and XRP priced at + 10^(7 − decimals) of market. SBE holds off pricing pools that contain + these tokens until we fix it. Root cause confirmed in code; decimals of + all seven contract tokens the API lists were read on mainnet, and none + is 7. Design for where decimals come from proposed below, not yet + decided. +--- + +# Soroban AMM amounts are scaled as if every token had 7 decimals + +## Summary + +`amm_trade_to_tick` turns both raw i128 legs of every Soroban swap into +`Decimal` with one fixed scale, `AMM_AMOUNT_SCALE = 7` +([soroban.rs:37](../../../packages/prices-ingest-core/src/soroban.rs), used at +:859–860). Seven is right for a SAC (classic assets are 7-decimal by protocol) +and wrong for a pure contract token, which has its own `decimals()`. Against a +7-decimal counter-leg the stored price is `true × 10^(7 − d)`. The constant came +in with 0060's sizing backfill as a "documented sizing-measurement +approximation" and never got replaced, though 0018 and 0048 §7.2 had both +specified reading `decimals()`. `amm-trades-schema.md` §3 still lists decimals +normalisation as "_Still open._" + +## Evidence (2026-10-05) + +`/v1/assets?type=soroban&min_volume_usd=0` lists 7 assets. Decimals were read +with `stellar contract invoke --network mainnet --send no -- decimals`. + +| Token | Contract | decimals | Our price | × 10^(d−7) | Corrected | +|---|---|---|---|---|---| +| SolvBTC | `CBIJBDNZ…M6VN` | 8 | 8 580.10 | ×10 | ~85 800 | +| XAUM | `CC2RBGYN…VAGO` | 9 | 41.92 | ×100 | ~4 192 | +| XRP | `CB7OOP3V…37XP` | 6 | 14.78 | ÷10 | ~1.48 | +| deJTRSY | `CBI7UCH5…IHRV` | 18 | 1.026e-11 | ×10¹¹ | ~1.03 | +| deJAAA | `CC64WBDG…YGSL` | 18 | 1.05e-11 | ×10¹¹ | ~1.05 | +| BnUSD | `CCT4ZYIY…X7KP` | 18 | 9.98e-12 | ×10¹¹ | ~0.998 | +| xSolvBTC | `CAUP7NFA…772J` | 8 | unpriced | — | — | + +**Not proven:** that the other leg in every priced pool is 7-decimal. This is +inferred only from the corrected prices matching the market. + +## Impact + +- OHLC, `close_usd` and therefore `price_usd_series*` (views over the candle + tables) are wrong for every contract-token asset, over its whole history. +- Base volume in token units is inflated by 10^(d − 7). USD volume is right, + because the price and quantity errors cancel. +- An asset quoted *in* a contract token gets a correct USD price (the two + errors multiply out), but its price in that token's units is wrong. +- 18-decimal tokens: a price near 1e-11 keeps only 3–4 significant digits in + `Decimal(38,14)`. A token under about $0.001 falls below 1e-14 and its fills + are classified non-price-forming (`price_survives_column_scale`). Rescaling + stored rows can recover neither case, so history needs a re-ingest, not an + `UPDATE`. +- `price_forming_i128` classifies the raw integer legs before scaling, so it + is unaffected. + +## Design: where decimals come from (accepted 2026-10-06) + +**Source: the token's SEP-41 `decimals()`, read once via RPC `simulateTransaction`.** +This is the only universal source. Swap events carry raw i128 only. Instance +storage layout depends on the implementation (soroban-token-sdk `METADATA`, +OpenZeppelin, custom tokens). The live ingest also never sees an old token's +instance entry in ledger meta. A SAC is 7 without a call. A SAC we still hold +as a Contract identity ([[0242]]) answers 7 over RPC anyway, so it needs no +special case. Decimals are fixed at deploy for every real token, which is the +same "absence, not staleness" rule [[0210]] applies to `prices.asset_symbol`: +fetch once, never refresh. + +**Store:** a new `prices.asset_decimals (contract_address, decimals, fetched_at)`, +ReplacingMergeTree. Live ingest and backfills read the same value. + +**Resolve in the ingest on a cache miss, not in asset-discovery.** The price is +fixed at write time, and asset-discovery runs hourly, so a new token's first +swaps would land before its decimals do. That leaves a guessed price or a hole +to re-price by hand for every new token. Load the table at run start (it holds +tens of rows). A miss costs one RPC call with the existing 5 s timeout and is +persisted for every later run. Reuse `build_simulate_envelope` from +asset-discovery, which is already generic over the function name, by moving it +into a shared crate. Lambdas have egress (no VPC), and `SOROBAN_RPC_URL` is +already the convention. + +**On failure, or decimals > 28 (rust_decimal's maximum scale): never fall back +to 7.** Drop the tick, record it next to `prices.unresolved_pools`, and repair +with `events-backfill` once the token resolves. A wrong price is worse than a +missing one. + +Rejected: +- **Hard-coded map of the 7 tokens.** Every new token is mispriced or unpriced + until a PR lands. Since the RPC code exists, the map is barely smaller. +- **Asset-discovery only.** Leaves an hourly gap and a manual re-price per new + token. +- **Instance storage from ledger meta.** Layout varies by implementation, and + old tokens never appear in the meta. + +## Plan + +1. Shared simulate helper and a `decimals()` resolver with the three-way + outcome from `symbols.rs`. +2. `prices.asset_decimals` DDL. +3. `amm_trade_to_tick` scales each leg by its own decimals: SAC = 7, unresolved + → drop and record. This is the only production call site of the scale. +4. Backfills (`events-backfill` and the soroban path of `sdex-backfill`) share + the same resolver. +5. **History rides [[0286]] phase 3 if timing allows.** Stages B/C re-ingest the + AMM era through `events-backfill`. If this fix is on `fishuser-hero` before + stage C reaches the months these tokens traded, nothing extra runs. + Otherwise, a targeted `events-backfill` plus pre-roll over those pools' + range. Coordinate with the phase-3 operator. +6. Tell Karol (SBE) when prices are correct. + +## Acceptance Criteria + +- [ ] Unit test: an 8-, 18- and 6-decimal leg against a 7-decimal leg gives the + true price and the true base volume. +- [ ] A token whose decimals are unresolved never produces a price. +- [ ] After deploy, all six priced tokens are within a few percent of an + external reference. +- [ ] History of the affected assets re-ingested; `price_usd_series*` right over + the full range, with significant digits restored for the 18-decimal + tokens. +- [ ] `amm-trades-schema.md` §3 closed; the `AMM_AMOUNT_SCALE` constant removed. From 02921dacc10de6f528ae3b8a34a9ec5789f6eccf Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:20:12 +0200 Subject: [PATCH 027/144] docs(lore-0329): record implementation, decisions and deploy order --- ...ban-token-amounts-assume-seven-decimals.md | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md index 157f0084..692a86d3 100644 --- a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md +++ b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md @@ -141,14 +141,66 @@ Rejected: range. Coordinate with the phase-3 operator. 6. Tell Karol (SBE) when prices are correct. +## Implementation (branch `fix/0329_soroban-token-amounts-assume-seven-decimals`) + +- `prices_ingest_core::soroban_rpc`: the `simulateTransaction` envelope and the + Absent/Transient boundary, moved from asset-discovery's `symbols.rs`, which now + delegates to it and re-exports the old names. +- `AssetRegistry::decimals_of`: classic identities and SACs → 7, a `Contract` + → its resolved decimals or `None`. `amm_trade_to_tick` scales each leg by its + own decimals. An unknown leg yields no tick, interns nothing, and lands in + `LedgerSoroban::missing_decimals`. +- `prices.asset_decimals` + `OhlcvWriter::{load,write}_decimals` + + `DecimalsResolver` (`decimals()` must be a `U32` ≤ 28). +- Callers decode, resolve what was missing, persist it, record it, and decode + that ledger again: the live reconcile loop, `events-backfill`, and + `sdex-backfill` in `combined` mode. + +### Design decisions + +#### Emerged + +1. **No `unresolved_pools`-style record for unresolved tokens.** That table is + keyed by pool. The resolver WARNs once per contract per retry window, and + `events-backfill` prints `trades dropped (decimals): N`, which the 0286 + runbook now tells the operator to read. The live path has no metric or alarm + for it yet. +2. **A failed contract is retried after 10 minutes, in memory.** Absent and + Transient are treated alike, with no `attempts` column. That costs one RPC + call per such token per 10 minutes, and stops a token that trades every + ledger from costing a 5 s timeout per ledger. +3. **Decode again rather than a two-phase tick.** Decoding is deterministic and + the registry inserts are idempotent map writes, so the first result is + discarded whole. +4. **`CandleSink::write_decimals` defaults to a no-op.** Only `ClickHouseSink` + persists. The test sinks need no change. +5. **`sdex-backfill` reads the table only in `combined` mode.** The 0286 phase-3 + `sdex-only` runs therefore do not need the table to exist. +6. **An `events-backfill` dry-run resolves but does not persist.** + +### Deploy order + +1. Apply `init.sql` on the box: it creates `prices.asset_decimals`. The live + processor's init fails without it. +2. Optional pre-seed, so the first cold start needs no RPC call (the Lambda has + a 60 s budget): + `INSERT INTO prices.asset_decimals (contract_address, decimals) VALUES ('CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN', 8), ('CBI7UCH5KGSVQRO5H4SUCZUTZABCITZLRHQQZTWL2TK4RZ72TAR6IHRV', 18), ('CB7OOP3VSAWBZOOTOG2YEFANVU45GVWYUUM5HI32DKLHVKUDOFVQ37XP', 6), ('CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO', 9), ('CC64WBDGS6QQP22QTTIACYIXT3WF7BBQEYOQPLTP7GTKYY7PZ74QYGSL', 18), ('CCT4ZYIYZ3TUO2AWQFEOFGBZ6HQP3GW5TA37CK7CRZVFRDXYTHTYX7KP', 18), ('CAUP7NFABXE5TJRL3FKTPMWRLC7IAXYDCTHQRFSCLR5TMGKHOOQO772J', 8)` +3. Deploy the ledger processor. +4. Repair history right away. Until it is repaired, `vwap_24h` blends old and + new candles for 24 h, and a `carried` price keeps the old candle. Run + `events-backfill` + pre-roll at least over a recent window, and the full + range via 0286 phase 3 if stage C has not passed these months yet. + ## Acceptance Criteria -- [ ] Unit test: an 8-, 18- and 6-decimal leg against a 7-decimal leg gives the - true price and the true base volume. -- [ ] A token whose decimals are unresolved never produces a price. +- [x] Unit test: an 8-, 18- and 6-decimal leg against a 7-decimal leg gives the + true price and the true base volume + (`each_leg_is_scaled_by_its_own_tokens_decimals`). +- [x] A token whose decimals are unresolved never produces a price + (`a_leg_with_unknown_decimals_prices_nothing_and_is_reported`). - [ ] After deploy, all six priced tokens are within a few percent of an external reference. - [ ] History of the affected assets re-ingested; `price_usd_series*` right over the full range, with significant digits restored for the 18-decimal tokens. -- [ ] `amm-trades-schema.md` §3 closed; the `AMM_AMOUNT_SCALE` constant removed. +- [x] `amm-trades-schema.md` §3 closed; the `AMM_AMOUNT_SCALE` constant removed. From d0c668c1cfb08d5b2ef3833ee559fa1c6c0192d3 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:46:46 +0200 Subject: [PATCH 028/144] docs(lore-0236): unblock and record the pre-deploy check The deploy date has passed. On prod the detector's own queries read 0 on all seven tiers, so the legacy _1w/_1M buckets are out of the window. --- ...-for-internally-inconsistent-ohlcv-rows.md | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) rename lore/1-tasks/{blocked => active}/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md (82%) diff --git a/lore/1-tasks/blocked/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md b/lore/1-tasks/active/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md similarity index 82% rename from lore/1-tasks/blocked/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md rename to lore/1-tasks/active/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md index deac377d..29c0f118 100644 --- a/lore/1-tasks/blocked/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md +++ b/lore/1-tasks/active/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md @@ -2,7 +2,8 @@ id: "0236" title: "Nothing detects an internally inconsistent `price_ohlcv_*` row — and 0229's clamp removed the one surface that used to surface them" type: BUG -status: blocked +status: active +assignee: akot related_adr: ["0011"] related_tasks: ["0229", "0120", "0182", "0227"] tags: ["priority-medium", "effort-small", "data-correctness", "observability", "ohlcv", "milestone-M2"] @@ -11,7 +12,7 @@ links: - "../../../packages/prices-api/src/assets/queries_ch.rs" - "../../../tools/scripts/conformance-0120.mjs" history: - - date: 2026-08-28 + - date: "2026-08-28" status: backlog who: okarcz note: > @@ -28,6 +29,7 @@ history: - date: "2026-09-29" status: blocked who: akot + by: ["0286"] note: > Detector, alarm, ITs and the ADR-0011 s3 amendment are built on fix/0236_ohlc-band-detector (not yet merged). Blocked on the deploy date: @@ -36,6 +38,14 @@ history: inherit the legacy `low = 0` (measured 22 in `_1w`, 109 in `_1M`), so the alarm would fire on deploy with nothing new broken. No floor in code (Adam's decision); the window moves past those buckets by 2026-10-04. + - date: "2026-10-06" + status: active + who: akot + note: > + Unblocked: the deploy date passed. The detector's own queries read 0 on + all seven tiers on prod, so the alarm will not fire on legacy rows. The + branch is rebased on develop, alongside 0139's and 0203's probe checks, + as block 5c. PR to follow; deploy is Adam's. --- # No detector for an internally inconsistent stored candle @@ -126,6 +136,27 @@ bound** — superseded RMT versions are included. - The legacy zero-price rows are pre-0286 residue that [[0286]] phase 3 re-ingests; they are recorded here, not filed as a separate task. +## Pre-deploy check — measured on prod 2026-10-06 + +Read-only, `dev_read`, 10:11 UTC. These are the detector's own queries +(`ohlc_band_queries()`: `FINAL`, 2 days widened by one bucket): + +| tier | band | nonpositive | scanned | oldest bucket in window | +|---|---:|---:|---:|---| +| `_1m` | 0 | 0 | 662,241 | 2026-10-04 10:12 | +| `_15m` | 0 | 0 | 296,212 | 2026-10-04 10:00 | +| `_1h` | 0 | 0 | 178,303 | 2026-10-04 10:00 | +| `_4h` | 0 | 0 | 98,634 | 2026-10-04 08:00 | +| `_1d` | 0 | 0 | 39,812 | 2026-10-04 | +| `_1w` | 0 | 0 | 44,609 | 2026-09-28 | +| `_1M` | 0 | 0 | 26,016 | 2026-10-01 | + +- The legacy `_1w` 2026-09-21 and `_1M` 2026-09-01 buckets have left the + window, so the first run after deploy should read 0. +- Cost: the seven reads together take 0.29 s of server time and read 3.7 M + rows. That is why the check runs as block 5c, before the time-budgeted + mismatch reads (0203), which stay last. + ## Decisions (Adam, 2026-09-28) 1. **Predicate**: band violations **and** priced rows with any price ≤ 0 — the @@ -145,10 +176,11 @@ construction and checked with zero tolerance where it is checked at all. ## Acceptance Criteria -- [ ] The number of internally inconsistent rows per table is **measured on +- [x] The number of internally inconsistent rows per table is **measured on prod** and recorded, before any alarm is designed. -- [ ] If the count is non-zero, the cause is identified and filed as its own - task rather than absorbed here. +- [x] If the count is non-zero, the cause is identified and filed as its own + task rather than absorbed here. Band violations are 0. The legacy + zero-price rows are recorded above and left to [[0286]] phase 3. - [ ] A recurring check exists wherever the other data-quality probes live, with its threshold justified by the measured baseline. - [ ] 0229's clamp is explicitly confirmed as the right behaviour for the read From e4fbc3ce3a3dfbae22050e1739dddd91cff7dbeb Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:51:17 +0200 Subject: [PATCH 029/144] docs(lore-0328): complete and archive the task --- ..._board-page-ignores-the-computed-assignee.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) rename lore/1-tasks/{active => archive}/0328_BUG_board-page-ignores-the-computed-assignee.md (73%) diff --git a/lore/1-tasks/active/0328_BUG_board-page-ignores-the-computed-assignee.md b/lore/1-tasks/archive/0328_BUG_board-page-ignores-the-computed-assignee.md similarity index 73% rename from lore/1-tasks/active/0328_BUG_board-page-ignores-the-computed-assignee.md rename to lore/1-tasks/archive/0328_BUG_board-page-ignores-the-computed-assignee.md index 031118b0..0e476ad4 100644 --- a/lore/1-tasks/active/0328_BUG_board-page-ignores-the-computed-assignee.md +++ b/lore/1-tasks/archive/0328_BUG_board-page-ignores-the-computed-assignee.md @@ -2,7 +2,7 @@ id: "0328" title: "The board page recomputes the assignee from the last history entry and ignores the one board.json carries" type: BUG -status: active +status: completed assignee: akot related_adr: [] related_tasks: ["0327", "0149", "0041"] @@ -11,6 +11,14 @@ links: - "../../board.html" - "../../../tools/scripts/generate-lore-board.mjs" history: + - date: "2026-10-06" + status: completed + who: akot + note: > + PR #394 merged (d0399fd2) and deployed to GitHub Pages. getTaskAssignee + returns t.assignee; a test in generate-lore-board.test.mjs pins it + (14/14, red on the old page). The deployed board shows 0149, 0242 and + 0139 as akot. - date: "2026-10-06" status: active who: akot @@ -46,7 +54,8 @@ carried the right value since. The page never reads it. ## Acceptance Criteria -- [ ] `board.html` reads `assignee` from `board.json` and has no assignee +- [x] `board.html` reads `assignee` from `board.json` and has no assignee logic of its own. -- [ ] On the deployed board, 0149, 0242 and 0139 show `@akot`. -- [ ] The assignee filter counts match `board.json`'s `assignee` values. +- [x] On the deployed board, 0149, 0242 and 0139 show `@akot`. +- [x] The assignee filter counts match `board.json`'s `assignee` values. By + construction: the filter counts go through the same `getTaskAssignee`. From fc93474dfe07fbcfd08496c824b80c9892e00e9e Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:12:29 +0200 Subject: [PATCH 030/144] fix(lore-0242): keep live ingest running when BE's is_sac set is unreadable Failing Init on an unreadable default.soroban_contracts stopped SDEX, oracle and AMM ingest over a guard that only covers SAC trades with no in-band proof (0 of 338,601 legs). The container now ingests without candidates, warns, and publishes SacCandidatesUnavailable on every run, with an alarm on it. PC3 reversed by Adam after review. --- infra/src/lib/stacks/observability-stack.ts | 35 +++++++++++++++++++ packages/prices-ledger-processor/src/main.rs | 25 ++++++++++--- .../prices-ledger-processor/src/metrics.rs | 32 +++++++++++++++++ .../prices-ledger-processor/src/sink/mod.rs | 34 +++++++++++++++--- .../tests/sac_identity.rs | 28 +++++++++++++-- 5 files changed, 143 insertions(+), 11 deletions(-) diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index 7eb9df8d..8e1e9f0a 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -344,6 +344,11 @@ export class ObservabilityStack extends cdk.Stack { * event proved, rather than minting it as a second identity. */ public readonly ledgerProcessorSacUnprovenAlarm: cloudwatch.Alarm; + /** + * Task 0242: a live container started without BE's `is_sac` set, so it mints + * an unproven SAC as a `Contract` identity instead of skipping it. + */ + public readonly ledgerProcessorSacCandidatesAlarm: cloudwatch.Alarm; /** * Task 0100: the weekly coverage sweep found swap/trade emitters in neither * `pool_registry` nor the committed allow-list (layer 3). @@ -1511,6 +1516,36 @@ export class ObservabilityStack extends cdk.Stack { this.ledgerProcessorSacUnprovenAlarm.addAlarmAction(snsAction); this.ledgerProcessorSacUnprovenAlarm.addOkAction(snsAction); + // Task 0242 — the cold start could not read BE's `is_sac` set and the + // processor runs on without it rather than failing Init (review of 0242). + // Each run of such a container publishes one `SacCandidatesUnavailable`; + // nothing is published while the set is loaded. + this.ledgerProcessorSacCandidatesAlarm = new cloudwatch.Alarm( + this, + 'LedgerProcessorSacCandidatesAlarm', + { + alarmName: `prices-${config.envName}-ledger-processor-sac-candidates-unavailable`, + alarmDescription: + 'The ledger-processor cold-started without BE\'s is_sac contract set (default.soroban_contracts unreadable as prices_writer: grant, rename or a BE migration). Ingest continues, but until a cold start reads the set again an unproven SAC is minted as a Contract identity instead of being skipped (task 0242). The WARN "is_sac contract set unreadable" carries the error. Fix the read, then force a cold start; SacContractIdentities shows any identity minted meanwhile. Runbook: docs/runbooks/0242-sac-identity-heal.md.', + metric: new cloudwatch.Metric({ + namespace: 'Prices/Ingest', + metricName: 'SacCandidatesUnavailable', + dimensionsMap: { Environment: config.envName }, + statistic: 'Sum', + period: cdk.Duration.minutes(5), + }), + threshold: 1, + evaluationPeriods: 1, + datapointsToAlarm: 1, + comparisonOperator: + cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + // Emitted only while the set is missing: "missing" is healthy. + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + }, + ); + this.ledgerProcessorSacCandidatesAlarm.addAlarmAction(snsAction); + this.ledgerProcessorSacCandidatesAlarm.addOkAction(snsAction); + // Task 0100 — layer 3 of the coverage model. Layer 2 (the alarm above) // sees only pools shaped like a venue we already index; the weekly coverage // sweep looks at EVERY contract emitting swap/trade-shaped events over a diff --git a/packages/prices-ledger-processor/src/main.rs b/packages/prices-ledger-processor/src/main.rs index 8d6209fb..400e2905 100644 --- a/packages/prices-ledger-processor/src/main.rs +++ b/packages/prices-ledger-processor/src/main.rs @@ -80,8 +80,10 @@ async fn main() -> Result<(), Error> { // - load_ingest_registry: the identities already in `prices.assets`, armed // with the contracts BE flags `is_sac` in `default.soroban_contracts`, read // with prices_writer's SELECT on that table (task 0242). A flagged token no - // SAC proof resolves is skipped, not minted, so a failed read fails Init: an - // empty set would mint (D2). `tests/sac_identity.rs` pins this call. + // SAC proof resolves is skipped, not minted. A failed BE read does NOT fail + // Init: ingest runs without candidates and every invocation publishes + // `SacCandidatesUnavailable` until a cold start reads the set again. + // `tests/sac_identity.rs` pins this call. // - load_pool_registry: discovered AMM pool classification from // `prices.pool_registry` (task 0078). The processor writes back only the // pools it learns from factory events (task 0291; `prices_writer` holds @@ -97,7 +99,7 @@ async fn main() -> Result<(), Error> { sink.load_ingest_registry(), sink.load_pool_registry(), ); - let registry = registry?; + let (registry, sac_candidates_loaded) = registry?; let pool_registry = pool_registry?; // Seed only on a genuinely empty table (true first run); any OTHER read error @@ -189,7 +191,17 @@ async fn main() -> Result<(), Error> { let r = reconciler.clone(); let cw = cw.clone(); let env_name = env_name.clone(); - async move { handler(event, r, max_iterations, cw, env_name).await } + async move { + handler( + event, + r, + max_iterations, + cw, + env_name, + sac_candidates_loaded, + ) + .await + } })) .await } @@ -200,6 +212,7 @@ async fn handler( max_iterations: usize, cw: Arc, env_name: Arc, + sac_candidates_loaded: bool, ) -> Result { let (payload, _ctx) = event.into_parts(); let mut batch_item_failures = Vec::new(); @@ -242,6 +255,10 @@ async fn handler( )); // Task 0242 — trades skipped on a SAC no proof resolved. m.extend(metrics::sac_unproven_metrics(stats.sac_unproven_skipped)); + // Task 0242 — this container runs without BE's `is_sac` set. + m.extend(metrics::sac_candidates_unavailable_metrics( + !sac_candidates_loaded, + )); if let Err(e) = metrics::publish(&cw, &env_name, &m).await { warn!(error = %e, "cloudwatch metric publish failed (non-fatal)"); } diff --git a/packages/prices-ledger-processor/src/metrics.rs b/packages/prices-ledger-processor/src/metrics.rs index 712be106..49971a1b 100644 --- a/packages/prices-ledger-processor/src/metrics.rs +++ b/packages/prices-ledger-processor/src/metrics.rs @@ -72,6 +72,14 @@ pub const UNREGISTERED_POOL_EVENTS: &str = "UnregisteredPoolEvents"; /// asset in the same transaction, a shape the resolver does not cover. pub const SAC_UNPROVEN_SKIPPED: &str = "SacUnprovenSkipped"; +/// One per run of a container whose cold start could not read BE's `is_sac` +/// set (task 0242). Such a container mints an unproven SAC as a `Contract` +/// identity instead of skipping it, until its next cold start. +/// +/// Emitted ONLY while the set is missing, so the alarm on it is `>= 1` over +/// `NOT_BREACHING`. +pub const SAC_CANDIDATES_UNAVAILABLE: &str = "SacCandidatesUnavailable"; + /// `PutMetricData` accepts at most 150 entries in a datum's `Values` array, so /// a run with more INSERTs than that spills into further datums of the same /// metric rather than being truncated (or, worse, aggregated back into @@ -193,6 +201,19 @@ pub fn sac_unproven_metrics(total: u64) -> Vec { }] } +/// The missing-candidates datapoint for one run (task 0242), or nothing when +/// the cold start read BE's `is_sac` set. +pub fn sac_candidates_unavailable_metrics(unavailable: bool) -> Vec { + if !unavailable { + return Vec::new(); + } + vec![Metric { + name: SAC_CANDIDATES_UNAVAILABLE, + unit: Unit::Count, + values: vec![1.0], + }] +} + /// Publish `metrics` to CloudWatch under [`METRIC_NAMESPACE`], tagged with an /// `Environment` dimension. One `PutMetricData` call for the whole batch. /// @@ -285,6 +306,17 @@ mod tests { assert_eq!(m[0].values, vec![3.0]); } + /// Task 0242: absent while the set is loaded, one count per run otherwise. + #[test] + fn missing_sac_candidates_publish_one_per_run() { + assert!(sac_candidates_unavailable_metrics(false).is_empty()); + let m = sac_candidates_unavailable_metrics(true); + assert_eq!(m.len(), 1); + assert_eq!(m[0].name, SAC_CANDIDATES_UNAVAILABLE); + assert_eq!(m[0].unit, Unit::Count); + assert_eq!(m[0].values, vec![1.0]); + } + #[test] fn a_forced_run_publishes_exactly_one_count_of_one() { let m = forced_partial_flush_metrics(true); diff --git a/packages/prices-ledger-processor/src/sink/mod.rs b/packages/prices-ledger-processor/src/sink/mod.rs index 05e3f24a..f7013e8d 100644 --- a/packages/prices-ledger-processor/src/sink/mod.rs +++ b/packages/prices-ledger-processor/src/sink/mod.rs @@ -30,6 +30,30 @@ pub fn with_sac_candidates( registry } +/// [`with_sac_candidates`] when BE's `is_sac` set was read, else the registry +/// as loaded, with no candidates; the flag says which (task 0242 PC3, +/// reversed in review). An unreadable BE table must not stop live ingest: with +/// no candidates an unproven SAC mints a `Contract` identity, which the probe's +/// `SacContractIdentities` counts and the 0242 heal removes, while SDEX, oracle +/// and AMM candles keep flowing. The caller publishes +/// `SacCandidatesUnavailable` while the flag is false. +pub fn arm_sac_candidates( + registry: AssetRegistry, + sac_contracts: Result, SinkError>, +) -> (AssetRegistry, bool) { + match sac_contracts { + Ok(set) => (with_sac_candidates(registry, set), true), + Err(e) => { + tracing::warn!( + error = %e, + "is_sac contract set unreadable: no SAC candidates until the next cold start, \ + an unproven SAC mints a Contract identity (task 0242)" + ); + (registry, false) + } + } +} + #[derive(Debug, thiserror::Error)] pub enum SinkError { #[error("sink write failed: {0}")] @@ -132,15 +156,15 @@ impl ClickHouseSink { } /// The registry the live processor ingests with: [`load_registry`] armed - /// with BE's `is_sac` set by [`with_sac_candidates`], the two reads joined. - /// Either failing fails the cold start: with no candidates an unproven SAC - /// would be minted as a `Contract` identity (task 0242 D2, PC3). + /// with BE's `is_sac` set by [`arm_sac_candidates`], the two reads joined, + /// and whether the set was read. Only the `prices.assets` read fails the + /// cold start; an unreadable BE set is a warning and a metric (task 0242). /// /// [`load_registry`]: ClickHouseSink::load_registry - pub async fn load_ingest_registry(&self) -> Result { + pub async fn load_ingest_registry(&self) -> Result<(AssetRegistry, bool), SinkError> { let (registry, sac_contracts) = tokio::join!(self.load_registry(), self.load_sac_contracts()); - Ok(with_sac_candidates(registry?, sac_contracts?)) + Ok(arm_sac_candidates(registry?, sac_contracts)) } /// Preload the discovered AMM pool registry from `prices.pool_registry` so the diff --git a/packages/prices-ledger-processor/tests/sac_identity.rs b/packages/prices-ledger-processor/tests/sac_identity.rs index 2bb3e123..2c98e95d 100644 --- a/packages/prices-ledger-processor/tests/sac_identity.rs +++ b/packages/prices-ledger-processor/tests/sac_identity.rs @@ -23,7 +23,7 @@ use prices_ledger_processor::{ galexie_key::ledger_s3_key, object_fetcher::{FetchError, ObjectFetcher}, reconcile::Reconciler, - sink::{CandleSink, SinkError, with_sac_candidates}, + sink::{CandleSink, SinkError, arm_sac_candidates, with_sac_candidates}, }; use stellar_xdr::{ ContractEvent, ContractEventBody, ContractEventType, ContractEventV0, ContractId, Hash, @@ -197,7 +197,31 @@ fn the_lambda_cold_start_arms_the_sac_candidates() { "a registry without candidates" ); let sink = include_str!("../src/sink/mod.rs"); - assert!(sink.contains("Ok(with_sac_candidates(registry?, sac_contracts?))")); + assert!(sink.contains("Ok(arm_sac_candidates(registry?, sac_contracts))")); + assert!(main.contains("sac_candidates_unavailable_metrics(")); +} + +/// Review of 0242 (PC3 reversed): an unreadable `is_sac` set does not stop +/// ingest. The flag drops, the SAC set is empty, and an unproven SAC trades +/// as a `Contract` identity, which the probe counts and the heal removes. +#[test] +fn an_unreadable_sac_set_ingests_without_candidates() { + let (mut assets, loaded) = arm_sac_candidates( + AssetRegistry::from_existing(vec![]), + Err(SinkError::Write("Code: 60. Unknown table".to_string())), + ); + assert!(!loaded); + let out = run(vec![trade(XLM_SAC, XCR_SAC)], &mut assets); + assert!(out.sac_unproven.is_empty(), "not a candidate, not skipped"); + assert_eq!(out.amm_ticks.len(), 1, "the trade still prices"); + assert!(holds_contract(&assets, XCR_SAC), "minted, for the probe"); + + let (assets, loaded) = arm_sac_candidates( + AssetRegistry::from_existing(vec![]), + Ok(HashSet::from([XCR_SAC.to_string()])), + ); + assert!(loaded); + assert!(assets.is_unproven_sac(XCR_SAC)); } /// The fixture constants are pinned by our own derivation, not trusted. From f9899c2c829006fbede29242462424ef7713b3de Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:12:29 +0200 Subject: [PATCH 031/144] refactor(lore-0242): read the SAC alias through identity_by_contract One SAC-to-classic resolver on the read side, so a fix to the view reaches the API as well. --- packages/prices-api/src/assets/queries_ch.rs | 24 ++++++++++++-------- packages/prices-api/tests/endpoints_it.rs | 8 +++++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index 5f13c7aa..7927c678 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -510,7 +510,7 @@ fn resolve_asset_id_sql(where_sql: &str) -> String { // SAC alias (task 0242, D5) // ---------------------------------------------------------------------------- -/// A classic row named by its SAC address. Positional: field order is the +/// A classic identity named by its SAC address. Positional: field order is the /// [`sac_alias_sql`] SELECT order. #[derive(Debug, clickhouse::Row, serde::Deserialize)] struct SacAliasRow { @@ -519,14 +519,18 @@ struct SacAliasRow { issuer_address: String, } -/// The classic rows whose stored `sac_address` is one of `n` bound addresses. -/// Read from the column only: every classic row carries it (XLM's included). +/// The classic identities (`native` / `credit`) of `n` bound SAC addresses, +/// read through `identity_by_contract`, the one SAC→classic resolver of the +/// read side (views.sql), so a fix there reaches the API too. The view reads +/// the stored `sac_address`; the ingest writes it on every classic row (0 of +/// 211,044 classic rows without it on 2026-10-06), and an address cannot be +/// turned back into its classic, so a read cannot derive it. fn sac_alias_sql(n: usize) -> String { let binds = vec!["?"; n].join(", "); format!( - "SELECT a.sac_address, a.asset_code, a.issuer_address \ - FROM assets AS a FINAL \ - WHERE a.contract_address = '' AND a.sac_address IN ({binds})" + "SELECT contract AS sac_address, asset_code, issuer_address \ + FROM identity_by_contract \ + WHERE asset_kind != 'contract' AND contract IN ({binds})" ) } @@ -2456,14 +2460,14 @@ mod tests { /// Task 0242: classic rows only, one bind per address, positional order /// matching [`SacAliasRow`]. #[test] - fn sac_alias_sql_reads_classic_rows_by_stored_sac_address() { + fn sac_alias_sql_reads_classic_rows_through_identity_by_contract() { use clickhouse::Row; let sql = sac_alias_sql(2); for needle in [ - "FROM assets AS a FINAL", - "a.contract_address = ''", - "a.sac_address IN (?, ?)", + "FROM identity_by_contract", + "asset_kind != 'contract'", + "contract IN (?, ?)", ] { assert!(sql.contains(needle), "missing {needle}: {sql}"); } diff --git a/packages/prices-api/tests/endpoints_it.rs b/packages/prices-api/tests/endpoints_it.rs index ba9c8633..bbeb9ecb 100644 --- a/packages/prices-api/tests/endpoints_it.rs +++ b/packages/prices-api/tests/endpoints_it.rs @@ -49,6 +49,14 @@ async fn setup(db: &str) -> Client { prices_clickhouse::apply_sql(&admin, &rewrite(prices_clickhouse::INIT_SQL, db)) .await .unwrap(); + // The SAC alias reads this view (task 0242); the rest of views.sql needs + // tables these tests do not seed. + let view = prices_clickhouse::VIEWS_SQL + .split(';') + .find(|s| s.contains("CREATE OR REPLACE VIEW prices.identity_by_contract")) + .expect("views.sql defines identity_by_contract"); + let view = &view[view.find("CREATE OR REPLACE VIEW").unwrap()..]; + admin.query(&rewrite(view, db)).execute().await.unwrap(); let assets = [ AssetFixture::new("XLM", "native", "", "").with_sac(XLM_SAC), From 5c564d26941a6d0a23c13e7c9a7cf5983046ad9d Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:12:29 +0200 Subject: [PATCH 032/144] fix(lore-0242): note a pre-0242 events-backfill and stop when it comes back A month without the unproven-sac line is noted instead of passing silently. Once a month of the state has printed it, a month without it means the host binary regressed and the run stops. --- tools/scripts/reingest_0286.py | 28 ++++++++++++++++++++++--- tools/scripts/reingest_0286_sac_test.py | 27 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/tools/scripts/reingest_0286.py b/tools/scripts/reingest_0286.py index d2234114..816584fc 100755 --- a/tools/scripts/reingest_0286.py +++ b/tools/scripts/reingest_0286.py @@ -679,7 +679,7 @@ def require_transport_flag(a): UNPROVEN_SAC_LABEL = "unproven sac swaps:" -def amm_summary(text, logfile, *, dry_run): +def amm_summary(text, logfile, *, dry_run, unproven_line_seen=False): """Read events-backfill's closing summary, or Stop naming what is missing. Called after the --dry-run pass as well as the write, so a binary whose @@ -687,6 +687,11 @@ def amm_summary(text, logfile, *, dry_run): Python AttributeError after the month's AMM candles are already in. `negative apply order:` replaced `events with no apply order:` in task 0304; a binary without it predates that fix and cannot read BE's events anyway. + + `unproven sac swaps:` (task 0242) may be absent: a binary built before 0242 + prints none, and phase 3 runs on one by design, the 0242 seed resolving its + SACs (D4, PC6). Absent after an earlier month of this state printed it + (`unproven_line_seen`), the host binary went back to a pre-0242 build: Stop. """ found = {} for key, label in (("fallbacks", "negative apply order:"), ("dropped", "swaps dropped (unresolved):")): @@ -698,6 +703,11 @@ def amm_summary(text, logfile, *, dry_run): found[key] = int(hit.group(1)) hit = re.search(rf"^{re.escape(UNPROVEN_SAC_LABEL)}\s*(\d+)", text, re.M) found["unproven_sacs"] = int(hit.group(1)) if hit else None + if found["unproven_sacs"] is None and unproven_line_seen: + raise Stop(f"events-backfill printed no `{UNPROVEN_SAC_LABEL}` line, but an earlier month of this " + "state did: the binary on the host is now one built before task 0242, which mints an " + f"unproven SAC as a second identity — see {logfile}. Put the 0242 build back " + "(docs/runbooks/0242-sac-identity-heal.md §4e) and `run` again") if found["unproven_sacs"]: m = Path(logfile).parent.name state = (f"This was the --dry-run pass: no AMM candle was written, but {m}'s 1m partition is " @@ -710,9 +720,19 @@ def amm_summary(text, logfile, *, dry_run): return found +def unproven_line_seen(st, m): + """Whether a month of this state other than `m` recorded `unproven sac swaps:` (task 0242).""" + return any(v.get("unproven_sacs") is not None for k, v in st.d["months"].items() if k != str(m)) + + def record_amm_summary(ms, m, summary): """Keep the write pass's figures on the month and note the ones that need reading.""" ms["fallbacks"], ms["dropped"] = summary["fallbacks"], summary["dropped"] + ms["unproven_sacs"] = summary["unproven_sacs"] + if ms["unproven_sacs"] is None: + note(f"{m}: events-backfill printed no `{UNPROVEN_SAC_LABEL}` line — a binary built before task 0242, " + "so its SACs resolve only through assets.sac_address (the 0242 seed), and an unseeded one " + "would mint a second identity (heal runbook §5a counts it)") if ms["dropped"]: note(f"{m}: {ms['dropped']} swaps dropped for unregistered pools — see prices.unresolved_pools") if ms["fallbacks"]: @@ -931,7 +951,8 @@ def done(): code, text = stream(cmd, env, mdir / "events-backfill.log") if code != 0 or "=== events-backfill complete ===" not in text: raise Stop(f"events-backfill {' '.join(flag)} exit {code} — see {mdir}/events-backfill.log") - summary = amm_summary(text, mdir / "events-backfill.log", dry_run=bool(flag)) + summary = amm_summary(text, mdir / "events-backfill.log", dry_run=bool(flag), + unproven_line_seen=unproven_line_seen(st, m)) record_amm_summary(ms, m, summary) elif a.amm == "ssh": for flag in (" --dry-run", ""): @@ -940,7 +961,8 @@ def done(): mdir / "events-backfill.log", stdin_text=pw) if code != 0 or "=== events-backfill complete ===" not in text: raise Stop(f"events-backfill{flag} exit {code} — see {mdir}/events-backfill.log") - summary = amm_summary(text, mdir / "events-backfill.log", dry_run=bool(flag)) + summary = amm_summary(text, mdir / "events-backfill.log", dry_run=bool(flag), + unproven_line_seen=unproven_line_seen(st, m)) record_amm_summary(ms, m, summary) else: marker = mdir / "amm.done" diff --git a/tools/scripts/reingest_0286_sac_test.py b/tools/scripts/reingest_0286_sac_test.py index 720f8a90..d4ef5156 100644 --- a/tools/scripts/reingest_0286_sac_test.py +++ b/tools/scripts/reingest_0286_sac_test.py @@ -51,6 +51,33 @@ def test_a_non_zero_count_stops_naming_it_and_what_the_month_holds(self): self.assertIn(part, msg) self.assertNotIn("nothing was written", msg) + def test_a_missing_line_stops_only_after_an_earlier_month_printed_it(self): + self.assertIsNone(r.amm_summary(summary(None), LOG, dry_run=True)["unproven_sacs"]) + with self.assertRaises(r.Stop) as cm: + r.amm_summary(summary(None), LOG, dry_run=True, unproven_line_seen=True) + msg = str(cm.exception) + for part in ("earlier month", "built before task 0242", str(LOG), "§4e"): + self.assertIn(part, msg) + self.assertEqual(r.amm_summary(summary(0), LOG, dry_run=True, unproven_line_seen=True) + ["unproven_sacs"], 0) + + def test_the_month_records_the_line_and_the_seen_test_reads_other_months(self): + class St: + d = {"months": {"202405": {"step": 9}}} + notes = [] + orig, r.note = r.note, notes.append + try: + ms = St.d["months"]["202405"] + r.record_amm_summary(ms, 202405, {"fallbacks": 0, "dropped": 0, "unproven_sacs": None}) + self.assertIsNone(ms["unproven_sacs"]) + self.assertTrue(any("built before task 0242" in n for n in notes), notes) + self.assertFalse(r.unproven_line_seen(St, 202406)) + r.record_amm_summary(ms, 202405, {"fallbacks": 0, "dropped": 0, "unproven_sacs": 0}) + self.assertTrue(r.unproven_line_seen(St, 202406)) + self.assertFalse(r.unproven_line_seen(St, 202405), "the month itself does not count") + finally: + r.note = orig + def test_a_missing_required_line_still_stops(self): with self.assertRaises(r.Stop) as cm: r.amm_summary(summary(0, fallbacks=False), LOG, dry_run=True) From 2f77c339621de14684fa0932e6f56fc10ad3c54f Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:12:29 +0200 Subject: [PATCH 033/144] refactor(lore-0242): share BE_DATABASE and drop the backfill's dead SAC learning events-backfill streams pool events only, so learning SACs from them did nothing; its proofs come from the run-start preload. --- packages/coverage-sweep-probe/src/lib.rs | 4 ++-- packages/prices-clickhouse/src/lib.rs | 4 ++++ packages/prices-ingest-core/src/canonical.rs | 6 ++++++ packages/prices-ingest-core/src/sac_proof.rs | 2 +- packages/prices-ingest-core/src/soroban.rs | 14 ++++---------- .../src/asset_id_uniqueness.rs | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/coverage-sweep-probe/src/lib.rs b/packages/coverage-sweep-probe/src/lib.rs index 9ea5d323..8c451ecc 100644 --- a/packages/coverage-sweep-probe/src/lib.rs +++ b/packages/coverage-sweep-probe/src/lib.rs @@ -39,9 +39,9 @@ pub const UNCLASSIFIED_SWAP_EVENTS: &str = "UnclassifiedSwapEvents"; pub const UNRESOLVED_SWAP_EMITTERS: &str = "UnresolvedSwapEmitters"; /// BE's database (`soroban_events`, `soroban_contracts`). -pub const BE_DATABASE: &str = "default"; +pub const BE_DATABASE: &str = prices_clickhouse::BE_DATABASE; /// Our database (`pool_registry`). -pub const PRICES_DATABASE: &str = "prices"; +pub const PRICES_DATABASE: &str = prices_clickhouse::PROD_DATABASE; /// Client-side `max_execution_time` for every statement, in seconds. The sweep /// runs as `prices_writer`, whose profile carries no execution bound. diff --git a/packages/prices-clickhouse/src/lib.rs b/packages/prices-clickhouse/src/lib.rs index c6fe2b1b..ad3f5ba1 100644 --- a/packages/prices-clickhouse/src/lib.rs +++ b/packages/prices-clickhouse/src/lib.rs @@ -212,6 +212,10 @@ pub const DEFAULT_USER: &str = "default"; /// The `prices` logical store. Every table in `schema/init.sql` lives here. pub const PROD_DATABASE: &str = "prices"; +/// BE's database on the shared cluster, holding `soroban_events` and +/// `soroban_contracts`. The one definition for every reader of BE's tables. +pub const BE_DATABASE: &str = "default"; + /// Canonical mainnet issuer of USDC (Circle). **Load-bearing join key** for the /// USD-close path: the backfill interns the USDC identity under this issuer, the /// enrichment peg tier and `resolve_reference_ids` match `prices.assets` on it, diff --git a/packages/prices-ingest-core/src/canonical.rs b/packages/prices-ingest-core/src/canonical.rs index 35191757..58570592 100644 --- a/packages/prices-ingest-core/src/canonical.rs +++ b/packages/prices-ingest-core/src/canonical.rs @@ -271,6 +271,12 @@ impl AssetRegistry { } /// The contracts BE flags `is_sac` (task 0242), replacing any earlier set. + /// + /// A snapshot: the live processor sets it once per cold start and never + /// refreshes it. A SAC deployed while a container stays warm is no + /// candidate there, so a trade of it with no in-band proof mints a + /// `Contract` identity. The probe's `SacContractIdentities` counts it (a + /// known gap, heal runbook §8). pub fn set_sac_candidates(&mut self, contracts: HashSet) { self.sac_candidates = contracts; } diff --git a/packages/prices-ingest-core/src/sac_proof.rs b/packages/prices-ingest-core/src/sac_proof.rs index b4fa66f7..d673e323 100644 --- a/packages/prices-ingest-core/src/sac_proof.rs +++ b/packages/prices-ingest-core/src/sac_proof.rs @@ -13,7 +13,7 @@ use crate::canonical::AssetRegistry; use crate::error::IngestError; /// BE's database, holding `soroban_contracts` and `soroban_events`. -pub const BE_DATABASE: &str = "default"; +pub const BE_DATABASE: &str = prices_clickhouse::BE_DATABASE; /// Ours, holding `assets`. pub const PRICES_DATABASE: &str = prices_clickhouse::PROD_DATABASE; diff --git a/packages/prices-ingest-core/src/soroban.rs b/packages/prices-ingest-core/src/soroban.rs index 111b400e..b4d95b29 100644 --- a/packages/prices-ingest-core/src/soroban.rs +++ b/packages/prices-ingest-core/src/soroban.rs @@ -511,16 +511,10 @@ pub fn process_soroban_event_rows( .all(|e| e.transaction_index == transaction_index), "events of transaction {tx_id} disagree on transaction_index" ); - // Kept identical to `process_ledger`, but in production it learns - // nothing: events-backfill streams pool events only, so a SAC's own - // transfer/mint/burn/clawback never reaches this seam. There the proofs - // come from `preload_sac_resolver` at run start. - learn_sacs( - events[tx_start..tx_end] - .iter() - .map(|ev| (ev.contract_id.as_str(), &ev.topics)), - assets, - ); + // No `learn_sacs` here, unlike `process_ledger`: events-backfill streams + // pool events only, so a SAC's own transfer/mint/burn/clawback never + // reaches this seam. Its proofs come from `preload_sac_resolver` at run + // start, before the first chunk (`execute`'s test pins the order). classify_amm_groups( amm_groups, transaction_index, diff --git a/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs b/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs index 19c371f1..f02eff5d 100644 --- a/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs +++ b/packages/rollup-freshness-probe/src/asset_id_uniqueness.rs @@ -44,7 +44,7 @@ pub const ASSET_ID_ORPHAN_CANDLES_METRIC: &str = "AssetIdOrphanCandles"; pub const SAC_CONTRACT_IDENTITIES_METRIC: &str = "SacContractIdentities"; /// BE's database, holding `soroban_contracts` (prices_writer has SELECT on it). -pub const BE_DATABASE: &str = "default"; +pub const BE_DATABASE: &str = prices_clickhouse::BE_DATABASE; /// The tier every writer of candles writes; the coarse tiers derive from it. pub const ORPHAN_CANDLE_TABLE: &str = "price_ohlcv_1m"; From 0d9657ef5892978175b2c50a2a50e97ee6bbe7c6 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:12:29 +0200 Subject: [PATCH 034/144] docs(lore-0242): deploy Compute after stage D and record the review changes The API alias needs the classic history in place (D9). The runbook also follows the fail-open cold start and the amended PC6. --- docs/runbooks/0242-sac-identity-heal.md | 88 +++++++++++++++++-------- 1 file changed, 59 insertions(+), 29 deletions(-) diff --git a/docs/runbooks/0242-sac-identity-heal.md b/docs/runbooks/0242-sac-identity-heal.md index 95078bcf..83d0a31d 100644 --- a/docs/runbooks/0242-sac-identity-heal.md +++ b/docs/runbooks/0242-sac-identity-heal.md @@ -11,7 +11,7 @@ deletes what is left. Observability stacks. **Read first:** [`0286-reingest-history.md`](0286-reingest-history.md) (phase 3, -its stages and its orchestrator) and lore task 0242 (decisions D1–D8, PC3, PC6, +its stages and its orchestrator) and lore task 0242 (decisions D1–D9, PC3, PC6, PC9). --- @@ -45,13 +45,13 @@ runbook heals the past. ## 1. Order of events -| # | Step | When | Section | -| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------- | -| 1 | Seed the 29 classic identities. Works with the `events-backfill` already on ch-prod-01 (D4) | After phase-3 stage B (202402–202404), before stage C | §3 | -| 2 | Re-plan phase 3 to 202609, run stage C with `--to-month 202609` (D4; the plan ended at 202608) | After the seed | §4a–§4d | -| 3 | Deploy the PR: Compute (live ledger processor + API), then EventBridge (probe). Not Observability (D8). Swap `events-backfill` on ch-prod-01 only between phase-3 stages, never while a month is at its `amm` step (D7) | Once merged. Independent of phase 3 (D4: phase 3 is not held for the code) | §4e | -| 4 | Residual cleanup (D3) | After stage D (`finish`) AND after the live slice is deployed with at least one cold start | §5 | -| 5 | Deploy Observability with the `SacContractIdentities` alarm actions on (D6, D8) | After §5's post-checks are green | §6 | +| # | Step | When | Section | +| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | ------- | +| 1 | Seed the 29 classic identities. Works with the `events-backfill` already on ch-prod-01 (D4) | After phase-3 stage B (202402–202404), before stage C | §3 | +| 2 | Re-plan phase 3 to 202609, run stage C with `--to-month 202609` (D4; the plan ended at 202608) | After the seed | §4a–§4d | +| 3 | Deploy the PR: Compute (live ledger processor + API), then EventBridge (probe). Not Observability (D8). Swap `events-backfill` on ch-prod-01 only between phase-3 stages, never while a month is at its `amm` step (D7) | After stage D (`finish`) and before §5 (D9: the API alias needs the classic history in place) | §4e | +| 4 | Residual cleanup (D3) | After stage D (`finish`) AND after the live slice is deployed with at least one cold start | §5 | +| 5 | Deploy Observability with the `SacContractIdentities` alarm actions on (D6, D8) | After §5's post-checks are green | §6 | Step 4 waits for the live deploy because until then a warm container that predates the seed, or a SAC the seed does not cover, can still mint a contract identity @@ -232,6 +232,9 @@ CLICKHOUSE_PASSWORD="$CH_PW" ~/events-backfill --start 51500460 --end 51500460 \ A 0242 binary prints `unproven sac swaps:` followed by a count; a binary built before 0242 prints nothing. Both are fine for stage C once the seed is in (PC6). +A month run on a pre-0242 binary carries the note `printed no unproven sac swaps: +line`. Once a month of the state has printed the line, a later month without it +is a STOP: the host binary went back to a pre-0242 build (review of 0242). **Post-check: U3's candles are back under the classic ids.** Once 202609 is done (CH shell): @@ -361,9 +364,19 @@ A STOP on the write pass is step 4's last case. ### 4e. Deploy the PR +**When: after stage D (`finish`), before §5 (D9).** The API alias answers a SAC +address as its classic. Until stage C has rebuilt a SAC's months under the +classic id, that classic holds only part of the history, while the `Contract` +row still holds the rest. Deployed earlier, `GET /v1/assets/{C…}/ohlcv` would +lose the pre-heal AMM series. Group B is the visible case: its classics were +seeded on 2026-10-06 and fill month by month as stage C runs. §5 still needs the +live slice deployed with one cold start, so the deploy goes between the two. + **Pre-check before the live deploy: `prices_writer` can read BE's SAC set.** The -live Lambda loads `default.soroban_contracts` at cold start as `prices_writer` -and fails Init when it cannot (PC3): live ingest would stop. +live Lambda loads `default.soroban_contracts` at cold start as `prices_writer`. +When it cannot, it does not fail Init (PC3, reversed in review): it ingests with +no SAC candidates, so an unproven SAC mints a `Contract` identity, and every run +publishes `SacCandidatesUnavailable`. Deploy with the read working. ```bash chq <<'SQL' @@ -386,7 +399,8 @@ curl -sS --fail-with-body --cert ~/prices-mtls/prices_writer.crt --key ~/prices- --data-binary "SELECT count() FROM (SELECT DISTINCT contract_id FROM default.soroban_contracts WHERE is_sac)" ``` -About 4,042 (M13b). An error or `0`: **do not deploy Compute**; fix the grant first. +About 4,042 (M13b). An error or `0`: **do not deploy Compute**; fix the grant +first. The deploy would not stop ingest, but it would run without candidates. **Record what runs now** (AWS shell), for the rollback: @@ -412,24 +426,28 @@ that holds this PR creates them, so tell the team: no `make deploy-production-observability` from `develop` until §6. A deploy that cannot wait shows the red tile; its description says it is expected. -Until §6 no alarm watches `SacUnprovenSkipped`. Read it by hand after the cold -start and at least daily until §6: +Until §6 no alarm watches `SacUnprovenSkipped` or `SacCandidatesUnavailable`. +Read both by hand after the cold start and at least daily until §6: ```bash -aws cloudwatch get-metric-statistics --namespace Prices/Ingest --metric-name SacUnprovenSkipped \ - --dimensions Name=Environment,Value=production \ - --start-time "$(date -u -d '-1 day' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" \ - --period 3600 --statistics Sum +for M in SacUnprovenSkipped SacCandidatesUnavailable; do + echo "== $M" + aws cloudwatch get-metric-statistics --namespace Prices/Ingest --metric-name "$M" \ + --dimensions Name=Environment,Value=production \ + --start-time "$(date -u -d '-1 day' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" \ + --period 3600 --statistics Sum +done ``` -No datapoints is healthy: the metric is published only when trades were -skipped. Any datapoint: §9. +No datapoints is healthy: both are published only when something is wrong. +Any datapoint: §9. **Post-checks.** - The live Lambda cold-started on the new build: `/aws/lambda/prices-production-ledger-processor` shows `loaded the is_sac contract set` with `sac_contracts` ≈ 4,042, then - `resumed cursor from ClickHouse`. No Init error. + `resumed cursor from ClickHouse`. No Init error, and no WARN + `is_sac contract set unreadable`. - The probe: `aws lambda invoke --function-name prices-production-rollup-freshness-probe ~/heal-0242/probe.json`, then `jq '.asset_id_uniqueness | {sac_contract_rows, sac_contract_scanned, be_sac_contracts}' ~/heal-0242/probe.json`. `sac_contract_rows` = 36 (the baseline until §5), `be_sac_contracts` ≈ 4,042. @@ -779,9 +797,10 @@ from the same identity. Keep the `bak_0242_*` tables until 0242 is closed; then ## 6. Deploy Observability with the alarm on (D6, D8) -Observability was held back in §4e (D8), so this deploy creates both 0242 -alarms, `prices-production-ledger-processor-sac-unproven` and -`prices-production-sac-contract-identities`, the second with its actions on. +Observability was held back in §4e (D8), so this deploy creates the three 0242 +alarms: `prices-production-ledger-processor-sac-unproven`, +`prices-production-ledger-processor-sac-candidates-unavailable` and +`prices-production-sac-contract-identities`, the last with its actions on. **Pre-check.** §5g is green and `SacContractIdentities` reads `0` for at least two periods (30 min): @@ -808,16 +827,17 @@ cd infra && make deploy-production-observability ```bash aws cloudwatch describe-alarms \ --alarm-names prices-production-sac-contract-identities prices-production-ledger-processor-sac-unproven \ + prices-production-ledger-processor-sac-candidates-unavailable \ --query 'MetricAlarms[].[AlarmName,ActionsEnabled,StateValue]' --output text ``` -Both alarms print `True`, then `OK` (`INSUFFICIENT_DATA` until the first -evaluation). The `INSUFFICIENT_DATA` → `OK` transition sends one OK -notification: expected. +All three print `True`, then `OK` (`INSUFFICIENT_DATA` until the first +evaluation). The `INSUFFICIENT_DATA` → `OK` transitions send OK notifications: +expected. **Rollback.** The same key back to `false` and the same deploy turns the actions off. Redeploying Observability from a commit without this PR removes -both alarms. +all three alarms. --- @@ -854,8 +874,10 @@ each SAC now resolves to one classic row. - **Known residual risk.** The live processor reads BE's `is_sac` set once per cold start. A SAC deployed after it, traded with no SAC event in the same transaction, is not a candidate yet and still mints a `Contract` identity; so - does a SAC BE does not flag. The in-band proof missed 0 of 338,601 sampled - legs, so this should stay rare. `SacContractIdentities` counts it, §9. + does a SAC BE does not flag, and so does every unproven SAC of a container + whose cold start could not read the set (`SacCandidatesUnavailable`). The + in-band proof missed 0 of 338,601 sampled legs, so this should stay rare. + `SacContractIdentities` counts it, §9. --- @@ -870,6 +892,14 @@ as in §4c step 2. A real SAC: seed its classic as §3 did; the next cold start resolves it. The skipped minutes stay without those trades until a re-ingest of their month. +**`prices-production-ledger-processor-sac-candidates-unavailable`** (live, +after §6). A container cold-started without BE's `is_sac` set. The WARN +`is_sac contract set unreadable` carries the error: usually the grant of §4e's +pre-check, a renamed table or a BE migration. Ingest is running. Fix the read, +then force a cold start (any configuration change of +`prices-production-ledger-processor`, or wait for the container to recycle). +Any SAC minted meanwhile shows in `SacContractIdentities`: treat it as below. + **`prices-production-sac-contract-identities`** (after §6). A contract row of `assets` is a SAC again. The alarm's description holds the query that names it. Treat it as a one-row §5: capture, gate its candles, copy, delete. Find which From 0a2814d12de7a0b63d9bafeb14129134d767e4a7 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:12:47 +0200 Subject: [PATCH 035/144] docs(lore-0242): record D9 and the PC3/PC6 changes From Oskar's review of PR #390: Compute deploys after stage D, the live cold start fails open on BE's is_sac set, and a missing unproven-sac line is noted, stopping only on a binary regression. --- ..._BUG_sac-minted-as-a-second-identity-splits-volume.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md index 86a90871..3a199d58 100644 --- a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md +++ b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md @@ -206,11 +206,20 @@ Prior art: soroban-block-explorer ADR 0051 (a SAC is a facet of the classic asse - Why: deployed earlier, that alarm would sit in ALARM on the dashboard's Row-0 alarm strip through stage C, stage D and the cleanup. That strip is [[0294]]'s SCF evidence (all alarms OK). - Cost: until §6 no alarm watches `SacUnprovenSkipped`. The runbook has the operator read the metric by hand after the cold start and at least daily. - Constraint on the team: no `make deploy-production-observability` from `develop` between the merge and §6. Such a deploy creates the alarms early, and the red tile's description says it is expected. +- **D9 Compute after stage D (Adam, 2026-10-06, from Oskar's review of PR #390).** Compute (live + API) deploys after phase-3 stage D and before the §5 cleanup. + - Why: the API alias answers a SAC address as its classic. Until stage C has rebuilt a SAC's months under the classic id, that classic holds only part of the history, so `GET /v1/assets/{C…}/ohlcv` would lose the pre-heal AMM series. Group B is the visible case: its classics were seeded on 2026-10-06. + - Cost: the new live code lands two or three days later. The seed already resolves every known SAC on the old live after its next cold start. Plan choices confirmed by Adam on 2026-10-05 (GSD plan `261005-htu`): - **PC3 Fail closed at the live cold start.** If the Lambda cannot read BE's `is_sac` set, Init fails, like the other cold reads. An empty set would mint `Contract` identities, which D2 forbids. The cost: if BE's `default.soroban_contracts` breaks, live ingest stops until it is fixed. - **PC6 A summary without the new line passes.** The orchestrator STOPs only when `unproven sac swaps:` is present and non-zero. A binary built before 0242 prints no such line, so phase 3 keeps running on it, which is why D4 holds. +Changed by Adam on 2026-10-06, after Oskar's review of PR #390: + +- **PC3 reversed: fail open.** An unreadable `is_sac` set no longer fails Init. The container ingests with no candidates, logs a WARN and publishes `SacCandidatesUnavailable` on every run, with an alarm on it (deployed in §6, D8). + - Why: the candidates guard only a SAC trade with no in-band proof, which happened 0 times in 338,601 legs. Failing closed stopped SDEX, oracle and AMM ingest over one BE table. + - The probe's `SacContractIdentities` counts any identity minted meanwhile. +- **PC6 amended.** A month without the `unproven sac swaps:` line gets a visible note. Once a month of the same state has printed the line, a later month without it is a STOP: the host binary went back to a pre-0242 build. Stage C on the pre-0242 binary is unaffected. - **PC9 Residual rows of a SAC with no classic.** Such a row is the only copy of its trades, so it is never deleted silently. The operator either re-runs that month through phase 3 or records the loss on this task. The rows are copied beside the 72 metadata rows for rollback. The 202404 Z/Q row, from stage B before the seed, is the expected case. Open: **U3**. About 75.6k Soroswap swaps on BLTA/BLTB/BLTC/PPRIME/LumenJoule produce no candle under any id, and the cause is not established. It must be settled before AC3 promises numbers for those assets. From e3d75b627953c6c31be7a851603e306613983b80 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:23:07 +0200 Subject: [PATCH 036/144] chore(lore-0205): activate task --- ...eploy-the-portal-gateway-mapping-and-verify-live.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) rename lore/1-tasks/{backlog => active}/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md (96%) diff --git a/lore/1-tasks/backlog/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md b/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md similarity index 96% rename from lore/1-tasks/backlog/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md rename to lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md index ff47b8a3..4a438013 100644 --- a/lore/1-tasks/backlog/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md +++ b/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md @@ -2,7 +2,7 @@ id: '0205' title: 'Deploy the portal gateway mapping and verify it live — three deploys 0184 merged but never shipped' type: CHORE -status: backlog +status: active related_adr: [] related_tasks: ['0184', '0183', '0185', '0186', '0194', '0141'] tags: @@ -69,6 +69,14 @@ history: both Cache-Control headers, the per-verb throttles, cookie-free access logs, /api-tokens/api/config and [[0185]]'s decision 13 still have no evidence. What remains is the live re-measurement, no deploy expected. + - date: "2026-10-06" + status: active + who: akot + note: > + Activated for the live re-measurement. The portal has since moved twice + ([[0194]]: backend on prices-api.sorobanscan…/api/, [[0326]]: bundle at + sorobanscan…/prices-api/), so each criterion is measured against what + replaced it. --- # Deploy the portal gateway mapping and verify it live From 69f003dd6c0809ad065e14de7840f3b4f965cbf3 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:24:35 +0200 Subject: [PATCH 037/144] fix(lore-0236): bound and parallelise the band reads, share the totals rule Review of #396: - each of the seven reads runs under block 6's deadline-derived bound, server- and client-side, in parallel; a stall is a recorded tier failure instead of a Lambda timeout that loses block 6 and the log; - ohlc_band_totals is the one completeness rule for metric and log, and rejects duplicate or unknown tier names; - ITs iterate OHLC_BAND_TIERS and use OhlcBandCounts::new; - types.ts counts six ladders on these rungs, not four; - the module doc states why pf_trade_count > 0 means priced on every tier (the ingest floor is the rollup floor, 0286 WR-03). --- infra/src/lib/types.ts | 13 +- packages/rollup-freshness-probe/src/main.rs | 63 ++++++++-- .../rollup-freshness-probe/src/ohlc_band.rs | 111 ++++++++++++++---- .../tests/rollup_freshness_it.rs | 67 ++++------- 4 files changed, 164 insertions(+), 90 deletions(-) diff --git a/infra/src/lib/types.ts b/infra/src/lib/types.ts index 7c191e2f..9609d23a 100644 --- a/infra/src/lib/types.ts +++ b/infra/src/lib/types.ts @@ -451,12 +451,13 @@ export interface EnvironmentConfig { * and the rationale below for `100` / `10000` is about the USDT metrics, not * about it. That ladder therefore pins its first rung at `1` in the stack * and borrows only the rungs above it from here, so raising the first rung - * to quiet a USDT ladder cannot hide a violation there. Since task 0236 - * they drive a FOURTH ladder, `CandleBandViolations`: all seven - * `price_ohlcv_*` tiers, `FINAL`, over a 48 h window widened by one bucket - * per tier. Its healthy value is also exactly 0, and it pins its first rung - * at `1` the same way. **Changing the higher rungs still re-tunes all four - * ladders.** + * to quiet a USDT ladder cannot hide a violation there. Task 0139 added + * two more on the same pinned ladder (`zeroLadder` in the stack): + * `AssetIdCollisions` and `AssetIdOrphanCandles`. Task 0236 adds the + * sixth, `CandleBandViolations`: all seven `price_ohlcv_*` tiers, `FINAL`, + * over a 48 h window widened by one bucket per tier, also healthy at + * exactly 0 and on `zeroLadder`. **Changing the higher rungs re-tunes all + * six ladders.** * * ⚠️ **Why a ladder and not a single `>= 1`.** A wrong `close_usd` is a * **standing condition** — it stays wrong until a person repairs it — so it diff --git a/packages/rollup-freshness-probe/src/main.rs b/packages/rollup-freshness-probe/src/main.rs index 118eb634..13183e2d 100644 --- a/packages/rollup-freshness-probe/src/main.rs +++ b/packages/rollup-freshness-probe/src/main.rs @@ -31,6 +31,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { }; use rollup_freshness_probe::ohlc_band::{ OHLC_BAND_TIERS, OhlcBandCounts, ohlc_band_detail, ohlc_band_metric, ohlc_band_queries, + ohlc_band_totals, }; use rollup_freshness_probe::reconcile_mismatch::{ MISMATCH_MIN_READ_SECS, MISMATCH_RESERVE, MismatchCount, mismatch_metric, mismatch_queries, @@ -462,15 +463,57 @@ async fn main() -> Result<(), lambda_runtime::Error> { // Seven INDEPENDENT reads, not one `UNION ALL`: a statement fails as a // unit, so one dropped tier or lost grant would hide the other six // counts, and the alarm is `NOT_BREACHING` on missing data. A failed - // tier is recorded and the loop moves on. + // tier is recorded and the others still count. + // + // Run in parallel, each under the bound block 6 uses: `min(10 s, + // time left - 5 s reserve)`, server-side and client-side. A stalled + // tier (a `FINAL` during a merge, a busy box during a re-ingest) + // is then a recorded failure, not a hard timeout that loses block 6 + // and the summary line. The block costs the slowest read, not the sum. // // Independent of block 5, although a priced `_1m` row with // `close = 0` fires both: that overlap is expected (see `ohlc_band`). let mut band_readings: Vec<(&'static str, OhlcBandCounts)> = Vec::new(); - for (table, sql) in ohlc_band_queries.iter() { - match ch.query(sql).fetch_one::().await { - Ok(counts) => band_readings.push((*table, counts)), - Err(e) => failures.push(format!("ohlc-band read {table}: {e}")), + let remaining = deadline + .duration_since(std::time::SystemTime::now()) + .unwrap_or_default(); + match mismatch_read_bound(remaining) { + None => failures.push(format!( + "ohlc-band skipped: {:.1} s left of the invocation, \ + under the {} s reserve + {MISMATCH_MIN_READ_SECS} s a read needs", + remaining.as_secs_f64(), + MISMATCH_RESERVE.as_secs(), + )), + Some(bound) => { + let mut reads = tokio::task::JoinSet::new(); + for (table, sql) in ohlc_band_queries.iter() { + let bounded = prices_clickhouse::with_execution_bound((*ch).clone(), bound); + let (table, sql) = (*table, sql.clone()); + reads.spawn(async move { + let read = tokio::time::timeout( + std::time::Duration::from_secs(bound + 2), + bounded.query(&sql).fetch_one::(), + ) + .await; + (table, read) + }); + } + while let Some(joined) = reads.join_next().await { + match joined { + Ok((table, Ok(Ok(counts)))) => band_readings.push((table, counts)), + Ok((table, Ok(Err(e)))) => failures + .push(format!("ohlc-band read {table} (bound {bound} s): {e}")), + Ok((table, Err(_))) => failures.push(format!( + "ohlc-band read {table}: no answer within {} s", + bound + 2 + )), + Err(e) => failures.push(format!("ohlc-band read task: {e}")), + } + } + // Completion order is arbitrary; log in tier order. + band_readings.sort_by_key(|(t, _)| { + OHLC_BAND_TIERS.iter().position(|(o, _)| o == t) + }); } } match ohlc_band_metric(&band_readings) { @@ -483,13 +526,9 @@ async fn main() -> Result<(), lambda_runtime::Error> { } Err(refusal) => failures.push(format!("ohlc-band: {refusal}")), } - // Totals only when every tier was read: a partial sum logged as a - // number would read as the whole chain. The detail names each tier. - let band_totals = (band_readings.len() == OHLC_BAND_TIERS.len()).then(|| { - band_readings.iter().fold((0u64, 0u64, 0u64), |(v, b, n), (_, c)| { - (v + c.violations(), b + c.band, n + c.nonpositive) - }) - }); + // Totals only for a complete run: a partial sum logged as a number + // would read as the whole chain. The detail names each tier. + let band_totals = ohlc_band_totals(&band_readings); let band_scanned_1m = band_readings .iter() .find(|(t, _)| *t == OHLC_BAND_TIERS[0].0) diff --git a/packages/rollup-freshness-probe/src/ohlc_band.rs b/packages/rollup-freshness-probe/src/ohlc_band.rs index b6dde0ec..d4bcb7da 100644 --- a/packages/rollup-freshness-probe/src/ohlc_band.rs +++ b/packages/rollup-freshness-probe/src/ohlc_band.rs @@ -34,6 +34,20 @@ //! `least(o, c) >= low > high`, and then `greatest(o, c) > high`); it is kept as //! the literal invariant the reader expects to find. //! +//! # `pf_trade_count > 0` means priced on every tier +//! +//! On a coarse tier `pf_trade_count` is a plain `sum` over the children, while +//! the four prices come only from children with `pf_trade_count > 0 AND close +//! >= 1e-12`. The two agree because the ingest counts a fill as price-forming +//! only when its price clears that same floor +//! (`prices_ingest_core::price::price_survives_column_scale`, pinned to +//! `prices_clickhouse::PRICE_FLOOR_LITERAL`). So no `_1m` row has +//! `pf_trade_count > 0` with a close below the floor, and a coarse row with +//! `pf_trade_count > 0` always has a child that lends it a price. On prod, from +//! 2026-09-23 to 2026-10-06, no row on any tier broke this. If one does, the +//! ingest and rollup floors have drifted apart (the 0286 review WR-03 defect), +//! and the `nonpositive` count reports it. +//! //! # The window is per tier, on bucket START //! //! `timestamp >= now() - 2 days - ` (`_1m` plain 2 days). A @@ -122,6 +136,15 @@ pub struct OhlcBandCounts { } impl OhlcBandCounts { + /// A reading from its three counts, for tests and fixtures. + pub const fn new(band: u64, nonpositive: u64, scanned: u64) -> Self { + Self { + band, + nonpositive, + scanned, + } + } + /// Rows breaking either shape. The classes are disjoint, so this is a row /// count, not a count of arms. pub fn violations(&self) -> u64 { @@ -193,12 +216,28 @@ impl std::fmt::Display for OhlcBandRefusal { impl std::error::Error for OhlcBandRefusal {} +/// `(violations, band, nonpositive)` summed over the run, or `None` unless every +/// tier of [`OHLC_BAND_TIERS`] was read exactly once. The one completeness rule +/// the metric and the log share: a duplicate or unknown table name is not a +/// complete run either. +pub fn ohlc_band_totals(readings: &[(&'static str, OhlcBandCounts)]) -> Option<(u64, u64, u64)> { + let complete = readings.len() == OHLC_BAND_TIERS.len() + && OHLC_BAND_TIERS + .iter() + .all(|(t, _)| readings.iter().filter(|(r, _)| r == t).count() == 1); + complete.then(|| { + readings.iter().fold((0, 0, 0), |(v, b, n), (_, c)| { + (v + c.violations(), b + c.band, n + c.nonpositive) + }) + }) +} + /// Shape the readings into the one datum to publish, or refuse. /// /// In order: a positive sum is always published (never a false OK, even with a -/// tier missing); a zero with any tier unread is [`OhlcBandRefusal::Incomplete`]; -/// a zero over an empty `_1m` window is [`OhlcBandRefusal::EmptyScan`]; otherwise -/// a healthy zero. +/// tier missing); a zero without a complete run ([`ohlc_band_totals`]) is +/// [`OhlcBandRefusal::Incomplete`]; a zero over an empty `_1m` window is +/// [`OhlcBandRefusal::EmptyScan`]; otherwise a healthy zero. pub fn ohlc_band_metric( readings: &[(&'static str, OhlcBandCounts)], ) -> Result { @@ -209,11 +248,14 @@ pub fn ohlc_band_metric( value: total as f64, }); } - let read = |table: &str| readings.iter().find(|(t, _)| *t == table).map(|(_, c)| c); - if OHLC_BAND_TIERS.iter().any(|(t, _)| read(t).is_none()) { + if ohlc_band_totals(readings).is_none() { return Err(OhlcBandRefusal::Incomplete); } - if read(OHLC_BAND_TIERS[0].0).is_some_and(|c| c.scanned == 0) { + let scanned_1m = readings + .iter() + .find(|(t, _)| *t == OHLC_BAND_TIERS[0].0) + .map(|(_, c)| c.scanned); + if scanned_1m == Some(0) { return Err(OhlcBandRefusal::EmptyScan); } Ok(SanityMetric { @@ -242,14 +284,6 @@ mod tests { use super::*; use crate::ROLLUP_TIERS; - fn counts(band: u64, nonpositive: u64, scanned: u64) -> OhlcBandCounts { - OhlcBandCounts { - band, - nonpositive, - scanned, - } - } - /// Seven readings, one per tier, all the given counts. fn all_tiers(c: OhlcBandCounts) -> Vec<(&'static str, OhlcBandCounts)> { OHLC_BAND_TIERS.iter().map(|(t, _)| (*t, c)).collect() @@ -362,25 +396,25 @@ mod tests { let readings: Vec<(&'static str, OhlcBandCounts)> = OHLC_BAND_TIERS .iter() .enumerate() - .map(|(i, (t, _))| (*t, counts(i as u64, 2 * i as u64, 100))) + .map(|(i, (t, _))| (*t, OhlcBandCounts::new(i as u64, 2 * i as u64, 100))) .collect(); // band 0+1+..+6 = 21, nonpositive 2*21 = 42 let metric = ohlc_band_metric(&readings).unwrap(); assert_eq!(metric.name, OHLC_BAND_METRIC); assert_eq!(metric.name, "CandleBandViolations"); assert_eq!(metric.value, 63.0); - assert_eq!(counts(2, 3, 9).violations(), 5); + assert_eq!(OhlcBandCounts::new(2, 3, 9).violations(), 5); } #[test] fn an_empty_1m_scan_is_refused_not_published_as_healthy() { - let refusal = ohlc_band_metric(&all_tiers(counts(0, 0, 0))).unwrap_err(); + let refusal = ohlc_band_metric(&all_tiers(OhlcBandCounts::new(0, 0, 0))).unwrap_err(); assert_eq!(refusal, OhlcBandRefusal::EmptyScan); } #[test] fn a_zero_over_a_missing_tier_is_refused_as_incomplete() { - let mut readings = all_tiers(counts(0, 0, 10)); + let mut readings = all_tiers(OhlcBandCounts::new(0, 0, 10)); readings.remove(3); assert_eq!( ohlc_band_metric(&readings).unwrap_err(), @@ -388,27 +422,52 @@ mod tests { ); } + /// Seven readings are not a complete run when one tier is read twice or an + /// unknown table stands in for a real one. The metric and the log share + /// this rule through `ohlc_band_totals`. + #[test] + fn a_duplicate_or_unknown_tier_is_not_a_complete_run() { + let complete = all_tiers(OhlcBandCounts::new(1, 2, 10)); + assert_eq!(ohlc_band_totals(&complete), Some((21, 7, 14))); + + let mut duplicate = complete.clone(); + duplicate[6].0 = duplicate[5].0; + let mut unknown = complete.clone(); + unknown[6].0 = "price_ohlcv_2h"; + for readings in [duplicate, unknown] { + assert_eq!(ohlc_band_totals(&readings), None); + let zero: Vec<_> = readings + .iter() + .map(|(t, _)| (*t, OhlcBandCounts::new(0, 0, 10))) + .collect(); + assert_eq!( + ohlc_band_metric(&zero).unwrap_err(), + OhlcBandRefusal::Incomplete + ); + } + } + /// A positive value is never a false OK, so a failed tier elsewhere does /// not hide the violations the other six found. #[test] fn a_positive_sum_is_published_even_with_a_tier_missing() { - let mut readings = all_tiers(counts(0, 0, 10)); + let mut readings = all_tiers(OhlcBandCounts::new(0, 0, 10)); readings.remove(6); - readings[2].1 = counts(1, 2, 10); + readings[2].1 = OhlcBandCounts::new(1, 2, 10); assert_eq!(ohlc_band_metric(&readings).unwrap().value, 3.0); } #[test] fn empty_coarse_tiers_beside_a_scanned_1m_publish_a_healthy_zero() { - let mut readings = all_tiers(counts(0, 0, 0)); - readings[0].1 = counts(0, 0, 500); + let mut readings = all_tiers(OhlcBandCounts::new(0, 0, 0)); + readings[0].1 = OhlcBandCounts::new(0, 0, 500); assert_eq!(ohlc_band_metric(&readings).unwrap().value, 0.0); } #[test] fn a_coarse_violation_is_published_even_when_1m_scanned_nothing() { - let mut readings = all_tiers(counts(0, 0, 0)); - readings[5].1 = counts(2, 0, 4); + let mut readings = all_tiers(OhlcBandCounts::new(0, 0, 0)); + readings[5].1 = OhlcBandCounts::new(2, 0, 4); assert_eq!(ohlc_band_metric(&readings).unwrap().value, 2.0); } @@ -432,8 +491,8 @@ mod tests { #[test] fn the_detail_names_every_reading() { let detail = ohlc_band_detail(&[ - ("price_ohlcv_1m", counts(1, 2, 6)), - ("price_ohlcv_1w", counts(0, 0, 0)), + ("price_ohlcv_1m", OhlcBandCounts::new(1, 2, 6)), + ("price_ohlcv_1w", OhlcBandCounts::new(0, 0, 0)), ]); assert_eq!( detail, diff --git a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs index 64d97999..6e2287c1 100644 --- a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs +++ b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs @@ -1932,21 +1932,13 @@ async fn an_empty_registry_or_window_is_refused_as_unreadable() { // ---- task 0236: stored candles outside the OHLC band, on all seven tiers ---- -/// Clear all seven tiers, `_1m` FIRST and then the coarse tiers finest to -/// coarsest. The rollup MVs run in this database: clearing a source before its -/// target stops a refresh landing between the two from re-feeding the tier just -/// cleared. Its own helper — [`reset_sanity_tables`] is shared and clears only -/// two tiers. +/// Clear every tier of `OHLC_BAND_TIERS`, in its order: `_1m` FIRST, then the +/// coarse tiers finest to coarsest. The rollup MVs run in this database: +/// clearing a source before its target stops a refresh landing between the two +/// from re-feeding the tier just cleared. Its own helper — [`reset_sanity_tables`] +/// is shared and clears only two tiers. async fn reset_ohlc_band_tables(c: &Client) { - for table in [ - "price_ohlcv_1m", - "price_ohlcv_15m", - "price_ohlcv_1h", - "price_ohlcv_4h", - "price_ohlcv_1d", - "price_ohlcv_1w", - "price_ohlcv_1M", - ] { + for (table, _) in rollup_freshness_probe::ohlc_band::OHLC_BAND_TIERS { exec(c, &format!("TRUNCATE TABLE prices.{table}")).await; } } @@ -2057,21 +2049,13 @@ async fn the_ohlc_band_scan_counts_only_rows_outside_the_band_on_every_tier() { assert_eq!(readings.len(), OHLC_BAND_TIERS.len()); assert_eq!( reading(&readings, m), - OhlcBandCounts { - band: 1, - nonpositive: 2, - scanned: 6 - }, + OhlcBandCounts::new(1, 2, 6), "_1m: one band violation, two non-positive rows (the double-shaped one counted once)" ); for (table, counts) in &readings[1..] { assert_eq!( *counts, - OhlcBandCounts { - band: 0, - nonpositive: 0, - scanned: 0 - }, + OhlcBandCounts::new(0, 0, 0), "{table} holds nothing" ); } @@ -2088,18 +2072,6 @@ async fn seed_healthy_1m(c: &Client) -> u32 { ts } -fn band_only( - band: u64, - nonpositive: u64, - scanned: u64, -) -> rollup_freshness_probe::ohlc_band::OhlcBandCounts { - rollup_freshness_probe::ohlc_band::OhlcBandCounts { - band, - nonpositive, - scanned, - } -} - /// The arm `high < greatest(open, close)`, and ONLY that arm: `o4 h4.5 l3 c5` /// has `low (3) <= least (4)` and `low <= high`. RED without the arm. #[tokio::test] @@ -2113,7 +2085,7 @@ async fn the_ohlc_band_scan_catches_a_high_below_the_close() { let readings = read_ohlc_band(&c).await; assert_eq!( reading(&readings, "price_ohlcv_1m"), - band_only(1, 0, 2), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 2), "high 4.5 below close 5 is outside the band" ); @@ -2133,7 +2105,7 @@ async fn the_ohlc_band_scan_catches_a_low_above_the_open() { let readings = read_ohlc_band(&c).await; assert_eq!( reading(&readings, "price_ohlcv_1m"), - band_only(1, 0, 2), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 2), "low 4 above open 3 is outside the band" ); @@ -2156,7 +2128,7 @@ async fn the_ohlc_band_scan_catches_a_low_above_the_high_once() { let readings = read_ohlc_band(&c).await; assert_eq!( reading(&readings, "price_ohlcv_1m"), - band_only(1, 0, 2), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 2), "a low above the high is one bad row" ); @@ -2178,7 +2150,7 @@ async fn the_ohlc_band_scan_catches_a_zero_low_beside_a_positive_close() { let readings = read_ohlc_band(&c).await; assert_eq!( reading(&readings, "price_ohlcv_1m"), - band_only(0, 1, 2), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(0, 1, 2), "a priced low of 0 is a non-positive price" ); @@ -2207,15 +2179,18 @@ async fn the_ohlc_band_scan_reads_each_coarse_tier_over_its_own_bucket_window() let readings = read_ohlc_band(&c).await; assert_eq!( reading(&readings, "price_ohlcv_1w"), - band_only(1, 0, 1), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 1), "a week bucket older than 2 days but inside its widened window" ); assert_eq!( reading(&readings, "price_ohlcv_1M"), - band_only(1, 0, 1), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 1), "a month bucket older than 2 days but inside its widened window" ); - assert_eq!(reading(&readings, "price_ohlcv_1m"), band_only(0, 0, 1)); + assert_eq!( + reading(&readings, "price_ohlcv_1m"), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(0, 0, 1) + ); assert_eq!(ohlc_band_metric(&readings).unwrap().value, 2.0); reset_ohlc_band_tables(&c).await; @@ -2256,12 +2231,12 @@ async fn the_ohlc_band_scan_ignores_violations_outside_the_window() { let readings = read_ohlc_band(&c).await; assert_eq!( reading(&readings, "price_ohlcv_1m"), - band_only(0, 0, 1), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(0, 0, 1), "only the healthy seed is inside the 1m window" ); assert_eq!( reading(&readings, "price_ohlcv_1w"), - band_only(0, 0, 0), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(0, 0, 0), "a week bucket starting before now - 2d - 1w is outside" ); for (table, counts) in &readings { @@ -2289,7 +2264,7 @@ async fn a_repaired_candle_stops_counting_in_the_ohlc_band_scan() { let readings = read_ohlc_band(&c).await; assert_eq!( reading(&readings, m), - band_only(1, 0, 3), + rollup_freshness_probe::ohlc_band::OhlcBandCounts::new(1, 0, 3), "the superseded version is neither counted nor scanned" ); From 65df56c8ebe5aabb32eef0b420001dc661ee205d Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:26:03 +0200 Subject: [PATCH 038/144] docs(lore-0205): record the live re-measurement Production no longer has the subject of most criteria: PortalHosting was deleted on 2026-08-31 and the backend moved to /api/{proxy+} on the API host, so the three deploys are moot. Measured each criterion against what replaced it; only the stale notes on the intermediate mapping remain. --- .../README.md} | 61 +++++++++----- .../notes/R-live-measurement-2026-10-06.md | 80 +++++++++++++++++++ 2 files changed, 122 insertions(+), 19 deletions(-) rename lore/1-tasks/active/{0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md => 0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md} (78%) create mode 100644 lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md diff --git a/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md b/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md similarity index 78% rename from lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md rename to lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md index 4a438013..ce2aadc6 100644 --- a/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live.md +++ b/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md @@ -4,7 +4,7 @@ title: 'Deploy the portal gateway mapping and verify it live — three deploys 0 type: CHORE status: active related_adr: [] -related_tasks: ['0184', '0183', '0185', '0186', '0194', '0141'] +related_tasks: ['0184', '0183', '0185', '0186', '0194', '0141', '0195', '0235', '0309', '0326'] tags: [ layer-infra, @@ -18,7 +18,8 @@ tags: ] milestone: 3 links: - - '../active/0184_FEATURE_portal-hosting-skeleton.md' + - '../../archive/0184_FEATURE_portal-hosting-skeleton.md' + - 'notes/R-live-measurement-2026-10-06.md' - '../../../docs/scf/api-endpoints.md' history: - date: "2026-08-14" @@ -77,10 +78,27 @@ history: ([[0194]]: backend on prices-api.sorobanscan…/api/, [[0326]]: bundle at sorobanscan…/prices-api/), so each criterion is measured against what replaced it. + - date: "2026-10-06" + status: active + who: akot + note: > + Measured live, read-only, about 12:10 UTC + ([R-live-measurement-2026-10-06](notes/R-live-measurement-2026-10-06.md)). + The three deploys are moot: PortalHosting was deleted on 2026-08-31 and + [[0235]]'s `/api/{proxy+}` sits under a new parent, so no + variable-child conflict. 10 of 11 criteria hold, or hold for what + replaced their subject. Open: the stale notes about the intermediate + mapping, in the gateway docblock, the OAuth deploy runbook and + [[0184]]'s record. --- # Deploy the portal gateway mapping and verify it live +> **2026-10-06: no deploy left to do.** Everything below describes production +> as of 2026-08-14. The portal moved off `/api-tokens` and our CloudFront since +> ([[0235]], [[0194]], [[0326]]); the criteria are marked against what replaced +> them — see [R-live-measurement-2026-10-06](notes/R-live-measurement-2026-10-06.md). + ## Summary [[0184]] is merged and its acceptance criteria were measured — but against the @@ -175,28 +193,33 @@ deferred criteria. ## Acceptance Criteria -- [ ] `/api-tokens` returns `302` to `/api-tokens/`, not `403 AccessDenied` -- [ ] `/api-tokens/index.html` carries +- [x] `/api-tokens` returns `302` to `/api-tokens/`, not `403 AccessDenied` + — prefix gone; `sorobanscan…/prices-api` answers `301 /prices-api/` +- [x] `/api-tokens/index.html` carries `Cache-Control: public, max-age=0, must-revalidate` — the entry document must revalidate, or a visitor keeps booting a stale app from a URL that - still resolves -- [ ] Everything under `/api-tokens/assets/` carries + still resolves — `/prices-api/index.html`, measured +- [x] Everything under `/api-tokens/assets/` carries `Cache-Control: public, max-age=31536000, immutable` — safe only because - those names are content-hashed, so a new build is a new URL -- [ ] `/api-tokens/api/a/b/c` returns an empty `404` — greedy matches any depth - again, so the depth-3 `403` is gone -- [ ] The deployed stage carries a throttle entry per verb (`GET`, `POST`, - `DELETE`) at 10 req/s burst 40, with caching off -- [ ] CloudFront access logs are landing in the log bucket, without cookies -- [ ] `/api-tokens/api/config` still answers `200 {"enabled":false}` with + those names are content-hashed, so a new build is a new URL — + `/prices-api/assets/`, all 5 files +- [x] `/api-tokens/api/a/b/c` returns an empty `404` — greedy matches any depth + again, so the depth-3 `403` is gone — `/api/a/b/c` on the API host +- [x] The deployed stage carries a throttle entry per verb (`GET`, `POST`, + `DELETE`) at 10 req/s burst 40, with caching off — and `OPTIONS` +- [x] CloudFront access logs are landing in the log bucket, without cookies + — our distribution is gone; the explorer's land, `IncludeCookies: false` +- [x] `/api-tokens/api/config` still answers `200 {"enabled":false}` with `no-store`, and every other path under the prefix still answers an empty - `404` — the flag is untouched by this task -- [ ] `/health`, `/api-docs-json` and `/v1/assets` (keyless → `403`) unchanged + `404` — the flag is untouched by this task — `/api/config`, now + `{"enabled":true,…}` after [[0194]] opened the portal +- [x] `/health`, `/api-docs-json` and `/v1/assets` (keyless → `403`) unchanged throughout — the data routes must not notice this happening - [ ] Both "ahead of the deploy" notes deleted -- [ ] The synthesized `PortalHosting` template shows the entry-document +- [x] The synthesized `PortalHosting` template shows the entry-document deployment depending on the asset deployment — neither is `DependsOn: null` - any more -- [ ] On a cold cache after the deploy, `/api-tokens/` loads and every asset it + any more — no template any more; `sync-portal-explorer` syncs `assets/*` + first, sequentially +- [x] On a cold cache after the deploy, `/api-tokens/` loads and every asset it references returns `200`; nothing under `/api-tokens/assets/` answers - `403 AccessDenied` + `403 AccessDenied` — per-object misses, no invalidation issued diff --git a/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md b/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md new file mode 100644 index 00000000..65bdef30 --- /dev/null +++ b/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md @@ -0,0 +1,80 @@ +--- +title: "Live re-measurement of the portal routes, 2026-10-06 — every criterion that still applies holds" +type: research +status: mature +tags: [portal, measurement, apigateway, cloudfront, production] +links: + - "../README.md" + - "../../../../../docs/scf/api-endpoints.md" +history: + - date: "2026-10-06" + status: mature + who: akot + note: > + Run read-only against production on 2026-10-06 at about 12:10 UTC: + public HTTP requests to both hosts, and AWS reads (API Gateway stage, + CloudFormation stack list, CloudFront distribution config) as + `AWS_PROFILE=stellar`. One point in time. +--- + +# Live re-measurement of the portal routes, 2026-10-06 + +## What changed since the task was written + +The task describes the 2026-08-14 production: the portal on our own +CloudFront (`PortalHosting`, `dojr4epgxo2qp.cloudfront.net`) under +`/api-tokens/`, the gateway mapping the intermediate `{proxy}` + +`{proxy}/{sub}` pair. Neither exists any more: + +- `Prices-production-PortalHosting` is `DELETE_COMPLETE` (2026-08-31 10:09 + UTC), deleted in [[0194]]'s cutover that moved the backend to the API's own + host; [[0195]] removed the stack from the code (`e8ed5c0e`). No CloudFront + distribution of ours serves the portal. +- The backend is `/api/{proxy+}` directly under the root on + `prices-api.sorobanscan.rumblefish.dev` — [[0235]] moved the prefix off + `/api-tokens` (`9d39207e`, 2026-08-28). `/api` is a new parent, so the + one-variable-child conflict that made this task three deploys never arose: + an ordinary gateway deploy created the greedy proxy and deleted the + `/api-tokens` tree. +- The bundle is on the block explorer's distribution `EA2TLS5SS5M87` + (`sorobanscan.rumblefish.dev`), under `/prices-api/` since [[0326]] + (2026-10-02), synced by `make -C infra sync-portal-explorer`. + +So the three deploys are not needed; what follows measures each criterion +against what replaced its subject. + +## Results + +| # | Criterion | Measured | Verdict | +|---|---|---|---| +| 1 | `/api-tokens` → `302` | `sorobanscan…/prices-api` → `301 Location: /prices-api/` (`FunctionGeneratedResponse`); `sorobanscan…/api/` → `301 /prices-api/`. `/api-tokens` on the API host → gateway `404 {"code":"not_found"}` | subject gone; replacement holds | +| 2 | entry document `public, max-age=0, must-revalidate` | `/prices-api/`, `/prices-api/index.html`, `/prices-api/dashboard`: `200 text/html`, 2738 B, `public, max-age=0, must-revalidate` | holds | +| 3 | `assets/*` `public, max-age=31536000, immutable` | the 2 assets `index.html` references and the 3 chunks the entry script loads lazily: `200`, `public, max-age=31536000, immutable` | holds | +| 4 | depth 3 → empty `404` | `/api/a`, `/api/a/b`, `/api/a/b/c`: `404`, 0 bytes (the handler's gate, not the gateway); `/api/auth/me` (depth 3, real route) `200 {"authenticated":false}` | holds | +| 5 | per-verb throttle 10/40, caching off | stage `production`, `api/{proxy+}/{GET,POST,DELETE,OPTIONS}`: rate 10, burst 40, `cachingEnabled: false`. `*/*` default 200/400 | holds | +| 6 | CloudFront access logs, no cookies | no distribution of ours. The explorer's `EA2TLS5SS5M87`: `Logging.Enabled: true`, `IncludeCookies: false`, bucket `production-soroban-explorer-cf-logs`, newest object `EA2TLS5SS5M87.2026-10-06-12.…gz` at 12:18 UTC | subject gone; explorer's holds | +| 7 | `/api/config` `200`, `no-store`; other paths empty `404` | `200 {"enabled":true,"rate_limit_per_second":1}`, `no-store`; with `Origin: https://sorobanscan.rumblefish.dev` adds `Access-Control-Allow-Origin` (that origin) + `Allow-Credentials: true`. `enabled: true` is [[0194]]'s opening, not a regression | holds for today's flag | +| 8 | `/health`, `/api-docs-json`, `/v1/assets` keyless | `200 {"status":"ok","stack":"prices-production"}`; `200` 58 863 B `public, max-age=300` (same at `/api/api-docs-json`); `403 {"message":"Forbidden"}` | holds | +| 9 | "ahead of the deploy" notes deleted | `docs/scf/api-endpoints.md`: already gone. Still present: `api-gateway-stack.ts` docblock of `PORTAL_API_METHODS`, `docs/runbooks/portal-oauth-deploy-prep.md` §preconditions, [[0184]]'s Implementation Notes | open at measurement | +| 10 | entry-document deployment `DependsOn` the asset one | no template. `sync-portal-explorer` runs the `assets/*` sync before the rest, sequentially in one recipe, then one invalidation — the race cannot occur | subject gone; ordering holds | +| 11 | cold cache: every asset `200`, none `403` | all 5 bundle files and the 3 icons `200`, most `Miss from cloudfront`. No invalidation was issued (a write), so "cold" is per-object misses, not a flushed distribution | holds | + +Also seen: `OPTIONS /api/config` with `Origin` + +`Access-Control-Request-Method: DELETE` → `204`, `allow-methods: GET,POST,DELETE`, +`allow-headers: Content-Type,Accept,X-Requested-With`, `max-age: 3600`, +credentials true — a MOCK answer, no invocation. A verb outside the list +(`PATCH /api/config`) gets the gateway's `404 {"code":"not_found"}` ([[0309]]). + +[[0185]]'s decision 13 (`/api-tokens/` in `distributionPaths`) has no subject: +the invalidation is now `'/prices-api/*'` on the explorer's distribution. + +## How it was measured + +`curl -s -D -` (GET; OPTIONS and PATCH where stated) against both hosts, +keeping the status line, `Location`, `Cache-Control`, `Content-Type`, +`Content-Length`, `Access-Control-*`, `X-Cache`, `x-amzn-ErrorType` and the +first bytes of the body. Asset list from the `src`/`href` attributes of +`/prices-api/`, lazy chunks from `assets/…` names inside the entry script. +AWS: `apigateway get-stages` on `02mabge71l`, `apigateway get-resources`, +`cloudformation list-stacks`, `cloudfront get-distribution-config +EA2TLS5SS5M87`, `s3api list-objects-v2` on the log bucket. The scripts stayed outside the repository. From fc407c1a978ca9363a41f5d6959d957ee8e35fe9 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:36:39 +0200 Subject: [PATCH 039/144] docs(lore-0274): complete and archive the task --- .../README.md | 23 +++++++++++++------ ...offer-priced-dust-population-2026-09-29.md | 0 2 files changed, 16 insertions(+), 7 deletions(-) rename lore/1-tasks/{active => archive}/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md (94%) rename lore/1-tasks/{active => archive}/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md (100%) diff --git a/lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md b/lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md similarity index 94% rename from lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md rename to lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md index 755cfb56..581adbe2 100644 --- a/lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md +++ b/lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/README.md @@ -2,7 +2,7 @@ id: "0274" title: "1-stroop offer-priced fills still publish a USD price — e.g. XAUa at $4,375 on $0.003 of volume" type: FEATURE -status: active +status: completed related_adr: ["0287"] related_tasks: ["0116", "0147", "0252", "0236", "0286", "0278", "0310", "0216"] tags: [layer-backend, layer-api, priority-medium, effort-medium, milestone-M3, data-quality, liquidity, api] @@ -12,7 +12,7 @@ links: - "../../../../packages/prices-clickhouse/schema/current.sql" - "notes/R-offer-priced-dust-population-2026-09-29.md" history: - - date: 2026-09-10 + - date: "2026-09-10" status: backlog who: okarcz note: > @@ -22,7 +22,7 @@ history: asset-level property. Filed rather than absorbed into 0116, because it is a different claim about a different subject — 0116 is about a bucket, this is about an asset. - - date: 2026-09-25 + - date: "2026-09-25" status: backlog who: okarcz note: > @@ -34,7 +34,7 @@ history: 103120 `USD` ($0.10 published vs a 0.5 XLM close) is out of scope here; it goes to its own new task. Retitled, because the old 93% headline describes a population that is now `unpriced`. Original text kept below. - - date: 2026-09-29 + - date: "2026-09-29" status: active who: akot note: > @@ -45,6 +45,14 @@ history: Decided: keep the price and publish what it rests on in a separate `price_basis` field (option C); no `unpriced`, no fourth `price_status` word. Converted to a directory for the measurement note. + - date: "2026-10-05" + status: completed + who: akot + note: > + PR #373 merged to develop as 586713f1: `price_basis` (`trades` / + `offer_dust` / '') on current_prices, the price, list and batch + endpoints, OpenAPI and portal samples. 782 Rust tests, 270 portal tests. + Closed before deploy; the rollout order in Implementation Notes is binding. --- # 1-stroop offer-priced fills still publish a USD price @@ -93,10 +101,11 @@ about $0.20). That is a different defect and has its own task. 0 of 224 thin-but-real assets (at most 5 priced minutes, at least $1 in 24 h) flagged. Known edge: BTC `GBVFOW…`, $86 in 24 h in trades of about 150 stroops each, reads `offer_dust` — see Implementation Notes.* -- [ ] A consumer can tell, without running their own aggregation, that such a +- [x] A consumer can tell, without running their own aggregation, that such a price does not rest on a real market (e.g. it is `unpriced`, or carries a - signal that says so). *Via `price_basis = "offer_dust"`.* -- [ ] `volume_quote_usd` and volume aggregates are explicitly unchanged. + signal that says so). *Via `price_basis = "offer_dust"`. Merged in #373 + (586713f1); reaches consumers only with the rollout, not deployed at close.* +- [x] `volume_quote_usd` and volume aggregates are explicitly unchanged. *By construction: candle tables, rollups and enrichment are not touched.* ## Design (decided 2026-09-29) diff --git a/lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md b/lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md similarity index 100% rename from lore/1-tasks/active/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md rename to lore/1-tasks/archive/0274_FEATURE_assets-with-no-meaningful-trading-are-published-as-priced/notes/R-offer-priced-dust-population-2026-09-29.md From 77903bdc860346ff02c316a0b2c45a9dfd77a76b Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:36:43 +0200 Subject: [PATCH 040/144] docs(lore-0139): complete and archive the task --- ...rice-usd-fans-out-on-duplicate-asset-id.md | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) rename lore/1-tasks/{active => archive}/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md (94%) diff --git a/lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md b/lore/1-tasks/archive/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md similarity index 94% rename from lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md rename to lore/1-tasks/archive/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md index 4999047e..eb50c3d0 100644 --- a/lore/1-tasks/active/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md +++ b/lore/1-tasks/archive/0139_BUG_current-price-usd-fans-out-on-duplicate-asset-id.md @@ -2,7 +2,7 @@ id: "0139" title: "current_price_usd returns duplicate rows — assets is keyed on natural identity, not asset_id" type: BUG -status: active +status: completed assignee: akot related_adr: [] related_tasks: ["0072", "0061", "0067", "0144", "0150", "0129"] @@ -11,6 +11,18 @@ tags: milestone: 2 links: [] history: + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed. The window ran 2026-10-02 (swap 12:46 UTC), PR #382 merged as + 44809656, W14 gap-verify and the W16 next-day checks all green. PR #387 + (phase-3 preflight on live tiers) merged. Checked read-only on prod + 2026-10-06: assets FINAL 211,103 rows = 211,103 distinct asset_ids; + current_price_usd 3,736 rows = current_prices 3,736; 0129's join + cross-check 94,907,504 = 94,907,504. Still owed at close: the second + pass over months ≤ 202202, dropping the __pre0139 tables after it, + W17 period-close after 2026-11-02, and the note to BE. - date: "2026-10-02" status: active who: akot @@ -553,8 +565,11 @@ stopgap. Spawn accordingly. - [x] Determined whether the 3,275 duplicated `asset_id`s are ID collisions or superseded natural-identity rows, with the measurement recorded. **2026-09-30: all 3,315 are collisions, 0 superseded.** -- [ ] `current_price_usd` returns exactly one row per `current_prices` row. -- [ ] Every other view in `views.sql` audited for the same join defect. +- [x] `current_price_usd` returns exactly one row per `current_prices` row. + *Prod 2026-10-06: 3,736 = 3,736.* +- [x] Every other view in `views.sql` audited for the same join defect. + *Fixed at the source instead: `assets` is unique on `asset_id`, so no + join on it can fan out (`asset_id_schema_it.rs`).* - [x] **O2 — the XLM-native and USDC `asset_id`s checked against the 3,279 duplicates** (one query; see the open section above). **2026-09-30: XLM (4), USDC (3) and USDT (111) are clean; no foreign candle is admitted.** @@ -563,15 +578,21 @@ stopgap. Spawn accordingly. the pivot tier's `xlm_usd` — invisibly, because uniform duplication leaves the weighted value unchanged. Low prior, but it must not be assumed. Raised by [[0144]] while answering BE's `volume_base` question. -- [ ] A test fails if the fan-out reappears. -- [ ] BE informed of the resolution. +- [x] A test fails if the fan-out reappears. + *`current_price_usd_returns_one_row_per_current_prices_row` in + `asset_id_schema_it.rs`.* +- [ ] BE informed of the resolution. *Not sent at close; owed.* Carried from [[0129]] when it closed into this task (2026-09-24) — they check the allocator, not the view, so a view-only dedupe does not satisfy them: -- [ ] `SELECT count(), countDistinct(asset_id) FROM prices.assets FINAL` returns - equal values in production. -- [ ] 0129's two-query cross-check (its §Evidence) agrees to the row. -- [ ] An invariant test or probe guards `asset_id` uniqueness going forward. +- [x] `SELECT count(), countDistinct(asset_id) FROM prices.assets FINAL` returns + equal values in production. *Prod 2026-10-06: 211,103 = 211,103.* +- [x] 0129's two-query cross-check (its §Evidence) agrees to the row. + *Prod 2026-10-06, `price_ohlcv_1h` 2024-02…2026-07, `volume_quote > 0`: + 94,907,504 without the join = 94,907,504 with it.* +- [x] An invariant test or probe guards `asset_id` uniqueness going forward. + *`rollup-freshness-probe/src/asset_id_uniqueness.rs` and its alarm.* - [ ] `GET /assets` verified to emit no duplicate asset across a full cursor - walk (extends 0074's pagination test). + walk (extends 0074's pagination test). *No full walk was run. It cannot + emit a duplicate while `asset_id` is unique on prod (above).* From 3ffdbbc778430cbf94222b071468956faa9bd70f Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:46:52 +0200 Subject: [PATCH 041/144] chore(0205): drop the stale notes on the intermediate proxy mapping (#397) * docs(lore-0205): drop the notes on the intermediate proxy mapping Both still described the {proxy} + {proxy}/{sub} pair as deployed and 0205 as the deploy that replaces it. Production has served the greedy /api/{proxy+} since the prefix moved (0235); measured on 2026-10-06. * docs(lore-0205): complete and archive the task Closed with no deploy: the portal 0205 was written for no longer exists, and every criterion holds against what replaced its subject. Also resolves the stale deploy note in 0184's record. --- docs/runbooks/portal-oauth-deploy-prep.md | 9 ++---- infra/src/lib/stacks/api-gateway-stack.ts | 10 ++---- .../0184_FEATURE_portal-hosting-skeleton.md | 27 ++++------------ .../README.md | 32 +++++++++++++++++-- .../notes/R-live-measurement-2026-10-06.md | 0 5 files changed, 42 insertions(+), 36 deletions(-) rename lore/1-tasks/{active => archive}/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md (88%) rename lore/1-tasks/{active => archive}/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md (100%) diff --git a/docs/runbooks/portal-oauth-deploy-prep.md b/docs/runbooks/portal-oauth-deploy-prep.md index f4b7f3e4..1b808501 100644 --- a/docs/runbooks/portal-oauth-deploy-prep.md +++ b/docs/runbooks/portal-oauth-deploy-prep.md @@ -297,12 +297,9 @@ finishing a slice. Before it happens, both must be true: which takes out `/v1` as well — the read is skipped entirely while the portal is closed, so this is a failure mode that only appears at the moment of opening). -- The gateway maps the portal prefix as a greedy `{proxy+}`. The four sign-in - routes are at **depth 3** (`auth/login`), and the deployed mapping at the time - of writing is the intermediate `{proxy}` + `{proxy}/{sub}` pair, which answers - `403 Missing Authentication Token` at that depth (`404 {"code": "not_found"}` - since task 0309). Task 0205 ships the committed shape. (Task 0187's `/key` is - at depth 3 as well, so it is the same deploy.) +- The gateway maps the portal prefix as a greedy `{proxy+}`, because the four + sign-in routes and task 0187's `/key` are at **depth 3** (`auth/login`). + Measured live on 2026-10-06 (task 0205). - **`ApiGatewayStack` has deployed at least once since task 0187 merged**, so that `/prices/production/pricing-api-free-plan-id` exists. See §7 — if it does not, opening the portal fails Lambda init and takes `/v1` down with it. diff --git a/infra/src/lib/stacks/api-gateway-stack.ts b/infra/src/lib/stacks/api-gateway-stack.ts index 3d144846..cc4ca08c 100644 --- a/infra/src/lib/stacks/api-gateway-stack.ts +++ b/infra/src/lib/stacks/api-gateway-stack.ts @@ -145,14 +145,8 @@ const PORTAL_API_RESOURCE_PATH = '/api/{proxy+}'; * `/auth/logout`; `DELETE` for task 0192's revoke if it prefers that shape to a * `POST`. * - * ⚠️ Task 0186's four routes sit at **depth 3** (`auth/login`, not `login`), - * which the greedy `{proxy+}` above covers and the intermediate - * `{proxy}` + `{proxy}/{sub}` pair that is CURRENTLY DEPLOYED does not — see - * task 0205, which ships this file's committed shape. Until that deploy runs, - * `/api/auth/login` answers the gateway's own - * `403 Missing Authentication Token` rather than reaching the handler. That is a - * deployment gap, not a defect in either task: the flag keeps the handler dark - * regardless, and `/config` at depth 1 is unaffected. + * Task 0186's four routes sit at **depth 3** (`auth/login`, not `login`), which + * is why the proxy is greedy rather than a fixed number of `{proxy}` levels. * * ⚠️ A verb that is NOT listed here never reaches the handler: it gets the * gateway's own `404 {"code": "not_found", …}` (task 0309's gateway response) diff --git a/lore/1-tasks/archive/0184_FEATURE_portal-hosting-skeleton.md b/lore/1-tasks/archive/0184_FEATURE_portal-hosting-skeleton.md index 8f9ba5fb..9eef1648 100644 --- a/lore/1-tasks/archive/0184_FEATURE_portal-hosting-skeleton.md +++ b/lore/1-tasks/archive/0184_FEATURE_portal-hosting-skeleton.md @@ -203,26 +203,13 @@ directly through execute-api after the failed first attempt (see below): `/health` `200`, `/api-docs-json` `200`, `/v1/assets` `403`, and `Prices-production-{ApiGateway,Compute}` both `UPDATE_COMPLETE`. -> **Production does not match this record, and the deploy is [[0205]].** Every -> acceptance criterion above holds live, but four properties that landed after -> the 2026-08-13 deploy exist only in code: -> -> - the gateway maps `ANY {proxy}` + `{proxy}/{sub}` with **no throttle** — -> neither the original `{proxy+}` nor decision 12's shape, left there by the -> 2026-08-14 deploy attempt. Behaviour is correct at depth 1-2 and `403` at -> depth 3. -> - the access-log bucket, `Cache-Control` on the uploaded objects and the -> trailing-slash redirect are code-only; `/api-tokens` answers `403 -> AccessDenied` rather than `302`. -> - `Prices-production-Compute` **was** deployed (2026-08-14 09:42) as a side -> effect of `cdk deploy` pulling in dependency stacks — the Makefile targets -> do not pass `--exclusively`. So [[0183]]'s handler and `PORTAL_ENABLED=false` -> are live ahead of that task's merge, and `/config` now answers -> `200 {"enabled":false}` with `no-store`. -> -> [[0205]] carries the three deploys, the reason there are three rather than -> one, and the probes to re-run. It also owns deleting this note and its twin in -> `docs/scf/api-endpoints.md`. +> **Resolved 2026-10-06 by [[0205]].** A note here described production as of +> 2026-08-14: the intermediate `{proxy}` + `{proxy}/{sub}` mapping with no +> throttle, and the access logs, upload `Cache-Control` and trailing-slash +> redirect still code-only. None of it is live any more — the prefix moved to +> `/api` ([[0235]]), the gateway serves the greedy `/api/{proxy+}` with the +> per-verb throttle, and the `PortalHosting` stack was deleted on 2026-08-31 +> ([[0194]], [[0195]]). See [[0205]]'s live re-measurement. ## Issues Encountered diff --git a/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md b/lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md similarity index 88% rename from lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md rename to lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md index ce2aadc6..2c41d93e 100644 --- a/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md +++ b/lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/README.md @@ -2,7 +2,7 @@ id: '0205' title: 'Deploy the portal gateway mapping and verify it live — three deploys 0184 merged but never shipped' type: CHORE -status: active +status: completed related_adr: [] related_tasks: ['0184', '0183', '0185', '0186', '0194', '0141', '0195', '0235', '0309', '0326'] tags: @@ -90,6 +90,15 @@ history: replaced their subject. Open: the stale notes about the intermediate mapping, in the gateway docblock, the OAuth deploy runbook and [[0184]]'s record. + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed with no deploy: the portal this task was written for no longer + exists. 11 of 11 criteria met — 8 against today's subject, 3 (redirect, + access logs, DependsOn) against what replaced theirs. The last one was + the stale notes: the gateway docblock, the OAuth deploy runbook and + [[0184]]'s record, on this branch. No code change, no follow-up tasks. --- # Deploy the portal gateway mapping and verify it live @@ -215,7 +224,9 @@ deferred criteria. `{"enabled":true,…}` after [[0194]] opened the portal - [x] `/health`, `/api-docs-json` and `/v1/assets` (keyless → `403`) unchanged throughout — the data routes must not notice this happening -- [ ] Both "ahead of the deploy" notes deleted +- [x] Both "ahead of the deploy" notes deleted — the `api-endpoints.md` one was + already gone; removed the gateway docblock's, the OAuth runbook's and + [[0184]]'s - [x] The synthesized `PortalHosting` template shows the entry-document deployment depending on the asset deployment — neither is `DependsOn: null` any more — no template any more; `sync-portal-explorer` syncs `assets/*` @@ -223,3 +234,20 @@ deferred criteria. - [x] On a cold cache after the deploy, `/api-tokens/` loads and every asset it references returns `200`; nothing under `/api-tokens/assets/` answers `403 AccessDenied` — per-object misses, no invalidation issued + +## Design Decisions + +### Emerged + +1. **Measured against what replaced each criterion's subject, not closed as + void.** Three criteria name things that are gone (`/api-tokens`, our + CloudFront, the `PortalHosting` template). Each has a successor with the + same job — the explorer's `/prices-api` redirect, its access logs, the + ordered `sync-portal-explorer` — so the property was checked there. +2. **The 0184 note was replaced by a short pointer, not deleted.** An archived + record that loses the paragraph explaining why its numbers differ from + production reads as wrong; three lines saying it was resolved, and by what, + keep it truthful. +3. **No cache invalidation for the cold-cache criterion.** It is a write on + another team's distribution; per-object `Miss from cloudfront` on the + bundle files was taken as enough. diff --git a/lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md b/lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md similarity index 100% rename from lore/1-tasks/active/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md rename to lore/1-tasks/archive/0205_CHORE_deploy-the-portal-gateway-mapping-and-verify-live/notes/R-live-measurement-2026-10-06.md From 683d1eafaf06e0683fe932967fb3178166c35ec8 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:05:25 +0200 Subject: [PATCH 042/144] docs(lore-0131): close as superseded by 0286 --- .../0131_FEATURE_preroll-usd-coverage-gate.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) rename lore/1-tasks/{backlog => archive}/0131_FEATURE_preroll-usd-coverage-gate.md (85%) diff --git a/lore/1-tasks/backlog/0131_FEATURE_preroll-usd-coverage-gate.md b/lore/1-tasks/archive/0131_FEATURE_preroll-usd-coverage-gate.md similarity index 85% rename from lore/1-tasks/backlog/0131_FEATURE_preroll-usd-coverage-gate.md rename to lore/1-tasks/archive/0131_FEATURE_preroll-usd-coverage-gate.md index 0c1ac361..daff27fd 100644 --- a/lore/1-tasks/backlog/0131_FEATURE_preroll-usd-coverage-gate.md +++ b/lore/1-tasks/archive/0131_FEATURE_preroll-usd-coverage-gate.md @@ -2,15 +2,26 @@ id: "0131" title: "0088 step-3 pre-roll: gate/warn when 1m USD coverage for the span is below a threshold" type: FEATURE -status: backlog +status: completed related_adr: [] -related_tasks: ["0114", "0088", "0144", "0147", "0145"] +related_tasks: ["0114", "0088", "0144", "0147", "0145", "0286"] tags: [clickhouse, enrichment, backfill, pre-roll, guard, priority-low, effort-small] links: - "../../../packages/prices-clickhouse/schema/preroll-incremental.sql" - "../../../docs/runbooks/continue-soroban-backfill.md" history: - - date: 2026-07-24 + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed as superseded by [[0286]]. The flow this guard was for is gone: + `preroll-incremental.sql` is marked HISTORICAL, DO NOT RUN, and fails on + the post-0286 column count. Phase 3 pre-rolls coarse tiers from + re-ingested 1m that carries close_usd = 0 by design. The coarse re-price + (re-ingest runbook §7b-2) then fixes them, and 0286's AC checks + `reachable_left` ~0 per tier. Nothing was built and there are no + follow-ups. + - date: "2026-07-24" status: backlog who: okarcz note: "Spawned from 0114 future work — the last 0114 AC; belongs in 0088's step-3 pre-roll flow, which hasn't run yet." @@ -81,3 +92,5 @@ exercised once that pre-roll runs — which is why it doesn't fit as a 0114 bloc - [ ] Below-threshold coverage **warns loudly** (hard-refuse opt-in), and the expected-low pre-Soroban tail does not spuriously block the recovery. - [ ] Documented in the 0088 pre-roll runbook. + +*None built: superseded by [[0286]] (see history, 2026-10-06).* From 86e32a3e1fc162e58bc5e7c140ba14dab068c675 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:06:56 +0200 Subject: [PATCH 043/144] docs(lore-0236): complete and archive the task --- ...-for-internally-inconsistent-ohlcv-rows.md | 68 +++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) rename lore/1-tasks/{active => archive}/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md (74%) diff --git a/lore/1-tasks/active/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md b/lore/1-tasks/archive/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md similarity index 74% rename from lore/1-tasks/active/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md rename to lore/1-tasks/archive/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md index 29c0f118..f62e2824 100644 --- a/lore/1-tasks/active/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md +++ b/lore/1-tasks/archive/0236_BUG_no-detector-for-internally-inconsistent-ohlcv-rows.md @@ -2,7 +2,7 @@ id: "0236" title: "Nothing detects an internally inconsistent `price_ohlcv_*` row — and 0229's clamp removed the one surface that used to surface them" type: BUG -status: active +status: completed assignee: akot related_adr: ["0011"] related_tasks: ["0229", "0120", "0182", "0227"] @@ -46,6 +46,14 @@ history: all seven tiers on prod, so the alarm will not fire on legacy rows. The branch is rebased on develop, alongside 0139's and 0203's probe checks, as block 5c. PR to follow; deploy is Adam's. + - date: "2026-10-06" + status: completed + who: akot + note: > + PR #396 merged (81a48f08) after review; Observability (3 ohlc-band + alarms) and EventBridge (probe Lambda only) deployed 2026-10-06. First + run 13:05 UTC: 7 tiers read, 0 violations, checks_failed 0, alarms OK. + Probe 104 unit tests, 40 ClickHouse ITs. --- # No detector for an internally inconsistent stored candle @@ -181,7 +189,59 @@ construction and checked with zero tolerance where it is checked at all. - [x] If the count is non-zero, the cause is identified and filed as its own task rather than absorbed here. Band violations are 0. The legacy zero-price rows are recorded above and left to [[0286]] phase 3. -- [ ] A recurring check exists wherever the other data-quality probes live, with - its threshold justified by the measured baseline. -- [ ] 0229's clamp is explicitly confirmed as the right behaviour for the read +- [x] A recurring check exists wherever the other data-quality probes live, with + its threshold justified by the measured baseline. Block 5c of + `rollup-freshness-probe`, every 15 min, alarm at >= 1 (baseline 0). +- [x] 0229's clamp is explicitly confirmed as the right behaviour for the read path, or changed — with the decision recorded in [[ADR-0011]] §3. + Confirmed, merged with #396. + +## Implementation Notes + +- `packages/rollup-freshness-probe/src/ohlc_band.rs`: seven per-tier `FINAL` + queries, the `band` and `nonpositive` classes, `ohlc_band_totals` (the one + completeness rule) and the `CandleBandViolations` metric with its two + refusals (`EmptyScan`, `Incomplete`). +- `main.rs` block 5c: the seven reads run in parallel on a `JoinSet`, each + under block 6's `mismatch_read_bound`, server- and client-side. +- Observability: `prices-{env}-ohlc-band-{1,100,10000}` on `zeroLadder`. +- ADR-0011 §3: the clamp stays; the as-stored arm does not clamp. + +## Deployed to production (2026-10-06) + +| step | result | +|---|---| +| merge | #396 → `develop` 81a48f08, 12:43 UTC | +| Observability | 3 alarms created, dashboard 86 alarms, 14:48 CEST | +| EventBridge | only `prices-production-rollup-freshness-probe` changed, 12:57 UTC | +| first run | 13:05:21 UTC: 7 tiers, 0 violations, `checks_failed` 0, 3.1–3.4 s, 51 MB | +| alarms | `ohlc-band-*` INSUFFICIENT_DATA → OK | + +## Design Decisions + +### From Plan + +1. **Predicate, scope, threshold**: Adam's decisions of 2026-09-28 above. + +### Emerged + +2. **Block 5c, not last**: block 6 (0203) is time-budgeted and stays last. + The seven reads cost 0.29 s of server time on prod. +3. **Bounded, parallel reads** (review of #396): a stalled tier is a + recorded failure, not a Lambda timeout that loses block 6 and the log. +4. **One completeness rule** (review): `ohlc_band_totals`, which also + rejects duplicate or unknown tier names. +5. **Review finding 1 declined**: `pf_trade_count > 0` means priced on every + tier, because the ingest floor is the rollup floor (0286 WR-03). Prod had + 0 rows that break this from 2026-09-23 to 2026-10-06. A break would be the + WR-03 defect, so paging on it is right. Documented in the module doc. +6. **The old blocked entry carries `by: ["0286"]`**: the validator requires + `by`, and the block was waiting for pre-0286 buckets to leave the window. + +## Issues Encountered + +- **Rebase onto 299 commits of develop**: 0139 (block 5b) and 0203 (block 6, + time budget, `%reading` log style) had grown the same probe. Four files + conflicted; the band block was fitted in between. +- **`client()` became async on develop**: it now stops the shared reconcile + MVs before a test seeds tiers. The 0236 ITs were adapted in c2eeddaf. From 34152397a5aec2ecc4806636e5580460f2a24cab Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:07:05 +0200 Subject: [PATCH 044/144] fix(lore-0329): count and alarm on trades dropped for decimals The decode now counts trades (not legs) dropped for unknown decimals. The live processor puts them on RunStats, WARNs with the contracts and publishes TradesMissingDecimals, alarmed like UnregisteredPoolEvents; sdex-backfill prints them in its summary like events-backfill. --- infra/src/lib/stacks/observability-stack.ts | 31 ++++++++++++++++ packages/events-backfill/src/run.rs | 6 ++-- packages/prices-ingest-core/src/soroban.rs | 12 +++++-- packages/prices-ledger-processor/src/main.rs | 5 +++ .../prices-ledger-processor/src/metrics.rs | 29 +++++++++++++++ .../prices-ledger-processor/src/reconcile.rs | 36 +++++++++++++++++++ packages/sdex-backfill/src/ingest.rs | 6 ++++ packages/sdex-backfill/src/run.rs | 8 +++++ 8 files changed, 128 insertions(+), 5 deletions(-) diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index 19ed6799..a2db8aef 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -339,6 +339,7 @@ export class ObservabilityStack extends cdk.Stack { public readonly ledgerProcessorForcedPartialFlushAlarm: cloudwatch.Alarm; /** Task 0291: live dropped trades from a pool missing from `pool_registry`. */ public readonly ledgerProcessorUnregisteredPoolAlarm: cloudwatch.Alarm; + public readonly ledgerProcessorTradesMissingDecimalsAlarm: cloudwatch.Alarm; /** * Task 0100: the weekly coverage sweep found swap/trade emitters in neither * `pool_registry` nor the committed allow-list (layer 3). @@ -1468,6 +1469,36 @@ export class ObservabilityStack extends cdk.Stack { this.ledgerProcessorUnregisteredPoolAlarm.addAlarmAction(snsAction); this.ledgerProcessorUnregisteredPoolAlarm.addOkAction(snsAction); + // Task 0329 — live dropped trades whose Soroban token decimals did not + // resolve. The ingest scales each AMM leg by its token's `decimals()`, read + // over RPC on first sight, and never guesses: until the call answers, the + // token's trades produce no candle. Published only when non-zero. + this.ledgerProcessorTradesMissingDecimalsAlarm = new cloudwatch.Alarm( + this, + 'LedgerProcessorTradesMissingDecimalsAlarm', + { + alarmName: `prices-${config.envName}-ledger-processor-trades-missing-decimals`, + alarmDescription: + 'The ledger-processor dropped AMM trades because a Soroban token\'s decimals() did not resolve over RPC (task 0329). Those trades produce no candle. Either the RPC (SOROBAN_RPC_URL, default mainnet.sorobanrpc.com) was failing, in which case the token is asked again after 10 minutes, or the token has no usable decimals(), in which case it is not asked again until a cold start. The WARN "dropped AMM trades whose token decimals did not resolve" lists the contracts. Fix: once decimals() answers (insert the row into prices.asset_decimals by hand if it never will), reprice the dropped minutes with events-backfill.', + metric: new cloudwatch.Metric({ + namespace: 'Prices/Ingest', + metricName: 'TradesMissingDecimals', + dimensionsMap: { Environment: config.envName }, + statistic: 'Sum', + period: cdk.Duration.minutes(5), + }), + threshold: 1, + evaluationPeriods: 1, + datapointsToAlarm: 1, + comparisonOperator: + cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + // Emitted only when trades were dropped: "missing" is healthy. + treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, + }, + ); + this.ledgerProcessorTradesMissingDecimalsAlarm.addAlarmAction(snsAction); + this.ledgerProcessorTradesMissingDecimalsAlarm.addOkAction(snsAction); + // Task 0100 — layer 3 of the coverage model. Layer 2 (the alarm above) // sees only pools shaped like a venue we already index; the weekly coverage // sweep looks at EVERY contract emitting swap/trade-shaped events over a diff --git a/packages/events-backfill/src/run.rs b/packages/events-backfill/src/run.rs index 6a756239..0b23251b 100644 --- a/packages/events-backfill/src/run.rs +++ b/packages/events-backfill/src/run.rs @@ -104,7 +104,7 @@ fn accumulate_ledger( raw_unresolved: &mut Vec, missing_decimals: &mut u64, ) { - *missing_decimals += out.missing_decimals.len() as u64; + *missing_decimals += out.trades_missing_decimals as u64; for (source, tick) in out.amm_ticks { accumulators.entry(source).or_default().merge(tick); *ticks_by_source.entry(source).or_default() += 1; @@ -260,8 +260,8 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { // not cover this ledger" — it can only be a corrupt or unexpected row, and // any non-zero count means those fills ARE in the wrong order. let mut apply_order_fallbacks: u64 = 0; - // Unknown-decimals legs of AMM trades the run could not reprice because - // the token's `decimals()` did not resolve (task 0329). + // AMM trades the run could not reprice because a token's `decimals()` did + // not resolve (task 0329) — trades, not legs. let mut missing_decimals: u64 = 0; // Run-level state (persists across chunks): one accumulator per source and diff --git a/packages/prices-ingest-core/src/soroban.rs b/packages/prices-ingest-core/src/soroban.rs index 7f40ff31..289fda48 100644 --- a/packages/prices-ingest-core/src/soroban.rs +++ b/packages/prices-ingest-core/src/soroban.rs @@ -194,9 +194,12 @@ pub struct LedgerSoroban { /// The Soroban token behind every AMM trade dropped because its /// `decimals()` is not known yet (task 0329): one entry per unknown leg of /// each dropped trade. A caller that can reach RPC resolves these and - /// decodes the ledger again; whatever is still here after that is lost - /// volume. + /// decodes the ledger again. pub missing_decimals: Vec, + /// How many AMM trades were dropped for that reason — trades, not legs. + /// Whatever is still counted after the caller's second decode is lost + /// volume. + pub trades_missing_decimals: u32, } fn collect_tx_metas(lcm: &LedgerCloseMeta) -> Vec<&TransactionMeta> { @@ -567,6 +570,7 @@ fn classify_amm_groups( match dispatch(&rows, ®.venue, ®.phoenix, reg.pair_registries()) { Ok(trades) => { for t in trades { + let unknown_legs = out.missing_decimals.len(); if let Some(tick) = amm_trade_to_tick( &t, transaction_index, @@ -575,6 +579,8 @@ fn classify_amm_groups( &mut out.missing_decimals, ) { out.amm_ticks.push((source, tick)); + } else if out.missing_decimals.len() > unknown_legs { + out.trades_missing_decimals += 1; } } } @@ -2476,6 +2482,7 @@ mod tests { ); assert!(out.amm_ticks.is_empty()); assert_eq!(out.missing_decimals, vec![SEAM_T1.to_string()]); + assert_eq!(out.trades_missing_decimals, 1); assert!(out.unresolved.is_empty(), "a known pool is not unresolved"); assets.set_decimals(SEAM_T1.to_string(), 7); @@ -2490,6 +2497,7 @@ mod tests { ); assert_eq!(out.amm_ticks.len(), 1); assert!(out.missing_decimals.is_empty()); + assert_eq!(out.trades_missing_decimals, 0); } } diff --git a/packages/prices-ledger-processor/src/main.rs b/packages/prices-ledger-processor/src/main.rs index efab6fc3..7402d658 100644 --- a/packages/prices-ledger-processor/src/main.rs +++ b/packages/prices-ledger-processor/src/main.rs @@ -235,6 +235,11 @@ async fn handler( m.extend(metrics::unregistered_pool_event_metrics( stats.unregistered_pool_events.values().sum(), )); + // Task 0329 — trades dropped because a token's decimals did not + // resolve. Silent otherwise: the run succeeds without them. + m.extend(metrics::trades_missing_decimals_metrics( + stats.trades_missing_decimals, + )); if let Err(e) = metrics::publish(&cw, &env_name, &m).await { warn!(error = %e, "cloudwatch metric publish failed (non-fatal)"); } diff --git a/packages/prices-ledger-processor/src/metrics.rs b/packages/prices-ledger-processor/src/metrics.rs index 6aade4f1..60e69912 100644 --- a/packages/prices-ledger-processor/src/metrics.rs +++ b/packages/prices-ledger-processor/src/metrics.rs @@ -63,6 +63,12 @@ pub const FORCED_PARTIAL_FLUSH: &str = "ForcedPartialFlushes"; /// factory, or a factory event missed before a cold start. pub const UNREGISTERED_POOL_EVENTS: &str = "UnregisteredPoolEvents"; +/// AMM trades a reconcile run dropped because a leg's Soroban token decimals +/// did not resolve over RPC (task 0329) — the RPC was down, or the token has no +/// usable `decimals()`. Emitted ONLY when non-zero, like +/// [`UNREGISTERED_POOL_EVENTS`]. +pub const TRADES_MISSING_DECIMALS: &str = "TradesMissingDecimals"; + /// `PutMetricData` accepts at most 150 entries in a datum's `Values` array, so /// a run with more INSERTs than that spills into further datums of the same /// metric rather than being truncated (or, worse, aggregated back into @@ -170,6 +176,19 @@ pub fn unregistered_pool_event_metrics(total: u64) -> Vec { }] } +/// The dropped-for-decimals datapoint for one run (task 0329), or nothing when +/// the run dropped none. +pub fn trades_missing_decimals_metrics(total: u64) -> Vec { + if total == 0 { + return Vec::new(); + } + vec![Metric { + name: TRADES_MISSING_DECIMALS, + unit: Unit::Count, + values: vec![total as f64], + }] +} + /// Publish `metrics` to CloudWatch under [`METRIC_NAMESPACE`], tagged with an /// `Environment` dimension. One `PutMetricData` call for the whole batch. /// @@ -242,6 +261,16 @@ mod tests { assert!(unregistered_pool_event_metrics(0).is_empty()); } + /// Task 0329: same absent-while-healthy contract. + #[test] + fn trades_missing_decimals_publish_only_when_some_were_dropped() { + assert!(trades_missing_decimals_metrics(0).is_empty()); + let m = trades_missing_decimals_metrics(3); + assert_eq!(m.len(), 1); + assert_eq!(m[0].name, TRADES_MISSING_DECIMALS); + assert_eq!(m[0].values, vec![3.0]); + } + #[test] fn dropped_unregistered_trades_publish_one_count_of_the_total() { let m = unregistered_pool_event_metrics(7); diff --git a/packages/prices-ledger-processor/src/reconcile.rs b/packages/prices-ledger-processor/src/reconcile.rs index 65a07878..9b0120ba 100644 --- a/packages/prices-ledger-processor/src/reconcile.rs +++ b/packages/prices-ledger-processor/src/reconcile.rs @@ -116,6 +116,10 @@ pub struct RunStats { /// missing from the registry — dropped, not priced (task 0291). Counted over /// the minutes this run WROTE, like `offer_lookups`. Empty when healthy. pub unregistered_pool_events: BTreeMap<&'static str, u64>, + /// AMM trades dropped because a leg's token decimals did not resolve over + /// RPC, even after the second decode (task 0329). Counted over the minutes + /// this run WROTE, like `unregistered_pool_events`. 0 when healthy. + pub trades_missing_decimals: u64, /// Candle-INSERT latency for this run, or `None` when the run wrote no /// candles at all. `None` rather than a zeroed struct so an idle run /// publishes no `ClickHouseWriteLatencyMs` datapoint instead of a 0 ms one @@ -261,6 +265,9 @@ where // minute closes. let mut ledger_unregistered: Vec<(u32, &'static str, u32)> = Vec::new(); let mut ledger_unregistered_contracts: Vec<(u32, String)> = Vec::new(); + // (minute_start, trades, contracts) dropped because a token's decimals + // did not resolve (task 0329). Per ledger for the same reason as above. + let mut ledger_missing_decimals: Vec<(u32, u32, Vec)> = Vec::new(); for _ in 0..max_iterations { let next = current + 1; @@ -319,6 +326,13 @@ where .iter() .map(|c| (minute, c.clone())), ); + if sob.trades_missing_decimals > 0 { + ledger_missing_decimals.push(( + minute, + sob.trades_missing_decimals, + sob.missing_decimals, + )); + } obj_max = obj_max.max(seq); } // Flag only this object's HIGHEST ledger as a valid cursor landing @@ -345,6 +359,7 @@ where rows_emitted: 0, pools_persisted: 0, unregistered_pool_events: BTreeMap::new(), + trades_missing_decimals: 0, // Nothing was persisted, so no INSERT happened: no datapoint. ch_write: None, }); @@ -505,6 +520,7 @@ where // Pools are dimension rows written above, like the assets. pools_persisted, unregistered_pool_events: BTreeMap::new(), + trades_missing_decimals: 0, ch_write: None, }); }; @@ -585,6 +601,25 @@ where "dropped trades from pools missing from prices.pool_registry (task 0291)" ); } + let flushed_missing_decimals: Vec<&(u32, u32, Vec)> = ledger_missing_decimals + .iter() + .filter(|(minute, _, _)| *minute < flush_boundary) + .collect(); + let trades_missing_decimals: u64 = flushed_missing_decimals + .iter() + .map(|(_, n, _)| *n as u64) + .sum(); + if trades_missing_decimals > 0 { + let contracts: BTreeSet<&str> = flushed_missing_decimals + .iter() + .flat_map(|(_, _, c)| c.iter().map(String::as_str)) + .collect(); + tracing::warn!( + trades_missing_decimals, + ?contracts, + "dropped AMM trades whose token decimals did not resolve (task 0329)" + ); + } info!( start, end = current, @@ -609,6 +644,7 @@ where rows_emitted, pools_persisted, unregistered_pool_events, + trades_missing_decimals, ch_write: (!ch_write.samples_ms.is_empty()).then_some(ch_write), }) } diff --git a/packages/sdex-backfill/src/ingest.rs b/packages/sdex-backfill/src/ingest.rs index 1f5d5a6f..4d65f3be 100644 --- a/packages/sdex-backfill/src/ingest.rs +++ b/packages/sdex-backfill/src/ingest.rs @@ -33,6 +33,10 @@ pub struct PartitionStats { pub skipped: usize, pub trade_ticks: usize, pub amm_ticks: usize, + /// AMM trades dropped because a token's decimals did not resolve over RPC + /// (task 0329). Their ledgers are still marked done, so a non-zero total + /// means a range to reprice. + pub trades_missing_decimals: usize, pub oracle_rows: usize, pub candles_written: usize, pub total_bytes: u64, @@ -416,6 +420,7 @@ pub async fn index_partition( } sob = process_ledger(lcm, reg, registry); } + stats.trades_missing_decimals += sob.trades_missing_decimals as usize; } for (source, tick) in sob.amm_ticks { accs.merge_amm(source, tick); @@ -485,6 +490,7 @@ pub async fn index_partition( skipped = stats.skipped, trade_ticks = stats.trade_ticks, amm_ticks = stats.amm_ticks, + trades_missing_decimals = stats.trades_missing_decimals, oracle_rows = stats.oracle_rows, candles = stats.candles_written, bytes = stats.total_bytes, diff --git a/packages/sdex-backfill/src/run.rs b/packages/sdex-backfill/src/run.rs index 69203f93..cc0593f4 100644 --- a/packages/sdex-backfill/src/run.rs +++ b/packages/sdex-backfill/src/run.rs @@ -212,6 +212,7 @@ pub async fn execute( totals.skipped += stats.skipped; totals.trade_ticks += stats.trade_ticks; totals.amm_ticks += stats.amm_ticks; + totals.trades_missing_decimals += stats.trades_missing_decimals; totals.oracle_rows += stats.oracle_rows; totals.candles_written += stats.candles_written; totals.total_bytes += stats.total_bytes; @@ -543,6 +544,13 @@ fn print_run_summary( println!("ledgers already in DB: {}", totals.skipped); println!("SDEX trade ticks: {}", totals.trade_ticks); println!("AMM trade ticks: {}", totals.amm_ticks); + // Always printed, 0 included. Non-zero: a token's decimals() did not + // resolve, its trades are NOT in the candles, and their ledgers are marked + // done anyway — reprice that range (task 0329). + println!( + "trades dropped (decimals): {}", + totals.trades_missing_decimals + ); println!("oracle rows: {}", totals.oracle_rows); println!("price_ohlcv_1m rows: {}", totals.candles_written); println!("total bytes downloaded: {}", totals.total_bytes); From de3ae5ac9a34d9c70741350e802e213c03355b40 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:07:09 +0200 Subject: [PATCH 045/144] fix(lore-0329): park absent tokens and resolve decimals concurrently A contract that answers with no usable scale is not asked again by the process; a Transient failure still retries after ten minutes. The calls for one ledger run concurrently, so N new tokens cost one RPC timeout. --- packages/prices-ingest-core/src/decimals.rs | 137 +++++++++++++----- .../tests/decimals_rpc_it.rs | 12 +- 2 files changed, 109 insertions(+), 40 deletions(-) diff --git a/packages/prices-ingest-core/src/decimals.rs b/packages/prices-ingest-core/src/decimals.rs index acc364bd..595e63ba 100644 --- a/packages/prices-ingest-core/src/decimals.rs +++ b/packages/prices-ingest-core/src/decimals.rs @@ -24,12 +24,24 @@ use crate::soroban_rpc::{Simulated, http_client, rpc_url_from_env, simulate}; /// represented, so it is treated as unresolved rather than truncated. pub const MAX_DECIMALS: u32 = 28; -/// How long a contract whose `decimals()` gave no usable answer is left alone -/// before it is asked again. Bounds the RPC cost of a token that trades in -/// every ledger and never answers, without parking a transient failure for a -/// warm container's whole lifetime. +/// How long a contract that gave no answer at all (a Transient failure) is left +/// alone before it is asked again. Bounds the RPC cost of an outage for a token +/// that trades in every ledger, without parking a blip for a warm container's +/// whole lifetime. pub const RETRY_AFTER: Duration = Duration::from_secs(600); +/// What one `decimals()` call told us. +#[derive(Debug, PartialEq, Eq)] +pub enum Decimals { + Known(u8), + /// The contract answered and the answer is not a usable scale: it has no + /// `decimals()`, errored, returned something other than a `u32`, or more + /// than [`MAX_DECIMALS`]. Repeats on every call. + Absent, + /// No answer about the contract (transport, JSON-RPC error, archived state). + Transient, +} + /// One `prices.asset_decimals` row. #[derive(Debug, Clone, PartialEq, Eq, Serialize, clickhouse::Row)] pub struct DecimalsRow { @@ -48,7 +60,10 @@ impl DecimalsRow { pub struct DecimalsResolver { http: reqwest::Client, rpc_url: String, - /// When each contract last failed to resolve. + /// Contracts whose answer was [`Decimals::Absent`]. Never asked again by + /// this process: a fact does not change, and a cold start asks once more. + absent: HashSet, + /// When each contract last gave a [`Decimals::Transient`] failure. failed: HashMap, } @@ -57,6 +72,7 @@ impl DecimalsResolver { Self { http: http_client(), rpc_url, + absent: HashSet::new(), failed: HashMap::new(), } } @@ -66,62 +82,92 @@ impl DecimalsResolver { Self::new(rpc_url_from_env()) } - /// Resolve each distinct contract in `missing` that has not failed within - /// [`RETRY_AFTER`]. Returns the answers, which the caller persists and then + /// Resolve each distinct contract in `missing` that is neither known + /// Absent nor inside [`RETRY_AFTER`] of a Transient failure. Returns the + /// answers, which the caller persists and then /// [`record`](DecimalsRow::record)s. Empty means nothing new was learned, /// so decoding again would price nothing more. + /// + /// The calls run concurrently, so a ledger naming N new tokens costs one + /// RPC timeout ([`crate::soroban_rpc::RPC_TIMEOUT_SECS`]) at worst, not N. pub async fn resolve(&mut self, missing: &[String]) -> Vec { - let mut rows = Vec::new(); let mut asked = HashSet::new(); + let mut calls = tokio::task::JoinSet::new(); for contract in missing { - if !asked.insert(contract) + let parked = self.absent.contains(contract) || self .failed .get(contract) - .is_some_and(|at| at.elapsed() < RETRY_AFTER) - { + .is_some_and(|at| at.elapsed() < RETRY_AFTER); + if parked || !asked.insert(contract) { continue; } - match decimals_of(&self.http, &self.rpc_url, contract).await { - Some(decimals) => { + let (http, rpc_url, contract) = + (self.http.clone(), self.rpc_url.clone(), contract.clone()); + calls.spawn(async move { + let answer = decimals_of(&http, &rpc_url, &contract).await; + (contract, answer) + }); + } + + let mut rows = Vec::new(); + while let Some(joined) = calls.join_next().await { + // A panicked call answered nothing; the token is asked again on its + // next trade. + let Ok((contract, answer)) = joined else { + continue; + }; + match answer { + Decimals::Known(decimals) => { tracing::info!(contract, decimals, "resolved soroban token decimals"); - self.failed.remove(contract); + self.failed.remove(&contract); rows.push(DecimalsRow { - contract_address: contract.clone(), + contract_address: contract, decimals, }); } - None => { + Decimals::Absent => { + tracing::warn!( + contract, + "decimals() has no usable answer — this token's AMM trades are dropped; \ + not asked again until the process restarts" + ); + self.absent.insert(contract); + } + Decimals::Transient => { tracing::warn!( contract, retry_after_secs = RETRY_AFTER.as_secs(), - "decimals() unresolved — this token's AMM trades are dropped until it resolves" + "decimals() got no answer — this token's AMM trades are dropped until it resolves" ); - self.failed.insert(contract.clone(), Instant::now()); + self.failed.insert(contract, Instant::now()); } } } + rows.sort_by(|a, b| a.contract_address.cmp(&b.contract_address)); rows } } -/// `decimals()` of `contract`, or `None` when there is no usable answer. -pub async fn decimals_of(http: &reqwest::Client, rpc_url: &str, contract: &str) -> Option { - match simulate(http, rpc_url, contract, "decimals").await { - Simulated::Value(v) => usable_decimals(&v, contract), - Simulated::Absent | Simulated::Transient => None, - } +/// `decimals()` of `contract`, classified. +pub async fn decimals_of(http: &reqwest::Client, rpc_url: &str, contract: &str) -> Decimals { + classify( + simulate(http, rpc_url, contract, "decimals").await, + contract, + ) } /// SEP-41 declares `decimals() -> u32`; anything else, or more than /// [`MAX_DECIMALS`], is not a scale we can apply. -fn usable_decimals(v: &ScVal, contract: &str) -> Option { - match v { - ScVal::U32(d) if *d <= MAX_DECIMALS => Some(*d as u8), - other => { +fn classify(simulated: Simulated, contract: &str) -> Decimals { + match simulated { + Simulated::Value(ScVal::U32(d)) if d <= MAX_DECIMALS => Decimals::Known(d as u8), + Simulated::Value(other) => { tracing::warn!(contract, value = ?other, "decimals() returned no usable scale"); - None + Decimals::Absent } + Simulated::Absent => Decimals::Absent, + Simulated::Transient => Decimals::Transient, } } @@ -130,13 +176,32 @@ mod tests { use super::*; #[test] - fn only_a_u32_within_the_decimal_scale_is_usable() { - assert_eq!(usable_decimals(&ScVal::U32(8), "C"), Some(8)); - assert_eq!(usable_decimals(&ScVal::U32(0), "C"), Some(0)); - assert_eq!(usable_decimals(&ScVal::U32(28), "C"), Some(28)); - assert_eq!(usable_decimals(&ScVal::U32(29), "C"), None); - assert_eq!(usable_decimals(&ScVal::I32(7), "C"), None); - assert_eq!(usable_decimals(&ScVal::Void, "C"), None); + fn only_a_u32_within_the_decimal_scale_is_known() { + let v = |v| classify(Simulated::Value(v), "C"); + assert_eq!(v(ScVal::U32(8)), Decimals::Known(8)); + assert_eq!(v(ScVal::U32(0)), Decimals::Known(0)); + assert_eq!(v(ScVal::U32(28)), Decimals::Known(28)); + assert_eq!(v(ScVal::U32(29)), Decimals::Absent); + assert_eq!(v(ScVal::I32(7)), Decimals::Absent); + assert_eq!(v(ScVal::Void), Decimals::Absent); + // The simulation's own verdict passes through: a fact stays a fact, + // no answer stays retryable. + assert_eq!(classify(Simulated::Absent, "C"), Decimals::Absent); + assert_eq!(classify(Simulated::Transient, "C"), Decimals::Transient); + } + + #[tokio::test] + async fn an_absent_contract_is_never_asked_again() { + // Port 1 would answer Transient; a contract already known Absent must + // not reach it at all. + let mut r = DecimalsResolver::new("http://127.0.0.1:1/".to_string()); + let c = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN".to_string(); + r.absent.insert(c.clone()); + assert!(r.resolve(std::slice::from_ref(&c)).await.is_empty()); + assert!( + !r.failed.contains_key(&c), + "not asked, so no Transient recorded" + ); } #[tokio::test] diff --git a/packages/prices-ingest-core/tests/decimals_rpc_it.rs b/packages/prices-ingest-core/tests/decimals_rpc_it.rs index 5ad4900b..c983f29e 100644 --- a/packages/prices-ingest-core/tests/decimals_rpc_it.rs +++ b/packages/prices-ingest-core/tests/decimals_rpc_it.rs @@ -8,7 +8,7 @@ //! PR. The parser is pinned offline in `decimals.rs` and `soroban_rpc.rs`; this //! proves the two ends meet against real tokens. -use prices_ingest_core::decimals::decimals_of; +use prices_ingest_core::decimals::{Decimals, decimals_of}; use prices_ingest_core::soroban_rpc::{http_client, rpc_url_from_env}; #[tokio::test] @@ -34,14 +34,18 @@ async fn reads_the_decimals_sbe_checked_on_chain() { ), ] { let got = decimals_of(&http_client(), &rpc_url_from_env(), contract).await; - assert_eq!(got, Some(want), "{name} ({contract})"); + assert_eq!(got, Decimals::Known(want), "{name} ({contract})"); } } #[tokio::test] #[ignore = "requires public network — third-party uptime; never gates a PR"] -async fn a_contract_that_was_never_deployed_has_no_decimals() { +async fn a_contract_that_was_never_deployed_is_absent() { let never_deployed = stellar_strkey::Contract([3u8; 32]).to_string(); let got = decimals_of(&http_client(), &rpc_url_from_env(), &never_deployed).await; - assert_eq!(got, None); + assert_eq!( + got, + Decimals::Absent, + "a fact about the contract, not a retry" + ); } From c1e0fb1c4d8bffffcb549cf484f0251998dcc899 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:07:10 +0200 Subject: [PATCH 046/144] fix(lore-0329): keep the phase-3 reconcile off contract-token volumes Per-token decimals move contract-token volumes by design, so the reconcile compares volumes on the other candles and trades on all. The script now requires and gates on events-backfill's decimals line. --- docs/runbooks/0286-reingest-history.md | 21 ++++++++-- tools/scripts/reingest_0286.py | 40 +++++++++++++++++--- tools/scripts/reingest_0286_post0139_test.py | 38 ++++++++++++++++++- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/docs/runbooks/0286-reingest-history.md b/docs/runbooks/0286-reingest-history.md index f7a556f6..9a181968 100644 --- a/docs/runbooks/0286-reingest-history.md +++ b/docs/runbooks/0286-reingest-history.md @@ -255,13 +255,26 @@ the DROP and keep the output: SELECT source, count() AS candles, sum(trade_count) AS trades, - sum(volume_base) AS volume_base, - sum(volume_quote) AS volume_quote + sumIf(volume_base, + asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '') + AND quote_asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '')) + AS volume_base, + sumIf(volume_quote, + asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '') + AND quote_asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '')) + AS volume_quote FROM prices.price_ohlcv_1m FINAL WHERE toYYYYMM(timestamp) = GROUP BY source ORDER BY source FORMAT TabSeparated; ``` +Volumes leave out every candle that touches a pure Soroban token (task 0329). +The ingest now scales such a token's legs by its own `decimals()`, so its +volume moves by design: an 18-decimal token's base volume falls by 10^11, a +6-decimal one's rises tenfold. Trades are still counted on every candle. +`reingest_0286.py` runs exactly this, and a month whose "before" was read +without it gets volumes left uncompared, as a FINDING. + (If you later need a row-level diff rather than these sums, attach the frozen parts to a scratch table with the `ATTACH PARTITION … FROM '/var/lib/clickhouse/ shadow/…'` idiom in `repair-coarse-usd-values.md` → "Rollback".) @@ -341,7 +354,9 @@ Re-run the step-4c query and compare, per source: | any source | `count()` may differ: a minute that traded only dust still exists, so candles are not lost — but a minute whose every fill was dropped upstream for a zero amount never existed in either shape. | Differences in `open`/`high`/`low`/`close` are the POINT of the exercise and are -not reconciled. Differences in volume are a defect. Record both. +not reconciled. Differences in volume (on the candles step 4c compares) are a +defect. Record both. So is a non-zero `trades dropped (decimals)` in §6: the +script reads it and marks the month DEFECT. ### 4g. DROP the overlapping coarse partitions and pre-roll the month diff --git a/tools/scripts/reingest_0286.py b/tools/scripts/reingest_0286.py index c9610307..57cff0ab 100755 --- a/tools/scripts/reingest_0286.py +++ b/tools/scripts/reingest_0286.py @@ -540,8 +540,20 @@ def gate(name, passed, detail): COLLIDING_IDS = "(SELECT new_id FROM prices.asset_id_map_0139 WHERE status = 'colliding')" MAP_RESTRICT = f" AND asset_id IN {COPIED_IDS} AND quote_asset_id IN {COPIED_IDS}" -SUMS = """SELECT source, count(), sum(trade_count), sum(volume_base), sum(volume_quote) -FROM prices.price_ohlcv_{tier} FINAL WHERE toYYYYMM(timestamp) = {m}{restrict} +# Task 0329: the AMM path now scales a pure Soroban token's legs by its own +# decimals, so a re-ingested month's volume moves BY DESIGN on every candle that +# touches a contract token (an 18-decimal token's base volume falls by 10^11, a +# 6-decimal one's rises tenfold). Volumes are therefore compared on the other +# candles only; trades are still compared on all of them. The subquery reads +# ids in whatever width `prices.assets` carries, so it holds on either side of +# the 0139 swap. +CONTRACT_IDS = "(SELECT asset_id FROM prices.assets WHERE contract_address != '')" +NO_CONTRACT = f"asset_id NOT IN {CONTRACT_IDS} AND quote_asset_id NOT IN {CONTRACT_IDS}" +VOL_SCOPE = "no-contract-0329" # recorded on `before`, so an older reading is never compared + +SUMS = f"""SELECT source, count(), sum(trade_count), + sumIf(volume_base, {NO_CONTRACT}), sumIf(volume_quote, {NO_CONTRACT}) +FROM prices.price_ohlcv_{{tier}} FINAL WHERE toYYYYMM(timestamp) = {{m}}{{restrict}} GROUP BY source ORDER BY source""" OUTSIDE = f"""SELECT source, @@ -682,14 +694,17 @@ def amm_summary(text, logfile): Python AttributeError after the month's AMM candles are already in. `negative apply order:` replaced `events with no apply order:` in task 0304; a binary without it predates that fix and cannot read BE's events anyway. + `trades dropped (decimals):` came with task 0329; a binary without it still + scales every Soroban token at 7 decimals and must not write a month. """ found = {} - for key, label in (("fallbacks", "negative apply order:"), ("dropped", "swaps dropped (unresolved):")): + for key, label in (("fallbacks", "negative apply order:"), ("dropped", "swaps dropped (unresolved):"), + ("decimals", "trades dropped (decimals):")): hit = re.search(rf"^{re.escape(label)}\s*(\d+)", text, re.M) if not hit: raise Stop(f"events-backfill printed no `{label}` line — " - f"see {logfile}. A binary without it predates task 0304: rebuild events-backfill " - "from develop (on the CH host for --amm ssh)") + f"see {logfile}. A binary without it predates task 0304 or 0329: rebuild " + "events-backfill from develop (on the CH host for --amm ssh)") found[key] = int(hit.group(1)) return found @@ -697,6 +712,7 @@ def amm_summary(text, logfile): def record_amm_summary(ms, m, summary): """Keep the write pass's figures on the month and note the ones that need reading.""" ms["fallbacks"], ms["dropped"] = summary["fallbacks"], summary["dropped"] + ms["decimals"] = summary["decimals"] if ms["dropped"]: note(f"{m}: {ms['dropped']} swaps dropped for unregistered pools — see prices.unresolved_pools") if ms["fallbacks"]: @@ -825,6 +841,7 @@ def done(): if not b: # cleanup dropped whole 1m months on 2026-07-18; the coarse copy is the survivor b, ms["ref"] = sums(ch, "1h", m, post), "1h" ms["before"], ms["ids"] = b, "u64" if post else "u32" + ms["vol_scope"] = VOL_SCOPE if post: ms["outside_before"] = outside(ch, ms["ref"], m) done() @@ -889,6 +906,7 @@ def done(): if step("amm"): ms["fallbacks"] = "-" + ms.pop("decimals", None) # only a parsed summary sets it; never carry a stale count if soroban and not ch.dry: base = f"~/events-backfill --start {S} --end {E} --clickhouse-url http://localhost:8123 --verbose" if a.amm == "mtls": @@ -951,10 +969,20 @@ def done(): "did not copy, back under their own identities — information, not reconciled") rank, amm_bits = 0, [] # 0 OK, 1 FINDING, 2 DEFECT damaged = m in LIVE_LOSS_MONTHS + scoped = ms.get("vol_scope") == VOL_SCOPE + if not scoped: + rank = 1 + note(f"{m}: its before volumes were read before task 0329 took contract-token candles out of " + "the comparison — volumes are NOT compared for this month, trades are") + if ms.get("decimals"): + rank = 2 + note(f"{m}: {ms['decimals']} AMM trades dropped because a token's decimals() did not resolve " + "— they are NOT in the candles; re-run the month once it resolves (task 0329)") for src in sorted(set(ms["before"]) | set(after)): b = ms["before"].get(src, {"trades": 0, "vb": "0", "vq": "0"}) x = after.get(src, {"trades": 0, "vb": "0", "vq": "0"}) - less = x["trades"] < b["trades"] or Decimal(x["vb"]) < Decimal(b["vb"]) or Decimal(x["vq"]) < Decimal(b["vq"]) + less = x["trades"] < b["trades"] or scoped and ( + Decimal(x["vb"]) < Decimal(b["vb"]) or Decimal(x["vq"]) < Decimal(b["vq"])) same = (x["trades"], Decimal(x["vb"]), Decimal(x["vq"])) == (b["trades"], Decimal(b["vb"]), Decimal(b["vq"])) gain = (x["trades"] - b["trades"]) / b["trades"] * 100 if b["trades"] else 0.0 if less: diff --git a/tools/scripts/reingest_0286_post0139_test.py b/tools/scripts/reingest_0286_post0139_test.py index 8113d203..c1d9598c 100644 --- a/tools/scripts/reingest_0286_post0139_test.py +++ b/tools/scripts/reingest_0286_post0139_test.py @@ -250,12 +250,13 @@ def setUp(self): def tearDown(self): self.tmp.cleanup() - def run_reconcile(self, after_copied, ids="u64", width="UInt64", accept=()): + def run_reconcile(self, after_copied, ids="u64", width="UInt64", accept=(), **month): st = state([202201], self.tmp.name) ms = st.month(202201) ms.update(step=r.STEPS.index("reconcile"), ref="1m", ids=ids, before={"sdex": {"candles": 9, "trades": 1000, "vb": "40", "vq": "50"}}, - outside_before={}) + outside_before={}, vol_scope=r.VOL_SCOPE) + ms.update(month) def sums(sql): return after_copied if RESTRICT in sql else SUMS_ALL @@ -306,6 +307,39 @@ def test_before_on_the_old_width_stops_unless_accepted(self): self.assertIsNone(err) self.assertEqual(ms["result"]["verdict"], "OK") + # ---- task 0329: per-token decimals move contract-token volumes by design ---- + + def test_volumes_leave_out_contract_token_candles_and_trades_do_not(self): + sql = " ".join(r.SUMS.format(tier="1m", m=202201, restrict="").split()) + self.assertIn("sum(trade_count)", sql) + no_contract = ("asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '') " + "AND quote_asset_id NOT IN (SELECT asset_id FROM prices.assets WHERE contract_address != '')") + self.assertIn(f"sumIf(volume_base, {no_contract})", sql) + self.assertIn(f"sumIf(volume_quote, {no_contract})", sql) + + def test_lower_scoped_volume_is_still_a_defect(self): + ms, _, err = self.run_reconcile([["sdex", "9", "1000", "39", "50"]]) + self.assertEqual(ms["result"]["verdict"], "DEFECT") + self.assertRegex(str(err), "reconciliation failed") + + def test_a_before_read_without_the_scope_compares_trades_not_volumes(self): + ms, _, err = self.run_reconcile([["sdex", "9", "1000", "39", "50"]], vol_scope=None) + self.assertIsNone(err) + self.assertEqual(ms["result"]["verdict"], "FINDING") + self.assertTrue(any("volumes are NOT compared" in n for n in r.DASH["notes"])) + + def test_trades_dropped_for_decimals_are_a_defect(self): + ms, _, err = self.run_reconcile(SUMS_COPIED, decimals=3) + self.assertEqual(ms["result"]["verdict"], "DEFECT") + self.assertTrue(any("3 AMM trades dropped" in n for n in r.DASH["notes"])) + + def test_the_summary_needs_the_decimals_line(self): + text = ("negative apply order: 0\nswaps dropped (unresolved):0\n" + "trades dropped (decimals): 2\n") + self.assertEqual(r.amm_summary(text, "log"), {"fallbacks": 0, "dropped": 0, "decimals": 2}) + with self.assertRaisesRegex(r.Stop, "trades dropped"): + r.amm_summary(text.replace("trades dropped (decimals): 2\n", ""), "log") + def test_pre0139_reconciles_unrestricted(self): ms, ch, _ = self.run_reconcile(SUMS_COPIED, ids="u32", width="UInt32") self.assertEqual(ms["result"]["verdict"], "DEFECT") # +10 % on the unrestricted sums From 2b70318aef85eb92280c07719ef213cc2d48db4b Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:07:13 +0200 Subject: [PATCH 047/144] docs(lore-0329): fix doc placement and the sac-row comments --- .../database-schema-overview.md | 16 +++++++----- ...ban-token-amounts-assume-seven-decimals.md | 25 ++++++++++++------- packages/prices-clickhouse/schema/init.sql | 6 +++-- packages/prices-ingest-core/src/canonical.rs | 8 +++--- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/docs/database-schema/database-schema-overview.md b/docs/database-schema/database-schema-overview.md index eac690f9..f86cce92 100644 --- a/docs/database-schema/database-schema-overview.md +++ b/docs/database-schema/database-schema-overview.md @@ -1484,9 +1484,11 @@ each of which rules out an easier home: ### 3.10b `prices.asset_decimals` — Soroban token decimals (task 0329) -A swap event carries each leg as a raw i128 in its token's own decimals. Classic -assets and their SACs are 7-decimal by protocol and have no row here; a pure -Soroban token's decimals are whatever its SEP-41 `decimals()` returns. +A swap event carries each leg as a raw i128 in its token's own decimals. A +classic identity is 7-decimal by protocol and needs no row here; a pure Soroban +token's decimals are whatever its SEP-41 `decimals()` returns. A SAC the ingest +does not yet know as one (its classic asset not seen yet, task 0242) is held as +a contract and gets a row too, with 7. ```sql CREATE TABLE prices.asset_decimals ( @@ -1505,9 +1507,11 @@ no row, the decode drops that trade rather than guessing a scale. The caller asks `decimals()` over Soroban RPC, writes the row, and decodes the ledger again. **Read by:** the same three, in full, at startup. -> **Only answers are stored.** A token whose call failed has no row and is asked -> again after ten minutes. Until then its trades are not priced, and -> `events-backfill` counts them in its `trades dropped (decimals)` summary line. +> **Only answers are stored.** A token whose call got no answer has no row and +> is asked again after ten minutes. One that answered with no usable scale is +> not asked again until the process restarts. Until it resolves, its trades are +> not priced. The live processor publishes them as `TradesMissingDecimals` +> (alarmed), and both backfills print them as `trades dropped (decimals)`. > Rows are never refreshed, because decimals are fixed at deploy. > ⚠️ **Deploy order.** The live processor's cold start reads this table and diff --git a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md index 692a86d3..4253efa1 100644 --- a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md +++ b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md @@ -160,15 +160,17 @@ Rejected: #### Emerged -1. **No `unresolved_pools`-style record for unresolved tokens.** That table is - keyed by pool. The resolver WARNs once per contract per retry window, and - `events-backfill` prints `trades dropped (decimals): N`, which the 0286 - runbook now tells the operator to read. The live path has no metric or alarm - for it yet. -2. **A failed contract is retried after 10 minutes, in memory.** Absent and - Transient are treated alike, with no `attempts` column. That costs one RPC - call per such token per 10 minutes, and stops a token that trades every - ledger from costing a 5 s timeout per ledger. +1. **Dropped trades are counted, not recorded in a table.** `unresolved_pools` + is keyed by pool, so there is no row for a token. Instead the live processor + counts them on `RunStats`, WARNs with the contracts, and publishes + `TradesMissingDecimals`, alarmed like `UnregisteredPoolEvents`. Both + backfills print `trades dropped (decimals)`, and `reingest_0286.py` marks a + month with a non-zero count DEFECT. Added after review (PR #395). +2. **Absent is terminal per process, Transient retries after 10 minutes.** A + contract that answered with no usable scale is not asked again until a cold + start, and is never persisted as a sentinel. A contract that gave no answer + is retried after 10 minutes. Calls run concurrently, so one ledger costs one + RPC timeout at worst. 3. **Decode again rather than a two-phase tick.** Decoding is deterministic and the registry inserts are idempotent map writes, so the first result is discarded whole. @@ -177,6 +179,11 @@ Rejected: 5. **`sdex-backfill` reads the table only in `combined` mode.** The 0286 phase-3 `sdex-only` runs therefore do not need the table to exist. 6. **An `events-backfill` dry-run resolves but does not persist.** +7. **Phase-3 volume reconcile leaves out contract-token candles.** Their volume + moves by design (an 18-decimal token's base volume falls by 10^11), so + `reingest_0286.py` compares volumes on the other candles only and trades on + all of them. A "before" read without that scope compares trades only, as a + FINDING. Added after review (PR #395). ### Deploy order diff --git a/packages/prices-clickhouse/schema/init.sql b/packages/prices-clickhouse/schema/init.sql index 94c73066..68342739 100644 --- a/packages/prices-clickhouse/schema/init.sql +++ b/packages/prices-clickhouse/schema/init.sql @@ -417,8 +417,10 @@ SETTINGS index_granularity = 8192; -- `decimals()` of each pure Soroban token an AMM swap has touched (task 0329). -- The ingest scales each swap leg's raw i128 amount by its token's decimals. -- Before this table it assumed 7 for every token, the classic convention, and --- priced every other token off by 10^(7 - decimals). Classic assets and their --- SACs are 7 by protocol and have no row. +-- priced every other token off by 10^(7 - decimals). A classic identity is 7 +-- by protocol and needs no row. A SAC the ingest does not yet know as one (its +-- classic asset not seen yet, task 0242) is held as a contract like any token: +-- it is resolved over RPC and gets a row, with 7. -- -- Written by the ingest itself (live processor, events-backfill, sdex-backfill) -- the first time a swap names a token it has no row for: one Soroban RPC diff --git a/packages/prices-ingest-core/src/canonical.rs b/packages/prices-ingest-core/src/canonical.rs index 4b9a35a1..c8148143 100644 --- a/packages/prices-ingest-core/src/canonical.rs +++ b/packages/prices-ingest-core/src/canonical.rs @@ -65,6 +65,10 @@ impl AssetIdentity { /// symbol resolves to the same identity used as a trade quote. pub(crate) use prices_clickhouse::{USDC_ISSUER, USDT_ISSUER}; +/// Decimals of every classic asset, and so of every SAC: fixed by the protocol +/// (stroops). A pure Soroban token has its own, read from `decimals()`. +pub const CLASSIC_DECIMALS: u32 = 7; + /// Mainnet (Public) network passphrase. A SAC contract id is **network-scoped**: /// it is `sha256(HashIdPreimage::ContractId { network_id, asset })`, and /// `network_id = sha256(passphrase)`. So the passphrase decides which network's @@ -80,10 +84,6 @@ pub(crate) use prices_clickhouse::{USDC_ISSUER, USDT_ISSUER}; /// liquidity when someone debugs it. To support a non-mainnet backfill, thread the /// network passphrase through config and into [`AssetRegistry::from_existing`] /// (the single bake-in point, below) rather than reading this const. -/// Decimals of every classic asset, and so of every SAC: fixed by the protocol -/// (stroops). A pure Soroban token has its own, read from `decimals()`. -pub const CLASSIC_DECIMALS: u32 = 7; - const MAINNET_PASSPHRASE: &str = "Public Global Stellar Network ; September 2015"; fn mainnet_network_id() -> [u8; 32] { From 5a1e0c7cb5aa49fcc479701db5df71c978e82958 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:05:28 +0200 Subject: [PATCH 048/144] chore(lore-0226): activate 0226 and 0140 as one PR --- ...0_PERF_asset-discovery-reemits-full-registry-hourly.md | 8 +++++++- ...worker-loads-whole-asset-registry-to-write-two-rows.md | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) rename lore/1-tasks/{backlog => active}/0140_PERF_asset-discovery-reemits-full-registry-hourly.md (97%) rename lore/1-tasks/{backlog => active}/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md (97%) diff --git a/lore/1-tasks/backlog/0140_PERF_asset-discovery-reemits-full-registry-hourly.md b/lore/1-tasks/active/0140_PERF_asset-discovery-reemits-full-registry-hourly.md similarity index 97% rename from lore/1-tasks/backlog/0140_PERF_asset-discovery-reemits-full-registry-hourly.md rename to lore/1-tasks/active/0140_PERF_asset-discovery-reemits-full-registry-hourly.md index 60a5d9f3..6481c9d5 100644 --- a/lore/1-tasks/backlog/0140_PERF_asset-discovery-reemits-full-registry-hourly.md +++ b/lore/1-tasks/active/0140_PERF_asset-discovery-reemits-full-registry-hourly.md @@ -2,7 +2,8 @@ id: "0140" title: "asset-discovery re-emits the whole asset registry every hour — 0132's defect in a second component" type: PERF -status: backlog +status: active +assignee: akot related_adr: [] related_tasks: ["0210", "0132", "0133", "0067", "0256", "0226", "0241"] tags: ["priority-medium", "effort-small", "cost", "write-amplification", "clickhouse"] @@ -44,6 +45,11 @@ history: that PR's review: `ensure_seed` still READS the whole registry hourly to check ~20 identities. What remains here is that read and the `oracle-worker` instance. + - date: "2026-10-06" + status: active + who: akot + note: > + Activated together with [[0226]]: one PR on one branch. The oracle-worker `write_assets` and the `ensure_seed` whole-registry read are the same shape as 0226's load, and 0226 removes the registry the oracle write depends on. --- # `asset-discovery` re-emits the full asset registry every hour diff --git a/lore/1-tasks/backlog/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md b/lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md similarity index 97% rename from lore/1-tasks/backlog/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md rename to lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md index 8b229945..60e60907 100644 --- a/lore/1-tasks/backlog/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md +++ b/lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md @@ -2,7 +2,8 @@ id: "0226" title: "The oracle worker loads all 620,615 assets into memory to write 2 rows, sits at its 256 MB ceiling and OOMs several times a day" type: BUG -status: backlog +status: active +assignee: akot related_adr: [] related_tasks: ["0223", "0222", "0167", "0112", "0132", "0256", "0241", "0140"] tags: [layer-infra, priority-high, effort-medium, oracle, lambda, memory, observability, ops] @@ -98,6 +99,11 @@ history: task is an efficiency task, not an outage fix — but one more un-merged copy of the registry puts the oracle back at the ceiling. Priority left at high for the operator to lower; not lowered here. + - date: "2026-10-06" + status: active + who: akot + note: > + Activated together with [[0140]]: one PR on one branch, because both rewrite the same oracle-worker pass (the whole-registry load and the `write_assets` at `lib.rs:569`) and both ship through the EventBridge stack, whose deploy carries the cleanup-rule hazard. --- # The oracle worker reads the entire asset registry on every run From d78c79262a84027c91b423e888803ed9545f1e7f Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:55:34 +0200 Subject: [PATCH 049/144] docs(lore-0226): record that 0139 fixed 0226 and 0140 --- ...-whole-asset-registry-to-write-two-rows.md | 76 +++++++++++++++++-- ...-discovery-reemits-full-registry-hourly.md | 57 ++++++++++++-- 2 files changed, 119 insertions(+), 14 deletions(-) rename lore/1-tasks/{active => archive}/0140_PERF_asset-discovery-reemits-full-registry-hourly.md (78%) diff --git a/lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md b/lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md index 60e60907..b63b151d 100644 --- a/lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md +++ b/lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md @@ -104,6 +104,15 @@ history: who: akot note: > Activated together with [[0140]]: one PR on one branch, because both rewrite the same oracle-worker pass (the whole-registry load and the `write_assets` at `lib.rs:569`) and both ship through the EventBridge stack, whose deploy carries the cleanup-rule hazard. + - date: "2026-10-06" + status: active + who: akot + note: > + Found already fixed by [[0139]] before any code was written: the oracle + no longer loads the registry (deployed 2026-10-02 12:30 UTC). Measured on + prod: 256 → 52 MB, 7–8 s → 0.75 s, 0 errors and 0 OOM since. ACs 1, 2, + 3 and 5 met; AC 4 (a week without an OOM) ends 2026-10-09 12:44 UTC, + close after it. [[0140]] closed the same day. --- # The oracle worker reads the entire asset registry on every run @@ -318,19 +327,70 @@ no `-errors` alarm at all, `createWorkerLambda` (`lambda-baseline.ts:309`) build does not, that is a separate question. - [[0223]]'s framing decision itself — only the scope gap above is recorded here. +## ✅ Resolved by [[0139]] — measured on prod 2026-10-06 + +No code in this task. [[0139]] (PR #382, adfe4ccc + a6ecd20d) removed the +registry from the oracle's pass: it now calls `OhlcvWriter::write_absent_assets` +for its 2 sampled identities (`oracle-worker/src/lib.rs:562`), one keyed +`count()` each, and the `write_assets` at the old `lib.rs:569` is gone with it +— the collision with [[0140]] described above resolved itself. That is option +1 in its narrowest form; `load_assets()` is untouched for its other callers. + +**Cause, not coincidence.** CloudTrail: `UpdateFunctionCode` at 2026-10-02 +12:30:16 UTC, after #382 merged (11:33 UTC). The last old-code run, 11:52 UTC, +loaded `existing_assets=842792` (four un-merged copies) and used 256 MB in +10.9 s. The schedule was paused for the 0139 window; the first run on the new +code, 12:44 UTC on a cold container, used **51 MB in 1.4 s**. No +`loaded asset registry` line from the oracle since 11:52 UTC. + +| | 2026-09-28 → 10-02 | 2026-10-03 → 10-06 | +|---|---|---| +| `Max Memory Used`, daily max / p50 | **256** / 151–222 MB | **52 / 51 MB**, every day | +| `Runtime.OutOfMemory` lines | 8 (09-24 → 10-02) | **0**; last 2026-10-02 09:17 UTC | +| `AWS/Lambda Errors`, daily | 0–3 | **0** | +| average duration | 6.9–8.2 s | **0.74–0.77 s** | +| `prices-production-oracle-errors` | ALARM ↔ OK several times a day | last fired 2026-10-02 09:18 UTC, OK since | + +- The load was **per invocation**: ~12 cold starts a day against 288 runs, + with p50 at 151–222 MB and one `loaded asset registry` line per run (288 a + day). +- `system.query_log`, last 24 h: the oracle's keyed check is part of 1,056 + runs of `SELECT count() FROM prices.assets WHERE asset_code = ? AND …` + (2 × 288 oracle + 20 × 24 asset-discovery), 6 ms average, ≤ 8,546 rows read. + None of the 53 whole-registry `SELECT`s of the last 3 days is the oracle's: + 35 are ledger-processor cold starts, 18 an operator machine running + backfills — and the oracle could not have paid one, since the read alone + costs ~148 MB. +- The memory is now independent of un-merged copies and of the registry's + size, so the [[0294]] risk "the oracle OOMs while the re-ingest re-emits the + registry" is gone with it. +- `memorySize` stays 256 MB; no EventBridge change was needed. + `prices-production-cleanup` reads `DISABLED` (2026-10-06). + +**Callers of `load_assets()`** (AC 3), all unchanged by this task: +`events-backfill/src/run.rs:185`, `sdex-backfill/src/run.rs:147` (via +`sink.rs:72`), `prices-ledger-processor/src/sink/mod.rs:103` (cold start, +`main.rs:92`; `bin/cli.rs:86`). + +**Still open:** AC 4. The week without an OOM runs from the first new-code run +to **2026-10-09 12:44 UTC**; close after checking it. + ## Acceptance Criteria -- [ ] Whether the registry load is per-invocation or per-cold-start is - **measured**, not inferred from the two log lines above. -- [ ] The remedy reduces **peak** memory on a real production invocation, with - `Max Memory Used` recorded before and after. -- [ ] Every other caller of `load_assets()` is enumerated, and the change is +- [x] Whether the registry load is per-invocation or per-cold-start is + **measured**, not inferred from the two log lines above — per + invocation (288 load lines a day against ~12 cold starts). +- [x] The remedy reduces **peak** memory on a real production invocation, with + `Max Memory Used` recorded before and after — 256 → 52 MB ([[0139]]). +- [x] Every other caller of `load_assets()` is enumerated, and the change is shown not to alter their behaviour — or the narrowing is confined to a new - entry point. + entry point — `write_absent_assets` is the new entry point; callers + listed above. - [ ] `prices-production-oracle-errors` shows **zero** `Runtime.OutOfMemory` for a full week after the change. -- [ ] If the deploy touches `eventbridge-stack.ts`, `describe-rule` output for - `prices-production-cleanup` is recorded **before and after**. +- [x] If the deploy touches `eventbridge-stack.ts`, `describe-rule` output for + `prices-production-cleanup` is recorded **before and after** — not + touched (no `memorySize` change); the rule reads `DISABLED` on 2026-10-06. ## Notes diff --git a/lore/1-tasks/active/0140_PERF_asset-discovery-reemits-full-registry-hourly.md b/lore/1-tasks/archive/0140_PERF_asset-discovery-reemits-full-registry-hourly.md similarity index 78% rename from lore/1-tasks/active/0140_PERF_asset-discovery-reemits-full-registry-hourly.md rename to lore/1-tasks/archive/0140_PERF_asset-discovery-reemits-full-registry-hourly.md index 6481c9d5..1314ab56 100644 --- a/lore/1-tasks/active/0140_PERF_asset-discovery-reemits-full-registry-hourly.md +++ b/lore/1-tasks/archive/0140_PERF_asset-discovery-reemits-full-registry-hourly.md @@ -2,7 +2,7 @@ id: "0140" title: "asset-discovery re-emits the whole asset registry every hour — 0132's defect in a second component" type: PERF -status: active +status: completed assignee: akot related_adr: [] related_tasks: ["0210", "0132", "0133", "0067", "0256", "0226", "0241"] @@ -50,6 +50,15 @@ history: who: akot note: > Activated together with [[0226]]: one PR on one branch. The oracle-worker `write_assets` and the `ensure_seed` whole-registry read are the same shape as 0226's load, and 0226 removes the registry the oracle write depends on. + - date: "2026-10-06" + status: completed + who: akot + note: > + Closed without code: [[0139]] (PR #382, deployed 2026-10-02 12:30 UTC) + removed both remaining whole-registry paths. Measured on prod: 0 assets + written in 99 asset-discovery runs, largest daily `prices.assets` part + 1–29 rows (was ~210 k), memory 250 → 50 MB. All five ACs met; no + follow-up tasks. --- # `asset-discovery` re-emits the full asset registry every hour @@ -223,10 +232,46 @@ Priority stays at `medium`: what remains is the oracle instance below. load and points here. Same shape as [[0226]] (the oracle, every 5 minutes — twelve times as often, so that one first). +## ✅ Resolved by [[0139]] — measured on prod 2026-10-06 + +No code in this task. [[0139]] (PR #382, a6ecd20d) replaced both remaining +whole-registry paths with `OhlcvWriter::write_absent_assets`, a keyed +`count()` per identity: `ensure_seed` checks only its ~20 seed identities, and +the oracle's `write_assets` behind `count() > known_before` is gone (the oracle +checks its 2 sampled identities). With ids derived by ClickHouse +(`xxh3(identity)`), no watermark is needed. Deployed with the 0139 window: +CloudTrail `UpdateFunctionCode` for both Lambdas at 2026-10-02 12:30:16 UTC. + +| | before (09-28 → 10-02) | after (10-03 → 10-06) | +|---|---|---| +| `prices.assets` largest new part per day (`system.part_log`) | ~210 k rows (whole registry) | **1–2 rows**; 29 on 10-06 = [[0242]]'s classic seed | +| `assets_written` by asset-discovery | — | **0 in 99 runs** since the deploy | +| asset-discovery `Max Memory Used` | ~250 MB | **47–50 MB** | +| asset-discovery average duration | ~4 s | **0.6 s** | +| `loaded asset registry` lines from asset-discovery | 24 a day | **none** after 2026-10-02 11:19 UTC | + +- `system.query_log`, last 24 h: the keyed check ran **1,056 times** = 20 × 24 + (asset-discovery) + 2 × 288 (oracle), 6 ms average, at most 8,546 rows read. +- `prices.assets` holds one copy: 211,103 rows = 211,103 identities. +- The whole-registry `SELECT` still runs ~18 times a day, from the + ledger-processor's cold start (35 in 3 days, one AWS address each, in the + INIT second) and from an operator machine running backfills (18, one + non-AWS address). Neither is this task's: `load_assets()` stays for them by + design ([[0226]] AC 3). +- `OhlcvWriter::write_assets` has no production caller left (one IT, + `asset-discovery/tests/enrichment_survives_it.rs`). Not removed: dead but + harmless, and not worth a PR while the project closes (Adam, 2026-10-06). + ## Acceptance Criteria -- [ ] A zero-discovery hourly run writes **no** `prices.assets` rows. -- [ ] Newly-discovered assets are still persisted (seed path unaffected). -- [ ] The comment sits with the guard it actually describes. -- [ ] Other `write_assets` callers audited for the same defect. -- [ ] `part_log` shows the hourly ~204k rows gone, measured on prod. +- [x] A zero-discovery hourly run writes **no** `prices.assets` rows — 0 in 99 + runs since 2026-10-02 12:30 UTC. +- [x] Newly-discovered assets are still persisted (seed path unaffected) — + `seed_it::seed_writes_only_the_absent_identities` and + `absent_assets_it`, green in #382's CI. +- [x] The comment sits with the guard it actually describes — moot, deleted + with `discover_window` ([[0256]], PR #331). +- [x] Other `write_assets` callers audited for the same defect — AC 4 audit + above; the oracle instance was removed by [[0139]]. +- [x] `part_log` shows the hourly ~204k rows gone, measured on prod — + largest daily part 1–29 rows since 2026-10-03. From 0731c6e010689e318e9f651cca24f8e5023fea82 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:09:46 +0200 Subject: [PATCH 050/144] fix(lore-0329): harden decimals resolution after review - checked_div on the AMM quotient: per-token scales can push it past Decimal::MAX, where the plain division panicked and stalled ingest. - A contract error from a node behind the decoded ledger (latestLedger) is Behind, asked again next trade, not parked as Absent. - Backfills retry a Transient failure inline (1/5/20 s) instead of parking it for ten minutes of wall clock. - One decode_resolving for all three callers, tested with a fake resolver; reconcile tests inject an offline resolver. - The alarm says a hand-inserted row needs a cold start. --- infra/src/lib/stacks/observability-stack.ts | 2 +- packages/asset-discovery/src/symbols.rs | 6 +- packages/events-backfill/src/run.rs | 39 ++-- packages/prices-ingest-core/src/decimals.rs | 175 +++++++++++---- packages/prices-ingest-core/src/lib.rs | 2 +- packages/prices-ingest-core/src/soroban.rs | 205 +++++++++++++++++- .../prices-ingest-core/src/soroban_rpc.rs | 55 ++++- .../tests/decimals_rpc_it.rs | 4 +- .../prices-ledger-processor/src/reconcile.rs | 44 ++-- .../tests/pool_registry_persist.rs | 6 +- .../tests/reconcile_drain.rs | 7 +- .../tests/reconcile_e2e.rs | 8 +- packages/sdex-backfill/src/ingest.rs | 30 ++- packages/sdex-backfill/src/run.rs | 2 +- 14 files changed, 468 insertions(+), 117 deletions(-) diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index a2db8aef..f1444bd8 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -1479,7 +1479,7 @@ export class ObservabilityStack extends cdk.Stack { { alarmName: `prices-${config.envName}-ledger-processor-trades-missing-decimals`, alarmDescription: - 'The ledger-processor dropped AMM trades because a Soroban token\'s decimals() did not resolve over RPC (task 0329). Those trades produce no candle. Either the RPC (SOROBAN_RPC_URL, default mainnet.sorobanrpc.com) was failing, in which case the token is asked again after 10 minutes, or the token has no usable decimals(), in which case it is not asked again until a cold start. The WARN "dropped AMM trades whose token decimals did not resolve" lists the contracts. Fix: once decimals() answers (insert the row into prices.asset_decimals by hand if it never will), reprice the dropped minutes with events-backfill.', + 'The ledger-processor dropped AMM trades because a Soroban token\'s decimals() did not resolve over RPC (task 0329). Those trades produce no candle. Either the RPC (SOROBAN_RPC_URL, default mainnet.sorobanrpc.com) was failing, in which case the token is asked again after 10 minutes, or the token has no usable decimals(), in which case it is not asked again until a cold start. The WARN "dropped AMM trades whose token decimals did not resolve" lists the contracts. Fix: once decimals() answers, reprice the dropped minutes with events-backfill. If it never will, insert the row into prices.asset_decimals by hand AND force a cold start (any configuration update of the ledger-processor function, e.g. aws lambda update-function-configuration --description): a warm processor reads that table only at cold start and keeps dropping the token until then. Reprice after the cold start.', metric: new cloudwatch.Metric({ namespace: 'Prices/Ingest', metricName: 'TradesMissingDecimals', diff --git a/packages/asset-discovery/src/symbols.rs b/packages/asset-discovery/src/symbols.rs index 2da7ac77..90043208 100644 --- a/packages/asset-discovery/src/symbols.rs +++ b/packages/asset-discovery/src/symbols.rs @@ -136,10 +136,12 @@ pub fn sanitize_symbol(raw: &str) -> Option { /// the caller has to make. The line between the two is /// [`prices_ingest_core::soroban_rpc::simulate`]'s. pub async fn resolve_symbol(http: &reqwest::Client, rpc_url: &str, contract: &str) -> Outcome { - match simulate(http, rpc_url, contract, "symbol").await { + match simulate(http, rpc_url, contract, "symbol", None).await { Simulated::Value(v) => symbol_outcome(&v, contract), Simulated::Absent => Outcome::Absent, - Simulated::Transient => Outcome::Transient, + // `as_of` is None, so this arm is unreachable in practice; retrying is + // the safe reading either way. + Simulated::Transient | Simulated::Behind => Outcome::Transient, } } diff --git a/packages/events-backfill/src/run.rs b/packages/events-backfill/src/run.rs index 0b23251b..4c3af165 100644 --- a/packages/events-backfill/src/run.rs +++ b/packages/events-backfill/src/run.rs @@ -21,9 +21,9 @@ use serde_json::Value; use tracing::{info, warn}; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, DEFAULT_BACKOFF_MS, DecimalsResolver, LedgerSoroban, - OhlcvCandle, OhlcvWriter, RawSorobanEvent, Registries, UnresolvedPool, UnresolvedPoolSwap, - process_soroban_event_rows, retry_with_backoff, + AssetRegistry, CandleAccumulator, DEFAULT_BACKOFF_MS, DecimalsResolver, DecimalsRow, + LedgerSoroban, OhlcvCandle, OhlcvWriter, RawSorobanEvent, Registries, UnresolvedPool, + UnresolvedPoolSwap, decode_resolving, process_soroban_event_rows, retry_with_backoff, }; use crate::cli::Cli; @@ -65,22 +65,23 @@ async fn decode_ledger( writer: &OhlcvWriter, dry_run: bool, ) -> Result { - let mut out = LedgerSoroban::default(); - process_soroban_event_rows(ledger, closed_at, events, reg, assets, &mut out); - if !out.missing_decimals.is_empty() { - let rows = decimals.resolve(&out.missing_decimals).await; - if !rows.is_empty() { - if !dry_run { - retry_write(|| async { writer.write_decimals(&rows).await }).await?; - } - for row in &rows { - row.record(assets); - } - out = LedgerSoroban::default(); + decode_resolving( + decimals, + assets, + ledger, + |assets| { + let mut out = LedgerSoroban::default(); process_soroban_event_rows(ledger, closed_at, events, reg, assets, &mut out); - } - } - Ok(out) + out + }, + async |rows: &[DecimalsRow]| { + if dry_run { + return Ok(()); + } + retry_write(|| async { writer.write_decimals(rows).await }).await + }, + ) + .await } /// Merge one decoded ledger's ticks into the **run-level** per-source @@ -218,7 +219,7 @@ pub async fn execute(cli: &Cli) -> Result<(), EventsBackfillError> { let existing_assets = writer.load_assets().await?; let mut assets = AssetRegistry::from_existing(existing_assets); writer.load_decimals(&mut assets).await?; - let mut decimals = DecimalsResolver::from_env(); + let mut decimals = DecimalsResolver::backfill_from_env(); let mut reg = reprice_registry(writer.load_pool_registry().await?)?; // Every AMM pool has a `venue` entry (the registry superset); its strkeys are diff --git a/packages/prices-ingest-core/src/decimals.rs b/packages/prices-ingest-core/src/decimals.rs index 595e63ba..97de8c4c 100644 --- a/packages/prices-ingest-core/src/decimals.rs +++ b/packages/prices-ingest-core/src/decimals.rs @@ -4,12 +4,10 @@ //! Classic assets and their SACs are 7 by protocol; a pure Soroban token's //! decimals are whatever its SEP-41 `decimals()` returns. Decoding is //! synchronous and never guesses, so a trade whose token is unknown is dropped -//! and reported ([`crate::LedgerSoroban::missing_decimals`]). The caller then -//! asks [`DecimalsResolver`], persists what it learned to -//! `prices.asset_decimals`, records it in the [`AssetRegistry`], and decodes -//! the ledger again. -//! -//! [`AssetRegistry`]: crate::AssetRegistry +//! and reported ([`crate::LedgerSoroban::missing_decimals`]). +//! [`decode_resolving`] is the step every caller runs around a decode: ask the +//! resolver, persist what it learned to `prices.asset_decimals`, record it in +//! the [`AssetRegistry`], and decode the ledger again. use std::collections::{HashMap, HashSet}; use std::time::{Duration, Instant}; @@ -18,17 +16,23 @@ use serde::Serialize; use stellar_xdr::ScVal; use crate::canonical::AssetRegistry; +use crate::soroban::LedgerSoroban; use crate::soroban_rpc::{Simulated, http_client, rpc_url_from_env, simulate}; /// The largest scale `rust_decimal` holds. A token claiming more cannot be /// represented, so it is treated as unresolved rather than truncated. pub const MAX_DECIMALS: u32 = 28; -/// How long a contract that gave no answer at all (a Transient failure) is left -/// alone before it is asked again. Bounds the RPC cost of an outage for a token -/// that trades in every ledger, without parking a blip for a warm container's -/// whole lifetime. -pub const RETRY_AFTER: Duration = Duration::from_secs(600); +/// How long the live processor leaves a contract that gave no answer at all (a +/// Transient failure) before asking again. Bounds the RPC cost of an outage for +/// a token that trades in every ledger, without parking a blip for a warm +/// container's whole lifetime. +pub const LIVE_RETRY_AFTER: Duration = Duration::from_secs(600); + +/// A backfill's inline retries of a Transient failure, in milliseconds. Ten +/// minutes of wall clock is days of ledgers to a backfill, so it does not park: +/// it retries on the spot and then asks again on the token's next trade. +pub const BACKFILL_BACKOFF_MS: [u64; 3] = [1_000, 5_000, 20_000]; /// What one `decimals()` call told us. #[derive(Debug, PartialEq, Eq)] @@ -40,6 +44,9 @@ pub enum Decimals { Absent, /// No answer about the contract (transport, JSON-RPC error, archived state). Transient, + /// The node has not reached the ledger being decoded, so a just-deployed + /// token may not exist for it yet. Asked again on the token's next trade. + Behind, } /// One `prices.asset_decimals` row. @@ -56,10 +63,49 @@ impl DecimalsRow { } } +/// What [`decode_resolving`] asks for missing decimals: [`DecimalsResolver`] in +/// production, a fake in tests. +#[allow(async_fn_in_trait)] // only ever used through generics, never as `dyn` +pub trait ResolveDecimals { + /// Answers for the contracts in `missing`, met while decoding `ledger`. + async fn resolve(&mut self, missing: &[String], ledger: u32) -> Vec; +} + +/// Decode once. If the decode dropped trades for unknown decimals, resolve +/// them, `persist` the answers, record them in `assets` — only after `persist` +/// succeeds — and decode again. Decoding is deterministic and the first result +/// is discarded whole, so nothing is counted twice. Whatever is still in +/// `trades_missing_decimals` afterwards is volume the caller did not price. +pub async fn decode_resolving( + resolver: &mut impl ResolveDecimals, + assets: &mut AssetRegistry, + ledger: u32, + mut decode: impl FnMut(&mut AssetRegistry) -> LedgerSoroban, + persist: impl AsyncFnOnce(&[DecimalsRow]) -> Result<(), E>, +) -> Result { + let out = decode(assets); + if out.missing_decimals.is_empty() { + return Ok(out); + } + let rows = resolver.resolve(&out.missing_decimals, ledger).await; + if rows.is_empty() { + return Ok(out); + } + persist(&rows).await?; + for row in &rows { + row.record(assets); + } + Ok(decode(assets)) +} + /// Asks Soroban RPC for the decimals a decode reported missing. pub struct DecimalsResolver { http: reqwest::Client, rpc_url: String, + /// How long a Transient failure parks a contract. + retry_after: Duration, + /// Inline retries of a Transient failure before parking it. + backoff_ms: &'static [u64], /// Contracts whose answer was [`Decimals::Absent`]. Never asked again by /// this process: a fact does not change, and a cold start asks once more. absent: HashSet, @@ -68,29 +114,46 @@ pub struct DecimalsResolver { } impl DecimalsResolver { - pub fn new(rpc_url: String) -> Self { + /// For the live processor: no inline retries, a Transient failure parks the + /// contract for [`LIVE_RETRY_AFTER`]. + pub fn live(rpc_url: String) -> Self { + Self::with(rpc_url, LIVE_RETRY_AFTER, &[]) + } + + /// For a backfill: [`BACKFILL_BACKOFF_MS`] inline, then no parking. + pub fn backfill(rpc_url: String) -> Self { + Self::with(rpc_url, Duration::ZERO, &BACKFILL_BACKOFF_MS) + } + + /// [`live`](Self::live) against `SOROBAN_RPC_URL`, or the public default. + pub fn live_from_env() -> Self { + Self::live(rpc_url_from_env()) + } + + /// [`backfill`](Self::backfill) against `SOROBAN_RPC_URL`, or the public default. + pub fn backfill_from_env() -> Self { + Self::backfill(rpc_url_from_env()) + } + + fn with(rpc_url: String, retry_after: Duration, backoff_ms: &'static [u64]) -> Self { Self { http: http_client(), rpc_url, + retry_after, + backoff_ms, absent: HashSet::new(), failed: HashMap::new(), } } - /// Against `SOROBAN_RPC_URL`, or the public default. - pub fn from_env() -> Self { - Self::new(rpc_url_from_env()) - } - - /// Resolve each distinct contract in `missing` that is neither known - /// Absent nor inside [`RETRY_AFTER`] of a Transient failure. Returns the - /// answers, which the caller persists and then - /// [`record`](DecimalsRow::record)s. Empty means nothing new was learned, - /// so decoding again would price nothing more. + /// Resolve each distinct contract in `missing` that is neither known Absent + /// nor parked after a Transient failure. Returns the answers, which the + /// caller persists and then [`record`](DecimalsRow::record)s. Empty means + /// nothing new was learned, so decoding again would price nothing more. /// /// The calls run concurrently, so a ledger naming N new tokens costs one /// RPC timeout ([`crate::soroban_rpc::RPC_TIMEOUT_SECS`]) at worst, not N. - pub async fn resolve(&mut self, missing: &[String]) -> Vec { + pub async fn resolve(&mut self, missing: &[String], ledger: u32) -> Vec { let mut asked = HashSet::new(); let mut calls = tokio::task::JoinSet::new(); for contract in missing { @@ -98,14 +161,22 @@ impl DecimalsResolver { || self .failed .get(contract) - .is_some_and(|at| at.elapsed() < RETRY_AFTER); + .is_some_and(|at| at.elapsed() < self.retry_after); if parked || !asked.insert(contract) { continue; } let (http, rpc_url, contract) = (self.http.clone(), self.rpc_url.clone(), contract.clone()); + let backoff_ms = self.backoff_ms; calls.spawn(async move { - let answer = decimals_of(&http, &rpc_url, &contract).await; + let mut answer = decimals_of(&http, &rpc_url, &contract, ledger).await; + for ms in backoff_ms { + if answer != Decimals::Transient { + break; + } + tokio::time::sleep(Duration::from_millis(*ms)).await; + answer = decimals_of(&http, &rpc_url, &contract, ledger).await; + } (contract, answer) }); } @@ -137,11 +208,19 @@ impl DecimalsResolver { Decimals::Transient => { tracing::warn!( contract, - retry_after_secs = RETRY_AFTER.as_secs(), + retry_after_secs = self.retry_after.as_secs(), "decimals() got no answer — this token's AMM trades are dropped until it resolves" ); self.failed.insert(contract, Instant::now()); } + // Not parked: the node catches up within a ledger or two. + Decimals::Behind => { + tracing::info!( + contract, + ledger, + "rpc node behind; decimals() asked again next trade" + ); + } } } rows.sort_by(|a, b| a.contract_address.cmp(&b.contract_address)); @@ -149,10 +228,21 @@ impl DecimalsResolver { } } -/// `decimals()` of `contract`, classified. -pub async fn decimals_of(http: &reqwest::Client, rpc_url: &str, contract: &str) -> Decimals { +impl ResolveDecimals for DecimalsResolver { + async fn resolve(&mut self, missing: &[String], ledger: u32) -> Vec { + DecimalsResolver::resolve(self, missing, ledger).await + } +} + +/// `decimals()` of `contract` as of `ledger`, classified. +pub async fn decimals_of( + http: &reqwest::Client, + rpc_url: &str, + contract: &str, + ledger: u32, +) -> Decimals { classify( - simulate(http, rpc_url, contract, "decimals").await, + simulate(http, rpc_url, contract, "decimals", Some(ledger)).await, contract, ) } @@ -168,6 +258,7 @@ fn classify(simulated: Simulated, contract: &str) -> Decimals { } Simulated::Absent => Decimals::Absent, Simulated::Transient => Decimals::Transient, + Simulated::Behind => Decimals::Behind, } } @@ -175,6 +266,8 @@ fn classify(simulated: Simulated, contract: &str) -> Decimals { mod tests { use super::*; + const SOLVBTC: &str = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN"; + #[test] fn only_a_u32_within_the_decimal_scale_is_known() { let v = |v| classify(Simulated::Value(v), "C"); @@ -188,16 +281,17 @@ mod tests { // no answer stays retryable. assert_eq!(classify(Simulated::Absent, "C"), Decimals::Absent); assert_eq!(classify(Simulated::Transient, "C"), Decimals::Transient); + assert_eq!(classify(Simulated::Behind, "C"), Decimals::Behind); } #[tokio::test] async fn an_absent_contract_is_never_asked_again() { // Port 1 would answer Transient; a contract already known Absent must // not reach it at all. - let mut r = DecimalsResolver::new("http://127.0.0.1:1/".to_string()); - let c = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN".to_string(); + let mut r = DecimalsResolver::live("http://127.0.0.1:1/".to_string()); + let c = SOLVBTC.to_string(); r.absent.insert(c.clone()); - assert!(r.resolve(std::slice::from_ref(&c)).await.is_empty()); + assert!(r.resolve(std::slice::from_ref(&c), 1).await.is_empty()); assert!( !r.failed.contains_key(&c), "not asked, so no Transient recorded" @@ -205,17 +299,18 @@ mod tests { } #[tokio::test] - async fn a_failed_contract_is_not_asked_again_inside_the_retry_window() { + async fn a_failed_contract_is_parked_for_its_retry_window() { // Port 1 refuses at once, so each attempt is a fast Transient. - let mut r = DecimalsResolver::new("http://127.0.0.1:1/".to_string()); - let c = "CBIJBDNZNF4X35BJ4FFZWCDBSCKOP5NB4PLG4SNENRMLAPYG4P5FM6VN".to_string(); - assert!(r.resolve(&[c.clone(), c.clone()]).await.is_empty()); + let mut r = DecimalsResolver::live("http://127.0.0.1:1/".to_string()); + let c = SOLVBTC.to_string(); + assert!(r.resolve(&[c.clone(), c.clone()], 1).await.is_empty()); let first = r.failed[&c]; - assert!(r.resolve(std::slice::from_ref(&c)).await.is_empty()); - assert_eq!(r.failed[&c], first, "not asked again inside RETRY_AFTER"); + assert!(r.resolve(std::slice::from_ref(&c), 1).await.is_empty()); + assert_eq!(r.failed[&c], first, "not asked again inside the window"); - r.failed.insert(c.clone(), Instant::now() - RETRY_AFTER); - assert!(r.resolve(std::slice::from_ref(&c)).await.is_empty()); + // A backfill's window is zero: the next trade asks again. + r.retry_after = Duration::ZERO; + assert!(r.resolve(std::slice::from_ref(&c), 1).await.is_empty()); assert!( r.failed[&c] > first, "asked again once the window has passed" diff --git a/packages/prices-ingest-core/src/lib.rs b/packages/prices-ingest-core/src/lib.rs index f0181ee4..e81e2523 100644 --- a/packages/prices-ingest-core/src/lib.rs +++ b/packages/prices-ingest-core/src/lib.rs @@ -46,7 +46,7 @@ pub mod writer; pub use bucket::{CandleAccumulator, OhlcvCandle}; pub use canonical::{AssetIdentity, AssetRegistry, CanonicalPair, canonicalise}; -pub use decimals::{DecimalsResolver, DecimalsRow}; +pub use decimals::{DecimalsResolver, DecimalsRow, ResolveDecimals, decode_resolving}; pub use decode::{decode_object, ledger_close_time, ledger_sequence}; pub use error::IngestError; pub use filter::{ diff --git a/packages/prices-ingest-core/src/soroban.rs b/packages/prices-ingest-core/src/soroban.rs index 289fda48..1d31771e 100644 --- a/packages/prices-ingest-core/src/soroban.rs +++ b/packages/prices-ingest-core/src/soroban.rs @@ -897,10 +897,14 @@ fn amm_trade_to_tick( return None; } + // Checked: with per-token scales the quotient carries a factor of up to + // 10^28 (a 28-decimal leg against a 0-decimal one), past `Decimal::MAX`, + // and the plain `/` panics on overflow. A price no `Decimal` can hold is + // not one the candle columns could store either, so the trade is dropped. let (price, volume_base, volume_quote) = if pair.inverted { - (amount_in / amount_out, amount_out, amount_in) + (amount_in.checked_div(amount_out)?, amount_out, amount_in) } else { - (amount_out / amount_in, amount_in, amount_out) + (amount_out.checked_div(amount_in)?, amount_in, amount_out) }; // The bound clears a fill whose two legs are both enormous and says nothing @@ -2437,6 +2441,203 @@ mod tests { } } + /// The mirror of the case above: USDC sold for the token, so the pair is + /// inverted (`amount_in / amount_out`, base volume from `amount_out`). A + /// swapped `decimals_in` / `decimals_out` would hide in this branch. + #[test] + fn the_inverted_branch_scales_each_leg_by_its_own_decimals() { + for (decimals, usdc_per_token) in [(8u32, 86_055i128), (18, 1), (6, 2)] { + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(SOLVBTC.to_string(), decimals); + let sell = sell_for_usdc(&assets, SOLVBTC, 0, 0); + let buy = extractors_core::TradeRow { + token_in: sell.token_out.clone(), + token_out: SOLVBTC.to_string(), + amount_in: usdc_per_token * 10_000_000, + amount_out: 10i128.pow(decimals), + ..sell + }; + + let tick = amm_trade_to_tick(&buy, 0, 1_700_000_000, &mut assets, &mut Vec::new()) + .unwrap_or_else(|| panic!("{decimals}-decimal leg prices")); + + assert_eq!(tick.base, AssetIdentity::Contract(SOLVBTC.to_string())); + assert_eq!( + tick.price, + Decimal::from(usdc_per_token), + "{decimals} decimals" + ); + assert_eq!(tick.volume_base, Decimal::ONE, "{decimals} decimals"); + assert_eq!( + tick.volume_quote, + Decimal::from(usdc_per_token), + "{decimals} decimals" + ); + } + } + + /// Review of PR #395: per-token scales can push the quotient past + /// `Decimal::MAX`. One raw unit of a 28-decimal token for 9.07e7 stroops is + /// ~9.07e28 XLM per token; the plain `/` panicked there and halted ingest + /// on that ledger for good. It must drop the trade instead. + #[test] + fn a_price_past_decimal_max_drops_the_trade_instead_of_panicking() { + const DUST28: &str = "CC2RBGYNCFBCVENIDL5BFBWPH4OUZM2UA3OD2K2N54GLMWCC4KWPVAGO"; + const WHOLE0: &str = "CB7OOP3VSAWBZOOTOG2YEFANVU45GVWYUUM5HI32DKLHVKUDOFVQ37XP"; + let mut assets = AssetRegistry::from_existing(vec![]); + assets.set_decimals(DUST28.to_string(), 28); + assets.set_decimals(WHOLE0.to_string(), 0); + let xlm_sac = assets + .sac_address_of(&AssetIdentity::Native) + .expect("XLM has a SAC"); + let swap = |token_in: &str, token_out: &str, amount_out: i128| extractors_core::TradeRow { + venue: Venue::Soroswap, + contract_id: "CDBBBNMCWRMWEIFHUD5BXBCRTW6QM33ZEXIOBGKKQNDSH3WEF7WVBGMI".to_string(), + transaction_id: "tx".to_string(), + ledger_sequence: 100, + first_event_index: 0, + token_in: token_in.to_string(), + token_out: token_out.to_string(), + amount_in: 1, + amount_out, + fee: None, + trader: None, + }; + + // (28, 7): one raw unit in, ~9.07 XLM out. + let t = swap(DUST28, &xlm_sac, 90_700_000); + assert!(amm_trade_to_tick(&t, 0, 1_700_000_000, &mut assets, &mut Vec::new()).is_none()); + // (28, 0), both directions. Which leg lands in the denominator is the + // pair's canonical order, so only the call returning is asserted. + for (token_in, token_out) in [(DUST28, WHOLE0), (WHOLE0, DUST28)] { + let t = swap(token_in, token_out, 9); + let _ = amm_trade_to_tick(&t, 0, 1_700_000_000, &mut assets, &mut Vec::new()); + } + } + + // ---- decode_resolving: the step every caller runs around a decode ------- + + /// Answers every request with `answer` and records what it was asked. + struct FakeResolver { + answer: Vec, + asked: Vec<(Vec, u32)>, + } + + impl crate::decimals::ResolveDecimals for FakeResolver { + async fn resolve(&mut self, missing: &[String], ledger: u32) -> Vec { + self.asked.push((missing.to_vec(), ledger)); + self.answer.clone() + } + } + + fn decode_seam(reg: &mut Registries, assets: &mut AssetRegistry) -> LedgerSoroban { + let mut out = LedgerSoroban::default(); + process_soroban_event_rows( + SEAM_SEQ, + SEAM_CLOSED_AT, + &[seam_swap("tx-a", 0, 0)], + reg, + assets, + &mut out, + ); + out + } + + #[tokio::test] + async fn a_resolved_token_is_persisted_then_recorded_then_priced() { + let mut reg = seam_registries(); + let mut assets = AssetRegistry::from_existing(vec![]); + let row = crate::DecimalsRow { + contract_address: SEAM_T1.to_string(), + decimals: 8, + }; + let mut fake = FakeResolver { + answer: vec![row.clone()], + asked: Vec::new(), + }; + let mut persisted = Vec::new(); + + let out = crate::decimals::decode_resolving( + &mut fake, + &mut assets, + SEAM_SEQ, + |assets| decode_seam(&mut reg, assets), + async |rows: &[crate::DecimalsRow]| { + persisted.extend_from_slice(rows); + Ok::<(), ()>(()) + }, + ) + .await + .unwrap(); + + assert_eq!(fake.asked, vec![(vec![SEAM_T1.to_string()], SEAM_SEQ)]); + assert_eq!(persisted, vec![row]); + assert_eq!( + assets.decimals_of(&AssetIdentity::Contract(SEAM_T1.to_string())), + Some(8) + ); + assert_eq!(out.amm_ticks.len(), 1, "the second decode prices the swap"); + assert_eq!(out.trades_missing_decimals, 0); + } + + #[tokio::test] + async fn an_unresolved_token_is_counted_and_nothing_is_persisted() { + let mut reg = seam_registries(); + let mut assets = AssetRegistry::from_existing(vec![]); + let mut fake = FakeResolver { + answer: Vec::new(), + asked: Vec::new(), + }; + let mut persisted: Vec = Vec::new(); + + let out = crate::decimals::decode_resolving( + &mut fake, + &mut assets, + SEAM_SEQ, + |assets| decode_seam(&mut reg, assets), + async |rows: &[crate::DecimalsRow]| { + persisted.extend_from_slice(rows); + Ok::<(), ()>(()) + }, + ) + .await + .unwrap(); + + assert_eq!(fake.asked.len(), 1); + assert!(persisted.is_empty()); + assert!(out.amm_ticks.is_empty()); + assert_eq!(out.trades_missing_decimals, 1, "the lost trade is counted"); + } + + #[tokio::test] + async fn a_failed_persist_records_nothing() { + let mut reg = seam_registries(); + let mut assets = AssetRegistry::from_existing(vec![]); + let mut fake = FakeResolver { + answer: vec![crate::DecimalsRow { + contract_address: SEAM_T1.to_string(), + decimals: 8, + }], + asked: Vec::new(), + }; + + let got = crate::decimals::decode_resolving( + &mut fake, + &mut assets, + SEAM_SEQ, + |assets| decode_seam(&mut reg, assets), + async |_: &[crate::DecimalsRow]| Err("insert failed"), + ) + .await; + + assert_eq!(got.err(), Some("insert failed")); + assert_eq!( + assets.decimals_of(&AssetIdentity::Contract(SEAM_T1.to_string())), + None, + "recorded only after a durable write, so the next run asks again" + ); + } + /// A pure Soroban token whose decimals are unknown is never guessed at 7: /// the trade yields no tick, interns nothing, and every unknown leg is /// reported so the caller can resolve it. diff --git a/packages/prices-ingest-core/src/soroban_rpc.rs b/packages/prices-ingest-core/src/soroban_rpc.rs index ae51c274..02b23de8 100644 --- a/packages/prices-ingest-core/src/soroban_rpc.rs +++ b/packages/prices-ingest-core/src/soroban_rpc.rs @@ -94,6 +94,10 @@ pub enum Simulated { /// No answer about the contract: transport failure, JSON-RPC error, /// archived state, an empty body. Ask again later. Transient, + /// The contract errored, but the node has not yet reached the ledger the + /// caller asked about, so a contract deployed since may simply be invisible + /// to it. Ask again on the next ledger. + Behind, } #[derive(serde::Deserialize)] @@ -116,6 +120,9 @@ struct RpcResult { /// a live contract, not one that lacks the function. #[serde(default, rename = "restorePreamble")] restore_preamble: Option, + /// The newest ledger the node had applied when it simulated. + #[serde(default, rename = "latestLedger")] + latest_ledger: Option, } #[derive(serde::Deserialize)] struct RpcInvokeResult { @@ -123,13 +130,18 @@ struct RpcInvokeResult { } /// Call the zero-argument `func()` on `contract` by simulation and classify -/// the result. Never returns an error: every failure is Absent or Transient, -/// which is exactly the decision a caller has to make. +/// the result. Never returns an error: every failure is Absent, Transient or +/// Behind, which is exactly the decision a caller has to make. +/// +/// `as_of` is the ledger at which the caller needs the contract to exist (the +/// one it is decoding). A contract error from a node that has not reached it is +/// [`Simulated::Behind`], not a fact. `None` skips that check. pub async fn simulate( http: &reqwest::Client, rpc_url: &str, contract: &str, func: &str, + as_of: Option, ) -> Simulated { let Some(envelope) = build_simulate_envelope(contract, func) else { tracing::warn!(contract, func, "not a well-formed contract address"); @@ -160,7 +172,7 @@ pub async fn simulate( } }; match resp.json::().await { - Ok(r) => classify(r, contract, func), + Ok(r) => classify(r, contract, func, as_of), Err(err) => { tracing::warn!(contract, func, error = %err, "simulate rpc body unparseable"); Simulated::Transient @@ -176,7 +188,7 @@ pub async fn simulate( /// state, an empty body — we learned nothing, so it is Transient. A wrong /// Transient costs a repeated call; a wrong Absent can make a caller give up /// on a contract for good. -fn classify(resp: RpcResponse, contract: &str, func: &str) -> Simulated { +fn classify(resp: RpcResponse, contract: &str, func: &str, as_of: Option) -> Simulated { // Arrives with HTTP 200, so `error_for_status` cannot see it. if let Some(err) = resp.error { tracing::warn!(contract, func, error = %err, "simulate returned a json-rpc error"); @@ -191,8 +203,22 @@ fn classify(resp: RpcResponse, contract: &str, func: &str) -> Simulated { return Simulated::Transient; } // The simulation ran and the contract itself errored — deterministic, so a - // fact. This is the one arm that stays permanent. + // fact. This is the one arm that stays permanent, unless the node is still + // behind the ledger the caller is decoding: a contract deployed in between + // does not exist for it yet (task 0329 review). if let Some(err) = result.error { + if let (Some(needed), Some(tip)) = (as_of, result.latest_ledger) + && tip < needed + { + tracing::debug!( + contract, + func, + tip, + needed, + "node behind the ledger being decoded" + ); + return Simulated::Behind; + } tracing::debug!(contract, func, error = %err, "contract errored in simulation"); return Simulated::Absent; } @@ -220,10 +246,15 @@ mod tests { use super::*; fn classify_json(body: &str) -> Simulated { + classify_json_as_of(body, None) + } + + fn classify_json_as_of(body: &str, as_of: Option) -> Simulated { classify( serde_json::from_str(body).expect("test body parses"), "C_TEST", "symbol", + as_of, ) } @@ -265,6 +296,20 @@ mod tests { ); } + #[test] + fn a_contract_error_from_a_node_behind_the_ledger_is_not_a_fact() { + let body = r#"{"result":{"results":[],"error":"contract not found","latestLedger":100}}"#; + // The node is at 100 and the caller decodes 101: the contract may have + // been deployed in 101. + assert_eq!(classify_json_as_of(body, Some(101)), Simulated::Behind); + // At or past the ledger, the error is a fact again. + assert_eq!(classify_json_as_of(body, Some(100)), Simulated::Absent); + assert_eq!(classify_json_as_of(body, None), Simulated::Absent); + // A node that does not report its tip cannot be judged behind. + let no_tip = r#"{"result":{"results":[],"error":"contract not found"}}"#; + assert_eq!(classify_json_as_of(no_tip, Some(101)), Simulated::Absent); + } + #[test] fn a_returned_value_is_decoded() { let xdr = base64::engine::general_purpose::STANDARD diff --git a/packages/prices-ingest-core/tests/decimals_rpc_it.rs b/packages/prices-ingest-core/tests/decimals_rpc_it.rs index c983f29e..f3c0e979 100644 --- a/packages/prices-ingest-core/tests/decimals_rpc_it.rs +++ b/packages/prices-ingest-core/tests/decimals_rpc_it.rs @@ -33,7 +33,7 @@ async fn reads_the_decimals_sbe_checked_on_chain() { 6, ), ] { - let got = decimals_of(&http_client(), &rpc_url_from_env(), contract).await; + let got = decimals_of(&http_client(), &rpc_url_from_env(), contract, 1).await; assert_eq!(got, Decimals::Known(want), "{name} ({contract})"); } } @@ -42,7 +42,7 @@ async fn reads_the_decimals_sbe_checked_on_chain() { #[ignore = "requires public network — third-party uptime; never gates a PR"] async fn a_contract_that_was_never_deployed_is_absent() { let never_deployed = stellar_strkey::Contract([3u8; 32]).to_string(); - let got = decimals_of(&http_client(), &rpc_url_from_env(), &never_deployed).await; + let got = decimals_of(&http_client(), &rpc_url_from_env(), &never_deployed, 1).await; assert_eq!( got, Decimals::Absent, diff --git a/packages/prices-ledger-processor/src/reconcile.rs b/packages/prices-ledger-processor/src/reconcile.rs index 9b0120ba..3d3eff5a 100644 --- a/packages/prices-ledger-processor/src/reconcile.rs +++ b/packages/prices-ledger-processor/src/reconcile.rs @@ -63,9 +63,10 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::Instant; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, DecimalsResolver, OfferLookupCounts, OracleSample, - PoolRegistryRow, Registries, decode_object, extract_trades_with_counts, ledger_close_time, - ledger_sequence, process_ledger, raw_trade_to_tick, + AssetRegistry, CandleAccumulator, DecimalsResolver, DecimalsRow, OfferLookupCounts, + OracleSample, PoolRegistryRow, Registries, decode_object, decode_resolving, + extract_trades_with_counts, ledger_close_time, ledger_sequence, process_ledger, + raw_trade_to_tick, }; use tokio::sync::Mutex; use tracing::info; @@ -168,7 +169,7 @@ fn initial_state(assets: AssetRegistry, mut registries: Registries) -> Processin assets, registries, persisted_pools, - decimals: DecimalsResolver::from_env(), + decimals: DecimalsResolver::live_from_env(), } } @@ -200,6 +201,13 @@ where } } + /// Replace the decimals resolver (task 0329). Tests point it at an address + /// that refuses connections, so no test can reach a public RPC. + pub fn with_decimals_resolver(mut self, resolver: DecimalsResolver) -> Self { + self.state.get_mut().decimals = resolver; + self + } + /// One reconcile run for a LONG-RUNNING caller (the Lambda). Minutes the /// run did not see the end of are held back for the next run. pub async fn run(&self, max_iterations: usize) -> Result { @@ -290,22 +298,18 @@ where for trade in trades { sdex.merge(raw_trade_to_tick(&trade, &mut state.assets)); } - // Soroban AMM trades + oracle samples. - let mut sob = process_ledger(lcm, &mut state.registries, &mut state.assets); - // Task 0329: a trade naming a token whose decimals are not known - // yet was dropped, not guessed. Resolve, persist, and decode the - // ledger again — decoding is deterministic, and the first - // result is discarded whole, so nothing is counted twice. - if !sob.missing_decimals.is_empty() { - let rows = state.decimals.resolve(&sob.missing_decimals).await; - if !rows.is_empty() { - self.sink.write_decimals(&rows).await?; - for row in &rows { - row.record(&mut state.assets); - } - sob = process_ledger(lcm, &mut state.registries, &mut state.assets); - } - } + // Soroban AMM trades + oracle samples. Task 0329: a trade naming a + // token whose decimals are not known yet is dropped, not guessed; + // `decode_resolving` resolves it, persists it, and decodes again. + let registries = &mut state.registries; + let sob = decode_resolving( + &mut state.decimals, + &mut state.assets, + ledger_sequence(lcm), + |assets| process_ledger(lcm, registries, assets), + async |rows: &[DecimalsRow]| self.sink.write_decimals(rows).await, + ) + .await?; for (source, tick) in sob.amm_ticks { amm.entry(source).or_default().merge(tick); } diff --git a/packages/prices-ledger-processor/tests/pool_registry_persist.rs b/packages/prices-ledger-processor/tests/pool_registry_persist.rs index bf1be7e2..22b73eff 100644 --- a/packages/prices-ledger-processor/tests/pool_registry_persist.rs +++ b/packages/prices-ledger-processor/tests/pool_registry_persist.rs @@ -16,7 +16,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use prices_ingest_core::{ - AssetRegistry, OhlcvCandle, OracleSample, PoolRegistryRow, Registries, process_ledger, + AssetRegistry, DecimalsResolver, OhlcvCandle, OracleSample, PoolRegistryRow, Registries, + process_ledger, }; use prices_ledger_processor::{ cursor::{Cursor, StubFileCursor}, @@ -237,7 +238,8 @@ async fn harness( sink.clone(), AssetRegistry::from_existing(Vec::new()), registries, - ); + ) + .with_decimals_resolver(DecimalsResolver::live("http://127.0.0.1:1/".to_string())); (reconciler, sink) } diff --git a/packages/prices-ledger-processor/tests/reconcile_drain.rs b/packages/prices-ledger-processor/tests/reconcile_drain.rs index a7ce7828..0ad991ec 100644 --- a/packages/prices-ledger-processor/tests/reconcile_drain.rs +++ b/packages/prices-ledger-processor/tests/reconcile_drain.rs @@ -27,8 +27,8 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use prices_ingest_core::{ - AssetIdentity, AssetRegistry, OfferLookupCounts, OhlcvCandle, OracleSample, PoolRegistryRow, - Registries, decode_object, extract_trades_with_counts, + AssetIdentity, AssetRegistry, DecimalsResolver, OfferLookupCounts, OhlcvCandle, OracleSample, + PoolRegistryRow, Registries, decode_object, extract_trades_with_counts, }; use prices_ledger_processor::{ cursor::{Cursor, StubFileCursor}, @@ -194,7 +194,8 @@ async fn harness(dir: &std::path::Path) -> (Harness, MemoryFetcher, RecordingSin sink.clone(), AssetRegistry::from_existing(Vec::new()), Registries::new(), - ); + ) + .with_decimals_resolver(DecimalsResolver::live("http://127.0.0.1:1/".to_string())); (reconciler, fetcher, sink) } diff --git a/packages/prices-ledger-processor/tests/reconcile_e2e.rs b/packages/prices-ledger-processor/tests/reconcile_e2e.rs index 6d87760c..6bf03625 100644 --- a/packages/prices-ledger-processor/tests/reconcile_e2e.rs +++ b/packages/prices-ledger-processor/tests/reconcile_e2e.rs @@ -14,7 +14,9 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use prices_ingest_core::{AssetRegistry, OhlcvCandle, OracleSample, PoolRegistryRow, Registries}; +use prices_ingest_core::{ + AssetRegistry, DecimalsResolver, OhlcvCandle, OracleSample, PoolRegistryRow, Registries, +}; use prices_ledger_processor::{ cursor::{Cursor, StubFileCursor}, object_fetcher::LocalDiskFetcher, @@ -58,6 +60,7 @@ fn reconciler( AssetRegistry::from_existing(Vec::new()), Registries::new(), ) + .with_decimals_resolver(DecimalsResolver::live("http://127.0.0.1:1/".to_string())) } /// Fault-injecting sink: fails the first `write_new_assets` call, succeeds after. @@ -117,7 +120,8 @@ async fn assets_from_a_failed_run_are_written_on_the_next_run() { sink.clone(), AssetRegistry::from_existing(Vec::new()), Registries::new(), - ); + ) + .with_decimals_resolver(DecimalsResolver::live("http://127.0.0.1:1/".to_string())); // Run 1: interns new assets, then the (first) asset write fails → the run // errors and the cursor is never advanced (the doorbell would redeliver). diff --git a/packages/sdex-backfill/src/ingest.rs b/packages/sdex-backfill/src/ingest.rs index 4d65f3be..43c3ca48 100644 --- a/packages/sdex-backfill/src/ingest.rs +++ b/packages/sdex-backfill/src/ingest.rs @@ -5,9 +5,9 @@ use std::time::{Duration, Instant}; use tracing::{info, warn}; use prices_ingest_core::{ - AssetRegistry, CandleAccumulator, DecimalsResolver, OhlcvCandle, Registries, TradeTick, - UnresolvedPoolSwap, extract_trades, ledger_sequence, offer_lookup_counts, process_ledger, - raw_trade_to_tick, + AssetRegistry, CandleAccumulator, DecimalsResolver, DecimalsRow, OhlcvCandle, Registries, + TradeTick, UnresolvedPoolSwap, decode_resolving, extract_trades, ledger_sequence, + offer_lookup_counts, process_ledger, raw_trade_to_tick, }; use crate::error::BackfillError; @@ -407,21 +407,17 @@ pub async fn index_partition( // Only in Combined mode — pre-Soroban ledgers carry no Soroban // events, so SdexOnly skips the decode entirely. if mode == ExtractMode::Combined { - let mut sob = process_ledger(lcm, reg, registry); // Task 0329: a trade naming a token whose decimals are unknown - // was dropped, not guessed. Resolve, persist, decode again — - // the same step as the live processor. - if !sob.missing_decimals.is_empty() { - let rows = decimals.resolve(&sob.missing_decimals).await; - if !rows.is_empty() { - sink.write_decimals(&rows).await?; - for row in &rows { - row.record(registry); - } - sob = process_ledger(lcm, reg, registry); - } - stats.trades_missing_decimals += sob.trades_missing_decimals as usize; - } + // is dropped, not guessed — the same step as the live processor. + let sob = decode_resolving( + decimals, + registry, + ledger_sequence(lcm), + |assets| process_ledger(lcm, reg, assets), + async |rows: &[DecimalsRow]| sink.write_decimals(rows).await, + ) + .await?; + stats.trades_missing_decimals += sob.trades_missing_decimals as usize; for (source, tick) in sob.amm_ticks { accs.merge_amm(source, tick); stats.amm_ticks += 1; diff --git a/packages/sdex-backfill/src/run.rs b/packages/sdex-backfill/src/run.rs index cc0593f4..aa4bc6fb 100644 --- a/packages/sdex-backfill/src/run.rs +++ b/packages/sdex-backfill/src/run.rs @@ -153,7 +153,7 @@ pub async fn execute( if mode == ExtractMode::Combined { sink.load_decimals(&mut registry).await?; } - let mut decimals = DecimalsResolver::from_env(); + let mut decimals = DecimalsResolver::backfill_from_env(); // Venue / pool registries. Preloaded from the persisted `pool_registry` // artifact (decision #4) so a window starting after activation still // resolves earlier-created pools; empty on a fresh full run. Then grown From a2bdd1a9af905b150e911cfe8ea8139468340019 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:09:49 +0200 Subject: [PATCH 051/144] fix(lore-0329): carry the decimals count through amm-done --amm wait recorded only the fallbacks, so a hand run with dropped trades could be marked done. amm-done now requires --decimals and the script refuses an amm.done without it. --- docs/runbooks/0286-reingest-history.md | 3 ++- tools/scripts/reingest_0286.py | 21 +++++++++++++++++--- tools/scripts/reingest_0286_post0139_test.py | 9 +++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/runbooks/0286-reingest-history.md b/docs/runbooks/0286-reingest-history.md index 9a181968..4f608b0a 100644 --- a/docs/runbooks/0286-reingest-history.md +++ b/docs/runbooks/0286-reingest-history.md @@ -482,7 +482,8 @@ A pure Soroban token's legs are scaled by its `decimals()`, read over Soroban RPC the first time the run meets it and kept in `prices.asset_decimals`. **Anything but `0` means a token did not resolve and its trades are NOT in the month's candles.** The run WARNs naming the contract. Re-run the month once it -resolves. The binary needs outbound HTTPS to `SOROBAN_RPC_URL` (default +resolves. With `--amm wait`, pass the line to `amm-done … --decimals N`: the +script refuses an `amm.done` without it and marks a non-zero count DEFECT. The binary needs outbound HTTPS to `SOROBAN_RPC_URL` (default `https://mainnet.sorobanrpc.com`), and `prices.asset_decimals` must exist. --- diff --git a/tools/scripts/reingest_0286.py b/tools/scripts/reingest_0286.py index 57cff0ab..ea40118d 100755 --- a/tools/scripts/reingest_0286.py +++ b/tools/scripts/reingest_0286.py @@ -947,12 +947,13 @@ def done(): else: marker = mdir / "amm.done" note(f"{m}: waiting for the host run — read -rs CH_PW; CLICKHOUSE_PASSWORD=\"$CH_PW\" {base}" - f" then: reingest_0286.py amm-done {m} --fallbacks ") + f" then: reingest_0286.py amm-done {m} --fallbacks " + "--decimals ") while not marker.exists(): DASH["step"] = "waiting for amm-done" RERENDER() time.sleep(15) - ms["fallbacks"] = int(marker.read_text().strip() or 0) + ms["fallbacks"], ms["decimals"] = read_amm_done(marker) done() if step("reconcile"): @@ -1095,7 +1096,17 @@ def cmd_status(a, ch, st): def cmd_amm_done(a, ch, st): (st.dir / str(a.month)).mkdir(exist_ok=True) - (st.dir / str(a.month) / "amm.done").write_text(str(a.fallbacks)) + (st.dir / str(a.month) / "amm.done").write_text(f"{a.fallbacks} {a.decimals}") + + +def read_amm_done(marker): + """(fallbacks, decimals) from an `amm.done`. A marker without the decimals + count predates task 0329 and is refused rather than read as zero.""" + parts = marker.read_text().split() + if len(parts) < 2: + raise Stop(f"{marker} carries no `trades dropped (decimals)` count — re-record it with " + "`amm-done --fallbacks N --decimals N` from the run's summary") + return int(parts[0]), int(parts[1]) def cmd_finish(a, ch, st): @@ -1183,6 +1194,8 @@ def build_parser(): p.add_argument("--yes", action="store_true") p.add_argument("--accept", action="append", help="YYYYMM whose failed reconciliation is understood and recorded") p.add_argument("--fallbacks", type=int, default=0) + p.add_argument("--decimals", type=int, + help="amm-done: the run's `trades dropped (decimals)` line (task 0329)") p.add_argument("--ack-phase1-measured", action="store_true") p.add_argument("--ack-0285", action="store_true") p.add_argument("--ack-0300-binary", action="store_true", @@ -1227,6 +1240,8 @@ def main(): a = p.parse_args() if a.command in ("amm-done", "rollback", "release") and not a.month: p.error(f"{a.command} needs a month") + if a.command == "amm-done" and a.decimals is None: + p.error("amm-done needs --decimals: the run's `trades dropped (decimals)` line (task 0329)") if a.min_excluded_rows and not a.second_pass: p.error("--min-excluded-rows applies to --second-pass only") global LOG diff --git a/tools/scripts/reingest_0286_post0139_test.py b/tools/scripts/reingest_0286_post0139_test.py index c1d9598c..0825b781 100644 --- a/tools/scripts/reingest_0286_post0139_test.py +++ b/tools/scripts/reingest_0286_post0139_test.py @@ -340,6 +340,15 @@ def test_the_summary_needs_the_decimals_line(self): with self.assertRaisesRegex(r.Stop, "trades dropped"): r.amm_summary(text.replace("trades dropped (decimals): 2\n", ""), "log") + def test_amm_done_carries_the_decimals_count(self): + st = state([202201], self.tmp.name) + r.cmd_amm_done(args("amm-done", "202201", "--fallbacks", "1", "--decimals", "40"), None, st) + marker = Path(self.tmp.name) / "202201" / "amm.done" + self.assertEqual(r.read_amm_done(marker), (1, 40)) + marker.write_text("0") # written before task 0329 + with self.assertRaisesRegex(r.Stop, "decimals"): + r.read_amm_done(marker) + def test_pre0139_reconciles_unrestricted(self): ms, ch, _ = self.run_reconcile(SUMS_COPIED, ids="u32", width="UInt32") self.assertEqual(ms["result"]["verdict"], "DEFECT") # +10 % on the unrestricted sums From f44ced6a52f4856fd3b9cd827f4d69466d76a365 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:09:52 +0200 Subject: [PATCH 052/144] docs(lore-0329): record the review refinements --- .../database-schema/database-schema-overview.md | 9 ++++++--- ...roban-token-amounts-assume-seven-decimals.md | 17 +++++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/docs/database-schema/database-schema-overview.md b/docs/database-schema/database-schema-overview.md index f86cce92..957a23bb 100644 --- a/docs/database-schema/database-schema-overview.md +++ b/docs/database-schema/database-schema-overview.md @@ -1507,9 +1507,12 @@ no row, the decode drops that trade rather than guessing a scale. The caller asks `decimals()` over Soroban RPC, writes the row, and decodes the ledger again. **Read by:** the same three, in full, at startup. -> **Only answers are stored.** A token whose call got no answer has no row and -> is asked again after ten minutes. One that answered with no usable scale is -> not asked again until the process restarts. Until it resolves, its trades are +> **Only answers are stored.** A token whose call got no answer has no row. The +> live processor asks again after ten minutes; a backfill retries on the spot +> (1 s, 5 s, 20 s) and then on the token's next trade. A token that answered +> with no usable scale is not asked again until the process restarts, unless +> the node was still behind the ledger being decoded, in which case it is asked +> again on the next trade. Until it resolves, its trades are > not priced. The live processor publishes them as `TradesMissingDecimals` > (alarmed), and both backfills print them as `trades dropped (decimals)`. > Rows are never refreshed, because decimals are fixed at deploy. diff --git a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md index 4253efa1..4ef2787d 100644 --- a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md +++ b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md @@ -166,11 +166,14 @@ Rejected: `TradesMissingDecimals`, alarmed like `UnregisteredPoolEvents`. Both backfills print `trades dropped (decimals)`, and `reingest_0286.py` marks a month with a non-zero count DEFECT. Added after review (PR #395). -2. **Absent is terminal per process, Transient retries after 10 minutes.** A +2. **Absent is terminal per process; Transient depends on the caller.** A contract that answered with no usable scale is not asked again until a cold - start, and is never persisted as a sentinel. A contract that gave no answer - is retried after 10 minutes. Calls run concurrently, so one ledger costs one - RPC timeout at worst. + start, and is never persisted as a sentinel. Exception: a node behind the + ledger being decoded (`latestLedger` from the simulate response) gives + `Behind`, which is asked again on the next trade. A contract that gave no + answer is parked for 10 minutes live; a backfill retries it inline + (1/5/20 s) and then on the next trade. Calls run concurrently, so one ledger + costs one RPC timeout at worst. Refined after Adam's review (PR #395). 3. **Decode again rather than a two-phase tick.** Decoding is deterministic and the registry inserts are idempotent map writes, so the first result is discarded whole. @@ -184,6 +187,12 @@ Rejected: `reingest_0286.py` compares volumes on the other candles only and trades on all of them. A "before" read without that scope compares trades only, as a FINDING. Added after review (PR #395). +8. **A price past `Decimal::MAX` drops the trade.** Per-token scales can carry + a factor of up to 10^28, so the quotient uses `checked_div`. Before the + review a plain `/` would panic and stall ingest on that ledger for good. +9. **One `decode_resolving` for all three callers**, generic over a + `ResolveDecimals` trait so its persist-then-record order is tested with a + fake. ### Deploy order From 5b647f17ade8831930bb7c49e070e45129f23204 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:19:22 +0200 Subject: [PATCH 053/144] docs(lore-0226): complete and archive the task Closed before AC 4's week ended (2026-10-09), on Adam's decision; four clean days measured, fixed by 0139. --- ...ds-whole-asset-registry-to-write-two-rows.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) rename lore/1-tasks/{active => archive}/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md (96%) diff --git a/lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md b/lore/1-tasks/archive/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md similarity index 96% rename from lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md rename to lore/1-tasks/archive/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md index b63b151d..87ae4ac4 100644 --- a/lore/1-tasks/active/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md +++ b/lore/1-tasks/archive/0226_BUG_oracle-worker-loads-whole-asset-registry-to-write-two-rows.md @@ -2,7 +2,7 @@ id: "0226" title: "The oracle worker loads all 620,615 assets into memory to write 2 rows, sits at its 256 MB ceiling and OOMs several times a day" type: BUG -status: active +status: completed assignee: akot related_adr: [] related_tasks: ["0223", "0222", "0167", "0112", "0132", "0256", "0241", "0140"] @@ -113,6 +113,15 @@ history: prod: 256 → 52 MB, 7–8 s → 0.75 s, 0 errors and 0 OOM since. ACs 1, 2, 3 and 5 met; AC 4 (a week without an OOM) ends 2026-10-09 12:44 UTC, close after it. [[0140]] closed the same day. + - date: "2026-10-07" + status: completed + who: akot + note: > + Closed before AC 4's week ended (2026-10-09 12:44 UTC), on the + operator's decision: 0 OOM and 0 errors from the first new-code run + (2026-10-02 12:44 UTC) to the 2026-10-06 measurement; not re-checked + since. Fixed by [[0139]], no code here. ACs 1, 2, 3 and 5 met; no + follow-up tasks. --- # The oracle worker reads the entire asset registry on every run @@ -372,8 +381,8 @@ code, 12:44 UTC on a cold container, used **51 MB in 1.4 s**. No `sink.rs:72`), `prices-ledger-processor/src/sink/mod.rs:103` (cold start, `main.rs:92`; `bin/cli.rs:86`). -**Still open:** AC 4. The week without an OOM runs from the first new-code run -to **2026-10-09 12:44 UTC**; close after checking it. +**AC 4 not completed:** the week without an OOM would have ended **2026-10-09 +12:44 UTC**. The task was closed on 2026-10-07, after four clean days. ## Acceptance Criteria @@ -388,6 +397,8 @@ to **2026-10-09 12:44 UTC**; close after checking it. listed above. - [ ] `prices-production-oracle-errors` shows **zero** `Runtime.OutOfMemory` for a full week after the change. + Closed early, 2026-10-07: zero through the 2026-10-06 check (four of + the seven days); the rest of the week was not checked. - [x] If the deploy touches `eventbridge-stack.ts`, `describe-rule` output for `prices-production-cleanup` is recorded **before and after** — not touched (no `memorySize` change); the rule reads `DISABLED` on 2026-10-06. From 4934aa018354b77ca56d65ef165a436d32a13d01 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:29:45 +0200 Subject: [PATCH 054/144] docs(lore-0200): decide cleanup stays disabled, archive Measured on prod 2026-10-07: no invocation since 2026-07-20, the rule held DISABLED through 8 EventBridge deploys, 1m+15m grow ~11 GiB/yr against 752 GiB free, oracle_prices is 19.7 MiB. --- .../README.md | 83 ++++++++++++++++--- 1 file changed, 72 insertions(+), 11 deletions(-) rename lore/1-tasks/{backlog => archive}/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md (66%) diff --git a/lore/1-tasks/backlog/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md b/lore/1-tasks/archive/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md similarity index 66% rename from lore/1-tasks/backlog/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md rename to lore/1-tasks/archive/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md index 3f1bb03f..970f93f4 100644 --- a/lore/1-tasks/backlog/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md +++ b/lore/1-tasks/archive/0200_RESEARCH_is-the-cleanup-worker-still-needed/README.md @@ -2,7 +2,8 @@ id: "0200" title: "Is the cleanup worker still needed at all? Decide whether prices-production-cleanup is enabled or disabled" type: RESEARCH -status: backlog +status: completed +assignee: akot related_adr: [] related_tasks: ["0088", "0090", "0111", "0167", "0174", "0046", "0063"] tags: @@ -53,10 +54,66 @@ history: stack can NOT silently re-enable cleanup. That acceptance criterion of this task is therefore already DONE, and what remains here is only the enable-or-disable decision itself, now due after M3. + - date: "2026-10-07" + status: completed + who: akot + note: > + DECIDED: prices-production-cleanup stays DISABLED for good; the worker + stays in the code, dark. Measured on prod today: rule DISABLED, last + invocation 2026-07-20 03:00 UTC, none since; 8 EventBridge deploys + since 09-14 left it off. 1m + 15m grow ~0.9 GiB/month (~11 GiB/yr) + against 752 GiB free; oracle_prices is 19.7 MiB. 1m is the source the + 0286 re-ingest is rebuilding, so a 7-day retention would delete it. + No follow-up tasks. --- # Is `prices-production-cleanup` still worth running? +## ✅ Decision — 2026-10-07: DISABLED for good + +**`prices-production-cleanup` stays DISABLED.** No table is expired by a job. +The worker and its rule stay in the code (`enabled: false`), dark; re-enabling +is a deliberate, reviewed edit to `eventbridge-stack.ts`, nothing else. + +Measured on production, 2026-10-07: + +| | measurement | +|---|---| +| rule | `DISABLED`, `cron(0 3 * * ? *)`; target still the cleanup Lambda | +| last invocation | 2026-07-20 03:00 UTC (CloudWatch `Invocations`, last log stream); **0 since** | +| CDK vs live | 8 EventBridge stack deploys 2026-09-14 → 10-05 (CloudTrail `UpdateFunctionCode` by CloudFormation), the rule still `DISABLED` after all of them | +| ClickHouse | no cleanup query in `system.query_log` (kept from 2026-09-08): no `toUInt32(partition) < toYYYYMM(now() - …)` listing, no `DROP PARTITION` on `oracle_prices`. Every 1m/15m `DROP PARTITION` in that window is the 0286 re-ingest (`prices_admin`) or the 0139 rekey (`dev_shared`, `__new` tables) | +| `price_ohlcv_1m` | 28.3 GiB, 903 M rows; live months 0.56–0.82 GiB (2026-07 → 09) | +| `price_ohlcv_15m` | 12.1 GiB; live months 0.30–0.35 GiB | +| `oracle_prices` | **19.7 MiB**, 624 k rows, from 202509 | +| free disk | **752 GiB of 1.72 TiB** (shared; `default` = 614 GiB, `prices` = 193 GiB, of which 132 GiB are re-ingest / 0139 backup tables) | + +Why: + +1. **OHLCV (1m, 15m): growth is not a problem.** ~0.9 GiB/month at the live + rate → ~11 GiB in 12 months, ~22 GiB in 24, against 752 GiB free. The + 0286 re-ingest refilling 1m for 2024-10 → 2026-06 adds a one-off amount + of the same order. The backup tables of the running migrations weigh + more than two years of growth. +2. **1m is now the source, not a cache.** The 0286 phase-3 re-ingest is + rebuilding every 1m month and re-rolling the coarse tiers from it. A + 7-day window would drop that work every night, the same way the + 2026-07-18 run dropped 1m for 2024-10 → 2026-06 (measured today: those + 21 months are absent from `price_ohlcv_1m`, the coarse tables are their + only copy until the re-ingest reaches them). +3. **`oracle_prices`, answered separately:** 19.7 MiB, ~1.5 MiB/month. The + 13-month expiry saves nothing. With no expiry, [[0167]]'s reason for + `usd_rate` (escaping `oracle_prices`' expiry) no longer bites, but + `usd_rate` stays out of `RETENTION` (the test in `cleanup-worker` still + guards it), so re-enabling cleanup later cannot take it. +4. **Replacement signal exists:** `prices-production-ch-disk-free` + (`observability-stack.ts:1145`, task 0204) watches the shared volume. + +Not done here, on purpose: the worker is not deleted (removal would need an +EventBridge deploy and buys nothing), and the "check `describe-rule` before +and after every deploy" lines in the runbooks are left in place. They are now +redundant, not wrong. + The rule has been **DISABLED since 2026-07-20** and prod has been healthy without it for over three weeks. It is time to decide deliberately rather than leave it off by inertia. @@ -148,24 +205,28 @@ should delete the need for it. ## Acceptance Criteria -- [ ] **DECISION RECORDED: `prices-production-cleanup` should be ENABLED or +- [x] **DECISION RECORDED: `prices-production-cleanup` should be ENABLED or DISABLED.** One of the two, stated plainly, with the measurement behind it. A third option (keep it, but change the retention windows or the eligibility predicate) is acceptable — but it must be written as a decision, not left as "needs more thought". -- [ ] Per-table size and growth measured on prod, projected 12/24 months, against - measured free space. -- [ ] `oracle_prices` answered separately from the OHLCV tables, with `usd_rate`'s - dependency on that expiry ([[0167]]) explicitly addressed. -- [ ] **The CDK matches the decision** — `eventbridge-stack.ts` no longer asserts + → DISABLED, 2026-10-07 (Decision section above). +- [x] Per-table size and growth measured on prod, projected 12/24 months, against + measured free space. → ~11 / ~22 GiB against 752 GiB free. +- [x] `oracle_prices` answered separately from the OHLCV tables, with `usd_rate`'s + dependency on that expiry ([[0167]]) explicitly addressed. → point 3. +- [x] **The CDK matches the decision** — `eventbridge-stack.ts` no longer asserts a state that differs from the live rule, so the "check `describe-rule` after every deploy" workaround can be retired from the runbooks and from [[0182]]. -- [ ] If cleanup is retired: a replacement signal (disk-headroom alarm) exists - before the worker is removed, not after. -- [ ] If cleanup is kept: the historical-write hazard is addressed, so a backfill + → `enabled: false` since 0204; held through 8 deploys. The runbook lines + are not removed, only made redundant. +- [x] If cleanup is retired: a replacement signal (disk-headroom alarm) exists + before the worker is removed, not after. → `prices-production-ch-disk-free`; + the worker is not removed. +- [x] (n/a, not kept) If cleanup is kept: the historical-write hazard is addressed, so a backfill or repair run can no longer have its output deleted as it lands. -## ⛔ Until this is decided +## ⛔ Until this is decided (superseded 2026-10-07: decided, DISABLED) **Cleanup stays DISABLED.** That is the deliberate current position, not an oversight — recorded here so the next session does not "fix" it. Do not enable From 63b7661cdf5817730cf347c162b6aa97c9d42915 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:59:09 +0200 Subject: [PATCH 055/144] docs(lore-0329): record the schema, seed and ledger-processor deploy Only the ledger-processor went out, via update-function-code: the Compute stack would also have shipped #373's API ahead of its schema. Records the hashes, the checks, the CloudFormation drift and the rollback. --- ...ban-token-amounts-assume-seven-decimals.md | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md index 4ef2787d..c55cfd2f 100644 --- a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md +++ b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md @@ -12,6 +12,18 @@ links: - "../../../packages/asset-discovery/src/symbols.rs" - "../../../docs/database-schema/amm-trades-schema.md" history: + - date: "2026-10-07" + status: active + who: stkrolikiewicz + note: > + PR #395 merged (24e07cc0). prices.asset_decimals created on prod and + seeded with the 7 tokens. Only the ledger-processor was deployed, through + `update-function-code` rather than `make deploy-production-compute`: the + Compute stack would also have shipped #373's prices-api (0274, + `price_basis`), whose schema is not on prod, and that breaks every + price/list/batch request. Live and healthy. See "Rollout log". Still + open: the first post-deploy price, the observability stack, the history + repair, and the reply to SBE. - date: "2026-10-06" status: active who: stkrolikiewicz @@ -207,6 +219,42 @@ Rejected: `events-backfill` + pre-roll at least over a recent window, and the full range via 0286 phase 3 if stage C has not passed these months yet. +### Rollout log (2026-10-07) + +- **Schema and seed.** Done before the merge (07:40 UTC) by the operator on + `deploy@ch-prod-01`, applying only the `CREATE TABLE` from init.sql. Before: + `EXISTS TABLE` returned 0. After: 7 rows (6, 8, 8, 9, 18, 18, 18), matching + the mainnet reads. `prices_writer` holds `SELECT, INSERT … ON prices.*`. +- **Why not the Compute stack.** `develop` carries #373 (0274) besides this + task. Its rollout order is binding: ALTER `current_prices.price_basis`, then + views.sql, then current.sql, then prices-api. On prod all three schema + checks returned 0 (the column, `current_price_usd`, `mv_current_prices`). + `cdk diff Prices-production-Compute` showed both the ledger-processor and + the api-handler code changing. +- **Ledger-processor only.** Built with `make build-lambdas` from `develop @ + 24e07cc0`, 12 bootstraps verified, rustc 1.97.1 / cargo-lambda 1.9.1. + Bootstrap sha256 (base64) `b+M4AVx6wL5PZAqZ/oMc3OoUNry3PZ/6SCoyn7u6Mlc=`. + It carries `prices.asset_decimals`, `TradesMissingDecimals` and + `simulateTransaction`. Zipped (bootstrap only) and uploaded with `aws lambda + update-function-code` at 07:54:51 UTC. `CodeSha256 + aMzmQuGWzyLcyExHr57QA2sB2AB7eReo/X//uYTaZIU=`, arm64, Successful/Active. The + SQS mapping targets `$LATEST` and there are no aliases. +- **Verified.** Cold start at 07:54:55 logged `loaded soroban token decimals + {tokens: 7}`. 12 invocations a minute before and after, 0 errors, 0 + throttles, max duration about 2.4 s. The first run wrote 113 rows (cursor + 64814771 → 64814783). No resolver warnings, no panics. +- **Not yet verified.** A correct price in the API needs a swap after the + deploy: SolvBTC still read 8 326 $ with `as_of 06:11Z`. +- **Drift, until the next Compute deploy.** The CloudFormation template still + names the old ledger-processor asset. The next `make + deploy-production-compute` reconciles it, but it must come after 0274's + schema rollout; Adam has been told. api-handler is untouched (pre-#373). +- **Rollback.** `aws lambda update-function-code --function-name + prices-production-ledger-processor --s3-bucket + cdk-hnb659fds-assets-750702271865-eu-central-1 --s3-key + 8df901778d1c4aaf4384269fc0d085ea512d0e30e6c9934e201f95afdd2c7cd4.zip` (the + package deployed on 2026-10-05). + ## Acceptance Criteria - [x] Unit test: an 8-, 18- and 6-decimal leg against a 7-decimal leg gives the From 17147d98d215651d304f5d442215edc9d7bceac7 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:14:40 +0200 Subject: [PATCH 056/144] docs(lore-0181): close, two indicators delivered by 0203 view_refreshes exceptions and the empty-1M hole are covered by 0203. Part-count and mutation-age indicators not built: 11 parts/partition and 0 pending mutations on prod 2026-10-07. --- ...FEATURE_rollup-leading-indicator-alarms.md | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) rename lore/1-tasks/{backlog => archive}/0181_FEATURE_rollup-leading-indicator-alarms.md (72%) diff --git a/lore/1-tasks/backlog/0181_FEATURE_rollup-leading-indicator-alarms.md b/lore/1-tasks/archive/0181_FEATURE_rollup-leading-indicator-alarms.md similarity index 72% rename from lore/1-tasks/backlog/0181_FEATURE_rollup-leading-indicator-alarms.md rename to lore/1-tasks/archive/0181_FEATURE_rollup-leading-indicator-alarms.md index 76c1fc10..891b4d55 100644 --- a/lore/1-tasks/backlog/0181_FEATURE_rollup-leading-indicator-alarms.md +++ b/lore/1-tasks/archive/0181_FEATURE_rollup-leading-indicator-alarms.md @@ -2,7 +2,8 @@ id: "0181" title: "Rollup leading indicators — pending-mutation age, part counts and view_refreshes exceptions" type: FEATURE -status: backlog +status: completed +assignee: akot related_adr: [] related_tasks: ["0137", "0136", "0109", "0134"] tags: @@ -39,10 +40,37 @@ history: since it would be redundant under the permissive model. `storage = users_xml` on both runtime users confirms [[0134]]: we cannot GRANT them. Batch the grant request with the 0136 note already owed to BE. + - date: "2026-10-07" + status: completed + who: akot + note: > + Closed on the operator's decision, two of four parts delivered + elsewhere. view_refreshes exceptions and the empty-1M hole are covered + by [[0203]] (MvRefreshFailingCount, RollupMismatchBuckets for 1M; all + four mv-refresh alarms OK, MvRefreshUnreadable 0 on 96 reads a day). + Part-count and mutation-age indicators NOT built: measured today max 11 + active parts per partition (threshold ~1,000) and 0 pending mutations + in prices. No follow-up tasks. --- # Rollup leading indicators +## ✅ Closed 2026-10-07: two parts delivered by 0203, two not built + +Verified in code and on production, 2026-10-07: + +| indicator | state | +|---|---| +| `view_refreshes` exceptions | **delivered by [[0203]]**: `refresh_waits.rs`, `MvRefreshFailingCount` (non-empty `exception`, or no success for > 2 own periods) plus `MvRefreshWaitingCount`, `MvRefreshDisabledCount`, `MvRefreshUnreadable`. Four `prices-production-mv-refresh-*` alarms, OK since 2026-10-02. `MvRefreshUnreadable` = 0 on 96 reads a day, so `prices_writer` holds `SELECT ON system.view_refreshes` (rollout check in `0142-rollup-mv-reapply.md:329`); the BE ask below is obsolete for this table | +| empty `1M` tier | **covered by [[0203]]**: `RollupMismatchBuckets` with `Table=price_ohlcv_1M` counts closed `1M` buckets missing against `1d`, so an empty `1M` reads > 0 | +| part counts | **not built.** Max active parts per partition in `prices` today: 11 (`price_ohlcv_1m`), against the ~1,000 `parts_to_delay_insert` threshold | +| pending-mutation age | **not built.** 0 pending mutations in `prices` today. Whether `prices_writer` can read `system.mutations` was not verified (`dev_read` cannot `SHOW GRANTS FOR` another user). [[0109]]'s own `system.mutations` preflight criterion is unaffected | + +Closed on the operator's decision without the last two: both read far from +any threshold today, and no follow-up task was spawned. If a 0136-style +mutation backlog recurs, the lagging signals (freshness, mismatch, +`mv-refresh-*`) still fire; only the early warning is missing. + ## Summary [[0137]] alarms when a rollup tier's newest bucket ages past its bound — the @@ -131,8 +159,12 @@ comparing `1M`'s newest bucket against `1w`'s. mTLS user is **measured** and recorded either way. ✅ **Done 2026-08-12** — neither is granted; only `system.parts` is. See §Context. - [ ] Part-count indicator ships (unblocked — build this first). + Not built; closed without it (max 11 parts/partition on 2026-10-07). - [ ] Mutation-age and `view_refreshes` indicators ship, **or** the two-line BE grant request in §Context is raised and its outcome recorded here. + `view_refreshes`: shipped by [[0203]], grant in place. Mutation age: + not built, grant unverified. - [ ] No duplication with [[0109]] — ownership of the `system.mutations` watch is - settled and written down in both tasks. + settled and written down in both tasks. Moot: no watch built here. - [ ] Thresholds recorded with rationale, consistent with [[0137]]'s bounds. + Moot for the unbuilt two; 0203's are in `observability-stack.ts`. From 8c3c559007487ab4e72ff242d31b4fbb35a5b830 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:12:34 +0200 Subject: [PATCH 057/144] docs(lore-0329): record observability deploy, seed and binary swap First post-deploy prices match the market. All 46 historical contract tokens are seeded and Hetzner's events-backfill is the #395 build, so 0286 phase 3 needs no redo; Oskar restarts on the new script before 2025-07. --- ...ban-token-amounts-assume-seven-decimals.md | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md index c55cfd2f..bafa78b2 100644 --- a/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md +++ b/lore/1-tasks/active/0329_BUG_soroban-token-amounts-assume-seven-decimals.md @@ -12,6 +12,18 @@ links: - "../../../packages/asset-discovery/src/symbols.rs" - "../../../docs/database-schema/amm-trades-schema.md" history: + - date: "2026-10-07" + status: active + who: stkrolikiewicz + note: > + Observability stack deployed (alarm OK). First post-deploy prices are + right: SolvBTC 82 512 $, 1.2 % under CoinGecko; deJTRSY 1.027 $. Seeded + prices.asset_decimals for all 46 historical contract tokens (7 → 53). + Swapped the Hetzner `~/events-backfill` for the #395 build, so 0286 phase + 3 picks it up at its next AMM step. Affected tokens trade only from + 2025-07, and phase 3 stands at 2024-09, so no month it already did needs + a redo; Oskar must restart on the new reingest_0286.py before 2025-07. + Recent-window repair (2026-09-01 → deploy) under way. - date: "2026-10-07" status: active who: stkrolikiewicz @@ -254,6 +266,47 @@ Rejected: cdk-hnb659fds-assets-750702271865-eu-central-1 --s3-key 8df901778d1c4aaf4384269fc0d085ea512d0e30e6c9934e201f95afdd2c7cd4.zip` (the package deployed on 2026-10-05). +- **Observability.** `make deploy-production-observability` ran at 08:06 UTC: + the new alarm `prices-production-ledger-processor-trades-missing-decimals` + and the dashboard strip (76 → 77). The diff also listed 39 alarm description + changes. They were mangled-character repairs in the diff's view only: + CloudFormation updated just the alarm, the dashboard and the metadata. The + alarm went to OK, and `TradesMissingDecimals` has never been published. +- **First prices after the deploy.** deJTRSY (18 decimals) read 1.027 $ at + 08:43 UTC, where it had read 1.03e-11 $. SolvBTC (8 decimals) read 82 512 $ + at 09:42 UTC, where it had read 8 326 $; CoinGecko said 83 538 $ (−1.2 %). +- **Seed of the whole history.** 46 contract tokens had a 1d candle and no + decimals row. 45 were read on mainnet; the public RPC answered 13 of them + with 429 at first, and they succeeded on retry. The 46th, + `CDPV3H7C…QTMW` (Axelar USD Coin), fails every call with "Missing Entry + __InterchainTokenMigrationData", so its 6 comes from SBE's + `default.soroban_contract_metadata`. The table went from 7 to 53 rows, by + decimals: 6 → 5, 7 → 39, 8 → 2, 9 → 1, 18 → 6. Besides the first seven, + five more tokens are not 7-decimal: two USDT variants and Axelar USDC at 6, + so their prices were 10× too high; STBL_USST and two test tokens at 18. +- **events-backfill on Hetzner.** Built from `develop @ 63b7661c` for x86_64 + (cargo-lambda/zig; needs glibc ≥ 2.30, the box has 2.39), sha256 + `10872b2393961c056c5dd959900ca509766f1c82c2ea0afe835c825170d70ca8`. + Uploaded as `~/events-backfill-0329`, smoke-tested with `--help`, then put + in place as `~/events-backfill` while no events-backfill process ran. The + old binary (`ee992dd9…`) is kept as `~/events-backfill.pre0329`; rollback + is `mv ~/events-backfill.pre0329 ~/events-backfill`. The box reaches Soroban + RPC (`getHealth` healthy). +- **0286 phase 3.** It stands at 2024-09: the SDEX step ended 2026-10-06 + 23:09 UTC and the AMM step has not run since. Candles of tokens with + decimals other than 7 exist only from 2025-07, so no month phase 3 has done + needs a redo. From 2024-09 on it uses the new binary. The running + orchestrator still holds the old reingest_0286.py in memory, so Oskar must + restart it on the new one before 2025-07. Otherwise months with 18-decimal + tokens stop as DEFECT on the volume drop, and nothing reads `trades dropped + (decimals)`. +- **Recent-window repair.** Scripts are in a local `repair-0329/`, not + committed. The window runs from ledger 64216185 (2026-09-01 00:00) to + 64814771 (minute 07:53, the last ledger the old code processed), with t_e + `2026-10-07 07:54:00`. Only rows of tokens with decimals other than 7 are + replaced, and only after a backup: delete, then events-backfill, then + enrichment, then the affected-only pre-roll. Lightweight `DELETE FROM` is + used because the corrected rows tie the stale ones on `version`. ## Acceptance Criteria From 462f76bb982e36a13ffd7c73e952859fb2c44a9a Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:28:46 +0200 Subject: [PATCH 058/144] docs(lore-0330): open task for the false BEHIND in the xdr watch The watch has reported BEHIND since stellar-xdr 30.0.0 was published on 2026-10-08 (issue #398). It counts 30 as a crate for protocol 29. Upstream skipped 29, and protocol 29 kept the XDR of 28. --- ...eads-a-later-major-as-the-missing-crate.md | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md diff --git a/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md b/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md new file mode 100644 index 00000000..71c55344 --- /dev/null +++ b/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md @@ -0,0 +1,95 @@ +--- +id: "0330" +title: "The XDR watch reports BEHIND because stellar-xdr 30 counts as a crate for protocol 29" +type: BUG +status: active +related_adr: [] +related_tasks: ["0325", "0319", "0098"] +tags: [ci, resilience, ingestion, priority-medium, effort-small] +links: + - "../../../tools/scripts/verify-xdr-protocol-gap.mjs" + - "../../../tools/scripts/verify-xdr-protocol-gap.test.mjs" + - "../../../docs/runbooks/xdr-protocol-watch.md" + - "https://github.com/rumblefishdev/stellar-prices-api/issues/398" + - "https://github.com/stellar/rs-stellar-xdr/releases/tag/v30.0.0" + - "https://github.com/stellar/stellar-core/pull/5478" +history: + - date: "2026-10-12" + status: active + who: stkrolikiewicz + note: > + Created from issue #398. The scheduled watch has been red since + 2026-10-09 and reports BEHIND (pin 28, mainnet 29, newest crate 30.0.0). + Nothing is broken: protocol 29 kept the XDR of 28 (0325), and every + production rollup-freshness and ledger-processor alarm is OK. +--- + +# The XDR watch reports BEHIND because stellar-xdr 30 counts as a crate for protocol 29 + +## Summary + +`verify-xdr-protocol-gap.mjs` asks crates.io whether a crate for the needed +protocol exists by comparing `max_stable_version`'s major with it +(`crate.major < needed`). Upstream skipped 29: `stellar-xdr` went from 28.0.1 +to 30.0.0. Since 30 ≥ 29, the script treats 30.0.0 as the crate for protocol +29 and fails BEHIND. A bump target must have a major above our pin and **not +above** the protocol it is for. Read every published version, not only the +newest. + +## Context + +- `stellar-xdr` 30.0.0 was published 2026-10-08. It carries protocol 30 XDR + (CAP-0084 and CAP-0087 ungated). No `stellar-xdr` 29 exists. +- Horizon mainnet and testnet both report current 29 and core-supported 29 + (stellar-core 29.0.0). There is no stellar-core v30 release, and no + testnet or mainnet date for protocol 30. stellar-core merged its "Bump to + protocol 30" PR (#5478) to master on 2026-10-09. +- The watch has been red every day since 2026-10-09 (runs on 10-09, 10-10 and + 10-11). Issue #398 is open at tier BEHIND. +- Production is healthy. `stellar-xdr =28.0.0` decodes P29 (0325), and all + `prices-production-rollup-freshness-*`, `ledger-processor-lag` and + `ledger-processor-no-invocations` alarms are OK. +- BE's `soroban-block-explorer` develop still pins `stellar-xdr = "28"`. + +## Implementation Plan + +### Step 1: Read the published versions + +`publishedCrate()` also returns the stable, unyanked versions from the +`versions` array of the same crates.io document (no second request). A +missing `versions` array is an unreadable document, which is the +`cannot read` path. + +### Step 2: Choose the bump target per protocol + +`bumpTarget(protocol)` = the newest stable version with +`pin < major ≤ protocol`. + +- BEHIND: `pin < current` and `bumpTarget(current)` exists. +- LAGGING: otherwise, `pin < supported` and `bumpTarget(supported)` exists. + This keeps the lead time when mainnet sits on a crate-less protocol (29) + while core already supports the next (30). +- WAITING: neither. The `— WAITING: no stellar-xdr` wording stays, because + master's workflow matches it to close the issue. + +### Step 3: Tests and runbook + +New tier cases: today's state (pin, current pin+1, crate pin+2 only → +WAITING) and the next one (current pin+1, core pin+2, crate pin+2 → +LAGGING). The crates.io mock serves `versions`. Update the runbook's +"published crate" reading and the header comment. + +## Acceptance Criteria + +- [ ] Live run against mainnet 29 and crates.io 30.0.0 exits 0 with + `— WAITING: no stellar-xdr 29`, in PR and `--watch` mode. +- [ ] Current pin+1, core pin+2, crate pin+2 → LAGGING (exit 1 under `--watch`). +- [ ] All existing tier cases still pass, and the pre-fix script fails the new + WAITING case. +- [ ] After merge to `develop`, a manual watch run on `master` is green and + closes #398 as "not actionable yet". + +## Notes + +No workflow change on `master` is needed: the workflow checks out `develop` +for the script. From df4f5e39ea0c7642a5a409e6aa7e0fa408e84866 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:31:20 +0200 Subject: [PATCH 059/144] fix(lore-0330): stop counting a later major as the missing crate stellar-xdr skipped 29 on crates.io (28.0.1 -> 30.0.0). The watch compared only max_stable_version with the needed protocol, so 30.0.0 counted as a crate for protocol 29 and it reported BEHIND (issue #398). A bump target for protocol N is now a stable, unyanked release with pin < major <= N, asked for mainnet and core separately. That keeps LAGGING once core supports 30 while mainnet sits on 29. Refs: #398 --- docs/runbooks/xdr-protocol-watch.md | 18 ++++-- ...eads-a-later-major-as-the-missing-crate.md | 16 ++++- tools/scripts/verify-xdr-protocol-gap.mjs | 63 +++++++++++++++---- .../scripts/verify-xdr-protocol-gap.test.mjs | 8 ++- 4 files changed, 82 insertions(+), 23 deletions(-) diff --git a/docs/runbooks/xdr-protocol-watch.md b/docs/runbooks/xdr-protocol-watch.md index 48c54ebf..157c46a3 100644 --- a/docs/runbooks/xdr-protocol-watch.md +++ b/docs/runbooks/xdr-protocol-watch.md @@ -35,12 +35,12 @@ after 9 min, `rollup-freshness-1m` after 22 min). Four readings, compared every run: -| Reading | Source | -| ------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| **our pin** | `stellar-xdr` in `[workspace.dependencies]` of `Cargo.toml` (and `Cargo.lock` must agree, with only one major resolved) | -| **mainnet current** | `current_protocol_version` from `https://horizon.stellar.org/` | -| **core supports** | `core_supported_protocol_version` from the same page — rises **weeks before** the vote, once validators run the new stellar-core | -| **published crate** | `max_stable_version` from `https://crates.io/api/v1/crates/stellar-xdr` — only read when our pin lags `core supports` | +| Reading | Source | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| **our pin** | `stellar-xdr` in `[workspace.dependencies]` of `Cargo.toml` (and `Cargo.lock` must agree, with only one major resolved) | +| **mainnet current** | `current_protocol_version` from `https://horizon.stellar.org/` | +| **core supports** | `core_supported_protocol_version` from the same page — rises **weeks before** the vote, once validators run the new stellar-core | +| **published crate** | `max_stable_version` and the `versions` list from `https://crates.io/api/v1/crates/stellar-xdr` — only read when our pin lags `core supports` | Why the crate is read separately: stellar-core builds its XDR straight from the `stellar/stellar-xdr` git repo, so SDF runs a new protocol **before** the Rust @@ -79,6 +79,12 @@ Rules that are easy to get wrong: so; there is no bump to ask for, so the issue is left alone. A watch that silently checks nothing is the failure it exists to prevent, which is why the run still fails. +- **A later major is not a crate for this protocol** (task 0330). A bump + target for protocol N is a stable major above our pin and not above N. + Upstream skipped 29 (crates.io went 28.0.1 → 30.0.0), and 30.0.0 carries + protocol 30's XDR, so on mainnet 29 the watch is WAITING. Each protocol is + checked on its own: mainnet on 29 with core on 30 and 30.0.0 published is + LAGGING. - **A pre-release does not count.** `29.0.0-rc.1` is named in the report, but the tier stays WAITING until a stable `29.x` exists. - **Unchanged tier = silent.** The body is rewritten every run (with the fresh diff --git a/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md b/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md index 71c55344..0399f04b 100644 --- a/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md +++ b/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md @@ -22,6 +22,16 @@ history: 2026-10-09 and reports BEHIND (pin 28, mainnet 29, newest crate 30.0.0). Nothing is broken: protocol 29 kept the XDR of 28 (0325), and every production rollup-freshness and ledger-processor alarm is OK. + - date: "2026-10-12" + status: active + who: stkrolikiewicz + note: > + Implemented on fix/0330_xdr-watch-reads-a-later-major-as-the-missing-crate. + A bump target for protocol N is now the newest stable, unyanked version + with pin < major ≤ N, asked for mainnet and core separately. Tier test: + 18/18 pass. The pre-fix script fails 5 of them, exactly the new cases. + Live run against mainnet 29 and crates.io 30.0.0 exits 0 with + "WAITING: no stellar-xdr 29" in both --watch and PR mode. --- # The XDR watch reports BEHIND because stellar-xdr 30 counts as a crate for protocol 29 @@ -81,10 +91,10 @@ LAGGING). The crates.io mock serves `versions`. Update the runbook's ## Acceptance Criteria -- [ ] Live run against mainnet 29 and crates.io 30.0.0 exits 0 with +- [x] Live run against mainnet 29 and crates.io 30.0.0 exits 0 with `— WAITING: no stellar-xdr 29`, in PR and `--watch` mode. -- [ ] Current pin+1, core pin+2, crate pin+2 → LAGGING (exit 1 under `--watch`). -- [ ] All existing tier cases still pass, and the pre-fix script fails the new +- [x] Current pin+1, core pin+2, crate pin+2 → LAGGING (exit 1 under `--watch`). +- [x] All existing tier cases still pass, and the pre-fix script fails the new WAITING case. - [ ] After merge to `develop`, a manual watch run on `master` is green and closes #398 as "not actionable yet". diff --git a/tools/scripts/verify-xdr-protocol-gap.mjs b/tools/scripts/verify-xdr-protocol-gap.mjs index 4ffa96ff..4a01fe13 100755 --- a/tools/scripts/verify-xdr-protocol-gap.mjs +++ b/tools/scripts/verify-xdr-protocol-gap.mjs @@ -61,11 +61,11 @@ // crates.io's newest `stellar-xdr` was 28.0.1 — the issue asked for a bump // nobody could make. So a LAGGING reading also asks crates.io: // -// WAITING no published stable major reaches core_supported. The protocol is +// WAITING no published stable major in (ours, core_supported]. The protocol is // announced, there is nothing to bump to yet. NOT fatal, even under // --watch: a red run and an issue are for something someone can // act on, and here nobody can. The report says so and exits 0. -// LAGGING a stable major that reaches core_supported is on crates.io. Bump +// LAGGING a stable major in (ours, core_supported] is on crates.io. Bump // now (after BE's xdr-parser). Fatal under --watch. // // BEHIND asks crates.io the same way (task 0325): mainnet on a protocol with no @@ -74,6 +74,13 @@ // that DOES change XDR and is voted before its crate ships stays green here; // that stall shows as frozen candles, which the rollup-freshness and // ledger-processor-no-invocations alarms catch within the hour. +// +// "A crate for protocol N" means a major ABOVE ours and NOT ABOVE N (task +// 0330). Upstream skipped 29: crates.io went 28.0.1 → 30.0.0, and 30.0.0 +// carries protocol 30's XDR. Reading only max_stable_version (30 ≥ 29) made +// the watch report BEHIND for a protocol 28 already decodes. Each protocol is +// asked separately: mainnet on crate-less 29 with core on 30 and a crate 30 +// published is LAGGING. // ponytail: compares protocol numbers; comparing the crate's XDR commit with // core's (`src/protocol-curr/xdr`) is the exact check if this misfires again. // @@ -199,20 +206,37 @@ const publishedCrate = async () => { }); if (!response.ok) throw new Error(`HTTP ${response.status}`); - const { crate } = await response.json(); + const { crate, versions } = await response.json(); const stable = crate?.max_stable_version; - const major = Number(String(stable).match(/^(\d+)\.\d+\.\d+$/)?.[1]); - if (!Number.isInteger(major)) - throw new Error(`unexpected crate document: max_stable_version=${stable}`); + if (!/^\d+\.\d+\.\d+$/.test(String(stable)) || !Array.isArray(versions)) + throw new Error( + `unexpected crate document: max_stable_version=${stable}, ` + + `versions=${Array.isArray(versions) ? versions.length : versions}`, + ); const newest = crate.max_version; return { stable, - major, preRelease: newest && newest !== stable ? newest : null, + releases: versions + .filter((v) => !v.yanked && /^\d+\.\d+\.\d+$/.test(v.num)) + .map((v) => v.num), }; }; +/** + * The newest stable release a bump for `protocol` could use: a major above + * our pin and NOT above `protocol`. A later major carries a later protocol's + * XDR (task 0330: upstream skipped 29, and 30.0.0 is protocol 30, not 29). + */ +const bumpTarget = (crate, protocol) => + crate.releases + .filter((v) => { + const major = Number(v.split('.')[0]); + return major > pinned && major <= protocol; + }) + .sort((a, b) => b.localeCompare(a, 'en', { numeric: true }))[0]; + // -------------------------------------------------------------------- run -- const pinned = pinnedMajor(); @@ -282,9 +306,14 @@ if (behind || pinned < network.supported) { process.exit(0); } + // Asked per protocol: mainnet can sit on one with no crate (29) while core + // already supports the next one, which has a crate (30). That is LAGGING. + const forMainnet = behind && bumpTarget(crate, network.current); + const forCore = bumpTarget(crate, network.supported); + // The `— WAITING: no stellar-xdr` wording is matched by master's // xdr-protocol-watch.yml to close an open issue as "not actionable yet". - if (crate.major < needed) { + if (!forMainnet && !forCore) { console.log( [ behind @@ -294,13 +323,21 @@ if (behind || pinned < network.supported) { ` ${summary} | newest on crates.io ${crate.stable}` + (crate.preRelease ? ` (pre-release ${crate.preRelease})` : ''), '', + ...(Number(crate.stable.split('.')[0]) > needed + ? [ + `stellar-xdr ${crate.stable} carries a later protocol's XDR, so it is not`, + `a bump target for ${needed}.`, + '', + ] + : []), ...(behind ? [ 'Mainnet has voted, but there is no crate to bump to: either the', 'protocol changed no XDR (29 kept the XDR of 28) or the crate is late.', 'A real decode wall shows as frozen candles — the rollup-freshness and', 'ledger-processor-no-invocations alarms. The watch fails and opens an', - `issue the day a stable stellar-xdr ${needed}.x appears on crates.io.`, + `issue the day a stable stellar-xdr ${needed}.x appears on crates.io,`, + 'or core supports a protocol whose crate is published.', ] : [ 'Nothing to do yet: there is no crate to bump to, and mainnet has not', @@ -312,11 +349,11 @@ if (behind || pinned < network.supported) { process.exit(0); } - if (behind) + if (forMainnet) die( `error: stellar-xdr ${pinned} is BEHIND mainnet protocol ${network.current}.`, '', - ` ${summary} | newest on crates.io ${crate.stable}`, + ` ${summary} | newest on crates.io ${crate.stable} | bump to ${forMainnet}`, '', 'This is the proto27 condition: the decode wall is live or one ledger', 'away, and a stalled ledger-processor drains its queue and logs nothing.', @@ -331,8 +368,8 @@ if (behind || pinned < network.supported) { '', ` ${summary} | newest on crates.io ${crate.stable}`, '', - `stellar-xdr ${crate.stable} is published — the bump is possible now.`, - 'Mainnet has not voted yet, so nothing is broken — this is the lead time.', + `stellar-xdr ${forCore} is published — the bump is possible now.`, + `Mainnet has not voted ${network.supported} yet, so nothing is broken — this is the lead time.`, 'Open the bump before the vote. BE must bump xdr-parser first, and check', "the crate's XDR commit, not only its version number.", ]; diff --git a/tools/scripts/verify-xdr-protocol-gap.test.mjs b/tools/scripts/verify-xdr-protocol-gap.test.mjs index 6591d372..2704b480 100644 --- a/tools/scripts/verify-xdr-protocol-gap.test.mjs +++ b/tools/scripts/verify-xdr-protocol-gap.test.mjs @@ -1,5 +1,5 @@ // Tier matrix for tools/scripts/verify-xdr-protocol-gap.mjs (tasks 0319, -// 0325), run against a local mock of Horizon and crates.io so it needs no +// 0325, 0330), run against a local mock of Horizon and crates.io so it needs no // network. Every case is relative to the real pin in Cargo.toml, so a // stellar-xdr bump needs no edit here. // @@ -45,6 +45,7 @@ const net = (current, supported) => ({ }); const published = (stable, max = stable) => ({ crate: { max_stable_version: stable, max_version: max }, + versions: [...new Set([max, stable])].map((num) => ({ num, yanked: false })), }); // Async, not spawnSync: the mock lives in this process and must keep serving. @@ -78,6 +79,11 @@ const cases = [ ['BEHIND, crate published, watch', net(p + 1, p + 1), published(`${p + 1}.0.0`), true, 1, 'is BEHIND mainnet protocol'], ['BEHIND, crate published, PR', net(p + 1, p + 1), published(`${p + 1}.0.0`), false, 1, 'is BEHIND mainnet protocol'], ['BEHIND, crate for mainnet but not core', net(p + 1, p + 2), published(`${p + 1}.0.0`), true, 1, 'is BEHIND mainnet protocol'], + ['voted, only a later crate (#398), watch', net(p + 1, p + 1), published(`${p + 2}.0.0`), true, 0, `— WAITING: no stellar-xdr ${p + 1}`], + ['voted, only a later crate (#398), PR', net(p + 1, p + 1), published(`${p + 2}.0.0`), false, 0, 'not\na bump target'], + ['before the vote, only a later crate', net(p, p + 1), published(`${p + 2}.0.0`), true, 0, '— WAITING: no stellar-xdr'], + ['voted without a crate, core next with one', net(p + 1, p + 2), published(`${p + 2}.0.0`), true, 1, `${p + 2}.0.0 is published — the bump is possible now`], + ['crates.io lists no versions, watch', net(p + 1, p + 1), { crate: { max_stable_version: `${p + 1}.0.0` } }, true, 1, 'error: cannot read'], ['voted, crates.io down, watch', net(p + 1, p + 1), null, true, 1, 'error: cannot read'], ['voted, crates.io down, PR', net(p + 1, p + 1), null, false, 0, 'crate availability not checked'], ['Horizon down, watch', null, published(`${p}.0.1`), true, 1, 'error: cannot reach'], From ef834a2480977455b54530c1383a3d3ac380a5da Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:47:30 +0200 Subject: [PATCH 060/144] fix(lore-0286): let coarse-repair drain price_ohlcv_1m The hourly Lambda prices at most 200 k rows of 1m history an hour, so the ~900 M rows the phase-3 re-ingest left at close_usd = 0 would take ~190 days. coarse-repair runs the same pass one-shot per month, but refused 1m. Accept price_ohlcv_1m in the fill-the-zeros mode only: no --reset-*, and --skip-snapshot required for a real run. --- .../src/bin/coarse-repair.rs | 101 +++++++++++++++--- 1 file changed, 89 insertions(+), 12 deletions(-) diff --git a/packages/enrichment-worker/src/bin/coarse-repair.rs b/packages/enrichment-worker/src/bin/coarse-repair.rs index 57891606..b2dfe106 100644 --- a/packages/enrichment-worker/src/bin/coarse-repair.rs +++ b/packages/enrichment-worker/src/bin/coarse-repair.rs @@ -23,6 +23,11 @@ //! mTLS env (same as sdex-backfill): CH_DOMAIN, MTLS_CERT_PATH, MTLS_KEY_PATH, //! MTLS_CA_PATH. Revert a partition from its FREEZE snapshot with //! `SYSTEM UNFREEZE WITH NAME ''` + `ALTER TABLE … ATTACH`. +//! +//! `price_ohlcv_1m` is accepted too, for a whole-history drain (task 0286 §7b): +//! the hourly Lambda prices at most 200 k rows an hour, which is months for a +//! re-ingested history. Only the fill-the-zeros mode is allowed there — no +//! `--reset-*` — and only with `--skip-snapshot`, see [`check_table`]. use std::path::PathBuf; @@ -69,7 +74,8 @@ struct Args { #[arg(long, env = "MTLS_CA_PATH")] mtls_ca_path: Option, - /// Coarse table to repair (e.g. `price_ohlcv_1h`). One table per run. + /// Table to repair (e.g. `price_ohlcv_1h`). One table per run. + /// `price_ohlcv_1m` takes only the fill-the-zeros mode (see `check_table`). #[arg(long)] table: String, @@ -264,6 +270,46 @@ fn validate_month(label: &str, m: u32) -> Result<(), String> { Ok(()) } +/// Which tables this tool may write, and how. +/// +/// The coarse rollups take every mode. `price_ohlcv_1m` is the live base table +/// the hourly Lambda owns, so it takes only the fill-the-zeros mode, which is +/// exactly what the Lambda runs: it writes rows still at zero and discards +/// nothing, so it can run beside the Lambda (task 0286 §7b). A reset there +/// would discard prices every other tier is rolled from, so it is refused. +/// FREEZE is refused too: `prices_writer` cannot take it, and a 1m snapshot +/// protects nothing in a mode that only fills zeros. +fn check_table( + table: &str, + resetting: bool, + skip_snapshot: bool, + dry_run: bool, +) -> Result<(), String> { + if !table.starts_with("price_ohlcv_") { + return Err(format!( + "--table {table} is not an OHLCV table (expected price_ohlcv_1m, _15m … _1M)" + )); + } + if table != "price_ohlcv_1m" { + return Ok(()); + } + if resetting { + return Err( + "--reset-* is refused on price_ohlcv_1m: a reset discards prices every \ + coarse tier is rolled from. On 1m only the fill-the-zeros mode runs." + .into(), + ); + } + if !skip_snapshot && !dry_run { + return Err( + "price_ohlcv_1m needs --skip-snapshot: prices_writer cannot FREEZE, and \ + the fill-the-zeros mode discards nothing a snapshot would protect." + .into(), + ); + } + Ok(()) +} + #[tokio::main] async fn main() -> Result<(), Box> { tracing_subscriber::fmt() @@ -284,16 +330,12 @@ async fn main() -> Result<(), Box> { ) .into()); } - if !args.table.starts_with("price_ohlcv_") || args.table == "price_ohlcv_1m" { - // The driver targets the coarse rollups; 1m is the live base table the - // scheduled Lambda owns and has live writers. Refuse it outright. - return Err(format!( - "--table {} is not a coarse OHLCV table (expected price_ohlcv_15m … _1M; \ - price_ohlcv_1m is the live base table and is off-limits here)", - args.table - ) - .into()); - } + check_table( + &args.table, + args.reset_quote_asset_id.is_some(), + args.skip_snapshot, + args.dry_run, + )?; // Reset mode is the only path here that discards data, so its two footguns // are refused rather than warned about. @@ -487,7 +529,9 @@ async fn main() -> Result<(), Box> { driver.preflight().await?; - if args.skip_snapshot && !args.dry_run { + if args.skip_snapshot && !args.dry_run && args.table == "price_ohlcv_1m" { + info!("snapshot disabled on price_ohlcv_1m: fill-the-zeros mode, nothing is discarded"); + } else if args.skip_snapshot && !args.dry_run { warn!( "snapshot DISABLED (--skip-snapshot): partitions will be repaired with no FREEZE \ backup — only safe when the 1m source can rebuild this coarse table" @@ -606,3 +650,36 @@ async fn main() -> Result<(), Box> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::check_table; + + #[test] + fn coarse_tables_take_every_mode() { + assert!(check_table("price_ohlcv_1h", true, false, false).is_ok()); + assert!(check_table("price_ohlcv_1M", false, true, false).is_ok()); + } + + #[test] + fn a_non_ohlcv_table_is_refused() { + assert!(check_table("assets", false, true, false).is_err()); + } + + #[test] + fn one_minute_takes_the_fill_the_zeros_mode_without_a_snapshot() { + assert!(check_table("price_ohlcv_1m", false, true, false).is_ok()); + assert!(check_table("price_ohlcv_1m", false, false, true).is_ok()); + } + + #[test] + fn one_minute_refuses_a_reset_even_in_a_dry_run() { + assert!(check_table("price_ohlcv_1m", true, true, false).is_err()); + assert!(check_table("price_ohlcv_1m", true, true, true).is_err()); + } + + #[test] + fn one_minute_refuses_a_real_run_with_a_snapshot() { + assert!(check_table("price_ohlcv_1m", false, false, false).is_err()); + } +} From 24d425fd6166de2c0b7a5cfbcca7db8c7223474b Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:47:30 +0200 Subject: [PATCH 061/144] docs(lore-0286): drain 1m from fishuser-hero, not the hourly Lambda Section 7b assumed the worker re-prices the whole history in hours. It is bounded to 200 k rows an hour. Add the per-month coarse-repair loop with --batch-size 50000 and the measured estimate, let 7b-2 run before the 1m drain (not beside it), and correct the finish step's reminder. --- docs/runbooks/0286-reingest-history.md | 39 +++++++++++++++++++++++--- tools/scripts/reingest_0286.py | 5 ++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/docs/runbooks/0286-reingest-history.md b/docs/runbooks/0286-reingest-history.md index 4f608b0a..055a8ea1 100644 --- a/docs/runbooks/0286-reingest-history.md +++ b/docs/runbooks/0286-reingest-history.md @@ -506,8 +506,36 @@ source). The MV `mv_ohlcv_1d_to_1M` keeps them current afterwards. `close_usd = 0` and `volume_quote_usd = 0`, so the enrichment worker will re-price the entire estate — this is the cost the ledger estimate in precondition 8 does NOT include. For scale: task 0268's campaign re-priced 9.94 M candles in ~24 min; -0228 estimated ~190 M candles at ~4 h 40 m. This is all of them, and it runs on -the same worker under the same concurrency. +0228 estimated ~190 M candles at ~4 h 40 m. Those were one-shot runs. + +**Do not leave this to the hourly Lambda.** Its historical drain is bounded: +one month per run, at most `BATCH_SIZE × MAX_BATCHES` = 200 000 rows, and +`ENRICHMENT_ONE_SHOT` does not lift that (`frontier.rs` sets `one_shot = false` +for every historical month). On 2026-10-12 the re-ingest left ~900 M priceable +`1m` rows (2022-05 → 2026-08). At 200 k an hour that is ~190 days; the Lambda +works ~75 s of each hour. + +Drain `1m` from fishuser-hero with `coarse-repair`, which takes `price_ohlcv_1m` +in its fill-the-zeros mode only (no `--reset-*`, `--skip-snapshot` required). +It runs the same pass as the Lambda, one month at a time to the end, and writes +only rows still at zero, so the Lambda can keep running beside it. Same `tmux` +and env as §7b-2 ①: + +```bash +for M in $(seq 202205 202212) $(seq 202301 202312) $(seq 202401 202412) \ + $(seq 202501 202512) $(seq 202601 "$END"); do + ./target/release/coarse-repair --transport hetzner --table price_ohlcv_1m \ + --start-month "$M" --end-month "$M" --skip-snapshot --batch-size 50000 \ + || { echo "STOPPED at $M"; break; } +done 2>&1 | tee -a /tmp/0286_drain_1m.log +``` + +One month per command keeps each month's enumeration to a single partition, and +a re-run skips what is already priced. Each batch scans the whole partition, so +a bigger batch costs little more than a small one: measured on 202205, a 10 k +batch took ~3.8 s, which models the range at ~2 days with 50 k and ~6–8 days +with the default 10 k. Time the first month and re-plan from it. If a statement +fails on memory, lower `--batch-size` and re-run from the month that stopped. **7b-2. Re-price the coarse tiers. The drain above never reaches them.** The enrichment worker prices `price_ohlcv_1m` and nothing else @@ -526,8 +554,11 @@ over the whole history on all six tables. It does not read `1m`. It prices each coarse row from that table's own reference candles, with the same tiers as the worker, including the measured USDC rate on the pivot. -- **Start only when the `1m` drain is finished.** Both passes load the same - shared cluster, and 7c/7e measure the finished state. **The frontier gauge +- **Do not run it at the same time as the `1m` drain.** It does not read `1m`, + so it may go first, and on a re-ingest it should: clients read the coarse + tiers. But both passes load the same shared cluster, and 7c/7e measure the + finished state, so run them one after the other and measure after both. + Before 7c/7e, confirm the `1m` drain is finished. **The frontier gauge alone cannot tell you this.** `EnrichmentFrontierMonthsPending` counts only months that are NOT marked `exhausted` in `prices.enrichment_frontier` (`frontier.rs`, `months_to_sweep`). Almost every historical month was marked diff --git a/tools/scripts/reingest_0286.py b/tools/scripts/reingest_0286.py index ea40118d..a5d7f87c 100755 --- a/tools/scripts/reingest_0286.py +++ b/tools/scripts/reingest_0286.py @@ -1135,8 +1135,9 @@ def cmd_finish(a, ch, st): print(f"acceptance 2 — OHLC order on {t}: {bad} violations", flush=True) print(""" Not automatic, by design (runbook §7b-7e, §8): - 7b the enrichment worker now re-prices the whole history on its own (hours). Capture - POST_RUN_0228_VERSION_BEFORE_1W / _1M BEFORE it starts. + 7b drain 1m with coarse-repair from this machine (days); the hourly Lambda alone takes + months. Then 7b-2, coarse-repair on 15m..1M. Capture + POST_RUN_0228_VERSION_BEFORE_1W / _1M BEFORE either starts. 7c cargo test -p enrichment-worker --test post_run_0228_it -- --ignored 7e the USDT-peg read (USDT by identity, runbook §7e): peg_written = 0, pivot_written > 0, on 1m and 1h 8.3 XLM/USDC 1d closes on the seven dust days within 5 % of Bitstamp From e3e674ee9863d297eb5c9d237b1bfefc710a6a9a Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:52:22 +0200 Subject: [PATCH 062/144] docs(lore-0286): what to do when the 1m drain hits the memory cap Memory follows the partition read, not the batch, so a smaller batch does not fix Code 241 on the 2023 months. Collapse the month's versions with OPTIMIZE ... FINAL first. Note the load on the shared cluster. --- docs/runbooks/0286-reingest-history.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/runbooks/0286-reingest-history.md b/docs/runbooks/0286-reingest-history.md index 055a8ea1..c64a851c 100644 --- a/docs/runbooks/0286-reingest-history.md +++ b/docs/runbooks/0286-reingest-history.md @@ -534,8 +534,24 @@ One month per command keeps each month's enumeration to a single partition, and a re-run skips what is already priced. Each batch scans the whole partition, so a bigger batch costs little more than a small one: measured on 202205, a 10 k batch took ~3.8 s, which models the range at ~2 days with 50 k and ~6–8 days -with the default 10 k. Time the first month and re-plan from it. If a statement -fails on memory, lower `--batch-size` and re-run from the month that stopped. +with the default 10 k. Time the first month and re-plan from it. + +Memory follows the rows a statement reads, not the batch: ~70 MB per million +rows read, ~2 GiB on 202205 (2026-10-12, `system.query_log`). The 2022-12 … +2023-05 partitions are 2.5× that, and each priced row adds a version that +`FINAL` reads until the merge, so they can reach the writer's ~7.5 GiB cap. On +`Code: 241` (`MEMORY_LIMIT_EXCEEDED`), collapse the month's versions as +`prices_admin`, then re-run the loop from that month: + +```bash +chadmin "OPTIMIZE TABLE prices.price_ohlcv_1m PARTITION $M FINAL" +``` + +`--batch-size 20000` trims only the written part of the statement. + +The cluster is shared with the block explorer, and the loop keeps it busy +around the clock. Check the explorer's latency while it runs. `Ctrl-C` is a +safe pause at any point. **7b-2. Re-price the coarse tiers. The drain above never reaches them.** The enrichment worker prices `price_ohlcv_1m` and nothing else From 1f4f87949bddd62371d8bbbc6647ac529c3df737 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:04:30 +0200 Subject: [PATCH 063/144] docs(lore-0330): close, watch green and #398 closed PR #399 merged as 78765f9e. A manual watch run on master (38240193493) reported WAITING for protocol 29 and closed #398 as not actionable yet. --- ...eads-a-later-major-as-the-missing-crate.md | 69 +++++++++++++++++-- 1 file changed, 63 insertions(+), 6 deletions(-) rename lore/1-tasks/{active => archive}/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md (59%) diff --git a/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md b/lore/1-tasks/archive/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md similarity index 59% rename from lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md rename to lore/1-tasks/archive/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md index 0399f04b..ba547277 100644 --- a/lore/1-tasks/active/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md +++ b/lore/1-tasks/archive/0330_BUG_xdr-watch-reads-a-later-major-as-the-missing-crate.md @@ -2,7 +2,7 @@ id: "0330" title: "The XDR watch reports BEHIND because stellar-xdr 30 counts as a crate for protocol 29" type: BUG -status: active +status: completed related_adr: [] related_tasks: ["0325", "0319", "0098"] tags: [ci, resilience, ingestion, priority-medium, effort-small] @@ -13,6 +13,8 @@ links: - "https://github.com/rumblefishdev/stellar-prices-api/issues/398" - "https://github.com/stellar/rs-stellar-xdr/releases/tag/v30.0.0" - "https://github.com/stellar/stellar-core/pull/5478" + - "https://github.com/rumblefishdev/stellar-prices-api/pull/399" + - "https://github.com/rumblefishdev/stellar-prices-api/actions/runs/38240193493" history: - date: "2026-10-12" status: active @@ -32,6 +34,15 @@ history: 18/18 pass. The pre-fix script fails 5 of them, exactly the new cases. Live run against mainnet 29 and crates.io 30.0.0 exits 0 with "WAITING: no stellar-xdr 29" in both --watch and PR mode. + - date: "2026-10-12" + status: completed + who: stkrolikiewicz + note: > + PR #399 merged to develop as 78765f9e at 11:02:55 UTC. 4 files, + +82 −23. 18 tier cases, 5 of them new. CI 4/4 green. A manual + xdr-protocol-watch run on master (38240193493, 11:03 UTC) was green and + reported "WAITING: no stellar-xdr 29 is published yet". It closed #398 + with the "Closed as not actionable yet" comment. --- # The XDR watch reports BEHIND because stellar-xdr 30 counts as a crate for protocol 29 @@ -96,10 +107,56 @@ LAGGING). The crates.io mock serves `versions`. Update the runbook's - [x] Current pin+1, core pin+2, crate pin+2 → LAGGING (exit 1 under `--watch`). - [x] All existing tier cases still pass, and the pre-fix script fails the new WAITING case. -- [ ] After merge to `develop`, a manual watch run on `master` is green and +- [x] After merge to `develop`, a manual watch run on `master` is green and closes #398 as "not actionable yet". -## Notes - -No workflow change on `master` is needed: the workflow checks out `develop` -for the script. +## Implementation Notes + +- `publishedCrate()` returns `releases`: the stable, unyanked versions from + the `versions` array of the crate document. The `major` field is gone. +- `bumpTarget(crate, protocol)` returns the newest release with + `pin < major ≤ protocol`. +- The tier check asks `bumpTarget` for mainnet (BEHIND) and then for core + (LAGGING). WAITING when both are empty. +- The BEHIND report names the target (`bump to …`). The LAGGING report names + the target instead of `max_stable_version`. +- No workflow change on `master` was needed: the workflow checks out + `develop` for the script. + +**Modified tests:** the `published()` mock now also serves `versions`. No +existing case changed. + +## Design Decisions + +### From Plan + +1. **Ask mainnet and core separately.** Mainnet can sit on a crate-less + protocol (29) while core supports the next one, which has a crate (30). + That reading is LAGGING, not WAITING. +2. **Keep the WAITING wording.** `— WAITING: no stellar-xdr` is what + master's workflow matches to close the issue. + +### Emerged + +3. **A range, not an exact major.** The target is `pin < major ≤ N`, not + `major == N`. In chat I first described it as "exactly 29". The range also + covers a pin two protocols behind (pin 27, mainnet 29, only 28 published: + BEHIND, bump to 28). An exact check would call that WAITING. +4. **Same request, `versions` from the crate document.** No call to the + paginated `/versions` endpoint. A document without `versions` is + unreadable, which is the `cannot read` path (red run, no issue). +5. **Yanked versions do not count.** No test covers this. +6. **The WAITING report explains a later major.** When the newest crate is + above the needed protocol, the report says it carries a later protocol's + XDR and is not a bump target. + +## Issues Encountered + +- **`lore_set-task` could not find 0330** (TASK_NOT_FOUND) from the + `issue-398-review` worktree. The MCP server seems to resolve tasks in the + main checkout. The task was never set as the session's current task. +- **The pre-push hook took 3 of 4 nx tasks from cache.** It is unclear + whether the infra `test` target re-ran the changed test file locally. The + test was run directly (18/18), and CI ran it on the PR. +- **Prettier re-padded the whole tier table** in the runbook, which made + the runbook diff larger than the change. From 3958201659c58464391407244ae1c7035567e1ab Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:16:38 +0200 Subject: [PATCH 064/144] docs(lore-0331): open pre-2021 usd pricing at par The M3 reviewer found XLM daily USD prices null before 2021 and asked for priced 2018 candles. No row before 2021-02-01 has a USD reference. Decided with Marek: count every dollar token at $1 before that date, on-chain, with a volume-weighted reference and an outlier filter. --- .../README.md | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 lore/1-tasks/backlog/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md diff --git a/lore/1-tasks/backlog/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/backlog/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md new file mode 100644 index 00000000..0e2349ce --- /dev/null +++ b/lore/1-tasks/backlog/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -0,0 +1,101 @@ +--- +id: "0331" +title: "No USD price before 2021-02 — price that history by counting every dollar token at par ($1)" +type: FEATURE +status: backlog +related_adr: [] +related_tasks: ["0286", "0201", "0172", "0212", "0267", "0294"] +tags: [layer-backend, priority-high, effort-medium, enrichment, clickhouse, data-correctness] +links: + - "../../../../docs/runbooks/0286-reingest-history.md" + - "../../../../packages/prices-ingest-core/src/canonical.rs" + - "../../../../packages/prices-clickhouse/schema/current.sql" + - "../../../../docs/scf/milestone-3-evidence.md" +history: + - date: "2026-10-13" + status: backlog + who: stkrolikiewicz + note: > + Opened after the M3 reviewer found XLM daily USD prices null over + 2018–2025 and asked for priced candles from 2018 and 2022. 2021-02 + onward is repaired by 0286 §7b-2/§7a. Before 2021-02 no row has a USD + reference. Decided with Marek: before 2021-02 every dollar token counts + as $1, on-chain, with the safeguards below. Pre-run report goes to notes/. +--- + +# No USD price before 2021-02 — price that history by counting every dollar token at par + +## Summary + +Every USD value we publish is anchored on canonical USDC (`GA5Z…`), priced with a +measured USDC/USD rate. USDC trades on Stellar from 2021-02-01, so nothing earlier has +a USD reference: candles from 2015-11 to 2021-01 carry volumes and no USD price, on +every tier. This task prices that period by counting dollar tokens at $1 before +2021-02, both where they are the quote and as the source of the XLM/USD rate that +everything quoted in XLM is converted through. + +## Context + +- **The gap.** The 0286 runbook calls these rows `before_reference` ("Nothing can price + them"). [[0201]] measured the first XLM/USDC candle at 2021-02-01 21:00. Adam puts + them at ~1.8 % of rows (2026-10-13, not measured here). They were unpriced before the + 0286 re-ingest too. +- **Why there is a USDC-only rule.** Stellar USDT (`GCQTGZ…`) was valued at $1 from + 2018-05 to 2026-08. It lost its peg in June 2022 and is worth ~$0.13 now, so its + quoted candles came out ~7.4× too high ([[0172]], [[0212]]). `current.sql` and + `views.sql` since fence the USD reference to USDC by name. +- **The depeg is one dated event.** The pivot prices Stellar USDT at monthly closes of + 0.977–1.005 from 2021-02 to 2022-05, then 0.363 in 2022-06. A par rule that ends at + 2021-02-01 never reaches it. +- **Trigger.** M3 review feedback (via Marek, 2026-10-13): XLM daily USD null over + 2018–2025; asked for priced 2018 and 2022 candles. 2022 is repaired; 2018 is this + task. + +## Decision (2026-10-13) + +1. Before 2021-02-01, **every dollar token** counts as $1. Not a list of issuers. +2. **On-chain only**, chosen over an external XLM/USD series. The trade-off is accepted: + a bucket stays unpriced when no dollar-token market traded within the pivot window. +3. **History only.** The change lives in the repricing pass run from fishuser-hero. The + live path and the USDC-only rule for 2021-02 onward stay as they are. + +## Safeguards + +Anyone can issue a token called `USD`, so "every dollar token" needs these: + +1. The XLM/USD reference is **volume-weighted** across all dollar-token markets in the + bucket, as the USDC reference already is. +2. A market whose implied XLM/USD deviates from the bucket's median by more than a + threshold is left out. Threshold to be set from the report. +3. **Pre-run report** (`notes/`): every dollar token traded before 2021-02, with + volume, implied XLM/USD and deviation. +4. A distinct `method` on these candles, so a client can tell an assumed par from a + measured rate. +5. A spot-check against Bitstamp XLM/USD where Bitstamp has data. + +## Open questions + +- Which codes make a dollar token: `USD`, `USDT`, `USDC`, others? +- `method` label: the existing `assumed-par`, or a new one? +- The outlier threshold. +- Coverage ceiling: `canonical.rs` ranks quotes USDC > USDT > XLM, so XLM's own daily + series before 2021 is the XLM/USDT market. It has candles on 21 days of 2018, 63 of + 2019 and 257 of 2020 (API, 2026-10-13). XLM's own series cannot exceed that. + +## Plan + +1. Pre-run report → `notes/`. +2. Decide codes, threshold and label; record them here. +3. Implement the par rule for buckets before 2021-02 in the repricing pass, with tests. +4. Dry run, then run over 2015-11 → 2021-01 on all tiers. After 0286 §7b (~2026-10-16): + one pass at a time on the cluster shared with the block explorer. +5. Spot-check against Bitstamp. Then the M3 reply and evidence. + +## Acceptance Criteria + +- [ ] Pre-run report in `notes/`, read by Marek or Adam +- [ ] Codes, threshold and label decided and recorded here +- [ ] XLM daily candles before 2021-02 priced wherever a dollar-token market traded + within the window; the remaining days explained +- [ ] Spot-check against Bitstamp XLM/USD within an agreed tolerance +- [ ] Rows from 2021-02-01 on unchanged From 281c223fc3931243969a65bbdbe1e11f111f8490 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:32:01 +0200 Subject: [PATCH 065/144] chore(lore-0331): activate task --- .../README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) rename lore/1-tasks/{backlog => active}/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md (94%) diff --git a/lore/1-tasks/backlog/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md similarity index 94% rename from lore/1-tasks/backlog/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md rename to lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md index 0e2349ce..0ee3dbfe 100644 --- a/lore/1-tasks/backlog/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -2,7 +2,8 @@ id: "0331" title: "No USD price before 2021-02 — price that history by counting every dollar token at par ($1)" type: FEATURE -status: backlog +status: active +assignee: stkrolikiewicz related_adr: [] related_tasks: ["0286", "0201", "0172", "0212", "0267", "0294"] tags: [layer-backend, priority-high, effort-medium, enrichment, clickhouse, data-correctness] @@ -12,6 +13,12 @@ links: - "../../../../packages/prices-clickhouse/schema/current.sql" - "../../../../docs/scf/milestone-3-evidence.md" history: + - date: "2026-10-13" + status: active + who: stkrolikiewicz + note: > + Activated to implement the pre-2021 par mode in coarse-repair on a + branch, for Adam to review and run after 0286 §7b (~2026-10-16). - date: "2026-10-13" status: backlog who: stkrolikiewicz From 49bf0bc771a9a606b5f8860ebab9a2f2fceff72d Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:50:13 +0200 Subject: [PATCH 066/144] docs(lore-0331): add the pre-run report on dollar tokens Eleven dollar tokens traded against XLM before 2021-02. No usable reference exists before 2018-08. Par agrees with Binance to a 0.3-1.9 % median by year, but an off-peg USDT holds most of the volume on 19 days and sets the reference there, which the median filter cannot stop. --- .../README.md | 21 +- .../data/collect.py | 194 ++++ .../data/daily.csv | 1006 +++++++++++++++++ .../data/summary.json | 52 + .../data/tokens.csv | 12 + ...re-run-report-dollar-tokens-before-2021.md | 82 ++ 6 files changed, 1365 insertions(+), 2 deletions(-) create mode 100644 lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/collect.py create mode 100644 lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/daily.csv create mode 100644 lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/summary.json create mode 100644 lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/tokens.csv create mode 100644 lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md index 0ee3dbfe..c573ca04 100644 --- a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -13,6 +13,15 @@ links: - "../../../../packages/prices-clickhouse/schema/current.sql" - "../../../../docs/scf/milestone-3-evidence.md" history: + - date: "2026-10-13" + status: active + who: stkrolikiewicz + note: > + Pre-run report in notes/: 11 dollar tokens traded against XLM before + 2021-02; no usable reference before 2018-08; par agrees with Binance to + 0.3–1.9 % median by year; an off-peg USDT sets the reference on 19 days, + which the median filter cannot stop. Implementation in draft PR #401 + (coarse-repair --par-*), integration test green on CH 26.3.10.60. - date: "2026-10-13" status: active who: stkrolikiewicz @@ -84,14 +93,22 @@ Anyone can issue a token called `USD`, so "every dollar token" needs these: - Which codes make a dollar token: `USD`, `USDT`, `USDC`, others? - `method` label: the existing `assumed-par`, or a new one? -- The outlier threshold. +- The outlier threshold. The report finds 0.10 and 0.25 give the same days; 0.25 + keeps the honest small markets that sit 3–4 % off. +- **Per-token exclusion** (from the [pre-run report](notes/R-pre-run-report-dollar-tokens-before-2021.md)). + USDT `GC2PJ6J3` is ~10x off every other market but holds most of the volume on 19 + days, so it becomes the median and the filter cannot drop it: ~20 days of XLM/USD + ~10x wrong in 2019–2020 with every dollar token, 1 day with four off-peg tokens left + out. Keep "every dollar token", or exclude by list or by a per-token rule? +- Coverage: before 2018 there is no usable reference (one token, $31 in two years). + Priced history starts in August 2018 (AnchorUSD) and is daily from 2019. - Coverage ceiling: `canonical.rs` ranks quotes USDC > USDT > XLM, so XLM's own daily series before 2021 is the XLM/USDT market. It has candles on 21 days of 2018, 63 of 2019 and 257 of 2020 (API, 2026-10-13). XLM's own series cannot exceed that. ## Plan -1. Pre-run report → `notes/`. +1. Pre-run report → `notes/` (done 2026-10-13, data in `data/`). 2. Decide codes, threshold and label; record them here. 3. Implement the par rule for buckets before 2021-02 in the repricing pass, with tests. 4. Dry run, then run over 2015-11 → 2021-01 on all tiers. After 0286 §7b (~2026-10-16): diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/collect.py b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/collect.py new file mode 100644 index 00000000..c163ec53 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/collect.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""0331 pre-run report: every dollar token (USD/USDT/USDC, any issuer) traded +against XLM before 2021-02-01, its implied XLM/USD at par, and how it compares +with the day's other dollar markets and with Binance XLM/USDT. + +Resumable: every fetch is cached under OUT. Usage: KEY=... collect.py OUT +""" +import csv, json, os, ssl, statistics, sys, time, urllib.error, urllib.request + +CTX = ssl.create_default_context(cafile="/etc/ssl/cert.pem") + +OUT = sys.argv[1] +API = "https://prices-api.sorobanscan.rumblefish.dev/v1" +KEY = os.environ["KEY"] +UA = {"User-Agent": "stellar-prices-skill/1"} +CODES = ["USD", "USDT", "USDC"] +CANON = { + ("USDC", "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"), + ("USDT", "GCQTGZQQ5G4PTM2GL7CDIFKUBIPEC52BROAQIAPW53XBRJVN6ZJVTG6V"), +} +START, END = "2015-11-01", "2021-01-31" +os.makedirs(f"{OUT}/daily", exist_ok=True) + + +def get(url, headers, pace=0.0): + for i in range(6): + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=90, context=CTX) as r: + data = json.load(r) + time.sleep(pace) + return data + except urllib.error.HTTPError as e: + time.sleep(max(pace, 1)) + if e.code == 404: + return None + if e.code == 400: + return {"_err": 400, "body": e.read().decode()[:300]} + if e.code in (429, 500, 502, 503, 504): + time.sleep(2 * (i + 1)) + continue + raise + except Exception: + time.sleep(3 * (i + 1)) + return {"_err": "retries"} + + +def api(path): + return get(API + path, {**UA, "x-api-key": KEY}, pace=1.05) + + +def cached(path, fn): + if os.path.exists(path): + with open(path) as f: + return json.load(f) + data = fn() + with open(path, "w") as f: + json.dump(data, f) + return data + + +# 1. Every issuer of a dollar code, from Horizon. +def enumerate_assets(): + out = [] + for code in CODES: + url = f"https://horizon.stellar.org/assets?asset_code={code}&limit=200" + while url: + page = get(url, UA) + recs = page["_embedded"]["records"] + out += [{"code": r["asset_code"], "issuer": r["asset_issuer"], + "accounts": r.get("accounts", {}).get("authorized")} for r in recs] + url = page["_links"]["next"]["href"] if len(recs) == 200 else None + return out + + +assets = cached(f"{OUT}/assets.json", enumerate_assets) +print(f"assets: {len(assets)}", flush=True) + +# 2. Monthly scan: which of them traded against XLM (token = base) before 2021-02. +mpath = f"{OUT}/monthly.jsonl" +done = set() +if os.path.exists(mpath): + with open(mpath) as f: + done = {json.loads(l)["id"] for l in f} +with open(mpath, "a") as f: + for i, a in enumerate(assets): + aid = f"{a['code']}:{a['issuer']}" + if aid in done or (a["code"], a["issuer"]) in CANON: + continue + r = api(f"/assets/{aid}/ohlcv?start={START}&end={END}&granularity=1M&base_currency=XLM") + months = [] if not r or "data" not in r else [ + {"m": d["timestamp"][:7], "vol": d["volume_base"], "n": d["trade_count"]} for d in r["data"]] + f.write(json.dumps({"id": aid, "months": months, "err": (r or {}).get("_err")}) + "\n") + f.flush() + if i % 50 == 0: + print(f"monthly {i}/{len(assets)}", flush=True) + +with open(mpath) as f: + monthly = [json.loads(l) for l in f] +traded = [m for m in monthly if m["months"]] +print(f"traded vs XLM before 2021-02: {len(traded)}", flush=True) + +# 3. Daily series for the traded ones. +daily = {} +for m in traded: + aid = m["id"] + p = f"{OUT}/daily/{aid.replace(':', '_')}.json" + daily[aid] = cached(p, lambda: api(f"/assets/{aid}/ohlcv?start={START}&end={END}&granularity=1d&base_currency=XLM")) + +# 4. External reference: Binance XLM/USDT daily (from 2018-05-31). +def binance(): + rows, t = [], 1514764800000 + while t < 1612137600000: + k = get(f"https://api.binance.com/api/v3/klines?symbol=XLMUSDT&interval=1d&startTime={t}&limit=1000", UA) + if not k: + break + rows += k + t = k[-1][0] + 86400000 + return {time.strftime("%Y-%m-%d", time.gmtime(r[0] / 1000)): float(r[7]) / float(r[5]) + for r in rows if float(r[5]) > 0} + +bn = cached(f"{OUT}/binance.json", binance) + +# 5. XLM's own daily series (XLM as base = the canonical XLM/USDT market before 2021). +xlm = cached(f"{OUT}/xlm.json", lambda: api(f"/assets/native/ohlcv?start={START}&end={END}&granularity=1d")) + +# 6. Analysis. +per_day = {} # date -> list of (aid, implied_xlm_usd, vol_usd) +for aid, r in daily.items(): + for d in (r or {}).get("data", []): + px = d.get("pf_vwap") or d.get("vwap") + if not px or float(px) <= 0: + continue + per_day.setdefault(d["timestamp"][:10], []).append((aid, 1 / float(px), float(d["volume_base"]))) + + +def wmedian(pairs): + pairs = sorted(pairs) + tot, acc = sum(w for _, w in pairs), 0.0 + for v, w in pairs: + acc += w + if acc >= tot / 2: + return v + return None + + +day_rows, tok = [], {} +for day in sorted(per_day): + mk = per_day[day] + vol = sum(v for _, _, v in mk) + med = wmedian([(x, v) for _, x, v in mk if v > 0]) if vol > 0 else statistics.median(x for _, x, _ in mk) + ext = bn.get(day) + day_rows.append({"date": day, "markets": len(mk), "vol_usd_at_par": round(vol, 2), + "ref_wmedian": round(med, 6), "binance_vwap": round(ext, 6) if ext else "", + "ref_vs_binance_pct": round((med / ext - 1) * 100, 2) if ext else ""}) + for aid, x, v in mk: + t = tok.setdefault(aid, {"days": 0, "vol": 0.0, "first": day, "last": day, "dev_med": [], "dev_bn": []}) + t["days"] += 1; t["vol"] += v; t["last"] = day + t["dev_med"].append(x / med - 1) + if ext: + t["dev_bn"].append(x / ext - 1) + +tok_rows = [] +for aid, t in sorted(tok.items(), key=lambda kv: -kv[1]["vol"]): + dm = [abs(x) for x in t["dev_med"]] + db = [abs(x) for x in t["dev_bn"]] + tok_rows.append({"asset": aid, "days": t["days"], "vol_usd_at_par": round(t["vol"], 2), + "first": t["first"], "last": t["last"], + "median_abs_dev_vs_day_pct": round(statistics.median(dm) * 100, 2), + "days_dev_gt_10pct": sum(x > 0.10 for x in dm), "days_dev_gt_25pct": sum(x > 0.25 for x in dm), + "median_abs_dev_vs_binance_pct": round(statistics.median(db) * 100, 2) if db else "", + "binance_days": len(db)}) + +for name, rows in (("tokens.csv", tok_rows), ("daily.csv", day_rows)): + with open(f"{OUT}/{name}", "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader(); w.writerows(rows) + +xlm_days = {} +for d in (xlm or {}).get("data", []): + y = d["timestamp"][:4] + xlm_days.setdefault(y, [0, 0.0]); xlm_days[y][0] += 1; xlm_days[y][1] += float(d["volume_base"]) +years = {} +for r in day_rows: + y = r["date"][:4] + years.setdefault(y, {"days_with_ref": 0, "vol_usd": 0.0}) + years[y]["days_with_ref"] += 1; years[y]["vol_usd"] += r["vol_usd_at_par"] +summary = {"assets_enumerated": len(assets), "scanned": len(monthly), + "errors": sum(1 for m in monthly if m["err"]), "traded_vs_xlm": len(traded), + "days_with_ref": len(day_rows), "years": years, + "xlm_own_series": {y: {"days": v[0], "vol_xlm": round(v[1], 2)} for y, v in xlm_days.items()}, + "binance_days": len(bn)} +with open(f"{OUT}/summary.json", "w") as f: + json.dump(summary, f, indent=1) +print(json.dumps(summary, indent=1)) diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/daily.csv b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/daily.csv new file mode 100644 index 00000000..92c1b794 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/daily.csv @@ -0,0 +1,1006 @@ +date,markets,vol_usd_at_par,ref_wmedian,binance_vwap,ref_vs_binance_pct +2016-10-18,1,0.02,0.0022,, +2017-06-17,1,0.73,0.036349,, +2017-06-18,1,19.22,0.038442,, +2017-06-19,1,1.01,0.037268,, +2017-06-20,1,3.02,0.040926,, +2017-06-24,1,0.22,0.037282,, +2017-07-09,1,0.02,0.021424,, +2017-07-14,1,0.02,0.0158,, +2017-07-18,1,1.01,0.018351,, +2017-07-22,1,1.01,0.019067,, +2017-07-29,1,1.01,0.015702,, +2017-08-01,1,1.01,0.015604,, +2017-08-04,1,1.01,0.018927,, +2017-08-07,1,2.08,0.021879,, +2017-08-10,1,0.02,0.022661,, +2018-04-11,1,0.01,0.18517,, +2018-05-03,1,0.2,0.438748,, +2018-05-04,1,2.05,0.429616,, +2018-05-07,1,2.0,0.409855,, +2018-05-11,1,2.95,0.340176,, +2018-05-13,1,202.76,0.350851,, +2018-05-14,1,0.36,0.357143,, +2018-05-16,1,0.3,0.337079,, +2018-05-17,1,3.02,0.334727,, +2018-05-18,1,0.8,0.314041,, +2018-05-19,1,0.3,0.316799,, +2018-05-20,1,70.52,0.327812,, +2018-05-21,1,34.66,0.308774,, +2018-05-22,1,56.6,0.313171,, +2018-05-23,1,10.49,0.290863,, +2018-05-24,1,0.21,0.282643,, +2018-05-25,1,148.63,0.287691,, +2018-05-26,1,40.43,0.294442,, +2018-05-27,1,78.69,0.280542,, +2018-05-28,1,65.62,0.265922,, +2018-05-29,1,438.81,0.266866,, +2018-05-30,1,355.04,0.279823,, +2018-05-31,1,120.53,0.291356,0.296407,-1.7 +2018-06-01,1,104.6,0.293346,0.289636,1.28 +2018-06-02,1,0.72,0.295582,0.296764,-0.4 +2018-06-03,1,0.18,0.298814,0.303709,-1.61 +2018-06-04,1,1.76,0.292247,0.295668,-1.16 +2018-06-05,1,1.58,0.291045,0.288565,0.86 +2018-06-06,1,0.3,0.298733,0.293302,1.85 +2018-06-10,1,187.83,0.255169,0.259165,-1.54 +2018-06-11,1,246.0,0.253174,0.24264,4.34 +2018-06-12,1,151.83,0.252875,0.239271,5.69 +2018-06-13,1,42.4,0.222766,0.220336,1.1 +2018-06-15,1,92.72,0.2337,0.236426,-1.15 +2018-06-17,1,0.02,0.245461,0.230318,6.58 +2018-06-18,1,0.24,0.241383,0.228649,5.57 +2018-06-19,1,0.31,0.220244,0.233785,-5.79 +2018-06-20,1,9.0,0.23438,0.229937,1.93 +2018-06-21,1,1.11,0.221578,0.230041,-3.68 +2018-06-22,1,43.12,0.210292,0.211786,-0.71 +2018-06-23,1,0.58,0.193873,0.199784,-2.96 +2018-06-24,1,0.38,0.190922,0.188255,1.42 +2018-06-25,1,2.07,0.185676,0.194254,-4.42 +2018-06-26,1,0.19,0.18677,0.190926,-2.18 +2018-06-27,1,1.78,0.178057,0.186617,-4.59 +2018-06-28,1,3.58,0.184384,0.187925,-1.88 +2018-06-29,1,0.51,0.17023,0.176768,-3.7 +2018-06-30,1,0.51,0.173896,0.194774,-10.72 +2018-07-01,1,2.68,0.178014,0.193928,-8.21 +2018-07-02,1,3.27,0.192152,0.205923,-6.69 +2018-07-03,1,0.95,0.206018,0.212632,-3.11 +2018-07-04,1,1.39,0.198702,0.212211,-6.37 +2018-07-05,1,0.4,0.202454,0.206962,-2.18 +2018-07-06,1,0.98,0.195272,0.20353,-4.06 +2018-07-07,1,0.59,0.197101,0.204032,-3.4 +2018-07-08,1,0.81,0.201661,0.210481,-4.19 +2018-07-09,1,0.8,0.199782,0.210498,-5.09 +2018-07-11,1,0.73,0.183357,0.187798,-2.36 +2018-07-12,1,0.52,0.174852,0.182888,-4.39 +2018-07-13,1,2.03,0.184522,0.193622,-4.7 +2018-07-14,1,1.36,0.194612,0.204012,-4.61 +2018-07-15,1,1.83,0.203226,0.216784,-6.25 +2018-07-16,1,1.94,0.215588,0.228765,-5.76 +2018-07-17,1,29.25,0.2378,0.241381,-1.48 +2018-07-18,1,976.54,0.268476,0.292679,-8.27 +2018-07-19,1,2.55,0.283245,0.303386,-6.64 +2018-07-20,1,0.56,0.280405,0.285046,-1.63 +2018-07-21,1,2.43,0.269667,0.282497,-4.54 +2018-07-22,1,1.69,0.282476,0.29488,-4.21 +2018-07-23,1,1.65,0.275831,0.292111,-5.57 +2018-07-24,1,2.24,0.279532,0.295907,-5.53 +2018-07-25,1,890.17,0.30888,0.31924,-3.25 +2018-07-26,1,129.23,0.322042,0.323974,-0.6 +2018-07-27,1,9.0,0.319914,0.310411,3.06 +2018-07-28,1,6.0,0.333926,0.316758,5.42 +2018-07-29,1,1.0,0.322581,0.310443,3.91 +2018-07-30,1,504.38,0.304056,0.292679,3.89 +2018-07-31,1,305.0,0.292042,0.276633,5.57 +2018-08-01,1,5.0,0.283782,0.273162,3.89 +2018-08-02,1,16.14,0.280436,0.268901,4.29 +2018-08-03,1,255.0,0.266531,0.254916,4.56 +2018-08-04,1,208.0,0.256676,0.24692,3.95 +2018-08-05,2,25.92,0.233852,0.240488,-2.76 +2018-08-06,1,4.0,0.251668,0.237617,5.91 +2018-08-07,2,9.0,0.245844,0.236674,3.87 +2018-08-08,2,358.0,0.220866,0.208841,5.76 +2018-08-10,2,13521.77,0.231133,0.223561,3.39 +2018-08-11,2,6349.63,0.216612,0.213527,1.45 +2018-08-12,1,0.2,0.223265,0.223,0.12 +2018-08-14,2,3788.94,0.205177,0.210822,-2.68 +2018-08-15,2,0.87,0.215816,0.22335,-3.37 +2018-08-16,2,71.97,0.220584,0.214148,3.01 +2018-08-17,2,871.29,0.229461,0.226541,1.29 +2018-08-18,2,938.12,0.226085,0.228271,-0.96 +2018-08-19,1,0.44,0.218516,0.226308,-3.44 +2018-08-20,2,1242.29,0.224176,0.22271,0.66 +2018-08-21,1,0.83,0.207883,0.216589,-4.02 +2018-08-22,2,4833.32,0.22221,0.218997,1.47 +2018-08-23,2,41.45,0.212602,0.211877,0.34 +2018-08-24,2,20000.83,0.219523,0.215223,2.0 +2018-08-25,2,147803.63,0.218418,0.217145,0.59 +2018-08-26,2,1918950.77,0.215656,0.212413,1.53 +2018-08-27,2,107772.6,0.21785,0.218546,-0.32 +2018-08-28,2,142126.55,0.22787,0.230796,-1.27 +2018-08-29,2,378978.48,0.230475,0.228862,0.7 +2018-08-30,2,577607.08,0.220139,0.219376,0.35 +2018-08-31,1,501946.73,0.218983,0.220373,-0.63 +2018-09-01,2,20558.87,0.223495,0.228804,-2.32 +2018-09-02,1,39365.29,0.230767,0.227209,1.57 +2018-09-03,2,42561.32,0.225179,0.222677,1.12 +2018-09-04,2,2042.98,0.224459,0.226074,-0.71 +2018-09-05,2,12987.44,0.228401,0.219043,4.27 +2018-09-06,2,32816.96,0.200822,0.198574,1.13 +2018-09-07,2,4.96,0.209773,0.205843,1.91 +2018-09-08,1,109955.45,0.204559,0.200819,1.86 +2018-09-09,2,183057.2,0.196541,0.194248,1.18 +2018-09-10,1,8.37,0.188897,0.190206,-0.69 +2018-09-11,2,210.83,0.18956,0.196514,-3.54 +2018-09-12,2,12984.65,0.201151,0.19671,2.26 +2018-09-13,2,1269.91,0.204667,0.20563,-0.47 +2018-09-14,2,83.55,0.203046,0.203076,-0.01 +2018-09-15,2,85.16,0.205673,0.20312,1.26 +2018-09-16,2,1234.9,0.206473,0.204616,0.91 +2018-09-17,1,10719.77,0.211988,0.202638,4.61 +2018-09-18,2,11.1,0.200019,0.204979,-2.42 +2018-09-20,1,0.42,0.21055,0.220526,-4.52 +2018-09-21,2,37597.37,0.251561,0.25124,0.13 +2018-09-22,1,0.24,0.236356,0.240144,-1.58 +2018-09-23,2,53.24,0.285404,0.282588,1.0 +2018-09-24,1,502.35,0.260043,0.265521,-2.06 +2018-09-25,2,6395.53,0.243543,0.243118,0.17 +2018-09-26,2,7869.73,0.250531,0.249129,0.56 +2018-09-27,3,3012.08,0.250184,0.255661,-2.14 +2018-09-28,3,3714.32,0.25936,0.255018,1.7 +2018-09-29,3,4452.62,0.252418,0.253825,-0.55 +2018-09-30,3,851.1,0.256767,0.258295,-0.59 +2018-10-01,2,27549.83,0.26217,0.259695,0.95 +2018-10-02,3,2246.88,0.253284,0.254283,-0.39 +2018-10-03,2,4830.92,0.242597,0.242413,0.08 +2018-10-04,3,11936.83,0.245376,0.245723,-0.14 +2018-10-05,3,27882.39,0.241649,0.244866,-1.31 +2018-10-06,1,9764.46,0.24201,0.243995,-0.81 +2018-10-07,2,7655.66,0.241385,0.241722,-0.14 +2018-10-08,3,16283.55,0.243845,0.246558,-1.1 +2018-10-09,3,10359.65,0.245944,0.246729,-0.32 +2018-10-10,3,11304.48,0.241793,0.243852,-0.84 +2018-10-11,3,122224.14,0.216606,0.218775,-0.99 +2018-10-12,3,30439.71,0.214804,0.216323,-0.7 +2018-10-13,2,36305.04,0.213393,0.217214,-1.76 +2018-10-14,3,8375.51,0.210683,0.216253,-2.58 +2018-10-15,3,30721.33,0.213625,0.230929,-7.49 +2018-10-16,2,41474.35,0.221625,0.231667,-4.33 +2018-10-17,3,33121.3,0.228599,0.245653,-6.94 +2018-10-18,3,19617.23,0.23603,0.246735,-4.34 +2018-10-19,2,17092.27,0.236894,0.246808,-4.02 +2018-10-20,3,1054.21,0.235895,0.2525,-6.58 +2018-10-21,3,2759.1,0.240365,0.25098,-4.23 +2018-10-22,2,11991.35,0.237506,0.247797,-4.15 +2018-10-23,3,8866.16,0.236222,0.247071,-4.39 +2018-10-24,3,11247.95,0.236221,0.244379,-3.34 +2018-10-25,3,6908.5,0.234589,0.239254,-1.95 +2018-10-26,3,23823.37,0.23507,0.236924,-0.78 +2018-10-27,2,15996.9,0.231123,0.233873,-1.18 +2018-10-28,2,3381.07,0.230888,0.233422,-1.09 +2018-10-29,3,8362.32,0.230093,0.226838,1.43 +2018-10-30,3,13356.98,0.236351,0.224626,5.22 +2018-10-31,3,12037.39,0.221733,0.223021,-0.58 +2018-11-01,3,31190.88,0.221749,0.225556,-1.69 +2018-11-02,3,16789.82,0.227312,0.233607,-2.69 +2018-11-03,3,9145.22,0.234523,0.239587,-2.11 +2018-11-04,3,3424.89,0.237078,0.244464,-3.02 +2018-11-05,2,7995.15,0.240081,0.244979,-2.0 +2018-11-06,3,81167.16,0.251292,0.25861,-2.83 +2018-11-07,3,20025.25,0.261917,0.260735,0.45 +2018-11-08,3,42545.69,0.256173,0.259355,-1.23 +2018-11-09,3,16287.38,0.2591,0.261084,-0.76 +2018-11-10,3,29256.13,0.259662,0.261832,-0.83 +2018-11-11,3,85415.23,0.265142,0.2733,-2.99 +2018-11-12,3,33448.7,0.267196,0.277167,-3.6 +2018-11-13,3,37853.06,0.264509,0.267056,-0.95 +2018-11-14,3,31796.83,0.228621,0.239471,-4.53 +2018-11-15,3,40063.54,0.230829,0.235271,-1.89 +2018-11-16,2,5020.46,0.240004,0.243947,-1.62 +2018-11-17,3,12930.52,0.242609,0.243576,-0.4 +2018-11-18,3,35193.99,0.247738,0.253197,-2.16 +2018-11-19,3,89484.27,0.231713,0.233274,-0.67 +2018-11-20,3,68022.64,0.195668,0.209009,-6.38 +2018-11-21,3,38070.42,0.200166,0.201872,-0.85 +2018-11-22,2,35873.36,0.192733,0.19788,-2.6 +2018-11-23,3,26276.5,0.181508,0.182226,-0.39 +2018-11-24,3,44245.66,0.172095,0.16377,5.08 +2018-11-25,3,142182.32,0.146905,0.147136,-0.16 +2018-11-26,3,210095.7,0.147489,0.152353,-3.19 +2018-11-27,3,151450.29,0.141305,0.143523,-1.55 +2018-11-28,3,218145.89,0.156383,0.158216,-1.16 +2018-11-29,3,72355.88,0.16246,0.1642,-1.06 +2018-11-30,3,46379.35,0.16119,0.16252,-0.82 +2018-12-01,3,36623.28,0.165805,0.164156,1.0 +2018-12-02,3,35308.9,0.163544,0.16298,0.35 +2018-12-03,3,8685.73,0.153292,0.153949,-0.43 +2018-12-04,3,4847.47,0.148389,0.149427,-0.69 +2018-12-05,3,39142.01,0.141809,0.140858,0.68 +2018-12-06,3,7109.04,0.127616,0.129843,-1.72 +2018-12-07,3,51267.02,0.109517,0.112162,-2.36 +2018-12-08,3,41883.07,0.115669,0.116777,-0.95 +2018-12-09,3,9043.1,0.118969,0.120425,-1.21 +2018-12-10,3,17127.14,0.12202,0.117431,3.91 +2018-12-11,3,22914.48,0.116289,0.113185,2.74 +2018-12-12,3,13662.67,0.115631,0.11272,2.58 +2018-12-13,3,28505.06,0.111296,0.108257,2.81 +2018-12-14,3,15517.14,0.106159,0.099511,6.68 +2018-12-15,3,23628.02,0.101777,0.094848,7.31 +2018-12-16,3,10599.7,0.098316,0.097624,0.71 +2018-12-17,3,32896.51,0.098873,0.104986,-5.82 +2018-12-18,3,53316.61,0.111051,0.108648,2.21 +2018-12-19,3,74017.26,0.117305,0.117528,-0.19 +2018-12-20,3,68072.03,0.119826,0.119423,0.34 +2018-12-21,3,81288.5,0.125455,0.123673,1.44 +2018-12-22,2,10846.42,0.116706,0.117723,-0.86 +2018-12-23,3,43366.89,0.124579,0.122809,1.44 +2018-12-24,3,63712.46,0.126898,0.133316,-4.81 +2018-12-25,3,24286.57,0.120589,0.11939,1.0 +2018-12-26,3,43672.64,0.117862,0.117089,0.66 +2018-12-27,3,34407.11,0.11318,0.112484,0.62 +2018-12-28,3,16175.95,0.114507,0.115356,-0.74 +2018-12-29,3,30817.06,0.118646,0.117373,1.09 +2018-12-30,3,6655.81,0.1155,0.114728,0.67 +2018-12-31,3,6005.23,0.112228,0.112514,-0.25 +2019-01-01,3,9855.8,0.111407,0.111187,0.2 +2019-01-02,3,24535.07,0.116119,0.114798,1.15 +2019-01-03,3,15852.04,0.115741,0.113012,2.42 +2019-01-04,3,8442.6,0.112281,0.111606,0.61 +2019-01-05,3,8901.37,0.113574,0.112669,0.8 +2019-01-06,3,15451.19,0.116567,0.115194,1.19 +2019-01-07,3,28570.31,0.120475,0.121563,-0.9 +2019-01-08,3,12666.83,0.122557,0.121249,1.08 +2019-01-09,3,13505.27,0.122277,0.121586,0.57 +2019-01-10,3,21494.17,0.113173,0.113582,-0.36 +2019-01-11,3,16412.45,0.106727,0.104181,2.44 +2019-01-12,3,4494.24,0.10479,0.103778,0.98 +2019-01-13,3,32997.94,0.103929,0.105487,-1.48 +2019-01-14,3,24317.19,0.106434,0.105025,1.34 +2019-01-15,3,34382.19,0.107077,0.105373,1.62 +2019-01-16,3,16591.07,0.106131,0.105437,0.66 +2019-01-17,3,42640.1,0.106539,0.105711,0.78 +2019-01-18,3,4290.1,0.106071,0.10586,0.2 +2019-01-19,3,6666.91,0.107775,0.107149,0.58 +2019-01-20,3,24838.16,0.10403,0.104228,-0.19 +2019-01-21,2,20826.34,0.103742,0.101847,1.86 +2019-01-22,3,12991.33,0.102431,0.101595,0.82 +2019-01-23,3,15376.18,0.101373,0.101398,-0.02 +2019-01-24,3,4400.7,0.100517,0.100356,0.16 +2019-01-25,3,10593.17,0.099129,0.099426,-0.3 +2019-01-26,3,11698.19,0.101081,0.099933,1.15 +2019-01-27,3,6902.03,0.097915,0.095857,2.15 +2019-01-28,3,26979.21,0.085054,0.087657,-2.97 +2019-01-29,3,24078.32,0.082696,0.082771,-0.09 +2019-01-30,3,18978.95,0.085264,0.084827,0.52 +2019-01-31,3,9091.35,0.083417,0.083943,-0.63 +2019-02-01,3,16588.43,0.080331,0.081016,-0.85 +2019-02-02,3,12212.37,0.081032,0.081758,-0.89 +2019-02-03,3,25143.36,0.079538,0.080971,-1.77 +2019-02-04,3,5016.0,0.078605,0.079632,-1.29 +2019-02-05,3,27535.78,0.076939,0.07737,-0.56 +2019-02-06,3,8769.61,0.074911,0.073999,1.23 +2019-02-07,3,3067.06,0.074319,0.074779,-0.62 +2019-02-08,3,18247.13,0.0739,0.078852,-6.28 +2019-02-09,3,9546.6,0.079772,0.080554,-0.97 +2019-02-10,3,8278.95,0.077932,0.078413,-0.61 +2019-02-11,3,2760.94,0.077403,0.077924,-0.67 +2019-02-12,3,5019.92,0.075608,0.0763,-0.91 +2019-02-13,3,22515.59,0.074263,0.076854,-3.37 +2019-02-14,3,2075.94,0.075043,0.07568,-0.84 +2019-02-15,3,13918.37,0.079598,0.078725,1.11 +2019-02-16,3,5558.32,0.077766,0.078305,-0.69 +2019-02-17,3,3047.92,0.077588,0.078432,-1.08 +2019-02-18,3,6914.73,0.08119,0.081585,-0.48 +2019-02-19,3,18470.97,0.089106,0.088919,0.21 +2019-02-20,3,30172.38,0.089186,0.088927,0.29 +2019-02-21,3,9736.66,0.09073,0.090052,0.75 +2019-02-22,3,11126.05,0.091375,0.089834,1.72 +2019-02-23,3,21865.51,0.095872,0.092327,3.84 +2019-02-24,3,22244.24,0.082124,0.089219,-7.95 +2019-02-25,3,16894.37,0.085696,0.085243,0.53 +2019-02-26,3,4657.38,0.084974,0.084546,0.51 +2019-02-27,3,7599.77,0.08371,0.08374,-0.04 +2019-02-28,3,5512.65,0.083663,0.083933,-0.32 +2019-03-01,3,2551.61,0.085395,0.084958,0.51 +2019-03-02,3,783.61,0.08306,0.08365,-0.71 +2019-03-03,3,8904.41,0.086567,0.087001,-0.5 +2019-03-04,3,6152.24,0.082198,0.082773,-0.7 +2019-03-05,3,1942.38,0.084843,0.084079,0.91 +2019-03-06,3,45131.06,0.085395,0.084599,0.94 +2019-03-07,3,2791.73,0.084552,0.085037,-0.57 +2019-03-08,3,10681.01,0.08738,0.087529,-0.17 +2019-03-09,3,22904.01,0.087314,0.089309,-2.23 +2019-03-10,3,37907.17,0.09893,0.097304,1.67 +2019-03-11,3,11053.81,0.101673,0.102744,-1.04 +2019-03-12,3,54338.52,0.102746,0.102904,-0.15 +2019-03-13,3,73345.36,0.103499,0.105817,-2.19 +2019-03-14,3,72408.22,0.109019,0.108292,0.67 +2019-03-15,3,3871.46,0.104439,0.105798,-1.29 +2019-03-16,3,3435.99,0.10782,0.108259,-0.41 +2019-03-17,3,1351.53,0.107717,0.107789,-0.07 +2019-03-18,3,19442.44,0.110385,0.113208,-2.49 +2019-03-19,3,12926.69,0.112183,0.113845,-1.46 +2019-03-20,3,1185.32,0.110321,0.110383,-0.06 +2019-03-21,3,21839.1,0.104223,0.105985,-1.66 +2019-03-22,3,13293.34,0.104189,0.10698,-2.61 +2019-03-23,3,3078.85,0.108153,0.107312,0.78 +2019-03-24,3,7797.1,0.103867,0.104893,-0.98 +2019-03-25,3,2605.45,0.101997,0.10197,0.03 +2019-03-26,3,13679.28,0.098962,0.100734,-1.76 +2019-03-27,3,19487.26,0.10218,0.105749,-3.38 +2019-03-28,3,6276.66,0.104529,0.105655,-1.07 +2019-03-29,3,9049.44,0.106951,0.107012,-0.06 +2019-03-30,3,2293.05,0.106549,0.106234,0.3 +2019-03-31,3,3308.46,0.106879,0.107288,-0.38 +2019-04-01,3,22539.22,0.109395,0.109927,-0.48 +2019-04-02,3,59577.79,0.115286,0.116313,-0.88 +2019-04-03,3,72032.73,0.128205,0.125298,2.32 +2019-04-04,3,47360.33,0.120319,0.119682,0.53 +2019-04-05,3,32230.78,0.125014,0.125256,-0.19 +2019-04-06,3,5556.86,0.125932,0.125073,0.69 +2019-04-07,3,7495.41,0.12874,0.129281,-0.42 +2019-04-08,3,45181.79,0.131691,0.130179,1.16 +2019-04-09,3,22479.83,0.126225,0.125555,0.53 +2019-04-10,3,14715.27,0.126157,0.12576,0.32 +2019-04-11,3,12594.42,0.119239,0.115962,2.83 +2019-04-12,3,1969.4,0.111633,0.112697,-0.94 +2019-04-13,3,1693.81,0.114447,0.114334,0.1 +2019-04-14,3,14014.2,0.117865,0.115589,1.97 +2019-04-15,3,13342.59,0.115624,0.11498,0.56 +2019-04-16,3,3501.28,0.114571,0.113036,1.36 +2019-04-17,3,27746.32,0.116818,0.115048,1.54 +2019-04-18,3,15316.15,0.117695,0.117361,0.28 +2019-04-19,3,51099.26,0.115983,0.115361,0.54 +2019-04-20,3,804.24,0.115828,0.115406,0.37 +2019-04-21,3,4524.95,0.110842,0.110629,0.19 +2019-04-22,3,7067.36,0.110651,0.111909,-1.12 +2019-04-23,3,13473.26,0.112161,0.111121,0.94 +2019-04-24,3,64061.88,0.103841,0.102366,1.44 +2019-04-25,3,10675.18,0.098695,0.101718,-2.97 +2019-04-26,2,21680.89,0.097067,0.099678,-2.62 +2019-04-27,2,8865.9,0.097033,0.100197,-3.16 +2019-04-28,2,2133.99,0.096271,0.099824,-3.56 +2019-04-29,2,7630.6,0.095292,0.097164,-1.93 +2019-04-30,3,8884.81,0.095578,0.098425,-2.89 +2019-05-01,3,1617.08,0.100447,0.101381,-0.92 +2019-05-02,3,11154.7,0.09882,0.100669,-1.84 +2019-05-03,3,17863.47,0.099912,0.101422,-1.49 +2019-05-04,3,2017.55,0.101474,0.099762,1.72 +2019-05-05,3,5447.14,0.096613,0.097988,-1.4 +2019-05-06,3,5740.01,0.094141,0.096219,-2.16 +2019-05-07,3,17739.65,0.095231,0.096272,-1.08 +2019-05-08,3,6586.56,0.091052,0.092549,-1.62 +2019-05-09,3,27074.73,0.089041,0.090011,-1.08 +2019-05-10,3,22061.7,0.092095,0.091447,0.71 +2019-05-11,3,63997.07,0.098477,0.099088,-0.62 +2019-05-12,3,84006.26,0.098957,0.100318,-1.36 +2019-05-13,3,36809.38,0.101806,0.100661,1.14 +2019-05-14,3,64780.36,0.11014,0.110678,-0.49 +2019-05-15,3,71445.97,0.126892,0.12707,-0.14 +2019-05-16,3,302607.93,0.145724,0.145651,0.05 +2019-05-17,3,163053.69,0.12439,0.124913,-0.42 +2019-05-18,3,44534.72,0.133387,0.133546,-0.12 +2019-05-19,3,25455.39,0.139786,0.139314,0.34 +2019-05-20,3,28770.25,0.134039,0.134428,-0.29 +2019-05-21,3,2438.6,0.134437,0.133686,0.56 +2019-05-22,3,28250.67,0.128189,0.129687,-1.15 +2019-05-23,3,10410.46,0.122313,0.121879,0.36 +2019-05-24,3,5930.48,0.127389,0.126017,1.09 +2019-05-25,3,18037.78,0.127536,0.125611,1.53 +2019-05-26,3,36870.85,0.134534,0.12649,6.36 +2019-05-27,3,19908.77,0.13753,0.135822,1.26 +2019-05-28,3,19625.35,0.139003,0.137233,1.29 +2019-05-29,3,8974.92,0.135821,0.136063,-0.18 +2019-05-30,3,50794.33,0.132941,0.136741,-2.78 +2019-05-31,3,25507.15,0.129388,0.128964,0.33 +2019-06-01,3,15991.11,0.131989,0.132093,-0.08 +2019-06-02,3,12294.72,0.135871,0.135719,0.11 +2019-06-03,3,36287.3,0.13389,0.13364,0.19 +2019-06-04,3,86410.63,0.123812,0.122904,0.74 +2019-06-05,3,22120.11,0.122152,0.122459,-0.25 +2019-06-06,4,20817.37,0.121368,0.121854,-0.4 +2019-06-07,3,21795.26,0.12847,0.126993,1.16 +2019-06-08,3,3541.83,0.125813,0.125138,0.54 +2019-06-09,3,4503.34,0.120449,0.120798,-0.29 +2019-06-10,3,13720.28,0.122812,0.121226,1.31 +2019-06-11,3,13634.33,0.119868,0.120898,-0.85 +2019-06-12,4,2204.55,0.125212,0.124683,0.42 +2019-06-13,3,105.79,0.12531,0.12593,-0.49 +2019-06-14,4,7566.75,0.124098,0.122698,1.14 +2019-06-15,3,3252.49,0.125157,0.125451,-0.23 +2019-06-16,3,14885.78,0.129809,0.129247,0.44 +2019-06-17,4,28151.4,0.131519,0.129078,1.89 +2019-06-18,3,7931.76,0.125836,0.125966,-0.1 +2019-06-19,3,3907.63,0.124717,0.12427,0.36 +2019-06-20,3,2833.24,0.121626,0.121845,-0.18 +2019-06-21,4,4951.67,0.123197,0.123039,0.13 +2019-06-22,3,39949.37,0.128531,0.128388,0.11 +2019-06-23,3,1194.32,0.129369,0.130843,-1.13 +2019-06-24,3,5258.35,0.126537,0.127301,-0.6 +2019-06-25,4,6824.21,0.125369,0.125442,-0.06 +2019-06-26,4,68238.29,0.124377,0.126809,-1.92 +2019-06-27,3,113037.83,0.108025,0.112969,-4.38 +2019-06-28,3,20174.35,0.110144,0.111045,-0.81 +2019-06-29,3,25675.12,0.111026,0.111997,-0.87 +2019-06-30,3,20107.13,0.108023,0.109324,-1.19 +2019-07-01,4,28620.15,0.104299,0.105723,-1.35 +2019-07-02,3,28216.19,0.102191,0.102482,-0.28 +2019-07-03,3,20122.06,0.103717,0.103663,0.05 +2019-07-04,4,46635.47,0.106073,0.102452,3.54 +2019-07-05,2,2909.14,0.100769,0.100157,0.61 +2019-07-06,3,9164.51,0.104673,0.103247,1.38 +2019-07-07,3,2110.07,0.103811,0.104064,-0.24 +2019-07-08,3,3556.63,0.104899,0.104365,0.51 +2019-07-09,4,6929.23,0.102034,0.10196,0.07 +2019-07-10,4,19381.88,0.096224,0.096424,-0.21 +2019-07-11,3,24806.12,0.092795,0.088881,4.4 +2019-07-12,3,37779.56,0.097646,0.096009,1.71 +2019-07-13,3,5210.26,0.09491,0.09588,-1.01 +2019-07-14,3,28961.21,0.088055,0.089768,-1.91 +2019-07-15,3,39037.84,0.085825,0.085413,0.48 +2019-07-16,3,28319.98,0.081896,0.08298,-1.31 +2019-07-17,4,34857.58,0.07909,0.080203,-1.39 +2019-07-18,4,68479.85,0.087022,0.086977,0.05 +2019-07-19,4,25315.42,0.09272,0.090283,2.7 +2019-07-20,3,8043.78,0.095338,0.094032,1.39 +2019-07-21,3,2507.72,0.09211,0.091547,0.62 +2019-07-22,4,4128.49,0.089029,0.08923,-0.22 +2019-07-23,3,3553.16,0.085157,0.085458,-0.35 +2019-07-24,3,6870.5,0.083977,0.083757,0.26 +2019-07-25,3,8020.88,0.086226,0.086242,-0.02 +2019-07-26,3,6921.54,0.085734,0.086232,-0.58 +2019-07-27,3,13337.37,0.085266,0.08564,-0.44 +2019-07-28,3,13745.92,0.083018,0.083642,-0.75 +2019-07-29,3,9783.65,0.083354,0.083882,-0.63 +2019-07-30,3,6518.58,0.083501,0.083331,0.2 +2019-07-31,3,2165.26,0.08407,0.083856,0.25 +2019-08-01,4,1856.44,0.082808,0.082736,0.09 +2019-08-02,3,12480.65,0.082099,0.082517,-0.51 +2019-08-03,3,1766.83,0.082585,0.082225,0.44 +2019-08-04,3,19623.47,0.082804,0.08168,1.38 +2019-08-05,4,3770.97,0.082587,0.082641,-0.07 +2019-08-06,3,3820.19,0.078416,0.07983,-1.77 +2019-08-07,3,8447.06,0.078226,0.078173,0.07 +2019-08-08,3,6064.49,0.076553,0.077089,-0.69 +2019-08-09,3,15994.0,0.074617,0.073914,0.95 +2019-08-10,3,18949.65,0.073622,0.073105,0.71 +2019-08-11,3,6009.5,0.077261,0.076832,0.56 +2019-08-12,3,1437.46,0.075987,0.076523,-0.7 +2019-08-13,3,3214.17,0.073694,0.074534,-1.13 +2019-08-14,3,25966.76,0.071876,0.070928,1.34 +2019-08-15,3,13530.35,0.067852,0.069202,-1.95 +2019-08-16,3,7198.5,0.068362,0.06862,-0.38 +2019-08-17,3,2232.68,0.067792,0.068059,-0.39 +2019-08-18,4,4500.94,0.069814,0.069807,0.01 +2019-08-19,3,2441.62,0.071154,0.071135,0.03 +2019-08-20,3,1684.49,0.069412,0.069662,-0.36 +2019-08-21,3,1247.65,0.066657,0.067053,-0.59 +2019-08-22,4,11224.01,0.06632,0.067727,-2.08 +2019-08-23,4,1835.25,0.068709,0.068524,0.27 +2019-08-24,3,5718.01,0.068666,0.069153,-0.7 +2019-08-25,3,3545.77,0.070083,0.070132,-0.07 +2019-08-26,4,2298.69,0.069959,0.069843,0.17 +2019-08-27,3,5581.11,0.068055,0.06816,-0.15 +2019-08-28,3,18751.86,0.066294,0.066155,0.21 +2019-08-29,3,10835.01,0.061536,0.061942,-0.66 +2019-08-30,3,11783.28,0.06264,0.062293,0.56 +2019-08-31,3,2378.38,0.061242,0.061722,-0.78 +2019-09-01,3,1511.68,0.061672,0.06183,-0.26 +2019-09-02,3,1580.33,0.062695,0.062334,0.58 +2019-09-03,3,3361.35,0.063134,0.063055,0.13 +2019-09-04,3,5619.94,0.062431,0.062034,0.64 +2019-09-05,3,1667.47,0.060831,0.060848,-0.03 +2019-09-06,3,10974.93,0.0591,0.059401,-0.51 +2019-09-07,3,3371.58,0.059111,0.059742,-1.06 +2019-09-08,3,2869.43,0.060891,0.061027,-0.22 +2019-09-09,3,4892.7,0.060288,0.060503,-0.36 +2019-09-10,4,8138.55,0.059904,0.06041,-0.84 +2019-09-11,3,4749.8,0.058501,0.058944,-0.75 +2019-09-12,4,3007.91,0.058302,0.058154,0.25 +2019-09-13,4,7073.15,0.058033,0.057972,0.11 +2019-09-14,4,4658.18,0.058483,0.058067,0.72 +2019-09-15,4,649.09,0.058477,0.058484,-0.01 +2019-09-16,4,1077.0,0.058589,0.058537,0.09 +2019-09-17,4,5946.98,0.063436,0.061906,2.47 +2019-09-18,4,134286.55,0.081251,0.077974,4.2 +2019-09-19,4,197834.2,0.081881,0.081546,0.41 +2019-09-20,4,31087.81,0.075788,0.074806,1.31 +2019-09-21,4,7301.42,0.071496,0.071461,0.05 +2019-09-22,4,13548.15,0.067943,0.068104,-0.24 +2019-09-23,4,21813.03,0.066619,0.066363,0.39 +2019-09-24,4,63372.45,0.055464,0.058473,-5.15 +2019-09-25,4,57718.03,0.055574,0.054888,1.25 +2019-09-26,4,49845.19,0.057285,0.05839,-1.89 +2019-09-27,4,23661.68,0.057922,0.05802,-0.17 +2019-09-28,4,11523.49,0.058309,0.058267,0.07 +2019-09-29,4,6736.92,0.057578,0.057655,-0.13 +2019-09-30,4,11362.4,0.05968,0.059803,-0.21 +2019-10-01,4,12886.56,0.05968,0.060025,-0.57 +2019-10-02,4,3690.88,0.05884,0.058619,0.38 +2019-10-03,4,8798.97,0.058783,0.058678,0.18 +2019-10-04,5,4668.05,0.059141,0.058888,0.43 +2019-10-05,4,5223.83,0.058788,0.058703,0.14 +2019-10-06,4,15352.48,0.058387,0.058298,0.15 +2019-10-07,4,20683.28,0.060975,0.06166,-1.11 +2019-10-08,4,6023.17,0.06235,0.061945,0.65 +2019-10-09,4,5633.61,0.063231,0.062966,0.42 +2019-10-10,3,2266.04,0.061976,0.061839,0.22 +2019-10-11,4,11591.44,0.060714,0.060584,0.21 +2019-10-12,4,6002.07,0.061273,0.061068,0.34 +2019-10-13,4,1773.72,0.06156,0.061771,-0.34 +2019-10-14,4,7107.21,0.063801,0.063582,0.34 +2019-10-15,4,10526.84,0.064192,0.064523,-0.51 +2019-10-16,4,18314.73,0.061846,0.0622,-0.57 +2019-10-17,4,23545.55,0.063937,0.063681,0.4 +2019-10-18,4,15756.5,0.063255,0.063043,0.34 +2019-10-19,4,7426.18,0.062982,0.062903,0.13 +2019-10-20,4,6871.32,0.062946,0.062789,0.25 +2019-10-21,5,3816.26,0.063543,0.063267,0.44 +2019-10-22,5,2750.38,0.064438,0.06413,0.48 +2019-10-23,4,14004.58,0.059316,0.059785,-0.79 +2019-10-24,5,8970.76,0.059776,0.060024,-0.41 +2019-10-25,4,22734.34,0.063764,0.063016,1.19 +2019-10-26,5,36346.38,0.065295,0.064419,1.36 +2019-10-27,5,25326.01,3.3423,0.063837,5135.67 +2019-10-28,5,91869.12,15.846795,0.066135,23861.41 +2019-10-29,5,281621.21,42.207145,0.066933,62958.38 +2019-10-30,4,712350.44,200.049985,0.064434,310372.05 +2019-10-31,5,229847.87,73.840114,0.063617,115970.6 +2019-11-01,5,47311.96,0.070173,0.069143,1.49 +2019-11-02,4,77750.94,268.351635,0.06994,383586.15 +2019-11-03,5,32318.07,0.069226,0.06937,-0.21 +2019-11-04,5,61878.16,130.198254,0.071051,183146.04 +2019-11-05,5,328297.78,0.082942,0.082356,0.71 +2019-11-06,5,117618.07,0.079629,0.079069,0.71 +2019-11-07,6,137243.64,0.074352,0.074568,-0.29 +2019-11-08,6,73273.26,0.071665,0.072795,-1.55 +2019-11-09,6,34995.36,0.072385,0.072382,0.0 +2019-11-10,6,97357.81,0.078173,0.077674,0.64 +2019-11-11,6,70085.66,0.078608,0.078749,-0.18 +2019-11-12,6,155681.23,0.078157,0.077981,0.23 +2019-11-13,5,322442.17,508.700213,0.076686,663256.68 +2019-11-14,6,95065.02,1740.876548,0.073704,2361880.71 +2019-11-15,6,439249.14,902.036746,0.072837,1238339.66 +2019-11-16,6,62397.48,659.570376,0.071615,920900.22 +2019-11-17,6,17056.53,0.071877,0.072136,-0.36 +2019-11-18,6,356526.73,350.630621,0.068361,512812.65 +2019-11-19,6,43992.37,0.064734,0.064839,-0.16 +2019-11-20,6,6731.38,0.064966,0.064852,0.17 +2019-11-21,6,45656.86,0.060064,0.06093,-1.42 +2019-11-22,6,77599.42,0.056861,0.057334,-0.83 +2019-11-23,6,42788.26,0.060365,0.060154,0.35 +2019-11-24,6,178809.47,561.203129,0.058453,959992.79 +2019-11-25,6,104479.68,0.057579,0.056823,1.33 +2019-11-26,5,31067.77,0.057129,0.057222,-0.16 +2019-11-27,6,41105.96,0.057369,0.057621,-0.44 +2019-11-28,5,19664.08,0.05842,0.058362,0.1 +2019-11-29,6,27126.23,0.058771,0.058883,-0.19 +2019-11-30,4,20251.05,0.057525,0.05806,-0.92 +2019-12-01,6,8405.12,0.05678,0.056447,0.59 +2019-12-02,4,28081.39,0.05574,0.056254,-0.91 +2019-12-03,6,11431.38,0.055767,0.055743,0.04 +2019-12-04,4,49393.27,0.055164,0.055329,-0.3 +2019-12-05,5,21358.09,0.054991,0.055038,-0.09 +2019-12-06,5,6699.26,0.055144,0.055162,-0.03 +2019-12-07,5,1445.38,0.05545,0.055571,-0.22 +2019-12-08,5,9757.44,665.5,0.055451,1200055.3 +2019-12-09,4,8040.23,0.055703,0.055179,0.95 +2019-12-10,6,16450.91,3332.21926,0.053564,6220962.53 +2019-12-11,5,3373.02,0.053024,0.053094,-0.13 +2019-12-12,5,8733.06,0.052157,0.052407,-0.48 +2019-12-13,5,2723.92,0.052252,0.052511,-0.49 +2019-12-14,5,7825.49,0.051255,0.051648,-0.76 +2019-12-15,4,10255.37,0.050978,0.050934,0.09 +2019-12-16,5,18527.77,0.048634,0.048694,-0.12 +2019-12-17,5,52059.06,0.044036,0.044705,-1.5 +2019-12-18,5,144087.0,0.044878,0.044792,0.19 +2019-12-19,5,36508.67,0.045481,0.045321,0.35 +2019-12-20,6,11404.15,0.045521,0.045668,-0.32 +2019-12-21,3,6175.14,0.045826,0.045923,-0.21 +2019-12-22,6,26819.81,0.046582,0.046339,0.52 +2019-12-23,3,28323.76,0.045855,0.046203,-0.75 +2019-12-24,5,9965.68,0.045038,0.045065,-0.06 +2019-12-25,4,10356.73,0.044235,0.044382,-0.33 +2019-12-26,3,9209.08,0.044772,0.044887,-0.26 +2019-12-27,4,3595.48,0.04459,0.045278,-1.52 +2019-12-28,4,4154.94,0.045935,0.046105,-0.37 +2019-12-29,4,4287.11,0.046486,0.046117,0.8 +2019-12-30,4,1533.89,0.045849,0.046113,-0.57 +2019-12-31,5,7986.78,0.044798,0.045209,-0.91 +2020-01-01,4,3976.72,0.044912,0.045185,-0.61 +2020-01-02,6,6974.54,2424.241,0.044456,5453049.32 +2020-01-03,4,18242.99,0.044821,0.044963,-0.32 +2020-01-04,5,13452.14,0.045556,0.045487,0.15 +2020-01-05,5,1513.61,0.045584,0.045751,-0.36 +2020-01-06,6,53732.57,0.048725,0.048316,0.85 +2020-01-07,4,23320.55,0.048754,0.049049,-0.6 +2020-01-08,6,26025.2,0.04818,0.048249,-0.14 +2020-01-09,4,9467.08,0.047491,0.047663,-0.36 +2020-01-10,5,35426.54,0.046836,0.04692,-0.18 +2020-01-11,5,3513.93,0.048531,0.048393,0.29 +2020-01-12,2,15312.27,0.048824,0.048772,0.11 +2020-01-13,5,21621.72,0.048006,0.048132,-0.26 +2020-01-14,5,157654.52,0.051478,0.051155,0.63 +2020-01-15,5,145694.52,0.05405,0.054308,-0.47 +2020-01-16,3,1117732.83,0.053691,0.053278,0.78 +2020-01-17,5,470708.59,0.055321,0.059352,-6.79 +2020-01-18,5,270555.86,0.062486,0.062304,0.29 +2020-01-19,3,149165.02,0.061392,0.061597,-0.33 +2020-01-20,3,164791.34,0.06287,0.063134,-0.42 +2020-01-21,5,51339.37,0.063028,0.063267,-0.38 +2020-01-22,4,36521.6,0.062081,0.062437,-0.57 +2020-01-23,6,69917.32,0.058544,0.059183,-1.08 +2020-01-24,4,53235.56,0.057065,0.056854,0.37 +2020-01-25,4,13483.44,0.056135,0.056317,-0.32 +2020-01-26,4,6976.01,0.057448,0.0575,-0.09 +2020-01-27,3,14605.11,0.059222,0.059401,-0.3 +2020-01-28,5,59754.14,0.060556,0.060439,0.19 +2020-01-29,5,59412.2,0.061313,0.061435,-0.2 +2020-01-30,5,46441.84,0.06048,0.06128,-1.31 +2020-01-31,4,12567.88,0.061275,0.061515,-0.39 +2020-02-01,3,13481.45,0.061502,0.06169,-0.31 +2020-02-02,4,36642.18,0.064328,0.06378,0.86 +2020-02-03,4,36263.42,0.064277,0.063948,0.51 +2020-02-04,4,22378.46,0.06499,0.064691,0.46 +2020-02-05,5,73940.9,0.068509,0.067835,0.99 +2020-02-06,5,184535.72,0.070149,0.069903,0.35 +2020-02-07,2,85058.54,0.071873,0.072176,-0.42 +2020-02-08,3,24020.68,0.069724,0.070587,-1.22 +2020-02-09,3,57013.55,0.072307,0.072662,-0.49 +2020-02-10,4,104281.84,0.070369,0.070877,-0.72 +2020-02-11,4,50424.67,0.071965,0.071991,-0.04 +2020-02-12,4,67404.55,0.077583,0.078804,-1.55 +2020-02-13,4,37234.19,0.082342,0.083451,-1.33 +2020-02-14,3,9341.18,0.082426,0.083982,-1.85 +2020-02-15,3,31287.57,0.081201,0.082282,-1.31 +2020-02-16,5,56901.5,0.07098,0.074341,-4.52 +2020-02-17,6,27815.77,0.070118,0.070296,-0.25 +2020-02-18,4,16873.46,0.073256,0.073517,-0.35 +2020-02-19,4,15155.19,0.073789,0.074523,-0.99 +2020-02-20,5,17576.19,0.069468,0.069666,-0.28 +2020-02-21,3,5546.1,0.070255,0.071368,-1.56 +2020-02-22,5,3532.46,2386.065378,0.070368,3390720.01 +2020-02-23,3,156.8,0.072157,0.072264,-0.15 +2020-02-24,3,3835.91,0.068584,0.070039,-2.08 +2020-02-25,3,11558.43,0.065875,0.066061,-0.28 +2020-02-26,5,25060.87,0.058988,0.059959,-1.62 +2020-02-27,3,48991.32,0.06035,0.060051,0.5 +2020-02-28,3,29690.76,0.057665,0.058604,-1.6 +2020-02-29,5,14300.0,0.058142,0.058345,-0.35 +2020-03-01,4,8355.61,0.055431,0.056782,-2.38 +2020-03-02,5,12934.01,0.056943,0.058573,-2.78 +2020-03-03,5,4445.71,0.058265,0.058767,-0.85 +2020-03-04,3,3335.98,0.057579,0.058379,-1.37 +2020-03-05,4,9888.39,0.058738,0.060171,-2.38 +2020-03-06,4,7112.69,0.059585,0.060191,-1.01 +2020-03-07,4,10154.23,0.058874,0.060005,-1.88 +2020-03-08,5,502184.08,6766.837175,0.053396,12672845.89 +2020-03-09,4,23672.79,0.050009,0.050079,-0.14 +2020-03-10,3,9036.87,0.05168,0.051891,-0.41 +2020-03-11,5,23774.53,0.049807,0.050323,-1.03 +2020-03-12,4,43497.46,0.039819,0.038955,2.22 +2020-03-13,4,58095.11,0.036534,0.034811,4.95 +2020-03-14,4,64546.09,0.038082,0.038386,-0.79 +2020-03-15,4,42704.01,0.037972,0.038085,-0.3 +2020-03-16,5,26229.93,0.03512,0.034561,1.62 +2020-03-17,5,29502.84,0.037416,0.036759,1.79 +2020-03-18,3,42395.83,0.036726,0.035883,2.35 +2020-03-19,4,44635.1,0.040315,0.039423,2.26 +2020-03-20,5,48353.94,0.04202,0.041228,1.92 +2020-03-21,3,62309.34,0.040015,0.039474,1.37 +2020-03-22,4,22791.37,0.038968,0.038394,1.5 +2020-03-23,5,13923.59,0.039592,0.03808,3.97 +2020-03-24,3,24642.35,0.040112,0.039754,0.9 +2020-03-25,4,860.01,0.039916,0.039935,-0.05 +2020-03-26,4,1291.67,0.041026,0.041016,0.03 +2020-03-27,3,22139.99,0.041485,0.042057,-1.36 +2020-03-28,4,3085.91,0.039393,0.039538,-0.37 +2020-03-29,3,22292.53,0.038966,0.039013,-0.12 +2020-03-30,5,46718.33,0.040136,0.039745,0.99 +2020-03-31,5,52484.79,0.040554,0.040479,0.18 +2020-04-01,4,35642.39,0.040084,0.039993,0.23 +2020-04-02,5,77619.55,0.041318,0.041284,0.08 +2020-04-03,4,47930.88,0.041479,0.041463,0.04 +2020-04-04,6,42530.6,0.041543,0.041385,0.38 +2020-04-05,5,31827.01,0.042134,0.042226,-0.22 +2020-04-06,5,132978.16,0.047765,0.047639,0.27 +2020-04-07,5,157355.69,0.050061,0.050009,0.11 +2020-04-08,6,68629.79,0.049691,0.049851,-0.32 +2020-04-09,7,12805.75,0.050855,0.050696,0.31 +2020-04-10,5,25225.12,0.047936,0.048327,-0.81 +2020-04-11,4,4239.21,0.048923,0.048976,-0.11 +2020-04-12,4,10479.67,0.04988,0.049864,0.03 +2020-04-13,5,4471.89,0.047674,0.047376,0.63 +2020-04-14,5,54849.34,0.048397,0.04831,0.18 +2020-04-15,4,94954.03,0.04762,0.047387,0.49 +2020-04-16,6,131715.31,0.047641,0.047946,-0.64 +2020-04-17,5,45230.95,0.048921,0.04885,0.15 +2020-04-18,5,39643.61,0.049889,0.049858,0.06 +2020-04-19,4,45597.59,0.049591,0.049439,0.31 +2020-04-20,4,126756.92,0.0508,0.051015,-0.42 +2020-04-21,4,79396.85,0.050305,0.050225,0.16 +2020-04-22,4,55553.71,0.053361,0.053412,-0.1 +2020-04-23,5,189685.43,0.060657,0.061407,-1.22 +2020-04-24,4,141910.31,0.062794,0.062942,-0.24 +2020-04-25,5,62369.87,0.062065,0.062147,-0.13 +2020-04-26,5,61919.38,0.062,0.061928,0.12 +2020-04-27,5,84998.01,0.064205,0.064594,-0.6 +2020-04-28,5,159905.19,0.068908,0.068709,0.29 +2020-04-29,6,157772.76,0.070961,0.070708,0.36 +2020-04-30,5,293181.12,0.070129,0.0706,-0.67 +2020-05-01,4,202281.83,0.071238,0.071568,-0.46 +2020-05-02,5,121197.32,0.074295,0.073846,0.61 +2020-05-03,5,204076.81,0.073819,0.073897,-0.11 +2020-05-04,6,276121.91,0.071129,0.0707,0.61 +2020-05-05,6,110227.12,0.072712,0.072508,0.28 +2020-05-06,5,106095.86,0.072158,0.072088,0.1 +2020-05-07,6,154289.52,0.072072,0.072049,0.03 +2020-05-08,5,98588.2,0.072586,0.072501,0.12 +2020-05-09,5,128552.45,0.072441,0.072472,-0.04 +2020-05-10,6,276346.56,0.063903,0.063738,0.26 +2020-05-11,5,242046.8,0.062944,0.06283,0.18 +2020-05-12,6,188217.82,0.06652,0.067173,-0.97 +2020-05-13,5,138915.99,0.069714,0.069625,0.13 +2020-05-14,6,72632.52,0.069553,0.069534,0.03 +2020-05-15,5,86549.72,0.068742,0.068594,0.22 +2020-05-16,6,45866.64,0.067757,0.067846,-0.13 +2020-05-17,6,937424.15,322580.645161,0.069759,462422156.43 +2020-05-18,5,48556.2,0.069996,0.069989,0.01 +2020-05-19,6,77282.43,0.069441,0.069419,0.03 +2020-05-20,6,148875.96,0.070254,0.070308,-0.08 +2020-05-21,5,239713.85,0.067865,0.06775,0.17 +2020-05-22,4,150351.11,0.067412,0.067642,-0.34 +2020-05-23,4,93995.03,0.067745,0.067715,0.04 +2020-05-24,4,78237.24,0.066623,0.066405,0.33 +2020-05-25,6,93997.42,0.065121,0.064959,0.25 +2020-05-26,6,52605.02,0.065338,0.065002,0.52 +2020-05-27,4,57543.96,0.066101,0.066277,-0.27 +2020-05-28,4,121160.03,0.067378,0.067274,0.15 +2020-05-29,4,120432.04,0.068012,0.068062,-0.07 +2020-05-30,3,178165.51,0.070319,0.070237,0.12 +2020-05-31,5,368857.24,0.073098,0.073374,-0.38 +2020-06-01,4,307412.65,0.07383,0.074247,-0.56 +2020-06-02,5,422910.06,0.079674,0.079452,0.28 +2020-06-03,3,324550.81,0.080216,0.080454,-0.3 +2020-06-04,4,372650.49,0.083214,0.08303,0.22 +2020-06-05,5,285874.18,0.081037,0.081023,0.02 +2020-06-06,4,151722.2,0.079795,0.07987,-0.09 +2020-06-07,4,110219.2,0.078255,0.078351,-0.12 +2020-06-08,6,60941.93,0.079206,0.079322,-0.15 +2020-06-09,6,68070.75,0.078598,0.078559,0.05 +2020-06-10,5,58387.67,0.078462,0.078471,-0.01 +2020-06-11,6,148662.22,0.073653,0.074193,-0.73 +2020-06-12,5,116233.38,0.072879,0.072952,-0.1 +2020-06-13,4,75692.62,0.073143,0.073195,-0.07 +2020-06-14,4,67996.07,0.072311,0.072225,0.12 +2020-06-15,5,79832.2,0.068813,0.068386,0.62 +2020-06-16,5,112614.68,0.071083,0.07098,0.15 +2020-06-17,5,110896.62,0.071823,0.071918,-0.13 +2020-06-18,5,89671.86,0.07052,0.070865,-0.49 +2020-06-19,6,138024.27,0.069658,0.069721,-0.09 +2020-06-20,6,131071.47,0.069341,0.069294,0.07 +2020-06-21,5,45655.46,0.069478,0.069464,0.02 +2020-06-22,5,152818.05,0.070709,0.070788,-0.11 +2020-06-23,5,222230.79,0.07114,0.07121,-0.1 +2020-06-24,5,156590.1,0.070158,0.069909,0.36 +2020-06-25,5,91272.42,0.067721,0.067514,0.31 +2020-06-26,4,112398.2,0.067143,0.067231,-0.13 +2020-06-27,5,66150.0,0.065038,0.063423,2.55 +2020-06-28,4,61640.91,0.063499,0.063615,-0.18 +2020-06-29,4,69754.35,0.064127,0.064109,0.03 +2020-06-30,4,46944.08,0.066323,0.066027,0.45 +2020-07-01,5,105257.94,0.067699,0.067743,-0.06 +2020-07-02,5,179269.03,0.069019,0.06893,0.13 +2020-07-03,4,129370.78,0.067358,0.067341,0.03 +2020-07-04,4,72065.66,0.067323,0.067297,0.04 +2020-07-05,6,74488.44,0.066788,0.066621,0.25 +2020-07-06,5,116642.97,0.070087,0.069587,0.72 +2020-07-07,5,269612.71,0.072048,0.072135,-0.12 +2020-07-08,5,545826.12,0.078771,0.079302,-0.67 +2020-07-09,5,859175.69,0.090558,0.091012,-0.5 +2020-07-10,5,427428.42,0.086796,0.086878,-0.09 +2020-07-11,6,409317.13,0.096176,0.095511,0.7 +2020-07-12,5,317181.38,0.093484,0.093599,-0.12 +2020-07-13,5,384018.43,0.093625,0.093771,-0.16 +2020-07-14,5,251674.11,0.089579,0.089658,-0.09 +2020-07-15,4,105352.22,0.092755,0.092766,-0.01 +2020-07-16,5,94473.06,0.095034,0.095503,-0.49 +2020-07-17,4,548006.28,0.103857,0.103781,0.07 +2020-07-18,4,248774.07,0.101583,0.101392,0.19 +2020-07-19,4,61179.01,0.100534,0.099704,0.83 +2020-07-20,5,83141.09,0.096053,0.09837,-2.35 +2020-07-21,5,262609.88,0.096409,0.096249,0.17 +2020-07-22,5,43364.79,0.095844,0.096225,-0.4 +2020-07-23,6,227346.68,0.099249,0.098924,0.33 +2020-07-24,5,153398.59,0.096623,0.096195,0.44 +2020-07-25,5,77025.42,0.10038,0.098598,1.81 +2020-07-26,5,322876.55,0.100969,0.101412,-0.44 +2020-07-27,5,157002.91,0.098182,0.094729,3.65 +2020-07-28,6,17167.25,0.095035,0.095423,-0.41 +2020-07-29,5,12296.57,0.097346,0.096918,0.44 +2020-07-30,5,17841.57,0.101953,0.095924,6.29 +2020-07-31,5,33190.24,0.096234,0.096213,0.02 +2020-08-01,5,279884.13,0.10542,0.103142,2.21 +2020-08-02,5,721224.23,0.105604,0.10542,0.17 +2020-08-03,5,695012.83,0.106455,0.106083,0.35 +2020-08-04,5,444012.36,0.108115,0.108265,-0.14 +2020-08-05,5,154989.76,0.10807,0.10821,-0.13 +2020-08-06,5,282147.62,0.106784,0.106709,0.07 +2020-08-07,5,185829.04,0.106236,0.104038,2.11 +2020-08-08,4,13813.94,0.103163,0.102813,0.34 +2020-08-09,5,320721.11,0.104701,0.104656,0.04 +2020-08-10,6,374172.18,0.105502,0.10514,0.34 +2020-08-11,5,234786.39,0.105135,0.10223,2.84 +2020-08-12,5,12846.59,0.098864,0.099883,-1.02 +2020-08-13,5,11518.14,0.099945,0.099732,0.21 +2020-08-14,5,344590.11,0.10513,0.104345,0.75 +2020-08-15,5,254746.35,0.105872,0.106,-0.12 +2020-08-16,5,314196.15,0.109124,0.109355,-0.21 +2020-08-17,5,333169.46,0.114208,0.114175,0.03 +2020-08-18,5,239185.58,0.111315,0.111241,0.07 +2020-08-19,5,298555.53,0.106078,0.103517,2.47 +2020-08-20,5,178445.63,0.105215,0.103866,1.3 +2020-08-21,5,353299.04,0.106262,0.104384,1.8 +2020-08-22,4,10949.03,0.100679,0.09997,0.71 +2020-08-23,4,1354.23,0.101247,0.100753,0.49 +2020-08-24,5,280466.82,0.104652,0.104044,0.58 +2020-08-25,4,159720.58,0.101187,0.098624,2.6 +2020-08-26,4,21382.44,0.098517,0.098119,0.41 +2020-08-27,5,127457.53,0.097046,0.094551,2.64 +2020-08-28,5,12542.54,0.095658,0.094798,0.91 +2020-08-29,5,54396.04,0.096359,0.09578,0.6 +2020-08-30,4,123674.94,0.097295,0.097351,-0.06 +2020-08-31,4,239267.1,0.097098,0.096908,0.2 +2020-09-01,5,720123.0,0.09854,0.098803,-0.27 +2020-09-02,5,69727.78,0.094561,0.093575,1.05 +2020-09-03,5,32689.78,0.083284,0.085097,-2.13 +2020-09-04,5,157122.49,0.078136,0.079058,-1.17 +2020-09-05,5,10292.64,0.077718,0.077317,0.52 +2020-09-06,5,12261.13,0.077678,0.075833,2.43 +2020-09-07,5,4011.99,0.076256,0.076612,-0.46 +2020-09-08,6,4976.27,0.076625,0.077208,-0.76 +2020-09-09,4,5638.31,0.078713,0.077978,0.94 +2020-09-10,4,1803.56,0.080513,0.080594,-0.1 +2020-09-11,5,3132.92,0.081259,0.081729,-0.57 +2020-09-12,3,3470.91,0.084463,0.083672,0.95 +2020-09-13,5,3886.31,0.079764,0.081691,-2.36 +2020-09-14,5,5172.83,0.081789,0.080599,1.48 +2020-09-15,4,4318.8,0.07905,0.079716,-0.84 +2020-09-16,5,8474.18,0.079858,0.079083,0.98 +2020-09-17,5,91220.73,0.079675,0.079962,-0.36 +2020-09-18,5,74031.48,0.079663,0.078946,0.91 +2020-09-19,7,19075.21,0.078862,0.078257,0.77 +2020-09-20,4,10402.85,0.078139,0.076977,1.51 +2020-09-21,5,48329.51,0.074969,0.072778,3.01 +2020-09-22,5,4767.04,0.070407,0.070788,-0.54 +2020-09-23,5,40076.45,0.071292,0.069778,2.17 +2020-09-24,5,7302.51,0.070574,0.06926,1.9 +2020-09-25,5,7982.73,0.074142,0.072393,2.42 +2020-09-26,4,37278.39,0.074827,0.07427,0.75 +2020-09-27,4,94639.56,0.073025,0.073058,-0.04 +2020-09-28,5,114810.34,0.074983,0.074587,0.53 +2020-09-29,5,8155.36,0.07309,0.073528,-0.6 +2020-09-30,4,11963.08,0.072971,0.07338,-0.56 +2020-10-01,5,155719.71,0.07528,0.074201,1.45 +2020-10-02,5,6188.24,0.071173,0.070777,0.56 +2020-10-03,4,5118.75,0.071257,0.071288,-0.04 +2020-10-04,3,3933.04,0.07325,0.071827,1.98 +2020-10-05,4,2373.3,0.07316,0.073172,-0.02 +2020-10-06,4,217128.75,0.072469,0.073324,-1.17 +2020-10-07,4,341760.46,0.071828,0.071988,-0.22 +2020-10-08,3,143946.53,0.072899,0.072005,1.24 +2020-10-09,4,329272.06,0.074779,0.074429,0.47 +2020-10-10,4,474925.02,0.077222,0.077088,0.17 +2020-10-11,4,364711.94,0.077576,0.077607,-0.04 +2020-10-12,4,576094.05,0.077558,0.077618,-0.08 +2020-10-13,4,375914.57,0.076609,0.076653,-0.06 +2020-10-14,4,337961.04,0.075215,0.075065,0.2 +2020-10-15,4,257547.79,0.073282,0.073285,-0.0 +2020-10-16,4,449797.41,0.075175,0.075011,0.22 +2020-10-17,6,498928.51,0.080677,0.081182,-0.62 +2020-10-18,4,92332.87,0.080557,0.080655,-0.12 +2020-10-19,5,304377.28,0.084109,0.084425,-0.37 +2020-10-20,4,276463.22,0.08202,0.083138,-1.34 +2020-10-21,5,409867.4,0.083655,0.083767,-0.13 +2020-10-22,4,177128.69,0.085775,0.085675,0.12 +2020-10-23,4,225867.85,0.084502,0.084784,-0.33 +2020-10-24,3,136436.06,0.084416,0.084657,-0.29 +2020-10-25,5,188807.02,0.084129,0.08396,0.2 +2020-10-26,5,157623.29,0.082542,0.082387,0.19 +2020-10-27,6,76253.74,0.081998,0.082044,-0.06 +2020-10-28,4,8817.96,0.080625,0.080142,0.6 +2020-10-29,6,162876.23,0.077361,0.077415,-0.07 +2020-10-30,5,70442.22,0.076973,0.076099,1.15 +2020-10-31,4,25742.73,0.07816,0.078576,-0.53 +2020-11-01,4,56065.31,0.077426,0.077427,-0.0 +2020-11-02,4,107853.01,0.077204,0.077176,0.04 +2020-11-03,5,70577.81,0.074665,0.074592,0.1 +2020-11-04,5,153624.78,0.074567,0.074208,0.48 +2020-11-05,5,533623.12,0.077968,0.077704,0.34 +2020-11-06,5,883464.1,0.083253,0.083122,0.16 +2020-11-07,4,952966.77,0.082564,0.082512,0.06 +2020-11-08,4,126163.98,0.081243,0.081116,0.16 +2020-11-09,4,4857.2,0.080257,0.080102,0.19 +2020-11-10,6,31737.99,0.08128,0.08063,0.81 +2020-11-11,4,48495.36,0.081345,0.081034,0.38 +2020-11-12,5,29138.59,0.08,0.08002,-0.02 +2020-11-13,5,7575.7,0.081142,0.081322,-0.22 +2020-11-14,4,585745.07,0.082194,0.08232,-0.15 +2020-11-15,4,305251.8,0.081285,0.080845,0.54 +2020-11-16,5,223894.43,0.082057,0.081223,1.03 +2020-11-17,4,933957.05,0.085443,0.085525,-0.1 +2020-11-18,4,327950.55,0.083993,0.084204,-0.25 +2020-11-19,4,343255.92,0.084244,0.083881,0.43 +2020-11-20,4,136225.16,0.086214,0.085995,0.25 +2020-11-21,5,2819062.05,0.098695,0.099475,-0.78 +2020-11-22,6,3756515.07,0.104749,0.104917,-0.16 +2020-11-23,4,3465816.52,0.113722,0.113838,-0.1 +2020-11-24,5,3488669.27,0.170352,0.170239,0.07 +2020-11-25,5,2011754.1,0.208895,0.206455,1.18 +2020-11-26,5,1716009.43,0.161948,0.168575,-3.93 +2020-11-27,4,2685684.37,0.184221,0.185881,-0.89 +2020-11-28,3,1410781.67,0.206686,0.207864,-0.57 +2020-11-29,4,1045778.65,0.195011,0.194877,0.07 +2020-11-30,4,1319950.95,0.198954,0.198925,0.01 +2020-12-01,4,638042.94,0.192994,0.191951,0.54 +2020-12-02,4,631918.44,0.183743,0.183582,0.09 +2020-12-03,4,491241.69,0.184164,0.184083,0.04 +2020-12-04,4,1314468.13,0.175984,0.175238,0.43 +2020-12-05,4,594257.5,0.169531,0.16904,0.29 +2020-12-06,4,314251.97,0.173536,0.173778,-0.14 +2020-12-07,4,375748.66,0.170088,0.1704,-0.18 +2020-12-08,5,455178.2,0.16077,0.159239,0.96 +2020-12-09,4,857622.71,0.158017,0.157995,0.01 +2020-12-10,5,436963.96,0.170973,0.169703,0.75 +2020-12-11,3,390136.08,0.158305,0.15821,0.06 +2020-12-12,4,408358.03,0.159232,0.158447,0.5 +2020-12-13,6,854450.12,0.171607,0.170891,0.42 +2020-12-14,6,586774.36,0.171681,0.171625,0.03 +2020-12-15,6,141360.84,0.170661,0.170297,0.21 +2020-12-16,4,808948.25,0.176993,0.17519,1.03 +2020-12-17,5,1540479.49,0.192224,0.191542,0.36 +2020-12-18,5,606977.65,0.183622,0.183613,0.01 +2020-12-19,4,271091.17,0.183856,0.183967,-0.06 +2020-12-20,5,209363.44,0.178792,0.178843,-0.03 +2020-12-21,6,110797.67,0.165743,0.169229,-2.06 +2020-12-22,6,686395.41,0.161586,0.161962,-0.23 +2020-12-23,6,1358208.29,0.138448,0.139618,-0.84 +2020-12-24,6,3309195.29,0.149675,0.144446,3.62 +2020-12-25,6,2680628.58,0.15681,0.156492,0.2 +2020-12-26,6,1502031.6,0.149247,0.148919,0.22 +2020-12-27,7,1599746.11,0.145272,0.145167,0.07 +2020-12-28,3,1658595.28,0.14975,0.149076,0.45 +2020-12-29,6,1307062.55,0.138606,0.138145,0.33 +2020-12-30,5,512763.77,0.133141,0.133093,0.04 +2020-12-31,5,337794.25,0.127025,0.126683,0.27 +2021-01-01,5,278530.29,0.132695,0.133345,-0.49 +2021-01-02,5,75256.46,0.130676,0.130017,0.51 +2021-01-03,6,80333.45,0.132328,0.132209,0.09 +2021-01-04,6,2760994.2,0.147975,0.149607,-1.09 +2021-01-05,6,2369735.15,0.173151,0.179432,-3.5 +2021-01-06,6,6855005.22,0.276063,0.29244,-5.6 +2021-01-07,7,2833847.68,0.352224,0.3499,0.66 +2021-01-08,6,1347696.57,0.292563,0.29932,-2.26 +2021-01-09,5,1813732.32,0.299529,0.301215,-0.56 +2021-01-10,6,1033741.49,0.298377,0.298583,-0.07 +2021-01-11,7,876806.95,0.251253,0.244923,2.58 +2021-01-12,6,1141171.7,0.282687,0.286041,-1.17 +2021-01-13,7,90495.62,0.286246,0.289611,-1.16 +2021-01-14,6,548360.66,0.29909,0.301654,-0.85 +2021-01-15,7,236476.41,0.296941,0.289219,2.67 +2021-01-16,7,191902.59,0.292881,0.295966,-1.04 +2021-01-17,6,88282.33,0.301534,0.302217,-0.23 +2021-01-18,5,684999.03,0.299094,0.300334,-0.41 +2021-01-19,6,1944375.76,0.306766,0.308369,-0.52 +2021-01-20,6,526319.07,0.290399,0.288098,0.8 +2021-01-21,7,2056013.22,0.276087,0.271456,1.71 +2021-01-22,5,315715.61,0.274408,0.257845,6.42 +2021-01-23,7,81952.38,0.275,0.27134,1.35 +2021-01-24,6,643360.45,0.273398,0.273804,-0.15 +2021-01-25,6,707664.27,0.272137,0.271649,0.18 +2021-01-26,5,385259.41,0.257662,0.257414,0.1 +2021-01-27,7,646169.09,0.242451,0.243293,-0.35 +2021-01-28,6,1343563.73,0.268674,0.270505,-0.68 +2021-01-29,7,3859434.25,0.315464,0.320839,-1.68 +2021-01-30,6,1375506.91,0.310444,0.315503,-1.6 +2021-01-31,6,2668396.74,0.321607,0.317921,1.16 diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/summary.json b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/summary.json new file mode 100644 index 00000000..1b8619d1 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/summary.json @@ -0,0 +1,52 @@ +{ + "assets_enumerated": 1194, + "scanned": 1192, + "errors": 0, + "traded_vs_xlm": 11, + "days_with_ref": 1005, + "years": { + "2016": { + "days_with_ref": 1, + "vol_usd": 0.02 + }, + "2017": { + "days_with_ref": 14, + "vol_usd": 31.390000000000004 + }, + "2018": { + "days_with_ref": 228, + "vol_usd": 7594050.72 + }, + "2019": { + "days_with_ref": 365, + "vol_usd": 10688274.440000005 + }, + "2020": { + "days_with_ref": 366, + "vol_usd": 96215632.27999999 + }, + "2021": { + "days_with_ref": 31, + "vol_usd": 39861099.01 + } + }, + "xlm_own_series": { + "2018": { + "days": 21, + "vol_xlm": 4581.33 + }, + "2019": { + "days": 63, + "vol_xlm": 8947.89 + }, + "2020": { + "days": 257, + "vol_xlm": 5238448.71 + }, + "2021": { + "days": 31, + "vol_xlm": 1362866.82 + } + }, + "binance_days": 1000 +} \ No newline at end of file diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/tokens.csv b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/tokens.csv new file mode 100644 index 00000000..2d868bd8 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/data/tokens.csv @@ -0,0 +1,12 @@ +asset,days,vol_usd_at_par,first,last,median_abs_dev_vs_day_pct,days_dev_gt_10pct,days_dev_gt_25pct,median_abs_dev_vs_binance_pct,binance_days +USD:GB2O5PBQJDAFCNM2U2DIMVAEI7ISOYL4UJDTLN42JYYXAENKBWY6OBKZ,427,127640403.73,2019-11-07,2021-01-31,0.0,40,15,0.52,427 +USD:GDUKMGUGDZQK6YHYA5Z6AY2G4XDSZPSZ3SW5UN3ARVMO6QSRDWP5YLEX,902,18663168.12,2018-08-05,2021-01-31,0.0,19,19,0.63,902 +USDT:GC2PJ6J37QM3DOQ63DVGCXTRC5QOKAJUBQO2XKBXOJCWTCOCP75KWUDX,137,4588203.63,2019-06-06,2021-01-13,931.64,112,107,115809.19,137 +USD:GDSRCV5VTM3U7Y3L6DFRP3PEGBNQMGOWSRTGSBWX6Z3H6C7JHRI4XFJP,544,3360485.07,2018-09-27,2021-01-24,0.73,84,73,1.01,544 +USD:GBNLJIYH34UWO5YZFA3A3HD3N76R6DOI33N4JONUOHEEYZYCAYTEJ5AK,316,72076.94,2019-10-04,2021-01-31,3.54,88,49,3.47,316 +USD:GBUYUAI75XXWDZEKLY66CFYKQPET5JR4EENXZBUZ3YXZ7DS56Z4OKOFU,976,19441.54,2016-10-18,2021-01-31,0.75,22,19,0.7,939 +USD:GAEDZ7BHMDYEMU6IJT3CTTGDUSLZWS5CQWZHGP4XUOIDG5ISH3AFAEK2,271,12275.9,2020-03-07,2021-01-31,1.12,7,1,1.05,271 +USDT:GCCK3X3KRXHZOO5OVQHGTJKMBWTTNRDTYB3F4NM3ZEUK37XFMYFCSCLX,30,2790.93,2020-12-21,2021-01-30,2.44,4,1,4.24,30 +USD:GD3LHBHY6NYJXETU6YO5OYI37F3X6L6C46ZWKBFBT35RBRKACDVMRF5K,17,207.93,2020-10-26,2021-01-31,65.74,9,9,65.64,17 +USDT:GC3YRTJQUXVQMG5STS7MSWVF3OF3Z5MVF6NEWZLKNQAJZX5UUFLMCFMA,15,17.05,2020-11-16,2021-01-21,24.58,15,6,24.27,15 +USD:GA5O7NTUIKYDPZSV3MNMIRFO2LCV5RJBNO7TNN7Y3MDO2IXRBSWDEO3T,2,17.0,2020-06-08,2020-11-12,312462.08,2,2,312385.12,2 diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md new file mode 100644 index 00000000..42e6df07 --- /dev/null +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md @@ -0,0 +1,82 @@ +--- +type: R +status: mature +date: 2026-10-13 +who: stkrolikiewicz +--- + +# R — Pre-run report: dollar tokens traded against XLM before 2021-02 + +Safeguard 3 of the task. Built from the public API and Binance, not from +ClickHouse (no prod read access in the session that made it). Data and the +script that produced it are in [`../data/`](../data/). + +## Method + +1. Every issuer of `USD`, `USDT` and `USDC` on Horizon: **1,194** assets. +2. For each, `GET /v1/assets/{id}/ohlcv?granularity=1M&base_currency=XLM` over + 2015-11 → 2021-01: the candles where the token is the base and XLM the quote, + which is how `canonical.rs` stores every dollar token against XLM except + canonical USDT and USDC. 1,192 scanned (the two canonical ones skipped), 0 + errors. +3. Daily candles of the tokens that traded. Implied XLM/USD at par = `1 / pf_vwap`. +4. Per day: the reference is the USD-weighted median of those markets; each + token's deviation from it; Binance XLM/USDT daily VWAP as an outside check + (Binance data starts 2018-05-31; Horizon and CoinGecko no longer serve this + period publicly). + +**Not covered:** canonical USDT, which is XLM's *quote* and so sits in XLM's own +series. The API has no raw XLM/USDT price for it before 2021 (USD fields are +null), only its volume: 21 days and 4,581 XLM in 2018, 63 days and 8,948 XLM in +2019, 257 days and 5.2 M XLM in 2020. + +## Findings + +**Eleven dollar tokens traded against XLM before 2021-02.** + +| token (issuer prefix) | days | USD volume at par | first → last | median dev. from the day | median dev. from Binance | +|---|---|---|---|---|---| +| USD `GB2O5PBQ` | 427 | 127.6 M | 2019-11 → 2021-01 | 0.0 % | 0.52 % | +| USD `GDUKMGUG` (AnchorUSD) | 902 | 18.7 M | 2018-08 → 2021-01 | 0.0 % | 0.63 % | +| **USDT `GC2PJ6J3`** | 137 | 4.6 M | 2019-06 → 2021-01 | **932 %** | **115,809 %** | +| USD `GDSRCV5V` | 544 | 3.4 M | 2018-09 → 2021-01 | 0.73 % | 1.01 % | +| USD `GBNLJIYH` | 316 | 72 k | 2019-10 → 2021-01 | 3.54 % | 3.47 % | +| USD `GBUYUAI7` | 976 | 19 k | 2016-10 → 2021-01 | 0.75 % | 0.70 % | +| USD `GAEDZ7BH` | 271 | 12 k | 2020-03 → 2021-01 | 1.12 % | 1.05 % | +| USDT `GCCK3X3K` | 30 | 2.8 k | 2020-12 → 2021-01 | 2.44 % | 4.24 % | +| **USD `GD3LHBHY`** | 17 | 208 | 2020-10 → 2021-01 | **65.7 %** | **65.6 %** | +| **USDT `GC3YRTJQ`** | 15 | 17 | 2020-11 → 2021-01 | **24.6 %** | **24.3 %** | +| **USD `GA5O7NTU`** | 2 | 17 | 2020-06 → 2020-11 | **312,462 %** | **312,385 %** | + +**Coverage of the reference** (days with at least one dollar market): + +| year | days | USD volume | median abs. dev. from Binance | p90 | days > 10 % off | +|---|---|---|---|---|---| +| 2016 | 1 | $0.02 | — | — | — | +| 2017 | 14 | $31 | — | — | — | +| 2018 | 228 | $7.6 M | 1.87 % | 5.57 % | 1 | +| 2019 | 365 | $10.7 M | 0.61 % | 2.47 % | 15 | +| 2020 | 366 | $96.2 M | 0.34 % | 1.88 % | 4 | +| 2021-01 | 31 | $39.9 M | 0.85 % | 2.67 % | 0 | + +- **Before 2018 there is no usable reference:** one token, $31 in two years. + Priced history in practice starts in **August 2018** (AnchorUSD), daily from + 2019. +- **Where it exists, par agrees with the market:** median 0.3–1.9 % from + Binance by year. +- **The median filter does not stop USDT `GC2PJ6J3`.** It is ~10x off every + other market, but it holds more than half the day's USD-at-par volume on **19 + days**; there it *is* the weighted median and sets the reference. Simulated + with `max_dev` 0.10 and 0.25 alike: **20 days more than 10 % off Binance** in + 2018–2020 with all eleven tokens, **1 day** with the four off-peg tokens + (bold above) left out. + +## So what + +"Every dollar token" as decided on 2026-10-13 gives ~20 days of XLM/USD about +10x wrong in 2019–2020, and with it every XLM-quoted price on those days. The +deviation threshold cannot fix it. It needs a per-token exclusion: either a +list from this report (the four bold tokens), or a rule that drops a token whose +median deviation over its history exceeds the threshold. `max_dev` itself barely +matters here (0.10 and 0.25 give the same days); 0.25 keeps the honest small +markets that sit 3–4 % off. From e9ce5123ce34d44fed8effcba342d934be0f1054 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:04:04 +0200 Subject: [PATCH 067/144] docs(lore-0331): exclude four off-peg tokens from par The pre-run report showed an off-peg USDT setting the reference on 19 days, which no deviation threshold can stop. Record the amended decision: every dollar token except four, excluded by identity, with max_dev 0.25. --- .../README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md index c573ca04..2657d7aa 100644 --- a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -74,6 +74,12 @@ everything quoted in XLM is converted through. a bucket stays unpriced when no dollar-token market traded within the pivot window. 3. **History only.** The change lives in the repricing pass run from fishuser-hero. The live path and the USDC-only rule for 2021-02 onward stay as they are. +4. **Amended the same day after the pre-run report (Stanisław):** every dollar token + except four off-peg ones, excluded by identity (`--par-exclude`, its default): + USDT `GC2PJ6J3`, USD `GD3LHBHY`, USDT `GC3YRTJQ`, USD `GA5O7NTU`. USDT `GC2PJ6J3` + holds most of the volume on 19 days and is then the median itself, which no + threshold can fix. With the four out, 1 day of 2018–2020 is more than 10 % off + Binance, against ~20 with all of them. Threshold `--par-max-dev 0.25`. ## Safeguards @@ -93,13 +99,8 @@ Anyone can issue a token called `USD`, so "every dollar token" needs these: - Which codes make a dollar token: `USD`, `USDT`, `USDC`, others? - `method` label: the existing `assumed-par`, or a new one? -- The outlier threshold. The report finds 0.10 and 0.25 give the same days; 0.25 - keeps the honest small markets that sit 3–4 % off. -- **Per-token exclusion** (from the [pre-run report](notes/R-pre-run-report-dollar-tokens-before-2021.md)). - USDT `GC2PJ6J3` is ~10x off every other market but holds most of the volume on 19 - days, so it becomes the median and the filter cannot drop it: ~20 days of XLM/USD - ~10x wrong in 2019–2020 with every dollar token, 1 day with four off-peg tokens left - out. Keep "every dollar token", or exclude by list or by a per-token rule? +- ~~The outlier threshold~~ and ~~per-token exclusion~~: decided, Decision 4, from the + [pre-run report](notes/R-pre-run-report-dollar-tokens-before-2021.md). - Coverage: before 2018 there is no usable reference (one token, $31 in two years). Priced history starts in August 2018 (AnchorUSD) and is daily from 2019. - Coverage ceiling: `canonical.rs` ranks quotes USDC > USDT > XLM, so XLM's own daily From 048ac03c387b582ef72f3bb9349212ef3f3c1196 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:10:21 +0200 Subject: [PATCH 068/144] docs(lore-0331): correct the off-peg token's deviation USDT GC2PJ6J3 is a median ~1,160x off Binance, not ~10x: it held par in mid-2019 and lost it later. --- .../R-pre-run-report-dollar-tokens-before-2021.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md index 42e6df07..5a9fd79a 100644 --- a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/notes/R-pre-run-report-dollar-tokens-before-2021.md @@ -64,17 +64,21 @@ null), only its volume: 21 days and 4,581 XLM in 2018, 63 days and 8,948 XLM in 2019. - **Where it exists, par agrees with the market:** median 0.3–1.9 % from Binance by year. -- **The median filter does not stop USDT `GC2PJ6J3`.** It is ~10x off every - other market, but it holds more than half the day's USD-at-par volume on **19 - days**; there it *is* the weighted median and sets the reference. Simulated +- **The median filter does not stop USDT `GC2PJ6J3`.** It held par in mid-2019 + (2019-06-06: implied XLM/USD 0.1203 against Binance 0.1219) and then lost it: its + implied XLM/USD is a median **~1,160x** Binance's, ~346 in January 2021. Valued at + $1 its volume is inflated by the same factor, so it holds more than half the + day's USD-at-par volume on **19 days**; there it *is* the weighted median and + sets the reference. (The 932 % in the table is against the day's median, which + it skews itself.) Simulated with `max_dev` 0.10 and 0.25 alike: **20 days more than 10 % off Binance** in 2018–2020 with all eleven tokens, **1 day** with the four off-peg tokens (bold above) left out. ## So what -"Every dollar token" as decided on 2026-10-13 gives ~20 days of XLM/USD about -10x wrong in 2019–2020, and with it every XLM-quoted price on those days. The +"Every dollar token" as decided on 2026-10-13 gives ~20 days of XLM/USD wrong by up +to three orders of magnitude in 2019–2020, and with it every XLM-quoted price on those days. The deviation threshold cannot fix it. It needs a per-token exclusion: either a list from this report (the four bold tokens), or a rule that drops a token whose median deviation over its history exceeds the threshold. `max_dev` itself barely From 0f8e769d2121d26432c15e5f5a18e8adfa02494c Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:32:56 +0200 Subject: [PATCH 069/144] docs(lore-0331): record Marek's acceptance of the exclusions --- .../README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md index 2657d7aa..20b47fee 100644 --- a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -80,6 +80,7 @@ everything quoted in XLM is converted through. holds most of the volume on 19 days and is then the median itself, which no threshold can fix. With the four out, 1 day of 2018–2020 is more than 10 % off Binance, against ~20 with all of them. Threshold `--par-max-dev 0.25`. + **Accepted by Marek the same day.** ## Safeguards From ebc965f6d17cb23b90ba1f227ea0ec32201a776a Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:52:48 +0200 Subject: [PATCH 070/144] docs(lore-0332): open task for USD candle high/low markets USD-mode /ohlcv takes high/low as max/min across every market, each scaled by one rate per bucket. USDT-quoted markets ride on a USD rate that cannot be measured, so their extremes are guesses and win the max/min (XLM 2022-06-18: $0.1767/$0.0815 vs Binance $0.1135/$0.1001). Agreed with Marek and Oskar: only USDC- and XLM-quoted markets set open/high/low/close; every market still counts for volume. --- .../README.md | 103 ++++++++++++++++++ .../notes/R-usdt-legs-widen-usd-candles.md | 82 ++++++++++++++ ...-price-setting-markets-are-usdc-and-xlm.md | 58 ++++++++++ 3 files changed, 243 insertions(+) create mode 100644 lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md create mode 100644 lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md create mode 100644 lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md new file mode 100644 index 00000000..3b6bfe5d --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md @@ -0,0 +1,103 @@ +--- +id: "0332" +title: "USD candles take high/low from USDT-quoted markets, whose USD rate we cannot measure — set the extremes from USDC- and XLM-quoted markets only" +type: BUG +status: active +assignee: akot +related_adr: ["0011", "0287"] +related_tasks: ["0170", "0229", "0236", "0172", "0212", "0286", "0331"] +tags: [layer-backend, priority-high, effort-small, api, ohlcv, methodology, data-correctness, defect] +links: + - "../../../../packages/prices-api/src/assets/queries_ch.rs" + - "../../../../packages/prices-api/src/assets/handlers.rs" + - "../../../../packages/prices-api/tests/ohlcv_it.rs" + - "../../../2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md" + - "../../../../web/portal/public/openapi.json" +history: + - date: "2026-10-14" + status: active + who: akot + note: > + Opened from Adam's analysis of USD-mode /ohlcv high/low. Methodology change + agreed on Slack the same day: Marek ("wyklucz usdt w liczeniu high/low") + and Oskar (same reason USDT was dropped from the USD reference). Scope is + step 1 only (read path); the per-minute rate (step 2) is a follow-up. +--- + +# USD candles take high/low from USDT-quoted markets + +## Summary + +USD-mode `/ohlcv` merges every market of an asset (pair × source) into one +candle: each market is scaled by one rate per bucket, then `high = max`, +`low = min`. USDT-quoted markets are scaled by a USDT/USD rate we cannot +measure, so their extremes are guesses and max/min picks the worst ones. XLM on +2022-06-18 comes back $0.1767 / $0.0815 against Binance's $0.1135 / $0.1001. +The stored data is correct; the read path's merge is not. Only USDC- and +XLM-quoted markets will set the price fields; every market still counts for +volume. + +- Analysis, case study, prior art: [R-usdt-legs-widen-usd-candles](notes/R-usdt-legs-widen-usd-candles.md) +- Decision: [S-price-setting-markets-are-usdc-and-xlm](notes/S-price-setting-markets-are-usdc-and-xlm.md) + +## Status: Active + +**Current state:** methodology agreed (Marek, Oskar, 2026-10-14). No code yet. + +## Implementation Plan + +All in `packages/prices-api/src/assets/queries_ch.rs` unless stated. + +1. **XLM id kept apart.** `UsdRefs` (~L703) gains the price-setting quote ids + (USDC + XLM when tracked). `handlers.rs` ~L560 already resolves XLM; today it + sits in `pivots` next to USDT. `pivots` and the `traded` label stay as is. +2. **`band` predicate + per-bucket fallback.** `usd_projection` (~L1003): + `band = valid AND quote_asset_id IN ()`. `usd_aggregates` + (~L979): `if(countIf(band) > 0, If(.., band), If(.., valid))` for + `o`, `h`, `l`, `c`, `meth`. Keep `CLOSE_EXACT` / `HIGH_PUBLISHED` / + `LOW_PUBLISHED` (~L783–799) as single named expressions so `vw`/`pfvw` still + clamp against what the caller sees; keep the `isNull` overflow guards. +3. **Tests, table-driven.** ITs in `tests/ohlcv_it.rs`: USDC+USDT bucket (USDT + extremes excluded, its volume kept); USDT with the larger volume (o/c/method + still from USDC); XLM+USDT (XLM sets the band); USDT-only (unchanged); vwap + clamped into the new band; the 2022-06-18 shape as a regression. + `ohlcv_converts_each_leg_before_merging_across_them` still discriminates. + Plus the ClickHouse-free SQL-string tests. +4. **Contract text.** `/ohlcv` description in `handlers.rs` ~L398 ("against + every quote asset — are merged") rewritten; `openapi.json` regenerated. +5. **ADR 0011 amendment** under §2b/§3: the allowlist, how a token joins it, the + per-bucket fallback. ⚠️ §1 forbids a denomination that varies with available + data; the amendment must state that the fallback keeps the denomination (USD) + and only changes which markets set the extremes. +6. **Measure and announce.** Read-only before/after on prod; client notice + drafted in `notes/` (sending is Adam's). + +## Acceptance Criteria + +- [ ] USD mode: a bucket with a valid USDC- or XLM-quoted row takes + `open`/`high`/`low`/`close`/`method` from those rows only. +- [ ] A bucket with no such row returns the same candle as before. +- [ ] Volumes and trade counts unchanged for every bucket. +- [ ] `vwap` and `pf_vwap` lie within the published `[low, high]`. +- [ ] `base_currency=XLM` (`QuoteLeg` arm) unchanged. +- [ ] XLM 1d 2022-06-18 returns high ≈ $0.1137, low ≈ $0.1005; XLM p95 high/low + deviation vs Binance recorded before/after and set against the analysis' + step 1+2 figures. +- [ ] Affected assets/buckets counted, including non-USDT legs the allowlist + also drops (the analysis' 419 counts USDT only). +- [ ] ITs pass on CH 26.3.10.60; full ignored suite green. +- [ ] OpenAPI description and ADR 0011 amended in the same PR as the code. +- [ ] Client notice drafted in `notes/`. + +## Follow-ups (not spawned) + +- **Step 2 — per-minute rate for USDC/XLM markets.** Needs `_1m` priced + end-to-end (0286 USD re-pricing) and a choice: read `_1m` at query time, or + store USD extremes in the rollups (schema + re-ingest). Decide once AC 6 shows + what step 1 alone delivers. + +## Notes + +- Pre-2021 ([[0331]]): USDT-quoted rows carry a par `close_usd` there; the + allowlist still keeps them out of the band when an XLM leg traded — intended. +- Execution: `/gsd-quick --full` with a BRIEF, in a worktree. diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md new file mode 100644 index 00000000..1c383b00 --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md @@ -0,0 +1,82 @@ +--- +title: "Why USDT-quoted markets widen USD candles, and how others pick price-setting trades" +type: research +status: mature +spawns: + - notes/S-price-setting-markets-are-usdc-and-xlm.md +tags: [ohlcv, usdt, methodology, high-low] +links: [] +history: + - date: "2026-10-14" + status: mature + who: akot + note: "Adam's analysis, moved out of the task README" +--- + +# Why USDT-quoted markets widen USD candles + +A market here is pair × source: XLM/USDC on SDEX, XLM/USDT on SDEX, XLM/USDC on +Aquarius, and so on. + +## How USD mode computes high/low today + +Each market is converted with one rate for the whole bucket, the day's close: +`rate = close_usd / close` of that market. Then `high_usd = high × rate`, +`low_usd = low × rate`, and `high = max`, `low = min` across markets. + +## Case: XLM, 1d, 2022-06-18 + +| market | share of volume | high | low | +|---|---:|---:|---:| +| XLM/USDC, SDEX | 96 % (28.1 M XLM) | $0.1137 | $0.1005 | +| XLM/USDT, SDEX | 4 % (1.2 M XLM) | 0.3242 USDT → **$0.1767** | 0.1496 USDT → **$0.0815** | +| API today (max/min of both) | — | **$0.1767** | **$0.0815** | +| Binance | — | $0.1135 | $0.1001 | + +The USDT market is converted at the end-of-day USDT rate ($0.5449). That day +USDT traded between $0.43 and $0.96, so an evening rate does not fit morning +trades. The API returns the USDT market's extremes. + +## Why max/min is right and its inputs are wrong + +High and low are, by definition, the extremes of real trades; volume plays no +part. A volume-weighted "average high" (weights 0.96 / 0.04) gave tidy numbers +but was rejected: the result is no longer a price anyone traded at. + +A trade on XLM/USDC has a price in USD. A trade on XLM/USDT has a price in USDT; +its USD value is our estimate, price × a USDT rate we compute ourselves. A +maximum of values measured with error is biased up and a minimum down. The more +estimated markets go in, the wider the candle, whatever the truth. + +## Why Stellar USDT has no usable USD rate + +- "USDT" on Stellar (`GCQTGZ…`) is not Tether but an Apay bridge receipt (issuer + domain `dead.apay.io`). It lost its peg in 2022-06; ~$0.15 today. +- Its only USDC market has turned over a median of 0–27 USDC a day since 2023. +- Within one minute it can disagree with the rate implied through XLM. On + 2022-06-18 USDT/USDC printed 0.43 at 00:49 and 0.957 at 01:12 on ~74 USDC an + hour, while the XLM legs implied 0.36–0.65. +- A per-minute rate would not help: the token has no single, reliable USD price. +- Same reason [[0172]]/[[0212]] fenced the USD reference to USDC (Oskar, + 2026-10-14). + +## How others do it + +- **US equities (CTA tape):** every trade has a sale-condition code that decides + whether it may set high/low. Some trades count for volume only. +- **CoinMarketCap:** drops outlier prices and, manually, markets whose price does + not reflect a free market (e.g. an exchange blocking deposits/withdrawals). +- **CoinGecko:** drops tickers with outlier prices (MAD). + +We do not filter by price; we filter by whether the quote asset's USD value is +known. + +## Measured effect (steps 1 and 2 together) + +- 419 assets traded at the same time in USDT and in USDC or XLM since 2021. +- XLM vs Binance, p95 deviation: high 8.45 % → 1.14 %, low 12.93 % → 2.19 %. +- XLM-quoted tokens: median error 2–4× lower (ETH 1.34 % → 0.57 %, XRP + 0.74 % → 0.18 %). + +The analysis does not separate step 1 (drop USDT from the band) from step 2 +(per-minute rate for USDC/XLM markets). diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md new file mode 100644 index 00000000..55ec6df3 --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md @@ -0,0 +1,58 @@ +--- +title: "Decision: only USDC- and XLM-quoted markets set USD open/high/low/close" +type: synthesis +status: mature +spawned_from: notes/R-usdt-legs-widen-usd-candles.md +spawns: [] +tags: [ohlcv, methodology, high-low, adr-0011] +links: + - "../../../../2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md" +history: + - date: "2026-10-14" + status: mature + who: akot + note: > + Agreed on Slack: Marek ("wyklucz usdt w liczeniu high/low"), Oskar (same + reason USDT was dropped from the USD reference). +--- + +# Decision: only USDC- and XLM-quoted markets set USD prices + +## Conclusion + +Keep the candle definition (high/low = extremes of real trades) but take them +only from trades whose USD value we know. + +1. **Allowlist, not a denylist.** Price-setting quote assets are canonical USDC + and XLM (XLM/USDC is liquid, so XLM's USD rate is well known). Every other + quote leg, USDT included, counts for volume only. A new token (e.g. USDT0) + joins the list once its rate is reliable. +2. **Per bucket.** If a price-setting market has a valid row, open, high, low, + close and `method` come from price-setting markets only. Otherwise the bucket + is computed as today. +3. **Volume, trade counts:** every market. **vwap, pf_vwap:** every convertible + market, clamped into the new `[low, high]`. +4. **Read path only** (`queries_ch.rs`). No stored value changes. +5. **Contract:** amend ADR 0011, the `/ohlcv` description, and tell clients the + methodology changed. + +## Reasoning + +- The defect is the input to max/min, not max/min itself. +- USDT's USD rate is not measurable (see the R- note), so any USDT extreme in + USD is a guess, and max/min selects the worst guesses. +- Open/close today come from the highest-volume market, which can be USDT; the + same argument applies to them. + +## Alternatives considered + +- **Volume-weighted high/low.** Rejected: not a traded price. +- **Per-minute USDT rate.** Rejected: USDT has no single USD price even within a + minute. +- **Leave as is.** Rejected by Marek and Oskar. + +## Step 2 (not decided here) + +Convert USDC/XLM markets at the rate of the trade's minute, not the bucket close. +Matters for XLM-quoted tokens, since XLM moves within the day. Needs `_1m` +priced end-to-end. Decided after step 1 is measured. From 9d91c36acf51583f0ceda521bd3464acb2384361 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:57:42 +0200 Subject: [PATCH 071/144] docs(lore-0332): record no fallback flag, no ADR change Fallback buckets (no USDC/XLM market) are not marked in the response; AC 7 counts them instead. ADR 0011 stays as is; the change ships as code, the /ohlcv description and a client notice. --- .../README.md | 28 ++++++++++++++----- ...-price-setting-markets-are-usdc-and-xlm.md | 4 +-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md index 3b6bfe5d..dfbf55c1 100644 --- a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md @@ -14,6 +14,12 @@ links: - "../../../2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md" - "../../../../web/portal/public/openapi.json" history: + - date: "2026-10-14" + status: active + who: akot + note: > + Decided: fallback buckets get no response flag (measured by AC 7 + instead); ADR 0011 is not amended. Implementation branch opened. - date: "2026-10-14" status: active who: akot @@ -65,11 +71,7 @@ All in `packages/prices-api/src/assets/queries_ch.rs` unless stated. Plus the ClickHouse-free SQL-string tests. 4. **Contract text.** `/ohlcv` description in `handlers.rs` ~L398 ("against every quote asset — are merged") rewritten; `openapi.json` regenerated. -5. **ADR 0011 amendment** under §2b/§3: the allowlist, how a token joins it, the - per-bucket fallback. ⚠️ §1 forbids a denomination that varies with available - data; the amendment must state that the fallback keeps the denomination (USD) - and only changes which markets set the extremes. -6. **Measure and announce.** Read-only before/after on prod; client notice +5. **Measure and announce.** Read-only before/after on prod; client notice drafted in `notes/` (sending is Adam's). ## Acceptance Criteria @@ -84,11 +86,23 @@ All in `packages/prices-api/src/assets/queries_ch.rs` unless stated. deviation vs Binance recorded before/after and set against the analysis' step 1+2 figures. - [ ] Affected assets/buckets counted, including non-USDT legs the allowlist - also drops (the analysis' 419 counts USDT only). + also drops (the analysis' 419 counts USDT only), and fallback buckets + (no USDC/XLM market) counted separately. - [ ] ITs pass on CH 26.3.10.60; full ignored suite green. -- [ ] OpenAPI description and ADR 0011 amended in the same PR as the code. +- [ ] OpenAPI description updated in the same PR as the code. - [ ] Client notice drafted in `notes/`. +## Decisions (2026-10-14, Adam) + +1. **No flag on fallback buckets.** A bucket with no USDC/XLM market is computed + as today and is not marked in the response: a new field changes the response + shape (OpenAPI, portal types, conformance suite, client notice). AC 7 counts + fallback buckets; a flag becomes its own task if there are many. +2. **ADR 0011 is not amended.** The change ships as code, the `/ohlcv` + description and the client notice. The fallback keeps the denomination (USD) + and only narrows which markets set the extremes — the same kind of + data-dependent input choice as today's highest-volume open/close. + ## Follow-ups (not spawned) - **Step 2 — per-minute rate for USDC/XLM markets.** Needs `_1m` priced diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md index 55ec6df3..d5ac1d75 100644 --- a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/S-price-setting-markets-are-usdc-and-xlm.md @@ -33,8 +33,8 @@ only from trades whose USD value we know. 3. **Volume, trade counts:** every market. **vwap, pf_vwap:** every convertible market, clamped into the new `[low, high]`. 4. **Read path only** (`queries_ch.rs`). No stored value changes. -5. **Contract:** amend ADR 0011, the `/ohlcv` description, and tell clients the - methodology changed. +5. **Contract:** the `/ohlcv` description and a client notice. ADR 0011 is not + amended (Adam, 2026-10-14). Fallback buckets carry no response flag. ## Reasoning From ace9f5f8dcc33612ba014feabdade85b0fa51b09 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:51:39 +0200 Subject: [PATCH 072/144] fix(lore-0332): take USD candle prices from USDC/XLM legs A bucket with a valid USDC- or XLM-quoted row takes open, high, low, close and method from those rows; other buckets render as before. --- packages/prices-api/src/assets/handlers.rs | 22 ++-- packages/prices-api/src/assets/queries_ch.rs | 101 ++++++++++++++++--- packages/prices-api/tests/ohlcv_it.rs | 51 ++++++++++ 3 files changed, 152 insertions(+), 22 deletions(-) diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index 336fc9c5..044324cc 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -572,19 +572,21 @@ pub async fn get_ohlcv( // be any candle's quote leg, so it only costs the `traded` // label. A lookup ERROR still fails — that is the database // misbehaving, not an absent asset. - let mut pivots = Vec::new(); - for found in [xlm, usdt] { - match found { - Ok(Some(id)) => pivots.push(id), - Ok(None) => tracing::debug!( + let optional = |found: Result, _>| match found { + Ok(Some(id)) => Ok(Some(id)), + Ok(None) => { + tracing::debug!( "ohlcv pivot reference not tracked; rows on this leg go unlabelled" - ), - Err(e) => return Err(e), + ); + Ok(None) } - } + Err(e) => Err(e), + }; + let (xlm, usdt) = (optional(xlm)?, optional(usdt)?); + let pivots = [xlm, usdt].into_iter().flatten().collect(); // USDC must resolve: see UsdRefs::usdc. match usdc { - Ok(Some(usdc)) => Ok(queries_ch::UsdRefs { usdc, pivots }), + Ok(Some(usdc)) => Ok(queries_ch::UsdRefs { usdc, xlm, pivots }), Ok(None) => Err(clickhouse::error::Error::Custom( "canonical USDC is not tracked".to_string(), )), @@ -643,7 +645,7 @@ pub async fn get_ohlcv( // alternative is that a legitimate USDC identity silently falls back to the // known-empty path, which is the defect this task exists to remove. let (peg_usdc, peg_xlm) = match &args.denomination { - queries_ch::Denomination::Usd(refs) => (Some(refs.usdc), refs.pivots.first().copied()), + queries_ch::Denomination::Usd(refs) => (Some(refs.usdc), refs.xlm), queries_ch::Denomination::QuoteLeg(x) => { match queries_ch::resolve_asset_id(state.ch(), &usdc_identifier()).await { Ok(Some(u)) => (Some(u), Some(*x)), diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index 9ef713d0..ce500c12 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -706,6 +706,9 @@ pub struct UsdRefs { /// indistinguishable from the anomalous same-signature rows on other legs /// and would be dropped. Its absence is a real server-side data gap. pub usdc: u64, + /// XLM: a price-setting quote and the peg series' anchor. Optional, like the + /// pivots. + pub xlm: Option, /// XLM and canonical USDT, the two pivot references. **Optional**: they only /// select the `traded` label. An untracked reference cannot be any candle's /// quote leg, so the branch simply never matches — refusing to serve the @@ -714,6 +717,14 @@ pub struct UsdRefs { pub pivots: Vec, } +impl UsdRefs { + /// The quote legs whose rows set a USD candle's prices. Never empty. + pub fn price_setting(&self) -> Vec { + // A token joins here once its USD rate is reliable (lore-0332). + std::iter::once(self.usdc).chain(self.xlm).collect() + } +} + /// What `base_currency` asks for. Per [ADR 0011] it **denominates**; it does not /// select a quote leg. #[derive(Debug, Clone)] @@ -780,7 +791,9 @@ pub struct OhlcvArgs { /// no clamp: it is derived through the same `rate` as `h_x`/`l_x`, and scaling by /// one positive factor is monotonic, so `l_x <= o_x <= h_x` holds within a row and /// survives `min`/`max` across rows. Only the exact/derived boundary breaks. -const CLOSE_EXACT: &str = "argMaxIf(c_x, (volume_base, quote_asset_id), valid)"; +fn close_exact(p: &str) -> String { + format!("argMaxIf(c_x, (volume_base, quote_asset_id), {p})") +} /// The published `high` / `low`, before rendering. Named because `vw` clamps /// against these rather than against the raw aggregates — the response has to be @@ -795,8 +808,20 @@ const CLOSE_EXACT: &str = "argMaxIf(c_x, (volume_base, quote_asset_id), valid)"; /// NULL the whole expression is NULL, so `greatest(vwap, NULL)` leaves vwap /// unclamped on that side — which is correct: a bound we could not compute must /// not be used to move a value we could. -const HIGH_PUBLISHED: &str = "if(isNull(maxIf(h_x, valid)), NULL, greatest(maxIf(h_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))"; -const LOW_PUBLISHED: &str = "if(isNull(minIf(l_x, valid)), NULL, least(minIf(l_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))"; +fn high_published(p: &str) -> String { + let c = close_exact(p); + format!("if(isNull(maxIf(h_x, {p})), NULL, greatest(maxIf(h_x, {p}), {c}))") +} +fn low_published(p: &str) -> String { + let c = close_exact(p); + format!("if(isNull(minIf(l_x, {p})), NULL, least(minIf(l_x, {p}), {c}))") +} + +/// `expr` over the USDC/XLM rows (`band`) when the bucket has one, else over +/// every valid row, as before lore-0332. +fn banded(expr: impl Fn(&str) -> String) -> String { + format!("if(countIf(band) > 0, {}, {})", expr("band"), expr("valid")) +} /// The volume-weighted mean before clamping — task 0229's review, finding 1. /// @@ -980,21 +1005,28 @@ const PF_ROWS: &str = "pf_trade_count > 0 AND close >= toDecimal128('0.000000000 /// can read it — see [`OUTER_ALIASES`] for why the column tail is not something /// to maintain by hand in three places. fn usd_aggregates() -> String { + // Bound once so `h`, `l`, `vw` and `pfvw` clamp against the same band. The + // `countIf(valid) = 0` guards stay unbanded: a USDT-only bucket keeps its price. + let open = banded(|p| format!("argMaxIf(o_x, (volume_base, quote_asset_id), {p})")); + let high_published = banded(high_published); + let low_published = banded(low_published); + let close_exact = banded(close_exact); + let meth = banded(|p| format!("argMaxIf(meth, (volume_base, quote_asset_id), {p})")); format!( - "if(countIf(valid) = 0, NULL, toString(argMaxIf(o_x, (volume_base, quote_asset_id), valid))) AS o, \ - if(countIf(valid) = 0, NULL, toString({HIGH_PUBLISHED})) AS h, \ - if(countIf(valid) = 0, NULL, toString({LOW_PUBLISHED})) AS l, \ - if(countIf(valid) = 0, NULL, toString({CLOSE_EXACT})) AS c, \ + "if(countIf(valid) = 0, NULL, toString({open})) AS o, \ + if(countIf(valid) = 0, NULL, toString({high_published})) AS h, \ + if(countIf(valid) = 0, NULL, toString({low_published})) AS l, \ + if(countIf(valid) = 0, NULL, toString({close_exact})) AS c, \ toString(sum(volume_base)) AS vb, \ toString(sum(volume_quote_usd)) AS vqu, \ if(countIf(valid) = 0 OR isNull({VWAP_RAW}), NULL, \ - toString(least(greatest({VWAP_RAW}, {LOW_PUBLISHED}), {HIGH_PUBLISHED}))) AS vw, \ + toString(least(greatest({VWAP_RAW}, {low_published}), {high_published}))) AS vw, \ toUInt64(sum(trade_count)) AS tc, \ - nullIf(if(countIf(valid) = 0, NULL, argMaxIf(meth, (volume_base, quote_asset_id), valid)), '') AS meth, \ + nullIf(if(countIf(valid) = 0, NULL, {meth}), '') AS meth, \ if(countIf(valid) = 0, NULL, toUInt8(1)) AS drv, \ toNullable(toUInt64(sum(pf_trade_count))) AS pftc, \ if(isNull({PF_VWAP_USD_RAW}), NULL, \ - toString(least(greatest({PF_VWAP_USD_RAW}, {LOW_PUBLISHED}), {HIGH_PUBLISHED}))) AS pfvw, \ + toString(least(greatest({PF_VWAP_USD_RAW}, {low_published}), {high_published}))) AS pfvw, \ {CLOSE_DIVERGENT_PLACEHOLDER}, \ {PROVENANCE_NULL_TAIL}" ) @@ -1010,6 +1042,8 @@ fn usd_projection(refs: &UsdRefs, granularity: Granularity) -> String { // The whole classification, extracted so its arm ORDER is testable // without a ClickHouse — see [`usd_method_expr`]. let meth_arm = usd_method_expr(usdc, &refs.pivots, granularity); + let ids: Vec = refs.price_setting().iter().map(|i| i.to_string()).collect(); + let price_setting = ids.join(", "); // ⚠️ `pf_trade_count > 0` is a conjunct of `valid` rather than a filter on // the subquery: `valid` gates the PRICE expressions only, while // `sum(volume_base)`, `sum(volume_quote_usd)` and `sum(trade_count)` @@ -1028,6 +1062,7 @@ fn usd_projection(refs: &UsdRefs, granularity: Granularity) -> String { (close >= {floor} AND close_usd >= {floor} \ AND (quote_asset_id = {usdc} OR close_usd != close)) AS convertible, \ (convertible AND pf_trade_count > 0) AS valid, \ + (valid AND quote_asset_id IN ({price_setting})) AS band, \ toFloat64(close_usd) / nullIf(toFloat64(close), 0) AS rate, \ toDecimal128OrNull(toString(toFloat64(open) * rate), 14) AS o_x, \ toDecimal128OrNull(toString(toFloat64(high) * rate), 14) AS h_x, \ @@ -1623,7 +1658,8 @@ pub(crate) fn usd_method_expr(usdc: u64, pivots: &[u64], granularity: Granularit /// Per-source rows are collapsed (`FINAL`) then merged per bucket: `high=max`, /// `low=min`, volumes + `trade_count` summed, `vwap` volume-weighted, and /// `open`/`close` from the highest-volume source (`argMax(.., volume_base)`). -/// Ascending by timestamp. +/// In USD mode open/high/low/close/method come from the USDC/XLM-quoted rows when +/// the bucket has a valid one (lore-0332). Ascending by timestamp. /// /// ## 🔑 In USD mode the conversion happens BEFORE the merge, and the order is /// load-bearing @@ -1730,7 +1766,7 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou // `c` is EXACT (`close_usd` as stored) while `h`/`l` are derived // through `toFloat64`, so the two are on different scales and can // cross — task 0229. `least`/`greatest` pull the derived extremes - // back over the exact close; see the CLOSE_EXACT note above. + // back over the exact close; see the `close_exact` note above. usd_aggregates(), ), // As stored: no conversion, so nothing is derived and there is no USD @@ -2889,10 +2925,47 @@ mod tests { fn usd_refs() -> UsdRefs { UsdRefs { usdc: 2, + xlm: Some(1), pivots: vec![1], } } + /// lore-0332: a bucket with a valid USDC/XLM row takes its prices from those + /// rows; any other bucket renders the pre-0332 text, byte for byte. + #[test] + fn usd_prices_come_from_the_price_setting_legs_when_the_bucket_has_one() { + let agg = usd_aggregates(); + // o 1 + h 3 (h, vw, pfvw) + l 3 + c 1 + meth 1. + assert_eq!(agg.matches("if(countIf(band) > 0, ").count(), 9, "{agg}"); + for p in ["band", "valid"] { + for expr in [ + format!("argMaxIf(o_x, (volume_base, quote_asset_id), {p})"), + format!("maxIf(h_x, {p})"), + format!("minIf(l_x, {p})"), + format!("argMaxIf(c_x, (volume_base, quote_asset_id), {p})"), + format!("argMaxIf(meth, (volume_base, quote_asset_id), {p})"), + ] { + assert!(agg.contains(&expr), "missing `{expr}`: {agg}"); + } + } + for before in [ + "argMaxIf(c_x, (volume_base, quote_asset_id), valid)", + "if(isNull(maxIf(h_x, valid)), NULL, greatest(maxIf(h_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))", + "if(isNull(minIf(l_x, valid)), NULL, least(minIf(l_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))", + ] { + assert!( + agg.contains(before), + "fallback drifted from `{before}`: {agg}" + ); + } + for raw in [VWAP_RAW, PF_VWAP_USD_RAW] { + assert!(!raw.contains("band"), "vwap inputs are unchanged: {raw}"); + } + let refs = usd_refs(); + assert_eq!(refs.price_setting(), vec![2, 1]); + assert_eq!(UsdRefs { xlm: None, ..refs }.price_setting(), vec![2]); + } + /// A candle's prices come only from the rows that held a price-forming /// fill. In USD mode that is one more conjunct on `valid`, which already /// gates every price expression; nothing else in that arm has to change. @@ -2907,6 +2980,10 @@ mod tests { valid.contains("pf_trade_count > 0"), "`valid` must exclude a row whose bucket formed no price: {valid}" ); + assert!( + projection.contains(") AS valid, (valid AND quote_asset_id IN (2, 1)) AS band, "), + "`band` narrows `valid` to the USDC/XLM legs (lore-0332): {projection}" + ); // The pf columns the aggregates weigh have to reach the outer SELECT. for col in ["pf_trade_count", "pf_volume", "pfpv_x"] { assert!( diff --git a/packages/prices-api/tests/ohlcv_it.rs b/packages/prices-api/tests/ohlcv_it.rs index 25cbbb59..cc453541 100644 --- a/packages/prices-api/tests/ohlcv_it.rs +++ b/packages/prices-api/tests/ohlcv_it.rs @@ -2919,3 +2919,54 @@ async fn ohlcv_the_usdc_peg_series_reports_no_price_forming_fields() { teardown(db).await; } + +/// A USDT leg's USD value rests on a rate nobody measures, so in a bucket that +/// also traded against USDC its extremes must not become the candle's (lore-0332). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_a_usdt_leg_no_longer_sets_the_usd_high_and_low() { + let db = "it_ohlcv_usdt_band_0332"; + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + exec(&admin, &assets_insert(db, &[BAZ, USDT])).await; + // Shaped like XLM 1d 2022-06-18: USDC 96 % at the peg, USDT 4 % at 0.5449. + exec( + &admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1d \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) VALUES \ + ('2022-06-18 00:00:00', {BAZ}, {USDC}, 'sdex', 0.105, 0.1137, 0.1005, 0.11, 96, 10.56, 10.56, 0.11, 0.11, 40, 1), \ + ('2022-06-18 00:00:00', {BAZ}, {USDT}, 'sdex', 0.2, 0.3242, 0.1496, 0.2, 4, 0.8, 0.43592, 0.10898, 0.2, 3, 1)" + ), + ) + .await; + + let uri = format!( + "/v1/assets/BAZ:{}/ohlcv?granularity=1d&start=2022-06-18T00:00:00Z\ + &end=2022-06-18T00:00:00Z&base_currency=USD", + iss() + ); + let (status, json) = get(client, &uri).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), 1, "body={json}"); + let c = &data[0]; + let num = |k: &str| c[k].as_str().unwrap().parse::().unwrap(); + assert!( + num("high") < 0.17, + "the USDT leg's high (0.1767) leaked in: {c}" + ); + assert!( + num("low") > 0.09, + "the USDT leg's low (0.0815) leaked in: {c}" + ); + approx(&c["high"], 0.1137); + approx(&c["low"], 0.1005); + approx(&c["open"], 0.105); + approx(&c["close"], 0.11); + approx(&c["volume_base"], 100.0); + assert_eq!(c["method"], "assumed-par", "{c}"); + + teardown(db).await; +} From f9b918001e6433c92d209527b446c2f78c42fcab Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:52:29 +0200 Subject: [PATCH 073/144] test(lore-0332): pin USD price-setting across five buckets USDC+USDT, USDT with the larger volume, XLM+USDT, USDT only (fallback) and a vwap pulled outside the new band. --- packages/prices-api/tests/ohlcv_it.rs | 142 ++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/packages/prices-api/tests/ohlcv_it.rs b/packages/prices-api/tests/ohlcv_it.rs index cc453541..7f5ca704 100644 --- a/packages/prices-api/tests/ohlcv_it.rs +++ b/packages/prices-api/tests/ohlcv_it.rs @@ -2970,3 +2970,145 @@ async fn ohlcv_a_usdt_leg_no_longer_sets_the_usd_high_and_low() { teardown(db).await; } + +/// BAZ on 2026-04-01, one 1h bucket per case. USDC legs sit at the peg (rate +/// 1), USDT legs at rate 0.5, the XLM leg at rate 0.25. +async fn seed_price_setting_0332(db: &str, admin: &Client) { + exec(admin, &assets_insert(db, &[BAZ, USDT])).await; + // lore-0332. 10:00 USDC + USDT; 11:00 USDT has the larger volume; + // 12:00 XLM + USDT; 13:00 USDT only; 14:00 USDT pulls vwap above the band. + exec( + admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1h \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) VALUES \ + ('2026-04-01 10:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.2, 0.9, 1.1, 100, 105, 105, 1.1, 1.05, 10, 1), \ + ('2026-04-01 10:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ + ('2026-04-01 11:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.2, 0.9, 1.1, 10, 11, 11, 1.1, 1.1, 2, 1), \ + ('2026-04-01 11:00:00', {BAZ}, {USDT}, 'sdex', 2.1, 2.2, 1.9, 2.1, 1000, 2100, 1050, 1.05, 2.1, 20, 1), \ + ('2026-04-01 12:00:00', {BAZ}, {XLM}, 'sdex', 10, 12, 9, 10, 100, 1000, 250, 2.5, 10, 7, 1), \ + ('2026-04-01 12:00:00', {BAZ}, {USDT}, 'sdex', 6, 8, 2, 5, 20, 100, 50, 2.5, 5, 3, 1), \ + ('2026-04-01 13:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ + ('2026-04-01 14:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.1, 0.9, 1.0, 10, 10, 10, 1.0, 1.0, 2, 1), \ + ('2026-04-01 14:00:00', {BAZ}, {USDT}, 'sdex', 4, 6, 2, 4, 990, 3960, 1980, 2.0, 4.0, 50, 1)" + ), + ) + .await; +} + +/// In USD mode USDC- and XLM-quoted rows set the prices when the bucket has +/// one; a USDT-only bucket is priced as before (lore-0332). Before-fix values +/// in the comments show what each case discriminates. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one() { + let db = "it_ohlcv_price_setting_0332"; + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + seed_price_setting_0332(db, &admin).await; + + let uri = format!( + "/v1/assets/BAZ:{}/ohlcv?granularity=1h&start=2026-04-01T10:00:00Z\ + &end=2026-04-01T14:00:00Z&base_currency=USD", + iss() + ); + let (status, json) = get(client, &uri).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), 5, "body={json}"); + let at = |hour: &str| { + let ts = format!("2026-04-01T{hour}:00:00Z"); + data.iter() + .find(|c| c["timestamp"] == ts.as_str()) + .unwrap_or_else(|| panic!("no bucket at {ts}: {json}")) + }; + + type Case = ( + &'static str, + &'static str, + [f64; 4], + &'static str, + f64, + Option, + ); + let cases: [Case; 5] = [ + // label, hour, [open, high, low, close], method, volume_base, vwap (= pf_vwap) + // was high 1.5, low 0.5 + ( + "1 usdc+usdt", + "10", + [1.0, 1.2, 0.9, 1.1], + "assumed-par", + 150.0, + Some(155.0 / 150.0), + ), + // was open 1.05, close 1.05, `traded` + ( + "2 usdt larger", + "11", + [1.0, 1.2, 0.9, 1.1], + "assumed-par", + 1010.0, + None, + ), + // was high 4.0, low 1.0 + ( + "3 xlm+usdt", + "12", + [2.5, 3.0, 2.25, 2.5], + "traded", + 120.0, + None, + ), + ( + "4 usdt only", + "13", + [1.0, 1.5, 0.5, 1.0], + "traded", + 50.0, + Some(1.0), + ), + // was high 3.0, close 2.0, vwap 1.99 + ( + "5 vwap clamp", + "14", + [1.0, 1.1, 0.9, 1.0], + "assumed-par", + 1000.0, + Some(1.1), + ), + ]; + for (label, hour, ohlc, method, vb, vwap) in cases { + let c = at(hour); + let num = |k: &str| c[k].as_str().map(|s| s.parse::().unwrap()); + for (field, want) in ["open", "high", "low", "close"].into_iter().zip(ohlc) { + let got = num(field).unwrap_or_else(|| panic!("case {label}: no {field}: {c}")); + assert!( + (got - want).abs() < 1e-6, + "case {label}: {field} {got} != {want}: {c}" + ); + } + assert_eq!(c["method"], method, "case {label}: {c}"); + assert!( + (num("volume_base").unwrap() - vb).abs() < 1e-6, + "case {label}: {c}" + ); + if let Some(want) = vwap { + for field in ["vwap", "pf_vwap"] { + let got = num(field).unwrap_or_else(|| panic!("case {label}: no {field}: {c}")); + assert!( + (got - want).abs() < 1e-6, + "case {label}: {field} {got} != {want}: {c}" + ); + } + } + } + // Case 1: the USDT leg still counts toward volume and trades. + let c1 = at("10"); + approx(&c1["volume_quote_usd"], 155.0); + assert_eq!(c1["trade_count"], 15, "case 1: {c1}"); + assert_eq!(c1["pf_trade_count"], 15, "case 1: {c1}"); + + teardown(db).await; +} From 43912c47aa6cac370dd405ffb64359b5ed21a75e Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:53:50 +0200 Subject: [PATCH 074/144] docs(lore-0332): say which markets set USD candle prices The /ohlcv description and the Candle open/high/low/close texts now name USDC- and XLM-quoted markets; openapi.json regenerated. --- packages/prices-api/src/assets/handlers.rs | 10 ++++++---- packages/prices-api/src/openapi/descriptions.rs | 12 ++++++++---- web/portal/public/openapi.json | 10 +++++----- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index 044324cc..c947f609 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -380,8 +380,8 @@ pub struct OhlcvParams { /// filters to the native quote and returns candles as stored. /// /// Price fields are **absent, not dropped**, on a bucket with no USD value. -/// Candles merge across sources — and, in USD mode, across quote legs — per -/// bucket. `backfill_note` appears only for `timeframe=all` while the backfill +/// Candles merge across sources and quote legs per bucket; in USD mode the +/// USDC/XLM legs set the prices when the bucket has one (lore-0332). `backfill_note` appears only for `timeframe=all` while the backfill /// is still running. #[utoipa::path( get, @@ -395,8 +395,10 @@ pub struct OhlcvParams { the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` \ only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but \ has no USD value yet is returned with its price fields `null`\nrather than omitted, so a \ - chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote \ - asset — are merged per bucket, in ascending time order.\n\nThe window is `timeframe`, ending \ + chart keeps its time axis. Trades from every venue and quote\nasset are merged per bucket, \ + in ascending time order, and all of them count toward the\nvolumes and trade counts. In USD \ + mode `open`, `high`, `low` and `close` come only from\nmarkets quoted in USDC or XLM when the \ + bucket has one, and from every market otherwise.\n\nThe window is `timeframe`, ending \ now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is \ rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the \ historical backfill is still running.", diff --git a/packages/prices-api/src/openapi/descriptions.rs b/packages/prices-api/src/openapi/descriptions.rs index 6a6a3f8a..9a9839a0 100644 --- a/packages/prices-api/src/openapi/descriptions.rs +++ b/packages/prices-api/src/openapi/descriptions.rs @@ -346,7 +346,8 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Closing price of the bucket, in `base_currency`: the LAST price-forming trade of \ this bucket, never one carried over from an earlier one. `null` when the bucket has \ no price — see `pf_trade_count`. Exact in USD mode — the other price fields are \ - scaled, see `derived`.", + scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when \ + the bucket has one.", ), ( "Candle", @@ -373,13 +374,15 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Candle", "high", "Highest price among this bucket's price-forming trades, in `base_currency`; `null` \ - when the bucket has no price — see `pf_trade_count`.", + when the bucket has no price — see `pf_trade_count`. In USD mode only markets \ + quoted in USDC or XLM set it when the bucket has one.", ), ( "Candle", "low", "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` \ - when the bucket has no price — see `pf_trade_count`.", + when the bucket has no price — see `pf_trade_count`. In USD mode only markets \ + quoted in USDC or XLM set it when the bucket has one.", ), ( "Candle", @@ -421,7 +424,8 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Candle", "open", "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of \ - this bucket. `null` when the bucket has no price — see `pf_trade_count`.", + this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD \ + mode only markets quoted in USDC or XLM set it when the bucket has one.", ), ( "Candle", diff --git a/web/portal/public/openapi.json b/web/portal/public/openapi.json index 4ff31a01..b5215761 100644 --- a/web/portal/public/openapi.json +++ b/web/portal/public/openapi.json @@ -319,7 +319,7 @@ "get": { "tags": ["prices"], "summary": "`GET /assets/{asset_identifier}/ohlcv` — candlestick history.", - "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", + "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue and quote\nasset are merged per bucket, in ascending time order, and all of them count toward the\nvolumes and trade counts. In USD mode `open`, `high`, `low` and `close` come only from\nmarkets quoted in USDC or XLM when the bucket has one, and from every market otherwise.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", "operationId": "get_ohlcv", "parameters": [ { @@ -1122,22 +1122,22 @@ }, "open": { "type": ["string", "null"], - "description": "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of this bucket. `null` when the bucket has no price — see `pf_trade_count`.", + "description": "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when the bucket has one.", "example": "0.21155612364244" }, "high": { "type": ["string", "null"], - "description": "Highest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`.", + "description": "Highest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when the bucket has one.", "example": "0.21336358305927" }, "low": { "type": ["string", "null"], - "description": "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`.", + "description": "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when the bucket has one.", "example": "0.21019557835832" }, "close": { "type": ["string", "null"], - "description": "Closing price of the bucket, in `base_currency`: the LAST price-forming trade of this bucket, never one carried over from an earlier one. `null` when the bucket has no price — see `pf_trade_count`. Exact in USD mode — the other price fields are scaled, see `derived`.", + "description": "Closing price of the bucket, in `base_currency`: the LAST price-forming trade of this bucket, never one carried over from an earlier one. `null` when the bucket has no price — see `pf_trade_count`. Exact in USD mode — the other price fields are scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when the bucket has one.", "example": "0.2104557806765" }, "volume_base": { From d2d257d68f1129f57dfa8a3c522b4791c335b21d Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:45:59 +0200 Subject: [PATCH 075/144] fix(lore-0332): address review of USD price-setting Restore the USD-mode scope in the /ohlcv text, name `method` among the banded fields and say "formed a price" instead of "has one". vwap, pf_vwap and close_divergent now say a non-price-setting market can push the mean to a bound. UsdRefs holds XLM once (usdt + pivots()), and an untracked XLM warns, since it now costs XLM its price-setting role until restart. --- packages/prices-api/src/assets/handlers.rs | 34 ++++++++++--------- packages/prices-api/src/assets/queries_ch.rs | 27 +++++++++------ .../prices-api/src/openapi/descriptions.rs | 31 +++++++++++------ web/portal/public/openapi.json | 18 +++++----- 4 files changed, 64 insertions(+), 46 deletions(-) diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index c947f609..c928a81b 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -380,9 +380,10 @@ pub struct OhlcvParams { /// filters to the native quote and returns candles as stored. /// /// Price fields are **absent, not dropped**, on a bucket with no USD value. -/// Candles merge across sources and quote legs per bucket; in USD mode the -/// USDC/XLM legs set the prices when the bucket has one (lore-0332). `backfill_note` appears only for `timeframe=all` while the backfill -/// is still running. +/// Candles merge across sources — and, in USD mode, across quote legs — per +/// bucket; in USD mode the USDC/XLM legs set the prices when one formed a price. +/// `backfill_note` appears only for `timeframe=all` while the backfill is still +/// running. #[utoipa::path( get, path = "/assets/{asset_identifier}/ohlcv", @@ -395,10 +396,11 @@ pub struct OhlcvParams { the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` \ only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but \ has no USD value yet is returned with its price fields `null`\nrather than omitted, so a \ - chart keeps its time axis. Trades from every venue and quote\nasset are merged per bucket, \ - in ascending time order, and all of them count toward the\nvolumes and trade counts. In USD \ - mode `open`, `high`, `low` and `close` come only from\nmarkets quoted in USDC or XLM when the \ - bucket has one, and from every market otherwise.\n\nThe window is `timeframe`, ending \ + chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote \ + asset — are merged per bucket, in ascending time order,\nand all of them count toward the \ + volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close` and `method` come \ + only from markets quoted in USDC or XLM when one of\nthem formed a price in the bucket, \ + and from every market otherwise.\n\nThe window is `timeframe`, ending \ now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is \ rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the \ historical backfill is still running.", @@ -571,24 +573,24 @@ pub async fn get_ohlcv( queries_ch::resolve_asset_id(state.ch(), &usdt_id), ); // The pivots are best-effort: an untracked reference cannot - // be any candle's quote leg, so it only costs the `traded` - // label. A lookup ERROR still fails — that is the database - // misbehaving, not an absent asset. - let optional = |found: Result, _>| match found { + // be any candle's quote leg. It costs the `traded` label and, + // for XLM, its place among the price-setting legs — until a + // restart, as this result is cached. A lookup ERROR still + // fails — that is the database misbehaving, not an absent asset. + let optional = |found: Result, _>, name: &str| match found { Ok(Some(id)) => Ok(Some(id)), Ok(None) => { - tracing::debug!( - "ohlcv pivot reference not tracked; rows on this leg go unlabelled" + tracing::warn!( + "ohlcv {name} reference untracked until restart; see UsdRefs::xlm" ); Ok(None) } Err(e) => Err(e), }; - let (xlm, usdt) = (optional(xlm)?, optional(usdt)?); - let pivots = [xlm, usdt].into_iter().flatten().collect(); + let (xlm, usdt) = (optional(xlm, "XLM")?, optional(usdt, "USDT")?); // USDC must resolve: see UsdRefs::usdc. match usdc { - Ok(Some(usdc)) => Ok(queries_ch::UsdRefs { usdc, xlm, pivots }), + Ok(Some(usdc)) => Ok(queries_ch::UsdRefs { usdc, xlm, usdt }), Ok(None) => Err(clickhouse::error::Error::Custom( "canonical USDC is not tracked".to_string(), )), diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index ce500c12..f6b4a8cc 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -706,18 +706,21 @@ pub struct UsdRefs { /// indistinguishable from the anomalous same-signature rows on other legs /// and would be dropped. Its absence is a real server-side data gap. pub usdc: u64, - /// XLM: a price-setting quote and the peg series' anchor. Optional, like the - /// pivots. + /// XLM and canonical USDT, the two pivot references. **Optional**: an + /// untracked reference cannot be any candle's quote leg, so its branch simply + /// never matches — refusing to serve the endpoint over it would turn a data + /// gap into an outage. XLM is also a price-setting quote and the peg series' + /// anchor. pub xlm: Option, - /// XLM and canonical USDT, the two pivot references. **Optional**: they only - /// select the `traded` label. An untracked reference cannot be any candle's - /// quote leg, so the branch simply never matches — refusing to serve the - /// endpoint over a missing label would turn a cosmetic dependency into an - /// outage. - pub pivots: Vec, + pub usdt: Option, } impl UsdRefs { + /// The pivots that select the `traded` label. + pub fn pivots(&self) -> Vec { + self.xlm.into_iter().chain(self.usdt).collect() + } + /// The quote legs whose rows set a USD candle's prices. Never empty. pub fn price_setting(&self) -> Vec { // A token joins here once its USD rate is reliable (lore-0332). @@ -818,7 +821,8 @@ fn low_published(p: &str) -> String { } /// `expr` over the USDC/XLM rows (`band`) when the bucket has one, else over -/// every valid row, as before lore-0332. +/// every valid row, as before lore-0332. A band extreme that overflows stays +/// NULL: the other legs do not stand in for it. fn banded(expr: impl Fn(&str) -> String) -> String { format!("if(countIf(band) > 0, {}, {})", expr("band"), expr("valid")) } @@ -1041,7 +1045,7 @@ fn usd_projection(refs: &UsdRefs, granularity: Granularity) -> String { let floor = PRECISION_FLOOR; // The whole classification, extracted so its arm ORDER is testable // without a ClickHouse — see [`usd_method_expr`]. - let meth_arm = usd_method_expr(usdc, &refs.pivots, granularity); + let meth_arm = usd_method_expr(usdc, &refs.pivots(), granularity); let ids: Vec = refs.price_setting().iter().map(|i| i.to_string()).collect(); let price_setting = ids.join(", "); // ⚠️ `pf_trade_count > 0` is a conjunct of `valid` rather than a filter on @@ -2926,7 +2930,7 @@ mod tests { UsdRefs { usdc: 2, xlm: Some(1), - pivots: vec![1], + usdt: None, } } @@ -2963,6 +2967,7 @@ mod tests { } let refs = usd_refs(); assert_eq!(refs.price_setting(), vec![2, 1]); + assert_eq!(refs.pivots(), vec![1]); assert_eq!(UsdRefs { xlm: None, ..refs }.price_setting(), vec![2]); } diff --git a/packages/prices-api/src/openapi/descriptions.rs b/packages/prices-api/src/openapi/descriptions.rs index 9a9839a0..2d858dca 100644 --- a/packages/prices-api/src/openapi/descriptions.rs +++ b/packages/prices-api/src/openapi/descriptions.rs @@ -347,7 +347,7 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ this bucket, never one carried over from an earlier one. `null` when the bucket has \ no price — see `pf_trade_count`. Exact in USD mode — the other price fields are \ scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when \ - the bucket has one.", + one of them formed a price in the bucket; otherwise every market does.", ), ( "Candle", @@ -357,8 +357,9 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ price-forming mean, so a wide gap marks a thin or one-sided bucket rather than an \ error: the close is still what it last traded at, and a poor summary of what it is \ worth. Prefer `pf_vwap` when ranking or valuing, and `close` when charting the last \ - price.\n\n`null` when either value is `null` — nothing was compared, which is not \ - the same as the two agreeing.", + price. In USD mode a gap can also come from a market that weighs into `pf_vwap` \ + but does not set `close` — one quoted in USDT, for example.\n\n`null` when either \ + value is `null` — nothing was compared, which is not the same as the two agreeing.", ), ( "Candle", @@ -375,14 +376,16 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "high", "Highest price among this bucket's price-forming trades, in `base_currency`; `null` \ when the bucket has no price — see `pf_trade_count`. In USD mode only markets \ - quoted in USDC or XLM set it when the bucket has one.", + quoted in USDC or XLM set it when one of them formed a price in the bucket; \ + otherwise every market does.", ), ( "Candle", "low", "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` \ when the bucket has no price — see `pf_trade_count`. In USD mode only markets \ - quoted in USDC or XLM set it when the bucket has one.", + quoted in USDC or XLM set it when one of them formed a price in the bucket; \ + otherwise every market does.", ), ( "Candle", @@ -418,14 +421,18 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ cases: when the price fields are `null`; always for \ `base_currency=XLM`, where nothing is converted; and for a USDC-quoted bucket \ below the oracle epoch that carries a converted rate no imported series covers \ - — nothing can attribute it, and a label would be a claim.", + — nothing can attribute it, and a label would be a claim.\n\nIn USD mode the label \ + follows the markets that set the prices: those quoted in USDC or XLM when one of \ + them formed a price in the bucket, otherwise every market. A bucket that traded \ + mostly against USDT can therefore report the `assumed-par` of its USDC market.", ), ( "Candle", "open", "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of \ this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD \ - mode only markets quoted in USDC or XLM set it when the bucket has one.", + mode only markets quoted in USDC or XLM set it when one of them formed a price in \ + the bucket; otherwise every market does.", ), ( "Candle", @@ -451,7 +458,9 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Volume-weighted average price of the bucket's price-forming trades, in \ `base_currency` — `vwap` with the dust left out, and the value `close_divergent` \ compares `close` against.\n\nDenominated exactly like `close` and bounded by the \ - published `low` and `high`, so the response stays self-consistent. `null` when the \ + published `low` and `high`, so the response stays self-consistent. In USD mode it \ + weighs every market, including ones that do not set `low` and `high` (USDT-quoted, \ + for example), so it can sit at one of those bounds. `null` when the \ bucket has no price-forming volume, and on the synthesized USDC self-series. Never \ `0`.", ), @@ -527,8 +536,10 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ zero.\n\nBounded by the published `low` and `high` so the response stays \ self-consistent. Those come from the price-forming trades only, so on a bucket \ where the trades that formed no price carry most of the volume the raw mean falls \ - outside that range and the value reported is the nearer bound. Compare it with \ - `pf_vwap` when the difference matters.", + outside that range and the value reported is the nearer bound. In USD mode the \ + same happens when a market that does not set `low` and `high` — one quoted in \ + USDT, for example — traded away from the ones that do. Compare it with `pf_vwap` \ + when the difference matters.", ), ( "ErrorEnvelope", diff --git a/web/portal/public/openapi.json b/web/portal/public/openapi.json index b5215761..b697c259 100644 --- a/web/portal/public/openapi.json +++ b/web/portal/public/openapi.json @@ -319,7 +319,7 @@ "get": { "tags": ["prices"], "summary": "`GET /assets/{asset_identifier}/ohlcv` — candlestick history.", - "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue and quote\nasset are merged per bucket, in ascending time order, and all of them count toward the\nvolumes and trade counts. In USD mode `open`, `high`, `low` and `close` come only from\nmarkets quoted in USDC or XLM when the bucket has one, and from every market otherwise.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", + "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close` and `method` come only from markets quoted in USDC or XLM when one of\nthem formed a price in the bucket, and from every market otherwise.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", "operationId": "get_ohlcv", "parameters": [ { @@ -1122,22 +1122,22 @@ }, "open": { "type": ["string", "null"], - "description": "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when the bucket has one.", + "description": "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does.", "example": "0.21155612364244" }, "high": { "type": ["string", "null"], - "description": "Highest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when the bucket has one.", + "description": "Highest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does.", "example": "0.21336358305927" }, "low": { "type": ["string", "null"], - "description": "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when the bucket has one.", + "description": "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does.", "example": "0.21019557835832" }, "close": { "type": ["string", "null"], - "description": "Closing price of the bucket, in `base_currency`: the LAST price-forming trade of this bucket, never one carried over from an earlier one. `null` when the bucket has no price — see `pf_trade_count`. Exact in USD mode — the other price fields are scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when the bucket has one.", + "description": "Closing price of the bucket, in `base_currency`: the LAST price-forming trade of this bucket, never one carried over from an earlier one. `null` when the bucket has no price — see `pf_trade_count`. Exact in USD mode — the other price fields are scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does.", "example": "0.2104557806765" }, "volume_base": { @@ -1152,7 +1152,7 @@ }, "vwap": { "type": ["string", "null"], - "description": "Volume-weighted average price of the bucket, in `base_currency`, over the trades it holds — including the ones too small to form a price. `pf_vwap` is the same mean with those left out. `null` when the bucket has no price.\n\nWHICH trades it weighs depends on the denomination, so the two can differ on a bucket that holds a source with no usable price. With `base_currency=XLM` it is every row of the bucket, as stored. With `USD` it is every row that has a usable rate to convert it with — a row whose own price is below the precision floor, or that has no USD value yet, cannot be restated in USD and is left out of the mean rather than counted at zero.\n\nBounded by the published `low` and `high` so the response stays self-consistent. Those come from the price-forming trades only, so on a bucket where the trades that formed no price carry most of the volume the raw mean falls outside that range and the value reported is the nearer bound. Compare it with `pf_vwap` when the difference matters.", + "description": "Volume-weighted average price of the bucket, in `base_currency`, over the trades it holds — including the ones too small to form a price. `pf_vwap` is the same mean with those left out. `null` when the bucket has no price.\n\nWHICH trades it weighs depends on the denomination, so the two can differ on a bucket that holds a source with no usable price. With `base_currency=XLM` it is every row of the bucket, as stored. With `USD` it is every row that has a usable rate to convert it with — a row whose own price is below the precision floor, or that has no USD value yet, cannot be restated in USD and is left out of the mean rather than counted at zero.\n\nBounded by the published `low` and `high` so the response stays self-consistent. Those come from the price-forming trades only, so on a bucket where the trades that formed no price carry most of the volume the raw mean falls outside that range and the value reported is the nearer bound. In USD mode the same happens when a market that does not set `low` and `high` — one quoted in USDT, for example — traded away from the ones that do. Compare it with `pf_vwap` when the difference matters.", "example": "0.2109568287069" }, "trade_count": { @@ -1165,7 +1165,7 @@ }, "method": { "type": ["string", "null"], - "description": "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* `external` — an imported, measured USDC/USD series supplied the rate.\n* `oracle` — a measured Reflector reading supplied the rate.\n* `traded` — priced through a reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`): no measured USDC/USD observation covered the bucket, so the $1 fallback was rendered. Never appears on a quote leg — there the same situation is `assumed-par`.\n\nEach value names the INPUT the rate came from, so `assumed-par` and `external` are never interchangeable: one is an assumption, the other a measurement that may sit percent off par.\n\nOn the USDC self-series a bucket that holds both a Reflector reading and an imported one reports `oracle`: a measured poll outranks an imported rate outright, whichever was observed first. Observation time only breaks ties between rows of the same kind. The imported series is loaded at HOURLY grain, so an hourly request over an imported day reports `external` — and that hour's own measured rate — for each of its twenty-four hours; on 2023-03-11 the 07:00 bucket reports the trough rather than the day's close. Where only a daily row exists for a day, that one row prices every bucket of its UTC day at every `granularity`, so an imported day mixes the two only where the oracle epoch falls inside it (2026-03-11): a bucket that extends past 14:00 UTC that day is not priced from the imported series, which ends at 13:00. The self-series carries the rate row's own `method`, so a measured rate that reads exactly 1.0 is `external` there.\n\nOn a quote leg the label is reconstructed at read time — the candle rows carry no provenance column — so one case is deliberately not separated: a bucket reading exactly 1.0 reports `assumed-par` whether the dollar was assumed or a measured rate happened to land on it. The two leave an identical row and carry the same number, so the label is the conservative one; `external` is reported only for a bucket whose value was actually scaled by an imported rate.\n\n`null` in three cases: when the price fields are `null`; always for `base_currency=XLM`, where nothing is converted; and for a USDC-quoted bucket below the oracle epoch that carries a converted rate no imported series covers — nothing can attribute it, and a label would be a claim.", + "description": "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* `external` — an imported, measured USDC/USD series supplied the rate.\n* `oracle` — a measured Reflector reading supplied the rate.\n* `traded` — priced through a reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`): no measured USDC/USD observation covered the bucket, so the $1 fallback was rendered. Never appears on a quote leg — there the same situation is `assumed-par`.\n\nEach value names the INPUT the rate came from, so `assumed-par` and `external` are never interchangeable: one is an assumption, the other a measurement that may sit percent off par.\n\nOn the USDC self-series a bucket that holds both a Reflector reading and an imported one reports `oracle`: a measured poll outranks an imported rate outright, whichever was observed first. Observation time only breaks ties between rows of the same kind. The imported series is loaded at HOURLY grain, so an hourly request over an imported day reports `external` — and that hour's own measured rate — for each of its twenty-four hours; on 2023-03-11 the 07:00 bucket reports the trough rather than the day's close. Where only a daily row exists for a day, that one row prices every bucket of its UTC day at every `granularity`, so an imported day mixes the two only where the oracle epoch falls inside it (2026-03-11): a bucket that extends past 14:00 UTC that day is not priced from the imported series, which ends at 13:00. The self-series carries the rate row's own `method`, so a measured rate that reads exactly 1.0 is `external` there.\n\nOn a quote leg the label is reconstructed at read time — the candle rows carry no provenance column — so one case is deliberately not separated: a bucket reading exactly 1.0 reports `assumed-par` whether the dollar was assumed or a measured rate happened to land on it. The two leave an identical row and carry the same number, so the label is the conservative one; `external` is reported only for a bucket whose value was actually scaled by an imported rate.\n\n`null` in three cases: when the price fields are `null`; always for `base_currency=XLM`, where nothing is converted; and for a USDC-quoted bucket below the oracle epoch that carries a converted rate no imported series covers — nothing can attribute it, and a label would be a claim.\n\nIn USD mode the label follows the markets that set the prices: those quoted in USDC or XLM when one of them formed a price in the bucket, otherwise every market. A bucket that traded mostly against USDT can therefore report the `assumed-par` of its USDC market.", "example": "oracle" }, "derived": { @@ -1182,12 +1182,12 @@ }, "pf_vwap": { "type": ["string", "null"], - "description": "Volume-weighted average price of the bucket's price-forming trades, in `base_currency` — `vwap` with the dust left out, and the value `close_divergent` compares `close` against.\n\nDenominated exactly like `close` and bounded by the published `low` and `high`, so the response stays self-consistent. `null` when the bucket has no price-forming volume, and on the synthesized USDC self-series. Never `0`.", + "description": "Volume-weighted average price of the bucket's price-forming trades, in `base_currency` — `vwap` with the dust left out, and the value `close_divergent` compares `close` against.\n\nDenominated exactly like `close` and bounded by the published `low` and `high`, so the response stays self-consistent. In USD mode it weighs every market, including ones that do not set `low` and `high` (USDT-quoted, for example), so it can sit at one of those bounds. `null` when the bucket has no price-forming volume, and on the synthesized USDC self-series. Never `0`.", "example": "0.21095682870692" }, "close_divergent": { "type": ["boolean", "null"], - "description": "Whether `close` sits more than 1% away from `pf_vwap`. The close is a SINGLE trade — the last one that formed a price — while `pf_vwap` is the whole bucket's price-forming mean, so a wide gap marks a thin or one-sided bucket rather than an error: the close is still what it last traded at, and a poor summary of what it is worth. Prefer `pf_vwap` when ranking or valuing, and `close` when charting the last price.\n\n`null` when either value is `null` — nothing was compared, which is not the same as the two agreeing.", + "description": "Whether `close` sits more than 1% away from `pf_vwap`. The close is a SINGLE trade — the last one that formed a price — while `pf_vwap` is the whole bucket's price-forming mean, so a wide gap marks a thin or one-sided bucket rather than an error: the close is still what it last traded at, and a poor summary of what it is worth. Prefer `pf_vwap` when ranking or valuing, and `close` when charting the last price. In USD mode a gap can also come from a market that weighs into `pf_vwap` but does not set `close` — one quoted in USDT, for example.\n\n`null` when either value is `null` — nothing was compared, which is not the same as the two agreeing.", "example": false }, "source": { From 3d644c8a0268bfa6ce14228c37d067090c0cfdd4 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:45:59 +0200 Subject: [PATCH 076/144] test(lore-0332): pin dust, minority XLM and overflow buckets A USDC leg that formed no price falls back; XLM sets open/close against a heavier USDT leg; an overflowing extreme stays null in both branches. --- packages/prices-api/tests/ohlcv_it.rs | 56 ++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/packages/prices-api/tests/ohlcv_it.rs b/packages/prices-api/tests/ohlcv_it.rs index 7f5ca704..efb72cab 100644 --- a/packages/prices-api/tests/ohlcv_it.rs +++ b/packages/prices-api/tests/ohlcv_it.rs @@ -2976,7 +2976,9 @@ async fn ohlcv_a_usdt_leg_no_longer_sets_the_usd_high_and_low() { async fn seed_price_setting_0332(db: &str, admin: &Client) { exec(admin, &assets_insert(db, &[BAZ, USDT])).await; // lore-0332. 10:00 USDC + USDT; 11:00 USDT has the larger volume; - // 12:00 XLM + USDT; 13:00 USDT only; 14:00 USDT pulls vwap above the band. + // 12:00 XLM + USDT; 13:00 USDT only; 14:00 USDT pulls vwap above the band; + // 15:00 USDC formed no price + USDT; 16:00 XLM + a larger USDT; + // 17:00 USDT only, overflowing; 18:00 USDC overflowing + USDT. exec( admin, &format!( @@ -2991,7 +2993,26 @@ async fn seed_price_setting_0332(db: &str, admin: &Client) { ('2026-04-01 12:00:00', {BAZ}, {USDT}, 'sdex', 6, 8, 2, 5, 20, 100, 50, 2.5, 5, 3, 1), \ ('2026-04-01 13:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ ('2026-04-01 14:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.1, 0.9, 1.0, 10, 10, 10, 1.0, 1.0, 2, 1), \ - ('2026-04-01 14:00:00', {BAZ}, {USDT}, 'sdex', 4, 6, 2, 4, 990, 3960, 1980, 2.0, 4.0, 50, 1)" + ('2026-04-01 14:00:00', {BAZ}, {USDT}, 'sdex', 4, 6, 2, 4, 990, 3960, 1980, 2.0, 4.0, 50, 1), \ + ('2026-04-01 15:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ + ('2026-04-01 16:00:00', {BAZ}, {XLM}, 'sdex', 10, 12, 9, 10, 10, 100, 25, 2.5, 10, 2, 1), \ + ('2026-04-01 16:00:00', {BAZ}, {USDT}, 'sdex', 6, 8, 2, 5, 200, 1000, 500, 2.5, 5, 9, 1), \ + ('2026-04-01 17:00:00', {BAZ}, {USDT}, 'sdex', 1000, 1000, 1000, 0.000000000001, 10, 0.00000000001, 10, 10000000000, 1000, 3, 1), \ + ('2026-04-01 18:00:00', {BAZ}, {USDC}, 'sdex', 1000, 1000, 1000, 0.000000000001, 10, 0.00000000001, 10, 10000000000, 1000, 3, 1), \ + ('2026-04-01 18:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1)" + ), + ) + .await; + // 15:00: a USDC leg that traded only dust is not `valid`, so not `band`. + exec( + admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1h \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, \ + pf_trade_count, pf_volume, pf_price_volume, version) VALUES \ + ('2026-04-01 15:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.2, 0.9, 1.1, 10, 11, 11, 1.1, 1.1, 2, \ + 0, 0, 0, 1)" ), ) .await; @@ -3010,13 +3031,13 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one let uri = format!( "/v1/assets/BAZ:{}/ohlcv?granularity=1h&start=2026-04-01T10:00:00Z\ - &end=2026-04-01T14:00:00Z&base_currency=USD", + &end=2026-04-01T18:00:00Z&base_currency=USD", iss() ); let (status, json) = get(client, &uri).await; assert_eq!(status, StatusCode::OK, "body={json}"); let data = json["data"].as_array().unwrap(); - assert_eq!(data.len(), 5, "body={json}"); + assert_eq!(data.len(), 9, "body={json}"); let at = |hour: &str| { let ts = format!("2026-04-01T{hour}:00:00Z"); data.iter() @@ -3032,7 +3053,7 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one f64, Option, ); - let cases: [Case; 5] = [ + let cases: [Case; 7] = [ // label, hour, [open, high, low, close], method, volume_base, vwap (= pf_vwap) // was high 1.5, low 0.5 ( @@ -3078,6 +3099,24 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one 1000.0, Some(1.1), ), + // was 1.0, 1.2, 0.9, 1.1 with `band` not requiring `valid` + ( + "6 usdc dust+usdt", + "15", + [1.0, 1.5, 0.5, 1.0], + "traded", + 60.0, + None, + ), + // was open 3.0, high 4.0, low 1.0 + ( + "7 xlm smaller", + "16", + [2.5, 3.0, 2.25, 2.5], + "traded", + 210.0, + None, + ), ]; for (label, hour, ohlc, method, vb, vwap) in cases { let c = at(hour); @@ -3104,6 +3143,13 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one } } } + // An extreme that overflows stays null in the fallback (8) and in the band + // (9, was high 3.0): the USDT leg does not stand in for the USDC one. + for (label, hour) in [("8 usdt overflow", "17"), ("9 usdc overflow+usdt", "18")] { + let c = at(hour); + assert!(c["high"].is_null(), "case {label}: {c}"); + assert_eq!(c["close"], "10000000000", "case {label}: {c}"); + } // Case 1: the USDT leg still counts toward volume and trades. let c1 = at("10"); approx(&c1["volume_quote_usd"], 155.0); From b1686b897332c813852c03300fb615b8cb24a2be Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:02:20 +0200 Subject: [PATCH 077/144] docs(lore-0331): reassign to Adam --- .../README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md index 20b47fee..773d5a23 100644 --- a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -3,7 +3,7 @@ id: "0331" title: "No USD price before 2021-02 — price that history by counting every dollar token at par ($1)" type: FEATURE status: active -assignee: stkrolikiewicz +assignee: akot related_adr: [] related_tasks: ["0286", "0201", "0172", "0212", "0267", "0294"] tags: [layer-backend, priority-high, effort-medium, enrichment, clickhouse, data-correctness] @@ -13,6 +13,12 @@ links: - "../../../../packages/prices-clickhouse/schema/current.sql" - "../../../../docs/scf/milestone-3-evidence.md" history: + - date: "2026-10-14" + status: active + who: akot + note: > + Reassigned from stkrolikiewicz to Adam, who takes the task over + (PR #401 included). - date: "2026-10-13" status: active who: stkrolikiewicz From fa12b068cb942ffffb84f9cee65b87d6c09b8396 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:03:31 +0200 Subject: [PATCH 078/144] docs(lore-0332): record step 1 measured on prod Step 1 alone gives the analysis' XLM figures on its 557-day window; 37,582 buckets in 419 assets lose a USDT leg. Remaining cheap lows are mostly single Aquarius XLM/USDC fills. Client notice drafted; PR #402. --- .../README.md | 49 ++++--- .../G-client-notice-usd-candle-prices.md | 72 +++++++++++ .../notes/R-step-1-measured-on-production.md | 121 ++++++++++++++++++ .../notes/R-usdt-legs-widen-usd-candles.md | 1 + 4 files changed, 228 insertions(+), 15 deletions(-) create mode 100644 lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md create mode 100644 lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-step-1-measured-on-production.md diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md index dfbf55c1..de88b03b 100644 --- a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md @@ -14,6 +14,14 @@ links: - "../../../2-adrs/0011_base-currency-is-a-denomination-not-a-pair-filter.md" - "../../../../web/portal/public/openapi.json" history: + - date: "2026-10-14" + status: active + who: akot + note: > + Code in PR #402 (5 commits, not merged). ITs green on CH 26.3.10.60: + ohlcv_it 45, ignored suite 318/0 over 40 targets, workspace 1437/0. + Step 1 measured on prod (AC 6, 7); client notice drafted (AC 10). + Left: merge, compute deploy, sending the notice. - date: "2026-10-14" status: active who: akot @@ -45,10 +53,12 @@ volume. - Analysis, case study, prior art: [R-usdt-legs-widen-usd-candles](notes/R-usdt-legs-widen-usd-candles.md) - Decision: [S-price-setting-markets-are-usdc-and-xlm](notes/S-price-setting-markets-are-usdc-and-xlm.md) +- Measured on prod: [R-step-1-measured-on-production](notes/R-step-1-measured-on-production.md) +- Client notice draft: [G-client-notice-usd-candle-prices](notes/G-client-notice-usd-candle-prices.md) ## Status: Active -**Current state:** methodology agreed (Marek, Oskar, 2026-10-14). No code yet. +**Current state:** PR #402 open, not merged; not deployed. Notice drafted, not sent. ## Implementation Plan @@ -76,21 +86,23 @@ All in `packages/prices-api/src/assets/queries_ch.rs` unless stated. ## Acceptance Criteria -- [ ] USD mode: a bucket with a valid USDC- or XLM-quoted row takes +- [x] USD mode: a bucket with a valid USDC- or XLM-quoted row takes `open`/`high`/`low`/`close`/`method` from those rows only. -- [ ] A bucket with no such row returns the same candle as before. -- [ ] Volumes and trade counts unchanged for every bucket. -- [ ] `vwap` and `pf_vwap` lie within the published `[low, high]`. -- [ ] `base_currency=XLM` (`QuoteLeg` arm) unchanged. -- [ ] XLM 1d 2022-06-18 returns high ≈ $0.1137, low ≈ $0.1005; XLM p95 high/low +- [x] A bucket with no such row returns the same candle as before. +- [x] Volumes and trade counts unchanged for every bucket. +- [x] `vwap` and `pf_vwap` lie within the published `[low, high]`. +- [x] `base_currency=XLM` (`QuoteLeg` arm) unchanged. +- [x] XLM 1d 2022-06-18 returns high ≈ $0.1137, low ≈ $0.1005; XLM p95 high/low deviation vs Binance recorded before/after and set against the analysis' - step 1+2 figures. -- [ ] Affected assets/buckets counted, including non-USDT legs the allowlist + step 1+2 figures. Prod: $0.1137 / $0.1005; on the analysis' 557 days step + 1 alone gives its 1.14 % / 2.16 % (see the R- note). +- [x] Affected assets/buckets counted, including non-USDT legs the allowlist also drops (the analysis' 419 counts USDT only), and fallback buckets - (no USDC/XLM market) counted separately. -- [ ] ITs pass on CH 26.3.10.60; full ignored suite green. -- [ ] OpenAPI description updated in the same PR as the code. -- [ ] Client notice drafted in `notes/`. + (no USDC/XLM market) counted separately. 37,582 buckets / 419 assets, + all USDT (no other quote is convertible); fallback 1,345 / 160. 1d only. +- [x] ITs pass on CH 26.3.10.60; full ignored suite green. +- [x] OpenAPI description updated in the same PR as the code. +- [x] Client notice drafted in `notes/`. ## Decisions (2026-10-14, Adam) @@ -107,8 +119,15 @@ All in `packages/prices-api/src/assets/queries_ch.rs` unless stated. - **Step 2 — per-minute rate for USDC/XLM markets.** Needs `_1m` priced end-to-end (0286 USD re-pricing) and a choice: read `_1m` at query time, or - store USD extremes in the rollups (schema + re-ingest). Decide once AC 6 shows - what step 1 alone delivers. + store USD extremes in the rollups (schema + re-ingest). AC 6: for XLM, step 1 + alone already gives the analysis' step 1+2 figures; any gain from step 2 + would come from XLM-quoted tokens (ETH, XRP in the analysis). +- **Cheap single fills on the USDC/XLM markets themselves.** After step 1, + most remaining XLM low outliers are Aquarius XLM/USDC fills far below the + market (62 of 121 days >5 % off, 2025–26, drifting from $0.20 to $0.03). This + looks like a stale pool that nobody arbitrages; which pool is not yet + established. The analysis' ±50 % band (rule 3) removed these prints. See the + R- note. ## Notes diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md new file mode 100644 index 00000000..977bcb74 --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md @@ -0,0 +1,72 @@ +--- +title: "Client notice draft: USD candle prices come from USDC- and XLM-quoted markets" +type: generation +status: developing +spawned_from: notes/R-step-1-measured-on-production.md +spawns: [] +tags: [ohlcv, client-notice, high-low] +links: + - "S-price-setting-markets-are-usdc-and-xlm.md" +history: + - date: "2026-10-14" + status: developing + who: akot + note: > + Draft for Adam to send. Effective date left open until the compute + deploy is scheduled. +--- + +# Client notice draft + +Adam sends this. Fill in the date once the deploy is scheduled. + +--- + +**Subject:** Prices API: USD candles now take prices only from USD-measurable markets (effective ``) + +From ``, `GET /assets/{asset}/ohlcv` in USD mode (`base_currency=USD`, +the default) changes which markets set a candle's prices. + +**What changes** + +A candle merges every market of an asset: each trading pair on each venue. +Each market is converted to USD before the merge. A market quoted in Stellar +USDT (the Apay bridge token, `GCQTGZ…`, not Tether) has no reliable USD rate, +so its converted high and low could land far from where the asset traded. On +XLM, 2022-06-18, the API returned a high of $0.1767 and a low of $0.0815; +Binance shows $0.1135 and $0.1001. + +From ``: + +- `open`, `high`, `low`, `close` and `method` come only from markets quoted + in USDC or XLM, when one of them formed a price in the bucket. That same + day now returns $0.1137 / $0.1005. +- A bucket in which no USDC- or XLM-quoted market formed a price is computed + exactly as before. +- `volume_base`, `volume_quote_usd`, `trade_count` and `pf_trade_count` are + unchanged and still count every market. +- `vwap` and `pf_vwap` still weigh every market and stay within the published + `low` and `high`. When a USDT-quoted market traded outside the USDC/XLM + range, the reported mean is the nearer bound. `close_divergent` can flag + such a bucket. +- `method` follows the markets that set the prices. A bucket that traded + mostly against USDT can now report the `assumed-par` of its USDC market. +- `base_currency=XLM` is unchanged. + +The response shape does not change: no field is added, removed or renamed. + +**Who is affected** + +Measured on daily candles since 2021-02: 0.34 % of priced buckets (37,582), +across 419 assets that traded against USDT and against USDC or XLM on the same +day. High or low moves in 16,136 of those buckets, across 340 assets. For XLM, +the 95th-percentile gap to Binance falls from 23 % to 1.2 % for the daily high +and from 44 % to 6.4 % for the daily low. + +**What you need to do** + +Nothing, unless you store candles. Candles you fetched before `` for +the affected assets can differ from what the API returns now; fetch them +again to get the new values. + +The `/ohlcv` description in the API reference states the rule. diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-step-1-measured-on-production.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-step-1-measured-on-production.md new file mode 100644 index 00000000..5d3249b2 --- /dev/null +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-step-1-measured-on-production.md @@ -0,0 +1,121 @@ +--- +title: "Step 1 measured on production: XLM vs Binance, affected buckets, and where the remaining cheap lows come from" +type: research +status: mature +spawned_from: notes/R-usdt-legs-widen-usd-candles.md +spawns: [] +tags: [ohlcv, usdt, high-low, measurement, aquarius] +links: + - "R-usdt-legs-widen-usd-candles.md" +history: + - date: "2026-10-14" + status: mature + who: akot + note: > + Read-only measurement on prod (dev_read) of PR #402's SQL against the + code it replaces, 1d buckets 2021-02-01..2026-10-13. AC 6 and AC 7. +--- + +# Step 1 measured on production + +## Method + +- The query is the one PR #402 generates (taken from `system.query_log` of + the `ohlcv_it` run), widened to every asset and grouped by asset × day. +- The old candle is the same query with `band` false everywhere: the fallback + branch is byte-identical to the pre-change SQL, so this is today's API. +- Both run on `prices.price_ohlcv_1d` as `dev_read`, month by month. The + 3.73 GiB per-query limit rejects the whole range at once. +- Binance: `XLMUSDT` 1d klines from `data-api.binance.vision`. +- Only `1d` was measured, not `1h` or `1m`. The scripts stayed in the + session scratchpad. + +## AC 6: XLM vs Binance + +XLM 1d 2022-06-18: $0.1767 / $0.0815 → **$0.1137 / $0.1005** (Binance +$0.1135 / $0.1001). + +p95 of |ours / Binance − 1|: + +| window | days | high: before → after | low: before → after | +|---|---:|---|---| +| the analysis' window | 557 | 8.45 % → **1.14 %** | 12.93 % → **2.16 %** | +| 2022-07 → 2026-08 | 1524 | 27.7 % → 1.33 % | 52.3 % → 8.18 % | +| all | 2081 | 23.2 % → 1.24 % | 43.5 % → 6.44 % | + +Medians after: 0.16 % (high), 0.19 % (low). + +**Step 1 alone reproduces the analysis' "steps 1 + 2" figures.** The analysis +measured 557 days, the ones where `1m` had prices: 2021-02 → 2022-06 and +2026-09 → 2026-10-12. On those days step 1 gives 1.14 % / 2.16 % against the +analysis' 1.14 % / 2.19 %. For XLM, step 2 adds almost nothing, because its +price-setting market is USDC, whose USD rate stays near 1 all day. The analysis' +gains for XLM-quoted tokens (ETH, XRP) are a separate measurement and still +stand. + +Its baseline was lower (8.45 % / 12.93 % against 23.2 % / 43.5 %) only because +the window left out 2022-07 → 2026-08, where most bad days are. + +## AC 7: what changes (11,114,887 priced 1d buckets) + +| | buckets | assets | +|---|---:|---:| +| a leg leaves the band | 37,582 (0.34 %) | 419 | +| … of which USDT | 37,582 | 419 | +| … of which another quote | 0 | 0 | +| high or low changes | 16,136 | 340 | +| open or close changes | 4,519 | — | +| `method` changes | 528 | — | +| band extreme newly `null` (review IN-03) | 0 | — | +| fallback: no valid USDC/XLM leg | 1,345 | 160 | + +- In practice the allowlist drops only USDT. A row on any other quote fails + `convertible`, so it never entered the price before either. +- Where the high moves, the median move is −3.0 %. At p95 the old high was + about 3× the new one. The low moves by a median +3.2 % and a p95 +57 %. +- Fallback buckets are few and falling: 405 in 2021, 17 in 2026. This + supports Decision 1 (no flag). +- Changed buckets per year: + + | 2021 | 2022 | 2023 | 2024 | 2025 | 2026 | + |---:|---:|---:|---:|---:|---:| + | 2,486 | 4,534 | 1,780 | 2,484 | 2,698 | 2,154 | + +## What step 1 leaves: cheap lows on the USDC markets themselves + +After step 1, 121 XLM days have a low more than 5 % off Binance. Market that +sets the low: + +| source | days | low vs Binance (median / worst) | that market's close vs Binance | years | +|---|---:|---|---|---| +| Aquarius | 62 | −56 % / −98 % | −0.03 % | 2025–26 | +| SDEX | 29 | +5.6 % / −34 % | −0.04 % | mostly 2021 | +| Phoenix | 12 | −8 % / −29 % | −0.7 % | 2024 | +| Soroswap | 9 | −18 % / −46 % | −12.7 % | 2024 | +| Sushiswap | 9 | −37 % / −99 % | −26.5 % | 2025–26 | + +- **SDEX**: mostly real cross-venue gaps on the 2021 crash days, which the + analysis also found. +- **Phoenix, Soroswap, Sushiswap**: thin pools trading away from the market. +- **Aquarius**: on a market of thousands of fills a day with a correct close, + single fills print far below the price. The cheap low drifts down steadily: + + | 2025-10 | 2026-03 | 2026-04 | 2026-06 | 2026-08 | 2026-10 | + |---:|---:|---:|---:|---:|---:| + | $0.20 | $0.14 | $0.07 | $0.056 | $0.045 | $0.032 | + + At `1m`, 2026-09-25 08:38 holds 2 fills and 1,001 XLM: 1,000 XLM at + 0.2188 and exactly 1 XLM for 0.0383677 USDC. The arithmetic closes, so this + is a real swap and not a decoding error. It looks like one stale pool that + nobody arbitrages. + + Aquarius lists four XLM/USDC pools: + - a concentrated pool and a constant-product pool at 0.1 %, both deep; + - a stable-curve pool, now empty (`CAQXW3…`); + - a constant-product pool at 0.3 % holding ~$40 (`CARCLMAP…`). + + Which of them printed the fill is not established. Candles carry no pool + id, and stellar.expert's event index for these pools ends at 2025-12-31. +- The analysis' "rule 3" (a ±50 % band around the day's median price) removed + exactly this kind of print: on its window, the worst high went from 395 % to + 12.7 % and the worst low from 85 % to 18.2 %. It was not part of 0332. diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md index 1c383b00..a6efced9 100644 --- a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/R-usdt-legs-widen-usd-candles.md @@ -4,6 +4,7 @@ type: research status: mature spawns: - notes/S-price-setting-markets-are-usdc-and-xlm.md + - notes/R-step-1-measured-on-production.md tags: [ohlcv, usdt, methodology, high-low] links: [] history: From d7953c9b376125c5df1181a002e55b2629970dbd Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:33:35 +0200 Subject: [PATCH 079/144] fix(lore-0332): take USD means from price-setting legs Review of PR #402. vwap and pf_vwap now weigh the same USDC/XLM markets as the prices, so a USDT leg no longer pins them to a band edge or sets close_divergent. A USDC/XLM leg that formed a price but has no USD value yet leaves the bucket unpriced instead of handing it to USDT. The picks are computed once in a WITH clause. An untracked XLM is no longer cached in AppState, since it changes prices; XLM and USDT get separate log lines. --- packages/prices-api/src/assets/handlers.rs | 140 ++++++------ packages/prices-api/src/assets/queries_ch.rs | 201 ++++++++++++------ .../prices-api/src/openapi/descriptions.rs | 21 +- packages/prices-api/src/state.rs | 3 +- web/portal/public/openapi.json | 8 +- 5 files changed, 224 insertions(+), 149 deletions(-) diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index c928a81b..dcf3af04 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -371,6 +371,42 @@ pub struct OhlcvParams { pub base_currency: Option, } +/// The USD references for `/ohlcv`. A lookup ERROR fails: that is the +/// database misbehaving, not an absent asset. +/// +/// USDC must be tracked (see `UsdRefs::usdc`). XLM and USDT are optional: an +/// untracked reference cannot be any candle's quote leg. A missing USDT only +/// loses the `traded` label. A missing XLM also drops XLM from the markets that +/// set USD prices, so the caller does not cache that result. +async fn resolve_usd_refs( + state: &AppState, +) -> Result { + let (usdc_id, xlm_id, usdt_id) = ( + usdc_identifier(), + AssetIdentifier::Native, + usdt_identifier(), + ); + let (usdc, xlm, usdt) = tokio::join!( + queries_ch::resolve_asset_id(state.ch(), &usdc_id), + queries_ch::resolve_asset_id(state.ch(), &xlm_id), + queries_ch::resolve_asset_id(state.ch(), &usdt_id), + ); + let xlm = xlm?; + if xlm.is_none() { + tracing::error!("ohlcv: XLM is untracked; USD prices come from USDC markets only"); + } + let usdt = usdt?; + if usdt.is_none() { + tracing::warn!("ohlcv: USDT is untracked; its rows go unlabelled"); + } + match usdc? { + Some(usdc) => Ok(queries_ch::UsdRefs { usdc, xlm, usdt }), + None => Err(clickhouse::error::Error::Custom( + "canonical USDC is not tracked".to_string(), + )), + } +} + /// `GET /assets/{asset_identifier}/ohlcv` — candlestick history. /// /// `base_currency` **denominates** the series; it does not select a quote leg @@ -381,7 +417,8 @@ pub struct OhlcvParams { /// /// Price fields are **absent, not dropped**, on a bucket with no USD value. /// Candles merge across sources — and, in USD mode, across quote legs — per -/// bucket; in USD mode the USDC/XLM legs set the prices when one formed a price. +/// bucket; in USD mode the USDC/XLM legs set the prices and both means when one +/// formed a price. /// `backfill_note` appears only for `timeframe=all` while the backfill is still /// running. #[utoipa::path( @@ -398,9 +435,10 @@ pub struct OhlcvParams { has no USD value yet is returned with its price fields `null`\nrather than omitted, so a \ chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote \ asset — are merged per bucket, in ascending time order,\nand all of them count toward the \ - volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close` and `method` come \ - only from markets quoted in USDC or XLM when one of\nthem formed a price in the bucket, \ - and from every market otherwise.\n\nThe window is `timeframe`, ending \ + volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` \ + and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a \ + price in the bucket, and from every market otherwise. If\nsuch a market has no USD value \ + yet, those fields are `null`.\n\nThe window is `timeframe`, ending \ now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is \ rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the \ historical backfill is still running.", @@ -550,72 +588,42 @@ pub async fn get_ohlcv( // // The three references are resolved by natural identity, mirroring the // enrichment worker's own resolve_reference_ids so read and write cannot - // disagree about which USDC is canonical. All three must be tracked: their - // absence is a server-side data gap, not "no candles", so it stays a 503 - // rather than being masked as an empty 200 (which looks like a healthy asset - // with no history). - let denomination = - match base_currency { - BaseCurrency::Usd => { - // Resolved once per AppState and shared thereafter — see - // AppState::usd_refs. The three identities are constants and the - // database derives their ids from them, so re-reading `assets - // FINAL` on every request is pure waste on a p95-bounded path. - let refs = state - .usd_refs() - .get_or_try_init(|| async { - // Concurrent: they do not depend on each other. - let (usdc_id, xlm_id, usdt_id) = - (usdc_identifier(), AssetIdentifier::Native, usdt_identifier()); - let (usdc, xlm, usdt) = tokio::join!( - queries_ch::resolve_asset_id(state.ch(), &usdc_id), - queries_ch::resolve_asset_id(state.ch(), &xlm_id), - queries_ch::resolve_asset_id(state.ch(), &usdt_id), - ); - // The pivots are best-effort: an untracked reference cannot - // be any candle's quote leg. It costs the `traded` label and, - // for XLM, its place among the price-setting legs — until a - // restart, as this result is cached. A lookup ERROR still - // fails — that is the database misbehaving, not an absent asset. - let optional = |found: Result, _>, name: &str| match found { - Ok(Some(id)) => Ok(Some(id)), - Ok(None) => { - tracing::warn!( - "ohlcv {name} reference untracked until restart; see UsdRefs::xlm" - ); - Ok(None) - } - Err(e) => Err(e), - }; - let (xlm, usdt) = (optional(xlm, "XLM")?, optional(usdt, "USDT")?); - // USDC must resolve: see UsdRefs::usdc. - match usdc { - Ok(Some(usdc)) => Ok(queries_ch::UsdRefs { usdc, xlm, usdt }), - Ok(None) => Err(clickhouse::error::Error::Custom( - "canonical USDC is not tracked".to_string(), - )), - Err(e) => Err(e), - } - }) - .await; - match refs { - Ok(r) => queries_ch::Denomination::Usd(r.clone()), - Err(e) => { - tracing::error!(error = %e, "ohlcv reference resolution failed"); - return errors::service_unavailable( - errors::QUOTE_UNAVAILABLE, - "pricing in the requested base_currency is unavailable", - ); + // disagree about which USDC is canonical. Only USDC must be tracked; see + // `resolve_usd_refs` for XLM and USDT. + let denomination = match base_currency { + BaseCurrency::Usd => { + // Resolved once per AppState and shared thereafter — see + // AppState::usd_refs. The three identities are constants and the + // database derives their ids from them, so re-reading `assets + // FINAL` on every request is pure waste on a p95-bounded path. + let refs = match state.usd_refs().get() { + Some(r) => Ok(r.clone()), + // Cached only once XLM resolves: a missing XLM changes + // prices, so it must not outlive the gap it came from. + None => resolve_usd_refs(&state).await.inspect(|r| { + if r.xlm.is_some() { + let _ = state.usd_refs().set(r.clone()); } + }), + }; + match refs { + Ok(r) => queries_ch::Denomination::Usd(r), + Err(e) => { + tracing::error!(error = %e, "ohlcv reference resolution failed"); + return errors::service_unavailable( + errors::QUOTE_UNAVAILABLE, + "pricing in the requested base_currency is unavailable", + ); } } - BaseCurrency::Xlm => { - match resolve_reference(state.ch(), AssetIdentifier::Native, base_currency).await { - Ok(id) => queries_ch::Denomination::QuoteLeg(id), - Err(resp) => return resp, - } + } + BaseCurrency::Xlm => { + match resolve_reference(state.ch(), AssetIdentifier::Native, base_currency).await { + Ok(id) => queries_ch::Denomination::QuoteLeg(id), + Err(resp) => return resp, } - }; + } + }; // The validated window is exactly what binds into SQL — the same eff_start // the point-count check measured, so there is one interpretation of the diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index f6b4a8cc..d14fa12e 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -722,6 +722,11 @@ impl UsdRefs { } /// The quote legs whose rows set a USD candle's prices. Never empty. + /// + /// ⚠️ XLM's USD value comes from the XLM/USDC pivot, which task 0286 + /// documents as quantised on dust days (e.g. 2023-03-11) until its + /// re-ingest and stage-D re-enrichment finish. Until then such a bucket + /// takes its prices from that XLM leg alone. pub fn price_setting(&self) -> Vec { // A token joins here once its USD rate is reliable (lore-0332). std::iter::once(self.usdc).chain(self.xlm).collect() @@ -820,11 +825,40 @@ fn low_published(p: &str) -> String { format!("if(isNull(minIf(l_x, {p})), NULL, least(minIf(l_x, {p}), {c}))") } -/// `expr` over the USDC/XLM rows (`band`) when the bucket has one, else over -/// every valid row, as before lore-0332. A band extreme that overflows stays -/// NULL: the other legs do not stand in for it. -fn banded(expr: impl Fn(&str) -> String) -> String { - format!("if(countIf(band) > 0, {}, {})", expr("band"), expr("valid")) +/// The USD candle's price inputs, computed once per bucket and named, so every +/// output column and both clamps read the same values (lore-0332). +/// +/// `_ub`: a USDC/XLM market formed a price in the bucket (`ps`). Then the +/// prices and both means come from those markets only (`band`, `bandc`); +/// otherwise from every market (`valid`, `convertible`), as before lore-0332. +/// Such a market with no USD value yet (`ps` but not `band`) leaves the bucket +/// unpriced (`_np`) rather than handing it to the other markets, and a band +/// extreme that overflows stays NULL for the same reason. +fn usd_with() -> String { + let pick = |f: &dyn Fn(&str) -> String, band: &str, all: &str| { + format!("if(_ub, {}, {})", f(band), f(all)) + }; + let open = pick( + &|p| format!("argMaxIf(o_x, (volume_base, quote_asset_id), {p})"), + "band", + "valid", + ); + let meth = pick( + &|p| format!("argMaxIf(meth, (volume_base, quote_asset_id), {p})"), + "band", + "valid", + ); + let high = pick(&high_published, "band", "valid"); + let low = pick(&low_published, "band", "valid"); + let close = pick(&close_exact, "band", "valid"); + let vwap = pick(&vwap_raw, "bandc", "convertible"); + let pf_vwap = pick(&pf_vwap_usd_raw, "band", "valid"); + format!( + "WITH countIf(ps) > 0 AS _ub, \ + (if(_ub, countIf(band), countIf(valid)) = 0) AS _np, \ + {open} AS _op, {high} AS _hi, {low} AS _lo, {close} AS _cl, \ + {meth} AS _me, {vwap} AS _vw, {pf_vwap} AS _pfvw " + ) } /// The volume-weighted mean before clamping — task 0229's review, finding 1. @@ -858,9 +892,13 @@ fn banded(expr: impl Fn(&str) -> String) -> String { /// bucket whose dust leg carries most of the volume the clamp will move the /// published value to `low` or `high` — a mean of all trades cannot always sit /// inside the range of some of them. -const VWAP_RAW: &str = "toDecimal128OrNull(toString( \ - sumIf(toFloat64(w_x) * toFloat64(volume_base), convertible) \ - / nullIf(sumIf(toFloat64(volume_base), convertible), 0)), 14)"; +fn vwap_raw(p: &str) -> String { + format!( + "toDecimal128OrNull(toString( \ + sumIf(toFloat64(w_x) * toFloat64(volume_base), {p}) \ + / nullIf(sumIf(toFloat64(volume_base), {p}), 0)), 14)" + ) +} /// Smallest `close` / `close_usd` a USD rate may be derived from — a /// **precision precondition**, not a plausibility band. @@ -905,9 +943,13 @@ const PRECISION_FLOOR: &str = prices_clickhouse::PRICE_FLOOR_SQL; /// reports: before task 0286's migration the column does not exist, and its /// DEFAULT is `volume_quote`, which a fixture or a legacy writer may never have /// filled. -const PF_VWAP_USD_RAW: &str = "nullIf(toDecimal128OrNull(toString( \ - sumIf(toFloat64(pfpv_x), valid) \ - / nullIf(sumIf(toFloat64(pf_volume), valid), 0)), 14), 0)"; +fn pf_vwap_usd_raw(p: &str) -> String { + format!( + "nullIf(toDecimal128OrNull(toString( \ + sumIf(toFloat64(pfpv_x), {p}) \ + / nullIf(sumIf(toFloat64(pf_volume), {p}), 0)), 14), 0)" + ) +} /// The quote-leg arm's price gate, spelled once. Unlike the USD arm there is no /// `valid` to hang it on: that expression exists to say "this row can be @@ -1009,28 +1051,23 @@ const PF_ROWS: &str = "pf_trade_count > 0 AND close >= toDecimal128('0.000000000 /// can read it — see [`OUTER_ALIASES`] for why the column tail is not something /// to maintain by hand in three places. fn usd_aggregates() -> String { - // Bound once so `h`, `l`, `vw` and `pfvw` clamp against the same band. The - // `countIf(valid) = 0` guards stay unbanded: a USDT-only bucket keeps its price. - let open = banded(|p| format!("argMaxIf(o_x, (volume_base, quote_asset_id), {p})")); - let high_published = banded(high_published); - let low_published = banded(low_published); - let close_exact = banded(close_exact); - let meth = banded(|p| format!("argMaxIf(meth, (volume_base, quote_asset_id), {p})")); + // The price inputs are the `usd_with` names, so `h`, `l`, `vw` and `pfvw` + // clamp against the same bounds the caller sees. format!( - "if(countIf(valid) = 0, NULL, toString({open})) AS o, \ - if(countIf(valid) = 0, NULL, toString({high_published})) AS h, \ - if(countIf(valid) = 0, NULL, toString({low_published})) AS l, \ - if(countIf(valid) = 0, NULL, toString({close_exact})) AS c, \ + "if(_np, NULL, toString(_op)) AS o, \ + if(_np, NULL, toString(_hi)) AS h, \ + if(_np, NULL, toString(_lo)) AS l, \ + if(_np, NULL, toString(_cl)) AS c, \ toString(sum(volume_base)) AS vb, \ toString(sum(volume_quote_usd)) AS vqu, \ - if(countIf(valid) = 0 OR isNull({VWAP_RAW}), NULL, \ - toString(least(greatest({VWAP_RAW}, {low_published}), {high_published}))) AS vw, \ + if(_np OR isNull(_vw), NULL, \ + toString(least(greatest(_vw, _lo), _hi))) AS vw, \ toUInt64(sum(trade_count)) AS tc, \ - nullIf(if(countIf(valid) = 0, NULL, {meth}), '') AS meth, \ - if(countIf(valid) = 0, NULL, toUInt8(1)) AS drv, \ + nullIf(if(_np, NULL, _me), '') AS meth, \ + if(_np, NULL, toUInt8(1)) AS drv, \ toNullable(toUInt64(sum(pf_trade_count))) AS pftc, \ - if(isNull({PF_VWAP_USD_RAW}), NULL, \ - toString(least(greatest({PF_VWAP_USD_RAW}, {low_published}), {high_published}))) AS pfvw, \ + if(_np OR isNull(_pfvw), NULL, \ + toString(least(greatest(_pfvw, _lo), _hi))) AS pfvw, \ {CLOSE_DIVERGENT_PLACEHOLDER}, \ {PROVENANCE_NULL_TAIL}" ) @@ -1060,13 +1097,19 @@ fn usd_projection(refs: &UsdRefs, granularity: Granularity) -> String { // price". The prices take `valid`; `vwap` takes `convertible`, because ADR // 0287 §7 weighs every fill (review C2). Spelling `valid` in terms of // `convertible` is what keeps them one decision rather than two. + // + // lore-0332: `band` and `bandc` are the same two predicates on the USDC/XLM + // legs; `ps` asks only whether such a leg formed a price, so an unconverted + // one still claims the bucket (see `usd_with`). format!( "timestamp, volume_base, volume_quote_usd, trade_count, quote_asset_id, \ pf_trade_count, pf_volume, \ (close >= {floor} AND close_usd >= {floor} \ AND (quote_asset_id = {usdc} OR close_usd != close)) AS convertible, \ (convertible AND pf_trade_count > 0) AS valid, \ - (valid AND quote_asset_id IN ({price_setting})) AS band, \ + (pf_trade_count > 0 AND quote_asset_id IN ({price_setting})) AS ps, \ + (valid AND ps) AS band, \ + (convertible AND quote_asset_id IN ({price_setting})) AS bandc, \ toFloat64(close_usd) / nullIf(toFloat64(close), 0) AS rate, \ toDecimal128OrNull(toString(toFloat64(open) * rate), 14) AS o_x, \ toDecimal128OrNull(toString(toFloat64(high) * rate), 14) AS h_x, \ @@ -1102,7 +1145,7 @@ fn quote_leg_aggregates() -> String { let ql_high = format!("maxIf(high, {PF_ROWS})"); let ql_low = format!("minIf(low, {PF_ROWS})"); // The price-forming mean, in the stored denomination. Same zero rule as - // `PF_VWAP_USD_RAW`. + // `pf_vwap_usd_raw`. let pf_vwap_raw = format!( "nullIf(toDecimal128OrNull(toString( \ sumIf(toFloat64(pf_price_volume), {PF_ROWS}) \ @@ -1757,14 +1800,15 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou // ascending for output. An ASC+LIMIT would instead return the OLDEST N and // silently drop the recent candles a chart actually wants. `ts` is ISO-8601 // (`%Y-%m-%dT%H:%i:%SZ`), so lexicographic `ts ASC` == chronological order. - let (projection, aggregates) = match args.denomination { + let (projection, with, aggregates) = match args.denomination { Denomination::Usd(ref refs) => ( // Per-row scaling — see the ordering note above. `valid` gates // both the arithmetic and the classification, so a row that // cannot be priced contributes to volume and trade_count but // never to a price or a method. usd_projection(refs, args.granularity), - // `countIf(valid) = 0` is what produces §5's price-less bucket: + usd_with(), + // `_np` is what produces §5's price-less bucket: // NULL across every price field, while the volume columns below // still aggregate over all rows. // `c` is EXACT (`close_usd` as stored) while `h`/`l` are derived @@ -1784,6 +1828,7 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou volume_quote_usd, trade_count, pf_trade_count, pf_volume, \ pf_price_volume" .to_string(), + String::new(), // ⚠️ `vw` is clamped into `[min(low), max(high)]` here too — task 0229's // review, finding 1. This arm applies no rate, so `o`/`h`/`l`/`c` are // the stored decimals and cannot cross; the merged vwap still can, @@ -1836,7 +1881,7 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou // `ohlcv_and_peg_series_project_the_same_alias_tail` is the guard. let sql = format!( "SELECT {OUTER_ALIASES} FROM ( \ - SELECT \ + {with}SELECT \ formatDateTime(timestamp, '%Y-%m-%dT%H:%i:%SZ') AS ts, \ {aggregates} \ FROM ( SELECT {projection} FROM {table} FINAL WHERE {conds} ) \ @@ -2934,36 +2979,48 @@ mod tests { } } - /// lore-0332: a bucket with a valid USDC/XLM row takes its prices from those - /// rows; any other bucket renders the pre-0332 text, byte for byte. + /// lore-0332: a bucket where a USDC/XLM market formed a price takes its + /// prices and both means from those markets; any other bucket renders the + /// pre-0332 expressions, byte for byte. #[test] fn usd_prices_come_from_the_price_setting_legs_when_the_bucket_has_one() { - let agg = usd_aggregates(); - // o 1 + h 3 (h, vw, pfvw) + l 3 + c 1 + meth 1. - assert_eq!(agg.matches("if(countIf(band) > 0, ").count(), 9, "{agg}"); - for p in ["band", "valid"] { - for expr in [ - format!("argMaxIf(o_x, (volume_base, quote_asset_id), {p})"), - format!("maxIf(h_x, {p})"), - format!("minIf(l_x, {p})"), - format!("argMaxIf(c_x, (volume_base, quote_asset_id), {p})"), - format!("argMaxIf(meth, (volume_base, quote_asset_id), {p})"), - ] { - assert!(agg.contains(&expr), "missing `{expr}`: {agg}"); - } + let with = usd_with(); + assert!(with.starts_with("WITH countIf(ps) > 0 AS _ub, "), "{with}"); + assert!( + with.contains("(if(_ub, countIf(band), countIf(valid)) = 0) AS _np"), + "an unconverted USDC/XLM leg leaves the bucket unpriced: {with}" + ); + // o, h, l, c, meth, vw, pfvw. + assert_eq!(with.matches("if(_ub, ").count(), 8, "{with}"); + for (band, all) in [("band", "valid"), ("bandc", "convertible")] { + let (b, a) = match band { + "band" => (high_published(band), high_published(all)), + _ => (vwap_raw(band), vwap_raw(all)), + }; + assert!(with.contains(&format!("if(_ub, {b}, {a})")), "{with}"); + } + for f in [low_published, close_exact, pf_vwap_usd_raw] { + assert!( + with.contains(&format!("if(_ub, {}, {})", f("band"), f("valid"))), + "{with}" + ); } for before in [ "argMaxIf(c_x, (volume_base, quote_asset_id), valid)", "if(isNull(maxIf(h_x, valid)), NULL, greatest(maxIf(h_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))", "if(isNull(minIf(l_x, valid)), NULL, least(minIf(l_x, valid), argMaxIf(c_x, (volume_base, quote_asset_id), valid)))", + "sumIf(toFloat64(w_x) * toFloat64(volume_base), convertible)", + "sumIf(toFloat64(pfpv_x), valid)", ] { assert!( - agg.contains(before), - "fallback drifted from `{before}`: {agg}" + with.contains(before), + "fallback drifted from `{before}`: {with}" ); } - for raw in [VWAP_RAW, PF_VWAP_USD_RAW] { - assert!(!raw.contains("band"), "vwap inputs are unchanged: {raw}"); + // The output columns read only the named inputs. + let agg = usd_aggregates(); + for gone in ["countIf(", "maxIf(", "minIf(", "argMaxIf(", "sumIf("] { + assert!(!agg.contains(gone), "`{gone}` belongs in usd_with: {agg}"); } let refs = usd_refs(); assert_eq!(refs.price_setting(), vec![2, 1]); @@ -2986,8 +3043,13 @@ mod tests { "`valid` must exclude a row whose bucket formed no price: {valid}" ); assert!( - projection.contains(") AS valid, (valid AND quote_asset_id IN (2, 1)) AS band, "), - "`band` narrows `valid` to the USDC/XLM legs (lore-0332): {projection}" + projection.contains( + ") AS valid, (pf_trade_count > 0 AND quote_asset_id IN (2, 1)) AS ps, \ + (valid AND ps) AS band, \ + (convertible AND quote_asset_id IN (2, 1)) AS bandc, " + ), + "`band`/`bandc` narrow `valid`/`convertible` to the USDC/XLM legs \ + (lore-0332): {projection}" ); // The pf columns the aggregates weigh have to reach the outer SELECT. for col in ["pf_trade_count", "pf_volume", "pfpv_x"] { @@ -3179,19 +3241,18 @@ mod tests { #[test] fn the_merged_vwap_weights_every_row_including_the_ones_that_formed_no_price() { let usd = usd_aggregates(); + let with = usd_with(); assert!( - VWAP_RAW.contains("sumIf(toFloat64(w_x) * toFloat64(volume_base), convertible)"), + with.contains(&format!( + "if(_ub, {}, {}) AS _vw", + vwap_raw("bandc"), + vwap_raw("convertible") + )), "the USD merge weights every row it can convert, not only the \ - price-forming ones: {VWAP_RAW}" + price-forming ones: {with}" ); assert!( - !VWAP_RAW.contains("valid"), - "`valid` carries `pf_trade_count > 0`, which is what this undoes: {VWAP_RAW}" - ); - assert!( - usd.contains(&format!( - "if(countIf(valid) = 0 OR isNull({VWAP_RAW}), NULL," - )), + usd.contains("if(_np OR isNull(_vw), NULL,"), "a bucket with no price-forming fill still publishes no vwap: {usd}" ); @@ -3242,9 +3303,16 @@ mod tests { /// price-forming volume to weigh: a zero here would be read as a price. #[test] fn the_price_forming_vwap_is_clamped_into_the_published_band_and_never_zero() { - for (name, agg) in [ - ("Denomination::Usd", usd_aggregates()), - ("Denomination::QuoteLeg", quote_leg_aggregates()), + // The USD arm names its mean in `usd_with` and clamps it in the list. + let usd_mean = usd_with() + .split(" AS _pfvw") + .next() + .and_then(|head| head.rsplit(" AS _vw, ").next()) + .unwrap_or_default() + .to_string(); + for (name, agg, mean) in [ + ("Denomination::Usd", usd_aggregates(), Some(usd_mean)), + ("Denomination::QuoteLeg", quote_leg_aggregates(), None), ] { let pfvw = agg .split(" AS pfvw") @@ -3256,6 +3324,7 @@ mod tests { pfvw.contains("least(greatest("), "{name}'s pf_vwap must be clamped into [low, high]: {pfvw}" ); + let pfvw = mean.unwrap_or(pfvw); assert!( pfvw.contains("nullIf("), "{name}'s pf_vwap must be NULL, not 0, with no price-forming \ diff --git a/packages/prices-api/src/openapi/descriptions.rs b/packages/prices-api/src/openapi/descriptions.rs index 2d858dca..bff3c6c2 100644 --- a/packages/prices-api/src/openapi/descriptions.rs +++ b/packages/prices-api/src/openapi/descriptions.rs @@ -357,9 +357,8 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ price-forming mean, so a wide gap marks a thin or one-sided bucket rather than an \ error: the close is still what it last traded at, and a poor summary of what it is \ worth. Prefer `pf_vwap` when ranking or valuing, and `close` when charting the last \ - price. In USD mode a gap can also come from a market that weighs into `pf_vwap` \ - but does not set `close` — one quoted in USDT, for example.\n\n`null` when either \ - value is `null` — nothing was compared, which is not the same as the two agreeing.", + price.\n\n`null` when either value is `null` — nothing was compared, which is not \ + the same as the two agreeing.", ), ( "Candle", @@ -459,8 +458,7 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ `base_currency` — `vwap` with the dust left out, and the value `close_divergent` \ compares `close` against.\n\nDenominated exactly like `close` and bounded by the \ published `low` and `high`, so the response stays self-consistent. In USD mode it \ - weighs every market, including ones that do not set `low` and `high` (USDT-quoted, \ - for example), so it can sit at one of those bounds. `null` when the \ + weighs the same markets that set those (see `/ohlcv`). `null` when the \ bucket has no price-forming volume, and on the synthesized USDC self-series. Never \ `0`.", ), @@ -531,15 +529,14 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ weighs depends on the denomination, so the two can differ on a bucket that holds a \ source with no usable price. With `base_currency=XLM` it is every row of the \ bucket, as stored. With `USD` it is every row that has a usable rate to convert it \ - with — a row whose own price is below the precision floor, or that has no USD value \ - yet, cannot be restated in USD and is left out of the mean rather than counted at \ - zero.\n\nBounded by the published `low` and `high` so the response stays \ + with, on the markets that set the prices (see `/ohlcv`) — a row whose own price is \ + below the precision floor, or that has no USD value yet, cannot be restated in USD \ + and is left out of the mean rather than counted at zero.\n\nBounded by the \ + published `low` and `high` so the response stays \ self-consistent. Those come from the price-forming trades only, so on a bucket \ where the trades that formed no price carry most of the volume the raw mean falls \ - outside that range and the value reported is the nearer bound. In USD mode the \ - same happens when a market that does not set `low` and `high` — one quoted in \ - USDT, for example — traded away from the ones that do. Compare it with `pf_vwap` \ - when the difference matters.", + outside that range and the value reported is the nearer bound. Compare it with \ + `pf_vwap` when the difference matters.", ), ( "ErrorEnvelope", diff --git a/packages/prices-api/src/state.rs b/packages/prices-api/src/state.rs index d602b83a..343af530 100644 --- a/packages/prices-api/src/state.rs +++ b/packages/prices-api/src/state.rs @@ -47,7 +47,8 @@ impl AppState { } } - /// Memoized reference ids; resolved once per state, then shared. + /// Memoized reference ids; resolved once per state, then shared. Not set + /// while XLM is untracked (see `handlers::resolve_usd_refs`). pub fn usd_refs(&self) -> &OnceCell { &self.usd_refs } diff --git a/web/portal/public/openapi.json b/web/portal/public/openapi.json index b697c259..1a4da391 100644 --- a/web/portal/public/openapi.json +++ b/web/portal/public/openapi.json @@ -319,7 +319,7 @@ "get": { "tags": ["prices"], "summary": "`GET /assets/{asset_identifier}/ohlcv` — candlestick history.", - "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close` and `method` come only from markets quoted in USDC or XLM when one of\nthem formed a price in the bucket, and from every market otherwise.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", + "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a price in the bucket, and from every market otherwise. If\nsuch a market has no USD value yet, those fields are `null`.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", "operationId": "get_ohlcv", "parameters": [ { @@ -1152,7 +1152,7 @@ }, "vwap": { "type": ["string", "null"], - "description": "Volume-weighted average price of the bucket, in `base_currency`, over the trades it holds — including the ones too small to form a price. `pf_vwap` is the same mean with those left out. `null` when the bucket has no price.\n\nWHICH trades it weighs depends on the denomination, so the two can differ on a bucket that holds a source with no usable price. With `base_currency=XLM` it is every row of the bucket, as stored. With `USD` it is every row that has a usable rate to convert it with — a row whose own price is below the precision floor, or that has no USD value yet, cannot be restated in USD and is left out of the mean rather than counted at zero.\n\nBounded by the published `low` and `high` so the response stays self-consistent. Those come from the price-forming trades only, so on a bucket where the trades that formed no price carry most of the volume the raw mean falls outside that range and the value reported is the nearer bound. In USD mode the same happens when a market that does not set `low` and `high` — one quoted in USDT, for example — traded away from the ones that do. Compare it with `pf_vwap` when the difference matters.", + "description": "Volume-weighted average price of the bucket, in `base_currency`, over the trades it holds — including the ones too small to form a price. `pf_vwap` is the same mean with those left out. `null` when the bucket has no price.\n\nWHICH trades it weighs depends on the denomination, so the two can differ on a bucket that holds a source with no usable price. With `base_currency=XLM` it is every row of the bucket, as stored. With `USD` it is every row that has a usable rate to convert it with, on the markets that set the prices (see `/ohlcv`) — a row whose own price is below the precision floor, or that has no USD value yet, cannot be restated in USD and is left out of the mean rather than counted at zero.\n\nBounded by the published `low` and `high` so the response stays self-consistent. Those come from the price-forming trades only, so on a bucket where the trades that formed no price carry most of the volume the raw mean falls outside that range and the value reported is the nearer bound. Compare it with `pf_vwap` when the difference matters.", "example": "0.2109568287069" }, "trade_count": { @@ -1182,12 +1182,12 @@ }, "pf_vwap": { "type": ["string", "null"], - "description": "Volume-weighted average price of the bucket's price-forming trades, in `base_currency` — `vwap` with the dust left out, and the value `close_divergent` compares `close` against.\n\nDenominated exactly like `close` and bounded by the published `low` and `high`, so the response stays self-consistent. In USD mode it weighs every market, including ones that do not set `low` and `high` (USDT-quoted, for example), so it can sit at one of those bounds. `null` when the bucket has no price-forming volume, and on the synthesized USDC self-series. Never `0`.", + "description": "Volume-weighted average price of the bucket's price-forming trades, in `base_currency` — `vwap` with the dust left out, and the value `close_divergent` compares `close` against.\n\nDenominated exactly like `close` and bounded by the published `low` and `high`, so the response stays self-consistent. In USD mode it weighs the same markets that set those (see `/ohlcv`). `null` when the bucket has no price-forming volume, and on the synthesized USDC self-series. Never `0`.", "example": "0.21095682870692" }, "close_divergent": { "type": ["boolean", "null"], - "description": "Whether `close` sits more than 1% away from `pf_vwap`. The close is a SINGLE trade — the last one that formed a price — while `pf_vwap` is the whole bucket's price-forming mean, so a wide gap marks a thin or one-sided bucket rather than an error: the close is still what it last traded at, and a poor summary of what it is worth. Prefer `pf_vwap` when ranking or valuing, and `close` when charting the last price. In USD mode a gap can also come from a market that weighs into `pf_vwap` but does not set `close` — one quoted in USDT, for example.\n\n`null` when either value is `null` — nothing was compared, which is not the same as the two agreeing.", + "description": "Whether `close` sits more than 1% away from `pf_vwap`. The close is a SINGLE trade — the last one that formed a price — while `pf_vwap` is the whole bucket's price-forming mean, so a wide gap marks a thin or one-sided bucket rather than an error: the close is still what it last traded at, and a poor summary of what it is worth. Prefer `pf_vwap` when ranking or valuing, and `close` when charting the last price.\n\n`null` when either value is `null` — nothing was compared, which is not the same as the two agreeing.", "example": false }, "source": { From d762cde25e4fd5196b5e8ed49acb63e70976a00e Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:33:35 +0200 Subject: [PATCH 080/144] test(lore-0332): pin banded means, unpriced leg, XLM cache Cases 1 and 5 now expect the USDC mean; case 10 is a USDC leg with no USD value beside USDT. A separate IT tracks XLM after a first request on the same AppState. --- packages/prices-api/tests/ohlcv_it.rs | 88 ++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 9 deletions(-) diff --git a/packages/prices-api/tests/ohlcv_it.rs b/packages/prices-api/tests/ohlcv_it.rs index efb72cab..a43c351f 100644 --- a/packages/prices-api/tests/ohlcv_it.rs +++ b/packages/prices-api/tests/ohlcv_it.rs @@ -2978,7 +2978,8 @@ async fn seed_price_setting_0332(db: &str, admin: &Client) { // lore-0332. 10:00 USDC + USDT; 11:00 USDT has the larger volume; // 12:00 XLM + USDT; 13:00 USDT only; 14:00 USDT pulls vwap above the band; // 15:00 USDC formed no price + USDT; 16:00 XLM + a larger USDT; - // 17:00 USDT only, overflowing; 18:00 USDC overflowing + USDT. + // 17:00 USDT only, overflowing; 18:00 USDC overflowing + USDT; + // 19:00 USDC formed a price but has no USD value yet + USDT. exec( admin, &format!( @@ -2999,7 +3000,9 @@ async fn seed_price_setting_0332(db: &str, admin: &Client) { ('2026-04-01 16:00:00', {BAZ}, {USDT}, 'sdex', 6, 8, 2, 5, 200, 1000, 500, 2.5, 5, 9, 1), \ ('2026-04-01 17:00:00', {BAZ}, {USDT}, 'sdex', 1000, 1000, 1000, 0.000000000001, 10, 0.00000000001, 10, 10000000000, 1000, 3, 1), \ ('2026-04-01 18:00:00', {BAZ}, {USDC}, 'sdex', 1000, 1000, 1000, 0.000000000001, 10, 0.00000000001, 10, 10000000000, 1000, 3, 1), \ - ('2026-04-01 18:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1)" + ('2026-04-01 18:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1), \ + ('2026-04-01 19:00:00', {BAZ}, {USDC}, 'sdex', 1.0, 1.2, 0.9, 1.1, 10, 11, 0, 0, 1.1, 2, 1), \ + ('2026-04-01 19:00:00', {BAZ}, {USDT}, 'sdex', 2.0, 3.0, 1.0, 2.0, 50, 100, 50, 1.0, 2.0, 5, 1)" ), ) .await; @@ -3018,6 +3021,66 @@ async fn seed_price_setting_0332(db: &str, admin: &Client) { .await; } +/// An untracked XLM is not cached: once it is tracked, the same `AppState` +/// lets XLM set the prices again without a restart (lore-0332, review). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_does_not_cache_an_untracked_xlm() { + let db = "it_ohlcv_untracked_xlm_0332"; + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + exec(&admin, &assets_insert(db, &[BAZ, USDT])).await; + // Case 3's shape: XLM sets 2.5–3.0, USDT would widen it to 1.0–4.0. + exec( + &admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1h \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) VALUES \ + ('2026-04-01 12:00:00', {BAZ}, {XLM}, 'sdex', 10, 12, 9, 10, 100, 1000, 250, 2.5, 10, 7, 1), \ + ('2026-04-01 12:00:00', {BAZ}, {USDT}, 'sdex', 6, 8, 2, 5, 20, 100, 50, 2.5, 5, 3, 1)" + ), + ) + .await; + exec( + &admin, + "ALTER TABLE assets DELETE WHERE asset_type = 'native' SETTINGS mutations_sync = 2", + ) + .await; + + let state = AppState::new(client); + let uri = format!( + "/v1/assets/BAZ:{}/ohlcv?granularity=1h&start=2026-04-01T12:00:00Z\ + &end=2026-04-01T12:00:00Z&base_currency=USD", + iss() + ); + let high = |state: AppState| { + let uri = uri.clone(); + async move { + let resp = app(&config(), state) + .oneshot(Request::builder().uri(&uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let json: Value = serde_json::from_slice(&bytes).unwrap(); + json["data"][0]["high"] + .as_str() + .unwrap() + .parse::() + .unwrap() + } + }; + // Untracked: XLM does not set the prices, so USDT widens the high. + assert!((high(state.clone()).await - 4.0).abs() < 1e-6); + exec(&admin, &assets_insert(db, &[XLM])).await; + // Tracked now, on the same state: XLM sets the high again. + assert!((high(state).await - 3.0).abs() < 1e-6); + + teardown(db).await; +} + /// In USD mode USDC- and XLM-quoted rows set the prices when the bucket has /// one; a USDT-only bucket is priced as before (lore-0332). Before-fix values /// in the comments show what each case discriminates. @@ -3031,13 +3094,13 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one let uri = format!( "/v1/assets/BAZ:{}/ohlcv?granularity=1h&start=2026-04-01T10:00:00Z\ - &end=2026-04-01T18:00:00Z&base_currency=USD", + &end=2026-04-01T19:00:00Z&base_currency=USD", iss() ); let (status, json) = get(client, &uri).await; assert_eq!(status, StatusCode::OK, "body={json}"); let data = json["data"].as_array().unwrap(); - assert_eq!(data.len(), 9, "body={json}"); + assert_eq!(data.len(), 10, "body={json}"); let at = |hour: &str| { let ts = format!("2026-04-01T{hour}:00:00Z"); data.iter() @@ -3055,14 +3118,14 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one ); let cases: [Case; 7] = [ // label, hour, [open, high, low, close], method, volume_base, vwap (= pf_vwap) - // was high 1.5, low 0.5 + // was high 1.5, low 0.5, vwap 155/150 ( "1 usdc+usdt", "10", [1.0, 1.2, 0.9, 1.1], "assumed-par", 150.0, - Some(155.0 / 150.0), + Some(1.05), ), // was open 1.05, close 1.05, `traded` ( @@ -3090,14 +3153,14 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one 50.0, Some(1.0), ), - // was high 3.0, close 2.0, vwap 1.99 + // was high 3.0, close 2.0, vwap 1.99; then 1.1, the USDT mean clamped ( - "5 vwap clamp", + "5 usdt mean", "14", [1.0, 1.1, 0.9, 1.0], "assumed-par", 1000.0, - Some(1.1), + Some(1.0), ), // was 1.0, 1.2, 0.9, 1.1 with `band` not requiring `valid` ( @@ -3150,6 +3213,13 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one assert!(c["high"].is_null(), "case {label}: {c}"); assert_eq!(c["close"], "10000000000", "case {label}: {c}"); } + // 10: a USDC leg with no USD value yet leaves the bucket unpriced rather + // than priced by USDT (was 1.0, 1.5, 0.5, 1.0); its volume still counts. + let c10 = at("19"); + for field in ["open", "high", "low", "close", "vwap", "pf_vwap", "method"] { + assert!(c10[field].is_null(), "case 10: {field}: {c10}"); + } + approx(&c10["volume_base"], 60.0); // Case 1: the USDT leg still counts toward volume and trades. let c1 = at("10"); approx(&c1["volume_quote_usd"], 155.0); From 2ccde637f1493d36bbd35404df9076ff8226257f Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:33:46 +0200 Subject: [PATCH 081/144] docs(lore-0332): record review changes in task and notice Means follow the price-setting markets; an unconverted USDC/XLM leg leaves the bucket unpriced. Notice updated to match. --- .../README.md | 5 +++++ .../notes/G-client-notice-usd-candle-prices.md | 16 ++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md index de88b03b..b3ffdacd 100644 --- a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/README.md @@ -59,6 +59,11 @@ volume. ## Status: Active **Current state:** PR #402 open, not merged; not deployed. Notice drafted, not sent. +Oskar's review (2026-10-14) addressed on the branch: the means follow the same +markets as the prices; a USDC/XLM leg that formed a price but has no USD value +yet leaves the bucket unpriced (0 such buckets in the stored 1d history); an +untracked XLM is no longer cached. The overflow-nulls-the-band trade-off was +kept (0 such buckets on prod). ## Implementation Plan diff --git a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md index 977bcb74..f2a2271c 100644 --- a/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md +++ b/lore/1-tasks/active/0332_BUG_usd-candle-high-low-set-by-markets-with-no-reliable-usd-rate/notes/G-client-notice-usd-candle-prices.md @@ -38,17 +38,16 @@ Binance shows $0.1135 and $0.1001. From ``: -- `open`, `high`, `low`, `close` and `method` come only from markets quoted - in USDC or XLM, when one of them formed a price in the bucket. That same - day now returns $0.1137 / $0.1005. +- `open`, `high`, `low`, `close`, `method`, `vwap` and `pf_vwap` come only + from markets quoted in USDC or XLM, when one of them formed a price in the + bucket. That same day now returns $0.1137 / $0.1005. +- If such a market has traded but its USD value is not computed yet, the + bucket's price fields are `null` until it is, rather than taken from the + other markets. This affects only the most recent buckets, and only briefly. - A bucket in which no USDC- or XLM-quoted market formed a price is computed exactly as before. - `volume_base`, `volume_quote_usd`, `trade_count` and `pf_trade_count` are unchanged and still count every market. -- `vwap` and `pf_vwap` still weigh every market and stay within the published - `low` and `high`. When a USDT-quoted market traded outside the USDC/XLM - range, the reported mean is the nearer bound. `close_divergent` can flag - such a bucket. - `method` follows the markets that set the prices. A bucket that traded mostly against USDT can now report the `assumed-par` of its USDC market. - `base_currency=XLM` is unchanged. @@ -59,7 +58,8 @@ The response shape does not change: no field is added, removed or renamed. Measured on daily candles since 2021-02: 0.34 % of priced buckets (37,582), across 419 assets that traded against USDT and against USDC or XLM on the same -day. High or low moves in 16,136 of those buckets, across 340 assets. For XLM, +day. High or low moves in 16,136 of those buckets, across 340 assets, and +`vwap`/`pf_vwap` can move in any of the 37,582. For XLM, the 95th-percentile gap to Binance falls from 23 % to 1.2 % for the daily high and from 44 % to 6.4 % for the daily low. From f61d78ebad7482d9fb2f59c973ee75725bbbd7ec Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:08:24 +0200 Subject: [PATCH 082/144] docs(lore-0333): open task for XLM/USD 2018-01-01 to 02-16 --- ...RE_xlm-usd-for-2018-01-01-to-2018-02-16.md | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md diff --git a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md b/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md new file mode 100644 index 00000000..0f70aa38 --- /dev/null +++ b/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md @@ -0,0 +1,71 @@ +--- +id: "0333" +title: "XLM/USD for 2018-01-01 → 2018-02-16" +type: FEATURE +status: backlog +related_adr: [] +related_tasks: ["0331"] +tags: [layer-backend, priority-medium, effort-small, enrichment, clickhouse, usd-reference] +links: + - "../active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md" +history: + - date: "2026-10-14" + status: backlog + who: akot + note: > + Spawned from 0331. The hourly XLM/USD series prepared for 0331 starts + 2018-02-16 23:00 UTC; the 47 days before it have no rate and stay + unpriced until this lands. Source not decided yet. +--- + +# XLM/USD for 2018-01-01 → 2018-02-16 + +## Summary + +[[0331]] prices pre-2021 candles from an hourly XLM/USD series loaded into +`prices.usd_rate`. The series prepared there starts at 2018-02-16 23:00 UTC. +The 47 days from 2018-01-01 to 2018-02-16 have no rate, so their XLM-quoted +candles stay at `close_usd = 0`. This task finds a rate for those days and +loads it in the same shape as the rest of the series. + +## Context + +- **Why the series stops there.** The exchange API the 0331 series was built + from cannot reach trades before 2018-02-16 23:03: a time-based `since` + anywhere from 2017-12 to 2018-02-16 lands on that first reachable trade, and + paging from the start jumps to it after the first 1,000 trades. Probed + 2026-10-14. +- **The market did trade then:** about 1,000 trades a day around 2018-01-01. +- **How the data will be handled is not decided.** The source and its terms + of use are still open (see 0331). + +## Options + +1. The same exchange's downloadable historical OHLCVT files, which cover the + period. They have to be fetched by hand. +2. The on-chain estimate from 0331's research: dollar tokens plus CNY/XCN/HKDT/ + EURT tokens at ECB reference rates, with a depeg guard and a smoother. + The on-chain witnesses (CNY, EURT, XCN) trade from 2018-01, so this is + possible for the period. It is not scored there: no ≥ 2-exchange truth + before 2018-05. +3. Leave the 47 days unpriced and say so in the API description. + +## Implementation Plan + +1. Pick the option and record why. +2. Build the hourly series for 2018-01-01 00:00 → 2018-02-16 22:00 UTC and + keep its provenance (file, hash, date). +3. Check the seam against the 0331 series from 2018-02-16 23:00 onward. +4. Insert in the same shape as 0331's rows (`('native','XLM','','')`, staged + method first, same verify queries), then promote together with them. +5. Re-run 0331's coarse-repair step for 201801–201802 on every tier. + +## Acceptance Criteria + +- [ ] Source chosen and recorded, with its terms of use. +- [ ] Hourly XLM/USD rows for every hour of 2018-01-01 → 2018-02-16 that has a + rate, provenance recorded. +- [ ] Agreement with the 0331 series on the overlap recorded (median and max + deviation). +- [ ] XLM 1d candles for 2018-01-01 → 2018-02-16 return a USD price, or the + gap is documented if option 3 is chosen. From be82ab352215b81bb4b890ab5fad2d27f2a3f684 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:10:38 +0200 Subject: [PATCH 083/144] feat(lore-0331): price pre-2021 candles from the XLM/USD rate Before 2021-02-01 Stellar has no USDC, so those candles carry no USD price. An opt-in pass mode prices XLM-quoted candles from the imported method = 'external' XLM/USD rate and dollar-token-quoted candles through that token's own XLM market. /ohlcv labels them external. --- packages/coarse-sweep-worker/src/main.rs | 2 + .../enrichment-worker/examples/run_ch_pass.rs | 1 + .../src/bin/coarse-repair.rs | 1 + packages/enrichment-worker/src/ch_enrich.rs | 337 +++++++++++++++++- packages/enrichment-worker/src/main.rs | 2 + .../enrichment-worker/tests/ch_enrich_it.rs | 141 ++++++++ packages/prices-api/src/assets/queries_ch.rs | 23 +- packages/prices-clickhouse/src/lib.rs | 13 + 8 files changed, 510 insertions(+), 10 deletions(-) diff --git a/packages/coarse-sweep-worker/src/main.rs b/packages/coarse-sweep-worker/src/main.rs index 5f09c2df..38eeb452 100644 --- a/packages/coarse-sweep-worker/src/main.rs +++ b/packages/coarse-sweep-worker/src/main.rs @@ -89,6 +89,8 @@ async fn main() -> Result<(), lambda_runtime::Error> { // Deliberately not env-driven — an operator names the quote leg and // epoch explicitly, via the `coarse-repair` binary. usd_reset: None, + // Opt-in coarse-repair mode only (task 0331). + pre_usdc_xlm_rate: false, }; // Cold start, mirroring the sibling workers: build and probe the mTLS client diff --git a/packages/enrichment-worker/examples/run_ch_pass.rs b/packages/enrichment-worker/examples/run_ch_pass.rs index eb4872f1..778df13d 100644 --- a/packages/enrichment-worker/examples/run_ch_pass.rs +++ b/packages/enrichment-worker/examples/run_ch_pass.rs @@ -30,6 +30,7 @@ async fn main() -> Result<(), Box> { one_shot: env_parse_or("ENRICHMENT_ONE_SHOT", false), time_window: None, usd_reset: None, + pre_usdc_xlm_rate: false, }; let pass = ChEnrichmentPass::new(cfg); diff --git a/packages/enrichment-worker/src/bin/coarse-repair.rs b/packages/enrichment-worker/src/bin/coarse-repair.rs index b2dfe106..ec3718b9 100644 --- a/packages/enrichment-worker/src/bin/coarse-repair.rs +++ b/packages/enrichment-worker/src/bin/coarse-repair.rs @@ -476,6 +476,7 @@ async fn main() -> Result<(), Box> { // clap's `requires` makes the mixed cases unreachable. _ => None, }, + pre_usdc_xlm_rate: false, }; let repair_cfg = CoarseRepairConfig { diff --git a/packages/enrichment-worker/src/ch_enrich.rs b/packages/enrichment-worker/src/ch_enrich.rs index dd21fca1..6ea66209 100644 --- a/packages/enrichment-worker/src/ch_enrich.rs +++ b/packages/enrichment-worker/src/ch_enrich.rs @@ -447,6 +447,9 @@ pub enum ChEnrichError { until a later run." )] ResetRequiresOneShot { quote_asset_id: u64 }, + + #[error("pre-2021 XLM/USD rate mode refused: {0}")] + PreUsdcXlmRateRefused(String), } /// Opt-in reset of **already-written** USD columns, so a corrected pricing tier @@ -999,6 +1002,9 @@ pub struct ChEnrichConfig { /// computed value if an operator explicitly names the quote leg and epoch. /// See [`UsdResetSpec`]. pub usd_reset: Option, + /// Opt-in coarse-repair mode (task 0331), `false` on every scheduled path. + /// When `true` the pass runs only the pre-2021 XLM/USD step. + pub pre_usdc_xlm_rate: bool, } impl Default for ChEnrichConfig { @@ -1016,6 +1022,7 @@ impl Default for ChEnrichConfig { one_shot: false, time_window: None, usd_reset: None, + pre_usdc_xlm_rate: false, } } } @@ -1509,6 +1516,9 @@ impl ChEnrichmentPass { /// pivot leg's `not_before` below the table's first priced reference /// candle is refused, because nothing could refill the rows in between. pub async fn assert_reset_is_admissible(&self) -> Result<(), ChEnrichError> { + if self.cfg.pre_usdc_xlm_rate { + return self.assert_pre_usdc_admissible().await.map(|_| ()); + } let Some(spec) = self.cfg.usd_reset.as_ref() else { return Ok(()); }; @@ -2028,6 +2038,96 @@ impl ChEnrichmentPass { Ok(()) } + /// The pre-2021 step's refusals; returns native XLM's id. The driver calls + /// this through [`Self::assert_reset_is_admissible`], so a dry run refuses too. + async fn assert_pre_usdc_admissible(&self) -> Result { + let refuse = |m: &str| Err(ChEnrichError::PreUsdcXlmRateRefused(m.to_string())); + if self.cfg.usd_reset.is_some() { + return refuse("it cannot be combined with a --reset-* spec"); + } + let Some(xlm) = self.resolve_reference_ids().await?.xlm else { + return refuse("native XLM does not resolve in assets"); + }; + let sql = format!( + "SELECT count() FROM ({}) WHERE rts < toDateTime({})", + usd_rate_series_sql(&self.cfg.database, XLM_RATE_IDENTITY), + prices_clickhouse::PRE_USDC_END_S, + ); + if self.client.query(&sql).fetch_one::().await? == 0 { + return refuse( + "usd_rate holds no method = 'external' XLM row before 2021-02-01 \ + (promote the staged series first)", + ); + } + Ok(xlm) + } + + fn pre_usdc_statements(&self, xlm: u64) -> (String, String) { + let pw = self.cfg.pivot_window_s; + // Bound the token-market scan to the month plus its look-back. + let ref_window = match self.cfg.time_window { + Some((s, e)) => format!( + " AND timestamp >= toDateTime({}) AND timestamp < toDateTime({e})", + s.saturating_sub(pw) + ), + None => String::new(), + }; + let db = &self.cfg.database; + let (cand, tbl) = (self.window_pred("timestamp"), &self.cfg.table); + pre_usdc_sql(db, tbl, xlm, pw, &cand, &ref_window) + } + + /// Rows a run with this config would write now (the dry-run figure). + pub async fn pre_usdc_pending(&self) -> Result { + let xlm = self.assert_pre_usdc_admissible().await?; + let watermark = self.watermark().await?; + let (_, count) = self.pre_usdc_statements(xlm); + Ok(self + .client + .query(&count) + .bind(watermark) + .fetch_one::() + .await?) + } + + /// The pre-2021 XLM/USD step (task 0331), looped to a fixed point like + /// [`Self::run_external_tier`], counting with its own pending query. + async fn run_pre_usdc(&self, watermark: u32) -> Result { + let start = std::time::Instant::now(); + let xlm = self.assert_pre_usdc_admissible().await?; + let (insert, count) = self.pre_usdc_statements(xlm); + let pending = || self.client.query(&count).bind(watermark).fetch_one::(); + let before = pending().await?; + let (mut remaining, mut batches) = (before, 0u32); + for _ in 0..self.effective_max_batches() { + if remaining == 0 { + break; + } + self.client + .query(&insert) + .bind(watermark) + .bind(self.cfg.batch_size) + .execute() + .await?; + let after = pending().await?; + batches += 1; + if after >= remaining { + remaining = after; + break; + } + remaining = after; + } + info!(before, remaining, batches, table = %self.cfg.table, "pre-2021 XLM/USD step done"); + Ok(ChPassStats { + batches, + candidates_before: before, + candidates_after: remaining, + rows_enriched: before.saturating_sub(remaining), + duration_ms: start.elapsed().as_millis() as u64, + ..Default::default() + }) + } + /// Tier 2 — the peg-pivot deep-history backbone, over the fixed `watermark` /// snapshot. Each per-batch pivot computes its volume-weighted reference/USDC /// rate **inline** ([`pivot_sql`]) rather than from a pre-materialized table, @@ -2174,6 +2274,9 @@ impl ChEnrichmentPass { /// pin the boundary and assert that candles newer than the snapshot are /// deferred, not enriched, in the current pass. pub async fn run_through(&self, watermark: u32) -> Result { + if self.cfg.pre_usdc_xlm_rate { + return self.run_pre_usdc(watermark).await; + } let start = std::time::Instant::now(); // Task 0182 — re-open already-written USD values so the tiers below can @@ -2597,6 +2700,24 @@ pub fn external_window_s(table: &str) -> u32 { bucket_width_s(table).max(86_400) } +/// `usd_rate` identity `(asset_kind, asset_code, issuer_address, contract_address)`. +const USDC_RATE_IDENTITY: [&str; 4] = ["credit", "USDC", USDC_ISSUER, ""]; +const XLM_RATE_IDENTITY: [&str; 4] = ["native", "XLM", "", ""]; + +/// The imported (`method = 'external'`) rate series of one `usd_rate` identity, +/// as an ASOF right side keyed `k = 1`. Equality on `method`: the staged +/// `external-candidate` rows share its prefix (`external_rate.rs`). +fn usd_rate_series_sql(db: &str, id: [&str; 4]) -> String { + let [kind, code, issuer, contract] = id; + format!( + "SELECT 1 AS k, timestamp AS rts, usd_rate AS usd \ + FROM {db}.usd_rate FINAL \ + WHERE asset_kind = '{kind}' AND asset_code = '{code}' \ + AND issuer_address = '{issuer}' AND contract_address = '{contract}' \ + AND method = 'external' AND usd_rate > 0" + ) +} + /// External statement (task 0268): a USDC-quoted candle the oracle tier left at /// `close_usd = 0` is priced from the MEASURED USDC/USD rate task 0267 imports /// into `prices.usd_rate`, instead of falling through to the peg tier's `× $1`. @@ -2702,6 +2823,7 @@ pub fn external_window_s(table: &str) -> u32 { /// `external_tier_never_overwrites_a_candle_the_oracle_tier_priced` proves the /// `close_usd` outcome end to end. fn external_sql(db: &str, tbl: &str, usdc_id: u64, window: &str) -> String { + let series = usd_rate_series_sql(db, USDC_RATE_IDENTITY); let bend = bucket_end_expr(tbl, "timestamp"); let epoch = prices_clickhouse::USDC_ORACLE_EPOCH_S; let columns = insert_columns(); @@ -2732,13 +2854,7 @@ fn external_sql(db: &str, tbl: &str, usdc_id: u64, window: &str) -> String { AND timestamp < toDateTime({epoch}) \ AND timestamp <= toDateTime(?){window} \ ) AS p \ - ASOF LEFT JOIN ( \ - SELECT 1 AS k, timestamp AS rts, usd_rate AS usd \ - FROM {db}.usd_rate FINAL \ - WHERE asset_kind = 'credit' AND asset_code = 'USDC' \ - AND issuer_address = '{USDC_ISSUER}' AND contract_address = '' \ - AND method = 'external' AND usd_rate > 0 \ - ) AS r \ + ASOF LEFT JOIN ( {series} ) AS r \ ON r.k = p.k AND r.rts < p.bend \ WHERE r.usd > 0 \ AND (toUInt32(p.bend) - toUInt32(r.rts)) <= ? \ @@ -2747,6 +2863,141 @@ fn external_sql(db: &str, tbl: &str, usdc_id: u64, window: &str) -> String { ) } +/// `prices.assets` rows of a dollar token other than canonical USDC (task 0331). +/// Keyed on code/issuer/contract, never `asset_type` (fixtures disagree on it). +fn dollar_token_asset_pred() -> String { + format!( + "contract_address = '' AND issuer_address != '' \ + AND asset_code IN ('USD', 'USDT', 'USDC') \ + AND NOT (asset_code = 'USDC' AND issuer_address = '{USDC_ISSUER}')" + ) +} + +/// Rows the pre-2021 XLM/USD step may write: before [`PRE_USDC_END_S`], quoted +/// in native XLM or a dollar token. Uncorrelated, so the driver's month +/// enumeration, the statement and its pending count share this one string. +/// +/// [`PRE_USDC_END_S`]: prices_clickhouse::PRE_USDC_END_S +pub fn pre_usdc_scope_pred(db: &str) -> String { + format!( + "timestamp < toDateTime({end}) AND volume_quote > 0 \ + AND quote_asset_id IN (SELECT asset_id FROM {db}.assets FINAL \ + WHERE (asset_code = 'XLM' AND issuer_address = '' AND contract_address = '') \ + OR ({dollar}))", + end = prices_clickhouse::PRE_USDC_END_S, + dollar = dollar_token_asset_pred(), + ) +} + +/// Pre-2021 step (task 0331): `close_usd = close × f × rate`, where `rate` is +/// the newest `external` XLM/USD row before the bucket end (within +/// [`external_window_s`]) and `f` is 1 on an XLM quote, else the dollar +/// token's XLM price from its own XLM markets (both orientations, pf columns) +/// at or before the bucket start, within `pivot_window_s`. +/// +/// Writes only rows whose computed pair differs from the stored one, so the +/// batch loop terminates, a re-run writes nothing, and a newer rate or a +/// narrower window re-prices (an unreachable row computes 0). +/// +/// Returns `(insert, pending_count)` from one body. Binds: the watermark, then +/// (insert only) the `LIMIT`. +fn pre_usdc_sql( + db: &str, + tbl: &str, + xlm: u64, + pivot_window_s: u32, + cand_window: &str, + ref_window: &str, +) -> (String, String) { + let end = prices_clickhouse::PRE_USDC_END_S; + let bend = bucket_end_expr(tbl, "timestamp"); + let stale = external_window_s(tbl); + let scope = pre_usdc_scope_pred(db); + let series = usd_rate_series_sql(db, XLM_RATE_IDENTITY); + let dollar = format!( + "SELECT asset_id FROM {db}.assets FINAL WHERE {}", + dollar_token_asset_pred() + ); + // Decimal256 sums: a Decimal(38, 14) sum wraps silently, a Float64 one + // depends on merge order and would make the differs filter flap. + let amt = |a: &str, b: &str| { + format!("toFloat64(sum(toDecimal256(if(asset_id = {xlm}, {a}, {b}), 14)))") + }; + let txlm = format!( + "{} / {}", + amt("pf_volume", "pf_price_volume"), + amt("pf_price_volume", "pf_volume") + ); + let xr = format!( + "if(r.usd > 0 AND (toUInt32(pt.bend) - toUInt32(r.rts)) <= {stale}, toFloat64(r.usd), 0.)" + ); + // Float64, then a NULL-on-overflow parse: a Decimal product wraps silently + // and an out-of-range CAST aborts the batch. + let usd = |col: &str| { + format!( + "ifNull(toDecimal128OrNull(toString(toFloat64(pt.{col}) * {xr} * pt.xf), 14), \ + toDecimal128(0, 14))" + ) + }; + let (new_cu, new_vqu) = (usd("close"), usd("volume_quote")); + let cols = prices_clickhouse::CANDLE_COLUMNS; + let from = |a: &str| { + cols.iter() + .map(|c| format!("{a}.{c} AS {c}")) + .collect::>() + .join(", ") + }; + let (p_cols, pt_cols, x_cols) = (from("p"), from("pt"), cols.join(", ")); + let out = cols + .iter() + .map(|c| match *c { + "close_usd" => "x.new_cu AS close_usd".to_string(), + "volume_quote_usd" => "x.new_vqu AS volume_quote_usd".to_string(), + "version" => "x.version + 1 AS version".to_string(), + c => format!("x.{c}"), + }) + .collect::>() + .join(", "); + let body = format!( + "SELECT {out} \ + FROM ( \ + SELECT {pt_cols}, {new_cu} AS new_cu, {new_vqu} AS new_vqu \ + FROM ( \ + SELECT {p_cols}, p.k AS k, p.bend AS bend, \ + if(p.quote_asset_id = {xlm}, 1., if(t.txlm > 0 \ + AND (toUInt32(p.timestamp) - toUInt32(t.tts)) <= {pivot_window_s}, \ + t.txlm, 0.)) AS xf \ + FROM ( \ + SELECT {x_cols}, 1 AS k, {bend} AS bend \ + FROM {db}.{tbl} FINAL \ + WHERE {scope} AND timestamp <= toDateTime(?){cand_window} \ + ) AS p \ + ASOF LEFT JOIN ( \ + SELECT if(asset_id = {xlm}, quote_asset_id, asset_id) AS tok, \ + timestamp AS tts, {txlm} AS txlm \ + FROM {db}.{tbl} FINAL \ + WHERE timestamp < toDateTime({end}){ref_window} \ + AND pf_trade_count > 0 AND pf_volume > 0 AND pf_price_volume > 0 \ + AND ((quote_asset_id = {xlm} AND asset_id IN ({dollar})) \ + OR (asset_id = {xlm} AND quote_asset_id IN ({dollar}))) \ + GROUP BY tok, tts \ + ) AS t \ + ON t.tok = p.quote_asset_id AND t.tts <= p.timestamp \ + ) AS pt \ + ASOF LEFT JOIN ( {series} ) AS r \ + ON r.k = pt.k AND r.rts < pt.bend \ + ) AS x \ + WHERE x.close_usd != x.new_cu OR x.volume_quote_usd != x.new_vqu" + ); + ( + format!( + "INSERT INTO {db}.{tbl} ({}) {body} ORDER BY x.timestamp LIMIT ?", + insert_columns() + ), + format!("SELECT count() FROM ({body})"), + ) +} + /// The count the 0268 reset mode refuses on: canonical USDC readings in /// `oracle_prices` — the table the oracle tier READS, not `usd_rate` — stamped /// below `USDC_ORACLE_EPOCH_S`. Pure, so the string is unit-testable; bound @@ -4190,8 +4441,14 @@ mod tests { /// `_1M` fixtures in `ch_enrich_it.rs`. Task file, Issues 10. #[test] fn every_timezone_sensitive_expression_pins_utc() { - for t in ["price_ohlcv_1d", "price_ohlcv_1w", "price_ohlcv_1M"] { - let sql = external_sql("prices", t, 3, ""); + let tables = ["price_ohlcv_1d", "price_ohlcv_1w", "price_ohlcv_1M"]; + for (t, sql) in tables.iter().flat_map(|t| { + [ + external_sql("prices", t, 3, ""), + pre_usdc_sql("prices", t, 9, 86_400, "", "").0, + ] + .map(|sql| (t, sql)) + }) { for f in ["addDays(", "addWeeks(", "addMonths(", "toDate("] { for (i, _) in sql.match_indices(f) { let tail = &sql[i..]; @@ -4710,6 +4967,10 @@ mod tests { reset_sql("prices", "price_ohlcv_1m", &spec, ""), ), ("pivot_sql", pivot_sql("prices", "price_ohlcv_1m", 5, 3, "")), + ( + "pre_usdc_sql", + pre_usdc_sql("prices", "price_ohlcv_1m", 9, 86_400, "", "").0, + ), ] } @@ -5383,4 +5644,62 @@ mod tests { "an unresolved USDC must be refused by the epoch guard itself, got {res:?}" ); } + + #[test] + fn external_sql_reads_the_usdc_external_series() { + let want = format!( + "ASOF LEFT JOIN ( SELECT 1 AS k, timestamp AS rts, usd_rate AS usd \ + FROM prices.usd_rate FINAL WHERE asset_kind = 'credit' AND asset_code = 'USDC' \ + AND issuer_address = '{USDC_ISSUER}' AND contract_address = '' \ + AND method = 'external' AND usd_rate > 0 ) AS r" + ); + assert!(external_sql("prices", "price_ohlcv_1h", 3, "").contains(&want)); + } + + #[test] + fn pre_usdc_sql_renders_its_contract() { + let end = prices_clickhouse::PRE_USDC_END_S; + let win = |s| format!(" AND timestamp >= toDateTime({s}) AND timestamp < toDateTime(9)"); + for (t, (cw, rw)) in GRAINS + .iter() + .flat_map(|t| [(t, (String::new(), String::new())), (t, (win(5), win(1)))]) + { + let (ins, cnt) = pre_usdc_sql("prices", t, 9, 86_400, &cw, &rw); + let must = [ + "asset_kind = 'native' AND asset_code = 'XLM' AND issuer_address = '' \ + AND contract_address = '' AND method = 'external' AND usd_rate > 0" + .to_string(), + "r.rts < pt.bend".into(), + format!("{} AS bend", bucket_end_expr(t, "timestamp")), + format!("<= {}, toFloat64(r.usd)", external_window_s(t)), + "<= 86400, t.txlm".into(), + "if(asset_id = 9, pf_volume, pf_price_volume)".into(), + "if(asset_id = 9, pf_price_volume, pf_volume)".into(), + "pf_trade_count > 0".into(), + dollar_token_asset_pred(), + "x.close_usd != x.new_cu OR x.volume_quote_usd != x.new_vqu".into(), + "toDecimal128OrNull(toString(".into(), + "toDecimal256(".into(), + format!("{cw} ) AS p"), + format!("{end}){rw} AND pf_trade_count"), + ]; + for needle in must { + assert!(ins.contains(&needle), "{t}: `{needle}` missing in {ins}"); + } + for needle in ["LIKE", "startsWith", "candidate", &win(1)] { + assert_eq!(ins.contains(needle), needle == rw, "{t}: `{needle}`"); + } + assert_eq!(ins.matches("method = ").count(), 1, "{ins}"); + assert!( + ins.matches(&format!("timestamp < toDateTime({end})")) + .count() + >= 2 + ); + assert_eq!((ins.matches('?').count(), cnt.matches('?').count()), (2, 1)); + let body = &cnt["SELECT count() FROM (".len()..cnt.len() - 1]; + let head = format!("INSERT INTO prices.{t} ({}) ", insert_columns()); + assert_eq!(ins, format!("{head}{body} ORDER BY x.timestamp LIMIT ?")); + assert!(cnt.starts_with("SELECT count() FROM (")); + } + } } diff --git a/packages/enrichment-worker/src/main.rs b/packages/enrichment-worker/src/main.rs index c9392813..25689095 100644 --- a/packages/enrichment-worker/src/main.rs +++ b/packages/enrichment-worker/src/main.rs @@ -69,6 +69,8 @@ async fn main() -> Result<(), lambda_runtime::Error> { // would re-zero and recompute the same rows forever. Deliberately not // env-driven — there is no ENRICHMENT_USD_RESET. usd_reset: None, + // Opt-in coarse-repair mode only (task 0331). + pre_usdc_xlm_rate: false, }; tracing::info!( diff --git a/packages/enrichment-worker/tests/ch_enrich_it.rs b/packages/enrichment-worker/tests/ch_enrich_it.rs index ac994a4a..a9deff1f 100644 --- a/packages/enrichment-worker/tests/ch_enrich_it.rs +++ b/packages/enrichment-worker/tests/ch_enrich_it.rs @@ -159,6 +159,7 @@ fn cfg(db: &str) -> ChEnrichConfig { one_shot: false, time_window: None, usd_reset: None, + pre_usdc_xlm_rate: false, } } @@ -5708,3 +5709,143 @@ async fn the_epoch_is_measured_on_the_pass_s_own_table() { .await .unwrap(); } + +// --- Task 0331: pre-2021 candles priced from the hourly XLM/USD rate --- + +const USDX: AssetFixture = AssetFixture::new("USD", "classic", "GUSDX", ""); +/// 2020-06-01 00:00 UTC. +const D0331: u32 = 1_590_969_600; +const H1: &str = "price_ohlcv_1h"; + +const fn h(n: u32) -> u32 { + D0331 + n * 3600 +} + +/// (ts, rate, method) of `('native','XLM','','')` rows in `usd_rate`. +type Rate = (u32, f64, &'static str); +/// (label, table, ts, asset, quote, close, volume_base, volume_quote, close_usd, volume_quote_usd) +#[rustfmt::skip] +type Leg = (&'static str, &'static str, u32, AssetFixture<'static>, AssetFixture<'static>, f64, f64, f64, f64, f64); + +const RATES_0331: &[Rate] = &[ + (h(10), 0.25, "external"), + (h(23), 0.20, "external"), + (h(24), 0.50, "external"), +]; + +#[rustfmt::skip] +const LEGS_0331: &[Leg] = &[ + ("FOO/XLM 1h", H1, h(10), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), + ("FOO/XLM 1d takes 23:00, not the next midnight", "price_ohlcv_1d", D0331, FOO, XLM, 10.0, 3.0, 30.0, 2.0, 6.0), + ("USDT/XLM", H1, h(10), USDT, XLM, 4.0, 10.0, 40.0, 1.0, 10.0), + ("XLM/USDX", H1, h(10), XLM, USDX, 0.25, 40.0, 10.0, 0.25, 10.0), + ("FOO/USDT via USDT/XLM", H1, h(10), FOO, USDT, 2.0, 3.0, 6.0, 2.0, 6.0), + ("FOO/USDX via XLM/USDX", H1, h(10), FOO, USDX, 2.0, 3.0, 6.0, 2.0, 6.0), +]; + +async fn seed_xlm_rate(client: &Client, db: &str, (ts, rate, method): Rate, version: u64) { + client + .query(&format!( + "INSERT INTO {db}.usd_rate \ + (asset_kind, asset_code, issuer_address, contract_address, \ + timestamp, usd_rate, method, reference_asset, hops, version) VALUES \ + ('native', 'XLM', '', '', {ts}, {rate}, '{method}', '', 0, {version})" + )) + .execute() + .await + .unwrap(); +} + +async fn seed_leg(client: &Client, db: &str, leg: &Leg) { + let (_, table, ts, asset, quote, close, vb, vq, _, _) = *leg; + client + .query(&format!( + "INSERT INTO {db}.{table} \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + VALUES ({ts}, {asset}, {quote}, 'sdex', {close}, {close}, {close}, {close}, \ + {vb}, {vq}, 0, 0, {close}, 1, 1)" + )) + .execute() + .await + .unwrap(); +} + +/// (close_usd, volume_quote_usd, version) of a leg, read FINAL. +async fn usd_of(client: &Client, db: &str, leg: &Leg) -> (f64, f64, u64) { + let (_, table, ts, asset, quote, ..) = *leg; + client + .query(&format!( + "SELECT toFloat64(close_usd), toFloat64(volume_quote_usd), version \ + FROM {db}.{table} FINAL \ + WHERE asset_id = {asset} AND quote_asset_id = {quote} AND timestamp = {ts}" + )) + .fetch_one::<(f64, f64, u64)>() + .await + .unwrap() +} + +async fn setup_0331(db: &str, rates: &[Rate], legs: &[Leg]) -> Client { + let client = setup_scratch(db).await; + client + .query(&assets_insert(db, &[XLM, USDC, USDT, FOO, USDX])) + .execute() + .await + .unwrap(); + for r in rates { + seed_xlm_rate(&client, db, *r, 1).await; + } + for l in legs { + seed_leg(&client, db, l).await; + } + client +} + +fn cfg_0331(db: &str, table: &str) -> ChEnrichConfig { + ChEnrichConfig { + table: table.to_string(), + one_shot: true, + pre_usdc_xlm_rate: true, + ..cfg(db) + } +} + +fn written_in(table: &str) -> u64 { + LEGS_0331 + .iter() + .filter(|l| l.1 == table && l.8 != 0.0) + .count() as u64 +} + +async fn assert_legs(client: &Client, db: &str, scale: f64) { + for leg in LEGS_0331 { + let (cu, vqu, _) = usd_of(client, db, leg).await; + assert!( + (cu - leg.8 * scale).abs() < 1e-9 && (vqu - leg.9 * scale).abs() < 1e-9, + "{}: got ({cu}, {vqu}), want ({}, {})", + leg.0, + leg.8 * scale, + leg.9 * scale + ); + } +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_prices_xlm_and_dollar_quoted_legs() { + let db = "test_ch_enrich_pre_usdc_prices"; + let client = setup_0331(db, RATES_0331, LEGS_0331).await; + for table in [H1, "price_ohlcv_1d"] { + let stats = ChEnrichmentPass::new(cfg_0331(db, table)) + .run() + .await + .unwrap(); + assert_eq!(stats.rows_enriched, written_in(table), "{table}: {stats:?}"); + } + assert_legs(&client, db, 1.0).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); +} diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index d14fa12e..44010648 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -1694,8 +1694,11 @@ pub(crate) fn usd_method_expr(usdc: u64, pivots: &[u64], granularity: Granularit quote_asset_id = {usdc} AND close_usd = close, 'assumed-par', \ quote_asset_id = {usdc} AND timestamp >= toDateTime({epoch}), 'oracle', \ quote_asset_id = {usdc}, '', \ + timestamp < toDateTime({pre}), 'external', \ {traded_arm}\ - '') AS meth" + '') AS meth", + // Task 0331: a priced pre-2021 non-USDC row took the XLM/USD rate. + pre = prices_clickhouse::PRE_USDC_END_S, ) } @@ -2167,6 +2170,24 @@ mod tests { ); } + /// Task 0331: a priced pre-2021 row outside the USDC arms is `external`, + /// and the arm sits after every USDC arm and before `traded`. + #[test] + fn usd_method_expr_labels_priced_pre_usdc_rows_external() { + let arm = format!( + "timestamp < toDateTime({}), 'external', ", + prices_clickhouse::PRE_USDC_END_S + ); + for pivots in [&[7u64][..], &[]] { + let sql = usd_method_expr(2, pivots, Granularity::H1); + let at = sql.find(&arm).unwrap_or_else(|| panic!("{sql}")); + assert!(at > sql.find("quote_asset_id = 2, '', ").unwrap(), "{sql}"); + if let Some(traded) = sql.find("'traded'") { + assert!(at < traded, "{sql}"); + } + } + } + /// Task 0139: every SELECT whose id lands in a Rust u64 field casts with /// `toUInt64`, so the readers work on the UInt32 schema and the UInt64 one. /// A bare `a.asset_id` read into a u64 misparses a UInt32 column. diff --git a/packages/prices-clickhouse/src/lib.rs b/packages/prices-clickhouse/src/lib.rs index ae0f3881..fe623cd3 100644 --- a/packages/prices-clickhouse/src/lib.rs +++ b/packages/prices-clickhouse/src/lib.rs @@ -247,6 +247,11 @@ pub const USDT_ISSUER: &str = "GCQTGZQQ5G4PTM2GL7CDIFKUBIPEC52BROAQIAPW53XBRJVN6 /// precondition 3 is the check that confirms it before the pass runs. pub const USDC_ORACLE_EPOCH_S: u32 = 1_773_237_600; +/// 2021-02-01T00:00:00Z: before it Stellar has no USDC, so candles are priced +/// from an imported XLM/USD rate (task 0331). A bucket edge on every tier, so +/// `timestamp <` it means the whole bucket lies before it. +pub const PRE_USDC_END_S: u32 = 1_612_137_600; + /// ClickHouse client configuration, sourced from environment with local-dev /// defaults. #[derive(Debug, Clone)] @@ -472,6 +477,14 @@ mod tests { ); } + #[test] + fn pre_usdc_end_is_2021_02_01t00_00_00z() { + assert_eq!( + i64::from(PRE_USDC_END_S), + days_from_civil(2021, 2, 1) * 86_400 + ); + } + #[test] fn split_statements_drops_line_comments_and_empty_chunks() { let sql = "-- top\n\ From 0d0a93c04d2b0b9d3c4f370dced987c7a482f099 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:13:33 +0200 Subject: [PATCH 084/144] feat(lore-0331): add --pre-usdc-xlm-rate to coarse-repair The driver enumerates months by the step's scope and its dry run reports the exact number of rows the real run writes. --- .../src/bin/coarse-repair.rs | 16 +++++- packages/enrichment-worker/src/repair.rs | 52 +++++++++++-------- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/packages/enrichment-worker/src/bin/coarse-repair.rs b/packages/enrichment-worker/src/bin/coarse-repair.rs index ec3718b9..af73cff9 100644 --- a/packages/enrichment-worker/src/bin/coarse-repair.rs +++ b/packages/enrichment-worker/src/bin/coarse-repair.rs @@ -118,6 +118,14 @@ struct Args { #[arg(long, env = "PIVOT_WINDOW_S", default_value_t = 86_400)] pivot_window_s: u32, + /// Before 2021-02-01 only (task 0331): price XLM-quoted candles from the + /// promoted `method = 'external'` XLM/USD rows, and dollar-token-quoted ones + /// through that token's own XLM market within `--pivot-window-s`. Writes only + /// rows whose values differ, so a re-run is idempotent. Refused until the + /// series is promoted. On `price_ohlcv_1m` it needs `--skip-snapshot`. + #[arg(long, conflicts_with = "reset_quote_asset_id")] + pre_usdc_xlm_rate: bool, + /// Re-open already-written USD values for this quote `asset_id` so the /// corrected tiers recompute them (task 0182). Requires /// `--reset-not-before`. @@ -476,7 +484,7 @@ async fn main() -> Result<(), Box> { // clap's `requires` makes the mixed cases unreachable. _ => None, }, - pre_usdc_xlm_rate: false, + pre_usdc_xlm_rate: args.pre_usdc_xlm_rate, }; let repair_cfg = CoarseRepairConfig { @@ -591,6 +599,12 @@ async fn main() -> Result<(), Box> { m.snapshot_name.as_deref().unwrap_or("-") ); } + if args.pre_usdc_xlm_rate { + println!( + "--pre-usdc-xlm-rate: zeros_before = rows to write, enriched = rows written, \ + zeros_after = rows still differing (> 0 after a capped run: re-run)" + ); + } println!( "{} month(s): {} enriched, {} left at the no_reference floor{}", summary.months.len(), diff --git a/packages/enrichment-worker/src/repair.rs b/packages/enrichment-worker/src/repair.rs index 7f03e481..3c8df7d4 100644 --- a/packages/enrichment-worker/src/repair.rs +++ b/packages/enrichment-worker/src/repair.rs @@ -32,7 +32,9 @@ use clickhouse::{Client, Row}; use serde::{Deserialize, Serialize}; use tracing::{debug, info, warn}; -use crate::ch_enrich::{ChEnrichConfig, ChEnrichError, ChEnrichmentPass, repair_target_pred}; +use crate::ch_enrich::{ + ChEnrichConfig, ChEnrichError, ChEnrichmentPass, pre_usdc_scope_pred, repair_target_pred, +}; /// One coarse-repair run over `[start_month, end_month]` (inclusive `YYYYMM`). #[derive(Debug, Clone)] @@ -206,6 +208,12 @@ impl CoarseRepairDriver { /// the month's first instant; `addMonths(..., 1)` is the exclusive upper /// bound. Both cast to `UInt32` to match [`MonthWindow`]. async fn months_with_zeros(&self) -> Result, ChEnrichError> { + let e = &self.cfg.enrich; + let pred = if e.pre_usdc_xlm_rate { + pre_usdc_scope_pred(&e.database) + } else { + repair_target_pred(&e.database, &e.table, e.usd_reset.as_ref()) + }; let sql = format!( "SELECT toYYYYMM(timestamp) AS month, \ toUInt32(toUnixTimestamp(toDateTime(min(toStartOfMonth(timestamp))))) AS start_ts, \ @@ -217,11 +225,6 @@ impl CoarseRepairDriver { GROUP BY month ORDER BY month", db = self.cfg.enrich.database, tbl = self.cfg.enrich.table, - pred = repair_target_pred( - &self.cfg.enrich.database, - &self.cfg.enrich.table, - self.cfg.enrich.usd_reset.as_ref(), - ), start = self.cfg.start_month, end = self.cfg.end_month, ); @@ -302,16 +305,31 @@ impl CoarseRepairDriver { break; } + // Partition-bounded pass over this one month. `one_shot` decides + // whether it drains the month fully (manual repair) or a bounded + // `max_batches` chunk with overflow deferring to the next run (the + // recurring hourly sweep) — see [`CoarseRepairConfig::one_shot`]. + let mut ecfg = self.cfg.enrich.clone(); + ecfg.time_window = Some((mw.start_ts, mw.end_ts)); + ecfg.one_shot = self.cfg.one_shot; + let pre_usdc = ecfg.pre_usdc_xlm_rate; + let pass = ChEnrichmentPass::with_client(self.client.clone(), ecfg); + if self.cfg.dry_run { + // Task 0331: report the rows the real run writes, not scope rows. + let zeros = if pre_usdc { + pass.pre_usdc_pending().await? + } else { + mw.zeros + }; info!( month = mw.month, - zeros = mw.zeros, - "coarse repair: DRY RUN — would repair" + zeros, "coarse repair: DRY RUN — would repair" ); summary.months.push(MonthRepair { month: mw.month, - zeros_before: mw.zeros, - zeros_after: mw.zeros, + zeros_before: zeros, + zeros_after: zeros, rows_enriched: 0, rows_reset: 0, snapshot_name: None, @@ -334,16 +352,7 @@ impl CoarseRepairDriver { None }; - // Partition-bounded pass over this one month. `one_shot` decides - // whether it drains the month fully (manual repair) or a bounded - // `max_batches` chunk with overflow deferring to the next run (the - // recurring hourly sweep) — see [`CoarseRepairConfig::one_shot`]. - let mut ecfg = self.cfg.enrich.clone(); - ecfg.time_window = Some((mw.start_ts, mw.end_ts)); - ecfg.one_shot = self.cfg.one_shot; - let stats = ChEnrichmentPass::with_client(self.client.clone(), ecfg) - .run() - .await?; + let stats = pass.run().await?; info!( month = mw.month, @@ -363,7 +372,8 @@ impl CoarseRepairDriver { }); // Defensive cross-check: the enumeration count and the pass's own // pre-count should agree (both are the month's zero population). - if mw.zeros != stats.candidates_before { + // In the 0331 mode the pass counts rows to write, not scope rows. + if !pre_usdc && mw.zeros != stats.candidates_before { warn!( month = mw.month, enumerated = mw.zeros, From d293f0da705333b0eac04c8e7fb2e58066d64125 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:13:33 +0200 Subject: [PATCH 085/144] test(lore-0331): pin the pre-2021 XLM/USD rewrite rule and preview --- .../enrichment-worker/tests/ch_enrich_it.rs | 181 ++++++++++++++++-- 1 file changed, 167 insertions(+), 14 deletions(-) diff --git a/packages/enrichment-worker/tests/ch_enrich_it.rs b/packages/enrichment-worker/tests/ch_enrich_it.rs index a9deff1f..6650109b 100644 --- a/packages/enrichment-worker/tests/ch_enrich_it.rs +++ b/packages/enrichment-worker/tests/ch_enrich_it.rs @@ -14,7 +14,7 @@ use enrichment_worker::repair::{ CoarseRepairConfig, CoarseRepairDriver, CoarseSweepConfig, run_coarse_sweep, }; use prices_clickhouse::asset_id::fixture::{AssetFixture, assets_insert, fetch_ids}; -use prices_clickhouse::{USDC_ISSUER, USDC_ORACLE_EPOCH_S, USDT_ISSUER}; +use prices_clickhouse::{PRE_USDC_END_S, USDC_ISSUER, USDC_ORACLE_EPOCH_S, USDT_ISSUER}; use std::fmt::Display; // The fixture assets. Each displays as its id, derived from its identity @@ -5731,6 +5731,9 @@ const RATES_0331: &[Rate] = &[ (h(10), 0.25, "external"), (h(23), 0.20, "external"), (h(24), 0.50, "external"), + (h(11), 9.0, "external-candidate"), + (h(81), 0.25, "external"), + (PRE_USDC_END_S - 3600, 0.30, "external"), ]; #[rustfmt::skip] @@ -5741,6 +5744,25 @@ const LEGS_0331: &[Leg] = &[ ("XLM/USDX", H1, h(10), XLM, USDX, 0.25, 40.0, 10.0, 0.25, 10.0), ("FOO/USDT via USDT/XLM", H1, h(10), FOO, USDT, 2.0, 3.0, 6.0, 2.0, 6.0), ("FOO/USDX via XLM/USDX", H1, h(10), FOO, USDX, 2.0, 3.0, 6.0, 2.0, 6.0), + ("FOO/XLM ignores the candidate's 9.0", H1, h(11), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), + ("FOO/XLM, newest rate 30 h old", H1, h(53), FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), + ("FOO/XLM, rate 2 h old", H1, h(82), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), + ("FOO/USDT, market 72 h old", H1, h(82), FOO, USDT, 2.0, 3.0, 6.0, 0.0, 0.0), + ("FOO/USDT, market 2 h old", H1, h(12), FOO, USDT, 2.0, 3.0, 6.0, 2.0, 6.0), + ("FOO/XLM at 2021-02-01", H1, PRE_USDC_END_S, FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), + ("XLM/USDC (canonical)", H1, h(10), XLM, USDC, 0.25, 40.0, 10.0, 0.0, 0.0), + ("FOO/USDC (canonical)", H1, h(10), FOO, USDC, 4.0, 1.0, 4.0, 0.0, 0.0), +]; + +/// The 1h legs whose bucket end resolves to the h(10) rate. +const ON_H10_RATE: &[&str] = &[ + "FOO/XLM 1h", + "USDT/XLM", + "XLM/USDX", + "FOO/USDT via USDT/XLM", + "FOO/USDX via XLM/USDX", + "FOO/XLM ignores the candidate's 9.0", + "FOO/USDT, market 2 h old", ]; async fn seed_xlm_rate(client: &Client, db: &str, (ts, rate, method): Rate, version: u64) { @@ -5817,17 +5839,27 @@ fn written_in(table: &str) -> u64 { .count() as u64 } -async fn assert_legs(client: &Client, db: &str, scale: f64) { +async fn assert_legs(client: &Client, db: &str, scale: impl Fn(&str) -> f64) -> Vec { + let mut versions = Vec::new(); for leg in LEGS_0331 { - let (cu, vqu, _) = usd_of(client, db, leg).await; + let (cu, vqu, v) = usd_of(client, db, leg).await; + let (want_cu, want_vqu) = (leg.8 * scale(leg.0), leg.9 * scale(leg.0)); assert!( - (cu - leg.8 * scale).abs() < 1e-9 && (vqu - leg.9 * scale).abs() < 1e-9, - "{}: got ({cu}, {vqu}), want ({}, {})", - leg.0, - leg.8 * scale, - leg.9 * scale + (cu - want_cu).abs() < 1e-9 && (vqu - want_vqu).abs() < 1e-9, + "{}: got ({cu}, {vqu}), want ({want_cu}, {want_vqu})", + leg.0 ); + versions.push(v); } + versions +} + +async fn drop_db(client: &Client, db: &str) { + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); } #[tokio::test] @@ -5842,10 +5874,131 @@ async fn pre_usdc_xlm_rate_prices_xlm_and_dollar_quoted_legs() { .unwrap(); assert_eq!(stats.rows_enriched, written_in(table), "{table}: {stats:?}"); } - assert_legs(&client, db, 1.0).await; - client - .query(&format!("DROP DATABASE {db}")) - .execute() - .await - .unwrap(); + let versions = assert_legs(&client, db, |_| 1.0).await; + // Only written rows bump their version; unreachable and out-of-scope ones stay at 1. + let want: Vec = LEGS_0331 + .iter() + .map(|l| 1 + u64::from(l.8 != 0.0)) + .collect(); + assert_eq!(versions, want); + drop_db(&client, db).await; +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_rewrites_only_values_that_differ() { + let db = "test_ch_enrich_pre_usdc_rewrite"; + let client = setup_0331(db, RATES_0331, LEGS_0331).await; + let run = |pivot_window_s| { + let pass = ChEnrichmentPass::new(ChEnrichConfig { + pivot_window_s, + ..cfg_0331(db, H1) + }); + async move { pass.run().await.unwrap().rows_enriched } + }; + assert_eq!(run(86_400).await, written_in(H1)); + let d1 = ChEnrichmentPass::new(cfg_0331(db, "price_ohlcv_1d")); + d1.run().await.unwrap(); + let first = assert_legs(&client, db, |_| 1.0).await; + + assert_eq!( + run(86_400).await, + 0, + "a re-run with the same rate writes nothing" + ); + assert_eq!( + assert_legs(&client, db, |_| 1.0).await, + first, + "and bumps no version" + ); + + seed_xlm_rate(&client, db, (h(10), 0.5, "external"), 2).await; + assert_eq!(run(86_400).await, ON_H10_RATE.len() as u64); + let doubled = |l: &str| if ON_H10_RATE.contains(&l) { 2.0 } else { 1.0 }; + assert_legs(&client, db, doubled).await; + + // A 1 h window can no longer reach the 2 h old USDT market: written back to 0. + assert_eq!(run(3600).await, 1); + let narrowed = |l: &str| { + if l == "FOO/USDT, market 2 h old" { + 0.0 + } else { + doubled(l) + } + }; + assert_legs(&client, db, narrowed).await; + drop_db(&client, db).await; +} + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_repair_refuses_unpromoted_rates_and_previews_exactly() { + let driver = |client: &Client, db: &str, dry_run| { + CoarseRepairDriver::with_client( + client.clone(), + CoarseRepairConfig { + enrich: cfg_0331(db, H1), + start_month: 202_001, + end_month: 202_103, + snapshot: false, + dry_run, + one_shot: true, + deadline: None, + }, + ) + }; + // (i) Only a staged row: refused before any write, in a dry run too. + let db = "test_ch_enrich_pre_usdc_refuse"; + let client = setup_0331(db, &[(h(10), 0.25, "external-candidate")], &LEGS_0331[..1]).await; + for dry_run in [true, false] { + let res = driver(&client, db, dry_run).run().await; + assert!( + matches!(res, Err(ChEnrichError::PreUsdcXlmRateRefused(_))), + "{res:?}" + ); + } + assert_eq!(usd_of(&client, db, &LEGS_0331[0]).await.2, 1); + // (ii) Never combined with a reset. + let spec = UsdResetSpec { + quote_asset_id: 1, + not_before: 1, + not_after: None, + require_external_rate: false, + require_pivot_usdc_rate: false, + }; + let res = ChEnrichmentPass::new(ChEnrichConfig { + usd_reset: Some(spec), + ..cfg_0331(db, H1) + }) + .assert_reset_is_admissible() + .await; + assert!( + matches!(res, Err(ChEnrichError::PreUsdcXlmRateRefused(_))), + "{res:?}" + ); + drop_db(&client, db).await; + + // (iii) The dry run reports exactly what the real run then writes. + let db = "test_ch_enrich_pre_usdc_preview"; + let client = setup_0331(db, RATES_0331, LEGS_0331).await; + let preview = driver(&client, db, true).run().await.unwrap(); + let months: Vec<(u32, u64)> = preview + .months + .iter() + .map(|m| (m.month, m.zeros_before)) + .collect(); + assert_eq!(months, [(202_006, written_in(H1))]); + assert_eq!( + usd_of(&client, db, &LEGS_0331[0]).await.2, + 1, + "the dry run wrote nothing" + ); + let real = driver(&client, db, false).run().await.unwrap(); + let months: Vec<(u32, u64, u64)> = real + .months + .iter() + .map(|m| (m.month, m.rows_enriched, m.zeros_after)) + .collect(); + assert_eq!(months, [(202_006, written_in(H1), 0)]); + drop_db(&client, db).await; } From 2be97bfa68b54773d0ccc85c5e094ac6ad081b55 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:15:49 +0200 Subject: [PATCH 086/144] docs(lore-0331): describe the pre-2021 XLM/USD rate on /ohlcv --- packages/prices-api/src/assets/handlers.rs | 5 ++++- packages/prices-api/src/openapi/descriptions.rs | 5 ++++- web/portal/public/openapi.json | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index dcf3af04..5a783961 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -438,7 +438,10 @@ async fn resolve_usd_refs( volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` \ and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a \ price in the bucket, and from every market otherwise. If\nsuch a market has no USD value \ - yet, those fields are `null`.\n\nThe window is `timeframe`, ending \ + yet, those fields are `null`. Before 2021-02-01, when Stellar had no USDC, candles \ + quoted in\nXLM or in another dollar token are converted with an imported, measured \ + XLM/USD rate\n(`method` is `external`); a dollar token is converted through its own XLM \ + market and\nstays unpriced without one.\n\nThe window is `timeframe`, ending \ now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is \ rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the \ historical backfill is still running.", diff --git a/packages/prices-api/src/openapi/descriptions.rs b/packages/prices-api/src/openapi/descriptions.rs index bff3c6c2..5544c651 100644 --- a/packages/prices-api/src/openapi/descriptions.rs +++ b/packages/prices-api/src/openapi/descriptions.rs @@ -391,7 +391,10 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "method", "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was \ measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* \ - `external` — an imported, measured USDC/USD series supplied the rate.\n* `oracle` — \ + `external` — an imported, measured USD series supplied the rate: USDC/USD on a \ + USDC-quoted candle; before 2021-02-01, when Stellar had no USDC, XLM/USD on a \ + candle quoted in XLM or in another dollar token (USD, USDT or a USDC other than \ + Circle's), the latter converted through that token's own XLM market.\n* `oracle` — \ a measured Reflector reading supplied the rate.\n* `traded` — priced through a \ reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series \ (`GET /assets/USDC:/ohlcv`): no measured USDC/USD observation covered the \ diff --git a/web/portal/public/openapi.json b/web/portal/public/openapi.json index 1a4da391..a6f3986f 100644 --- a/web/portal/public/openapi.json +++ b/web/portal/public/openapi.json @@ -319,7 +319,7 @@ "get": { "tags": ["prices"], "summary": "`GET /assets/{asset_identifier}/ohlcv` — candlestick history.", - "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a price in the bucket, and from every market otherwise. If\nsuch a market has no USD value yet, those fields are `null`.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", + "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a price in the bucket, and from every market otherwise. If\nsuch a market has no USD value yet, those fields are `null`. Before 2021-02-01, when Stellar had no USDC, candles quoted in\nXLM or in another dollar token are converted with an imported, measured XLM/USD rate\n(`method` is `external`); a dollar token is converted through its own XLM market and\nstays unpriced without one.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", "operationId": "get_ohlcv", "parameters": [ { @@ -1165,7 +1165,7 @@ }, "method": { "type": ["string", "null"], - "description": "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* `external` — an imported, measured USDC/USD series supplied the rate.\n* `oracle` — a measured Reflector reading supplied the rate.\n* `traded` — priced through a reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`): no measured USDC/USD observation covered the bucket, so the $1 fallback was rendered. Never appears on a quote leg — there the same situation is `assumed-par`.\n\nEach value names the INPUT the rate came from, so `assumed-par` and `external` are never interchangeable: one is an assumption, the other a measurement that may sit percent off par.\n\nOn the USDC self-series a bucket that holds both a Reflector reading and an imported one reports `oracle`: a measured poll outranks an imported rate outright, whichever was observed first. Observation time only breaks ties between rows of the same kind. The imported series is loaded at HOURLY grain, so an hourly request over an imported day reports `external` — and that hour's own measured rate — for each of its twenty-four hours; on 2023-03-11 the 07:00 bucket reports the trough rather than the day's close. Where only a daily row exists for a day, that one row prices every bucket of its UTC day at every `granularity`, so an imported day mixes the two only where the oracle epoch falls inside it (2026-03-11): a bucket that extends past 14:00 UTC that day is not priced from the imported series, which ends at 13:00. The self-series carries the rate row's own `method`, so a measured rate that reads exactly 1.0 is `external` there.\n\nOn a quote leg the label is reconstructed at read time — the candle rows carry no provenance column — so one case is deliberately not separated: a bucket reading exactly 1.0 reports `assumed-par` whether the dollar was assumed or a measured rate happened to land on it. The two leave an identical row and carry the same number, so the label is the conservative one; `external` is reported only for a bucket whose value was actually scaled by an imported rate.\n\n`null` in three cases: when the price fields are `null`; always for `base_currency=XLM`, where nothing is converted; and for a USDC-quoted bucket below the oracle epoch that carries a converted rate no imported series covers — nothing can attribute it, and a label would be a claim.\n\nIn USD mode the label follows the markets that set the prices: those quoted in USDC or XLM when one of them formed a price in the bucket, otherwise every market. A bucket that traded mostly against USDT can therefore report the `assumed-par` of its USDC market.", + "description": "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* `external` — an imported, measured USD series supplied the rate: USDC/USD on a USDC-quoted candle; before 2021-02-01, when Stellar had no USDC, XLM/USD on a candle quoted in XLM or in another dollar token (USD, USDT or a USDC other than Circle's), the latter converted through that token's own XLM market.\n* `oracle` — a measured Reflector reading supplied the rate.\n* `traded` — priced through a reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`): no measured USDC/USD observation covered the bucket, so the $1 fallback was rendered. Never appears on a quote leg — there the same situation is `assumed-par`.\n\nEach value names the INPUT the rate came from, so `assumed-par` and `external` are never interchangeable: one is an assumption, the other a measurement that may sit percent off par.\n\nOn the USDC self-series a bucket that holds both a Reflector reading and an imported one reports `oracle`: a measured poll outranks an imported rate outright, whichever was observed first. Observation time only breaks ties between rows of the same kind. The imported series is loaded at HOURLY grain, so an hourly request over an imported day reports `external` — and that hour's own measured rate — for each of its twenty-four hours; on 2023-03-11 the 07:00 bucket reports the trough rather than the day's close. Where only a daily row exists for a day, that one row prices every bucket of its UTC day at every `granularity`, so an imported day mixes the two only where the oracle epoch falls inside it (2026-03-11): a bucket that extends past 14:00 UTC that day is not priced from the imported series, which ends at 13:00. The self-series carries the rate row's own `method`, so a measured rate that reads exactly 1.0 is `external` there.\n\nOn a quote leg the label is reconstructed at read time — the candle rows carry no provenance column — so one case is deliberately not separated: a bucket reading exactly 1.0 reports `assumed-par` whether the dollar was assumed or a measured rate happened to land on it. The two leave an identical row and carry the same number, so the label is the conservative one; `external` is reported only for a bucket whose value was actually scaled by an imported rate.\n\n`null` in three cases: when the price fields are `null`; always for `base_currency=XLM`, where nothing is converted; and for a USDC-quoted bucket below the oracle epoch that carries a converted rate no imported series covers — nothing can attribute it, and a label would be a claim.\n\nIn USD mode the label follows the markets that set the prices: those quoted in USDC or XLM when one of them formed a price in the bucket, otherwise every market. A bucket that traded mostly against USDT can therefore report the `assumed-par` of its USDC market.", "example": "oracle" }, "derived": { From 75d629e79ba1c0f51b07291a4311e346a00697b5 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:15:49 +0200 Subject: [PATCH 087/144] test(lore-0331): label pre-2021 candles external on /ohlcv --- packages/prices-api/tests/ohlcv_it.rs | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/prices-api/tests/ohlcv_it.rs b/packages/prices-api/tests/ohlcv_it.rs index a43c351f..bf19c68e 100644 --- a/packages/prices-api/tests/ohlcv_it.rs +++ b/packages/prices-api/tests/ohlcv_it.rs @@ -3228,3 +3228,48 @@ async fn ohlcv_usd_prices_come_from_usdc_and_xlm_markets_when_the_bucket_has_one teardown(db).await; } + +/// Before 2021-02-01 a priced candle quoted in XLM or a dollar token took the +/// imported XLM/USD rate, so it is labelled `external` (lore-0331). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_labels_pre_usdc_candles_external() { + let db = "it_ohlcv_pre_usdc_0331"; + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + exec(&admin, &assets_insert(db, &[USDT])).await; + // (day, quote, close, close_usd, method). close_usd != close: an exact-par + // dollar row is dropped as a peg signature. + let cases = [ + ("2021-01-30", USDT, 2.0, 1.98, "external"), + ("2021-01-31", XLM, 10.0, 2.5, "external"), + ("2021-02-01", XLM, 10.0, 3.0, "traded"), + ]; + for (day, quote, close, cu, _) in cases { + exec( + &admin, + &format!( + "INSERT INTO {db}.price_ohlcv_1d \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + VALUES ('{day} 00:00:00', {FOO}, {quote}, 'sdex', {close}, {close}, {close}, {close}, \ + 1, {close}, {cu}, {cu}, {close}, 1, 1)" + ), + ) + .await; + } + let uri = format!( + "/v1/assets/FOO:{}/ohlcv?granularity=1d&start=2021-01-30T00:00:00Z\ + &end=2021-02-01T00:00:00Z&base_currency=USD", + iss() + ); + let (status, json) = get(client, &uri).await; + assert_eq!(status, StatusCode::OK, "body={json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), cases.len(), "body={json}"); + for (c, (day, _, _, cu, method)) in data.iter().zip(cases) { + assert_eq!(c["method"], method, "{day}: {c}"); + approx(&c["close"], cu); + } + teardown(db).await; +} From d900697991f7a7ef7e88462351ebaba177543129 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:09:15 +0200 Subject: [PATCH 088/144] docs(lore-0333): widen the XLM/USD range to 2017-01-17 The exchange's XLM/USD market opened 2017-01-17. Its OHLCVT files cover the gap before the 0331 series (2018-02-16), which is 99.6 % of the unpriced in-scope 1h rows. The 68 rows before 2017-01-17 stay unpriced. --- ...RE_xlm-usd-for-2017-01-17-to-2018-02-16.md | 98 +++++++++++++++++++ ...RE_xlm-usd-for-2018-01-01-to-2018-02-16.md | 71 -------------- 2 files changed, 98 insertions(+), 71 deletions(-) create mode 100644 lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2017-01-17-to-2018-02-16.md delete mode 100644 lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md diff --git a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2017-01-17-to-2018-02-16.md b/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2017-01-17-to-2018-02-16.md new file mode 100644 index 00000000..15ea20f0 --- /dev/null +++ b/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2017-01-17-to-2018-02-16.md @@ -0,0 +1,98 @@ +--- +id: "0333" +title: "XLM/USD for 2017-01-17 → 2018-02-16" +type: FEATURE +status: backlog +related_adr: [] +related_tasks: ["0331"] +tags: [layer-backend, priority-medium, effort-small, enrichment, clickhouse, usd-reference] +links: + - "../active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md" +history: + - date: "2026-10-15" + status: backlog + who: akot + note: > + Scope widened from 2018-01-01 → 02-16 to 2017-01-17 → 2018-02-16, the + start of the exchange's XLM/USD market. The downloadable OHLCVT files + are the chosen source. The 68 in-scope 1h rows before 2017-01-17 stay + unpriced. + - date: "2026-10-14" + status: backlog + who: akot + note: > + Spawned from 0331. The hourly XLM/USD series prepared for 0331 starts + 2018-02-16 23:00 UTC; the 47 days before it have no rate and stay + unpriced until this lands. Source not decided yet. +--- + +# XLM/USD for 2017-01-17 → 2018-02-16 + +## Summary + +[[0331]] prices pre-2021 candles from an hourly XLM/USD series loaded into +`prices.usd_rate`. That series starts at 2018-02-16 23:00 UTC, so XLM- and +dollar-quoted candles before it stay at `close_usd = 0`. This task extends the +series back to 2017-01-17, when the same exchange opened its XLM/USD market, +and loads it in the same shape. That prices almost all of the history before +2018-02-16. + +## Context + +- **Why the 0331 series stops at 2018-02-16.** The exchange's trades API + cannot reach trades before 2018-02-16 23:03 when paged. A time-based `since` + anywhere from 2017-12 to 2018-02-16 lands on that first reachable trade, and + paging from the start jumps to it after the first 1,000 trades. Probed + 2026-10-14. +- **When the exchange's markets start** (public trades API, `since=0`, probed + 2026-10-15): XLM/USD on 2017-01-17 20:17 UTC (first price $0.00225), and + XLM/BTC on 2016-07-19. +- **What is left to price** (prod, 1h rows with volume quoted in XLM or a + dollar token, all at `close_usd = 0`, 2026-10-15): + + | Period | 1h rows | Quote volume (M XLM) | + |---|---|---| + | 2015-11-18 → 2016-07-19 | 32 | 0.6 | + | 2016-07-19 → 2017-01-17 | 36 | 0.2 | + | 2017-01-17 → 2017-12-31 | 12,751 | 383.6 | + | 2018-01-01 → 2018-02-16 | 6,432 | 119.2 | + + XLM/USD from 2017-01-17 covers 19,183 of these 19,251 rows (99.6 %). +- **Thin market early on.** The first three XLM/USD trades are spread over + three days. 0331's rate window on tiers ≤ 1h is 24 h, so a candle with no + trade on the exchange in the preceding 24 h stays unpriced. + +## Decision (2026-10-15, Adam) + +- **Source:** the exchange's downloadable historical OHLCVT files, which + cover each pair "from the beginning of each market". This is option 1 of + the original task. They have to be fetched by hand. +- **Range:** 2017-01-17 → 2018-02-16 22:00 UTC. +- **Left unpriced:** the 68 in-scope 1h rows before 2017-01-17. Pricing them + would need XLM/BTC × BTC/USD or another exchange, which is not worth it for + that volume. + +## Implementation Plan + +1. Fetch the OHLCVT archive and record its provenance (file, sha256, date) + and the terms of use. +2. Build the hourly XLM/USD series for 2017-01-17 → 2018-02-16 22:00 UTC from + the 60-minute XLM/USD file, using the same rate definition as the 0331 rows. +3. Check the seam: compare it with the 0331 series where both have a value + (from 2018-02-16 23:00), and compare the 0331 range itself against the file. +4. Insert in the same shape as 0331's rows (`('native','XLM','','')`, staged + `external-candidate` first, same verify queries), then promote together with + them or on top of them. +5. Re-run 0331's `coarse-repair --pre-usdc-xlm-rate` for 201701–201802: 1h + first, then the wider tiers, which 0331 builds from 1h. + +## Acceptance Criteria + +- [ ] Source recorded with provenance and terms of use. +- [ ] Hourly XLM/USD rows for every hour of 2017-01-17 → 2018-02-16 that has a + trade on the exchange. +- [ ] Agreement with the 0331 series on the overlap recorded (median and max + deviation). +- [ ] XLM 1d candles from 2017-01-18 on return a USD price wherever the + exchange traded within the window; the remaining days counted and + explained, including the 68 rows before 2017-01-17. diff --git a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md b/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md deleted file mode 100644 index 0f70aa38..00000000 --- a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -id: "0333" -title: "XLM/USD for 2018-01-01 → 2018-02-16" -type: FEATURE -status: backlog -related_adr: [] -related_tasks: ["0331"] -tags: [layer-backend, priority-medium, effort-small, enrichment, clickhouse, usd-reference] -links: - - "../active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md" -history: - - date: "2026-10-14" - status: backlog - who: akot - note: > - Spawned from 0331. The hourly XLM/USD series prepared for 0331 starts - 2018-02-16 23:00 UTC; the 47 days before it have no rate and stay - unpriced until this lands. Source not decided yet. ---- - -# XLM/USD for 2018-01-01 → 2018-02-16 - -## Summary - -[[0331]] prices pre-2021 candles from an hourly XLM/USD series loaded into -`prices.usd_rate`. The series prepared there starts at 2018-02-16 23:00 UTC. -The 47 days from 2018-01-01 to 2018-02-16 have no rate, so their XLM-quoted -candles stay at `close_usd = 0`. This task finds a rate for those days and -loads it in the same shape as the rest of the series. - -## Context - -- **Why the series stops there.** The exchange API the 0331 series was built - from cannot reach trades before 2018-02-16 23:03: a time-based `since` - anywhere from 2017-12 to 2018-02-16 lands on that first reachable trade, and - paging from the start jumps to it after the first 1,000 trades. Probed - 2026-10-14. -- **The market did trade then:** about 1,000 trades a day around 2018-01-01. -- **How the data will be handled is not decided.** The source and its terms - of use are still open (see 0331). - -## Options - -1. The same exchange's downloadable historical OHLCVT files, which cover the - period. They have to be fetched by hand. -2. The on-chain estimate from 0331's research: dollar tokens plus CNY/XCN/HKDT/ - EURT tokens at ECB reference rates, with a depeg guard and a smoother. - The on-chain witnesses (CNY, EURT, XCN) trade from 2018-01, so this is - possible for the period. It is not scored there: no ≥ 2-exchange truth - before 2018-05. -3. Leave the 47 days unpriced and say so in the API description. - -## Implementation Plan - -1. Pick the option and record why. -2. Build the hourly series for 2018-01-01 00:00 → 2018-02-16 22:00 UTC and - keep its provenance (file, hash, date). -3. Check the seam against the 0331 series from 2018-02-16 23:00 onward. -4. Insert in the same shape as 0331's rows (`('native','XLM','','')`, staged - method first, same verify queries), then promote together with them. -5. Re-run 0331's coarse-repair step for 201801–201802 on every tier. - -## Acceptance Criteria - -- [ ] Source chosen and recorded, with its terms of use. -- [ ] Hourly XLM/USD rows for every hour of 2018-01-01 → 2018-02-16 that has a - rate, provenance recorded. -- [ ] Agreement with the 0331 series on the overlap recorded (median and max - deviation). -- [ ] XLM 1d candles for 2018-01-01 → 2018-02-16 return a USD price, or the - gap is documented if option 3 is chosen. From 46448ad513e935a860f61d0ebd01ee900aa68e8e Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:52:16 +0200 Subject: [PATCH 089/144] docs(lore-0265): replace the exchange name with third-party Archived USDC source research (0247, 0265, 0267) named one of the exchanges; refer to it as third-party instead. --- ...l-usdc-usd-history-from-an-external-anchor.md | 2 +- .../README.md | 6 +++--- .../analysis/guardrails.py | 10 +++++----- .../data/sources.csv | 2 +- .../notes/R-peer-stablecoins.md | 2 +- .../notes/R-source-candidates.md | 10 +++++----- .../notes/S-composition-rule.md | 6 +++--- .../notes/S-guardrails.md | 16 ++++++++-------- .../notes/memo.md | 4 ++-- ...sdc-measured-history-from-chainlink-anchor.md | 4 ++-- 10 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lore/1-tasks/archive/0247_FEATURE_backfill-usdc-usd-history-from-an-external-anchor.md b/lore/1-tasks/archive/0247_FEATURE_backfill-usdc-usd-history-from-an-external-anchor.md index b7f1ae38..12cd257f 100644 --- a/lore/1-tasks/archive/0247_FEATURE_backfill-usdc-usd-history-from-an-external-anchor.md +++ b/lore/1-tasks/archive/0247_FEATURE_backfill-usdc-usd-history-from-an-external-anchor.md @@ -98,7 +98,7 @@ Load an external USDC/USD daily (or hourly) series into `prices.usd_rate` for th pre-oracle window, keyed on the canonical Stellar identity. - **Source:** USDC/USD on centralized venues trades against actual dollars and - reaches back to 2018 — CoinGecko, Kraken, CryptoCompare. Pick one, record which + reaches back to 2018 — CoinGecko, third-party, CryptoCompare. Pick one, record which and at what granularity; a daily close is enough for a peg. - **`method` must be its own value**, NOT `'oracle'`. `'oracle'` means a Reflector reading we polled ourselves. Something like `'external'`, added to diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/README.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/README.md index ba562663..befbce20 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/README.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/README.md @@ -160,7 +160,7 @@ Read in this order; every number is reproducible with `analysis/run.sh`. | phase | note | one line | |---|---|---| | 0 | [notes/S-phase0-root-cause.md](notes/S-phase0-root-cause.md) | the 1.0 is ours: `handlers.rs:643` → `queries_ch.rs:933`; 232-asset sweep: USDC is the only fully synthetic series, but `method: peg` sits on 135 assets incl. `native` | -| 1 | [notes/R-source-candidates.md](notes/R-source-candidates.md), [data/sources.csv](data/sources.csv) | 19 candidates; Chainlink primary, Bitstamp fallback, Kraken cross-check; USDT-quoted venues sit 20–30 bps off in stress | +| 1 | [notes/R-source-candidates.md](notes/R-source-candidates.md), [data/sources.csv](data/sources.csv) | 19 candidates; Chainlink primary, Bitstamp fallback, third-party cross-check; USDT-quoted venues sit 20–30 bps off in stress | | 2 | [notes/R-peer-stablecoins.md](notes/R-peer-stablecoins.md) | no peer is a proxy: DAI corr 0.994 (collateral), USDT −0.81 (flight-to), the rest idiosyncratic | | 3 | [notes/S-composition-rule.md](notes/S-composition-rule.md) | primary + flagged fallback/dispute, no averaging, no forward fill; `source`/`quality`/`n_obs` on the wire | | 4 | [notes/S-guardrails.md](notes/S-guardrails.md), `analysis/guardrails.py` | `trade_count = 0` is the only standalone discriminator; thresholds 2–4× outside the noisiest real window; 6 tests green | @@ -175,7 +175,7 @@ measured 6 bps, ES 25 bps, worst day 300 bps. ### From Plan -1. **Chainlink rounds as the primary anchor**, Bitstamp fallback, Kraken +1. **Chainlink rounds as the primary anchor**, Bitstamp fallback, third-party cross-check — chosen on measured dispersion, not on preference. 2. **Backfill via [[0247]]'s design** (`usd_rate`, `method='external'`, view first) rather than a second mechanism. @@ -188,7 +188,7 @@ measured 6 bps, ES 25 bps, worst day 300 bps. has one source and one method; hourly-heartbeat feeds sampled every 2nd–4th round (`stride` column), daily closes unaffected. 5. **Coin Metrics kept as analysis-only** because of CC BY-NC. -6. **Large reproducible CSVs gitignored** (`data/.gitignore`: Kraken trades +6. **Large reproducible CSVs gitignored** (`data/.gitignore`: third-party trades 28 MB, four peer round files ≈ 1 MB each); everything else in `data/` is versioned so the notes' numbers can be checked without refetching. 7. **The `.gitignore` root entry `.claude/`** was added on request; the four diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/analysis/guardrails.py b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/analysis/guardrails.py index ccc86db0..8b69591a 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/analysis/guardrails.py +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/analysis/guardrails.py @@ -3,7 +3,7 @@ thresholds derived from real data and the tests that prove they separate. Thresholds (why each number — measured 2026-09-04 with the snippet at the -bottom of this file over Chainlink USDC/USDT/DAI, Bitstamp, Kraken, OKX daily +bottom of this file over Chainlink USDC/USDT/DAI, Bitstamp, third-party, OKX daily bars; see notes/S-guardrails.md for the table): ZERO_TRADES_SHARE_30D = 0.9 @@ -21,17 +21,17 @@ EXACT_PEG_SHARE_30D = 0.9 share of closes exactly == 1.0 in a 30-day window. Real max over any - window: 0.67 (Kraken, which prints 4 decimals), 0.47 (OKX), 0.40 + window: 0.67 (third-party, which prints 4 decimals), 0.47 (OKX), 0.40 (Bitstamp, Chainlink). Ours: 1.00 for 1 864 consecutive days. 0.9 sits above the noisiest real window with margin and below ours by 0.1. CONSTANT_CLOSE_RUN_DAYS = 30 longest run of identical consecutive closes. Real max: 18 days - (Kraken, 4 dp), 10 (OKX), 4 (Chainlink, Bitstamp). Ours: 1 864. + (third-party, 4 dp), 10 (OKX), 4 (Chainlink, Bitstamp). Ours: 1 864. MIN_RVOL_30D_BPS = 0.1 30-day realized vol of daily log returns. Real minimum over every - window: 0.23 bps (Bitstamp), 0.33 (Chainlink), 0.58 (Kraken), 0.85 + window: 0.23 bps (Bitstamp), 0.33 (Chainlink), 0.58 (third-party), 0.85 (OKX); no real window is ever exactly zero. Ours: 0 on every window before 2026-03-11. 0.1 is less than half the quietest real window. @@ -140,7 +140,7 @@ def test_our_native_control_passes_price_checks() -> None: def test_real_usd_sources_pass() -> None: - for name in ["chainlink_usdc_1d.csv", "bitstamp_usd_1d.csv", "kraken_usd_1d_last720.csv", "okx_usdt_1d.csv", + for name in ["chainlink_usdc_1d.csv", "bitstamp_usd_1d.csv", "third_party_usd_1d_last720.csv", "okx_usdt_1d.csv", "chainlink_usdt_1d.csv", "chainlink_dai_1d.csv", "composed_usdc_usd_1d.csv"]: df = _load(name) if "trades" in df: diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/data/sources.csv b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/data/sources.csv index 31abf50a..972a8917 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/data/sources.csv +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/data/sources.csv @@ -1,6 +1,6 @@ source,pair,quote,coverage_start,granularity,gaps,latency_update_rule,cost,licence_redistribution,api_stability,methodology,integration_effort,keyless_history,march2023_min_close_1h,march2023_close_1d,verdict Bitstamp,USDC/USD,USD,2021-01-01 (probed; API silent before),1m..1d OHLC,none 2021-01..2026-09 (0 missing days),exchange candles; realtime,free public API,exchange ToS; not verified for redistribution,stable v2 API,exchange order book (real USD),low,yes,0.8627 @ 2023-03-11 07:00,0.9685,candidate primary (USD; reaches 2021 not 2018) -Kraken,USDC/USD,USD,OHLC endpoint: last 720 candles only (2024-09-14 for daily); trades endpoint: full history paginated,trades / 1m..1d OHLC (720 cap),none in OHLC window; 580 028 trades fetched for 2023-03,exchange; realtime,free public API (~1 req/s),exchange ToS; not verified for redistribution,stable v0 API,exchange order book (real USD),medium (history must be rebuilt from trades),yes but slow (trades),0.9010 @ 2023-03-11 12:00 (hourly close from trades; low 0.8740),0.9670,candidate fallback / cross-check (USD) +third-party,USDC/USD,USD,OHLC endpoint: last 720 candles only (2024-09-14 for daily); trades endpoint: full history paginated,trades / 1m..1d OHLC (720 cap),none in OHLC window; 580 028 trades fetched for 2023-03,exchange; realtime,free public API (~1 req/s),exchange ToS; not verified for redistribution,stable v0 API,exchange order book (real USD),medium (history must be rebuilt from trades),yes but slow (trades),0.9010 @ 2023-03-11 12:00 (hourly close from trades; low 0.8740),0.9670,candidate fallback / cross-check (USD) Binance,USDC/USDT,USDT,2018-12-15,1m..1d klines,165 missing days: delisted 2022-09-26 03:00 UTC to relist 2023-03-11 14:00 UTC (announcements fetched) — the gap covers the depeg onset and trough,exchange; realtime,free via data-api.binance.vision (already used in this repo),exchange ToS; not verified for redistribution,stable v3 API,exchange order book (USDT-quoted),low,yes,0.9128 @ 2023-03-11 15:00 (first hour after relist),0.9592 (partial day),cross-check only: USDT quote + gap across the event OKX,USDC/USDT,USDT,2020-01-01,1m..1d (1Dutc),none; one idiosyncratic print 2020-03-12 close 0.9395 (610 bps from Binance),exchange; realtime,free public API (100 rows/call),exchange ToS; not verified for redistribution,stable v5 API,exchange order book (USDT-quoted),low,yes,0.8787 @ 2023-03-11 07:00,0.9607,cross-check only (USDT quote) Bybit,USDC/USDT,USDT,2021-10-22,1m..1d,none,exchange; realtime,free public API,exchange ToS; not verified for redistribution,stable v5 API,exchange order book (USDT-quoted),low,yes,0.8840 @ 2023-03-11 07:00,0.9611,cross-check only (USDT quote; starts 2021-10) diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-peer-stablecoins.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-peer-stablecoins.md index 647d7102..b2088643 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-peer-stablecoins.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-peer-stablecoins.md @@ -94,7 +94,7 @@ None of that needs a peer's price to be published as USDC's. ## Consequence for phase 3 -Primary and fallback stay what phase 1 chose (Chainlink, Bitstamp, Kraken): +Primary and fallback stay what phase 1 chose (Chainlink, Bitstamp, third-party): all three *are* USDC/USD. The peer basket enters only as a validation input. The [[0172]] point stands separately: canonical Stellar USDT (`GCQTGZQQ…TG6V`) is an IOU that traded at ~0.13, and its `/ohlcv` series is diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-source-candidates.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-source-candidates.md index 4964a8f2..e5ebfde8 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-source-candidates.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/R-source-candidates.md @@ -34,7 +34,7 @@ result the brief expected. [data] | source | quote | hourly min close (UTC) | 2023-03-11 daily close | daily low | |---|---|---|---|---| | Bitstamp | USD | **0.8627** @ 03-11 07:00 | 0.9685 | 0.8053 | -| Kraken (from 580 028 trades) | USD | 0.9010 @ 03-11 12:00 | 0.9670 | 0.8740 | +| third-party (from 580 028 trades) | USD | 0.9010 @ 03-11 12:00 | 0.9670 | 0.8740 | | Chainlink on-chain (3 240 rounds read, 2021-02-17 → today) | USD | **0.8800** @ 03-11 07:51:23, 243 rounds that day | 0.9681 | 0.8800 | | OKX | USDT | 0.8787 @ 03-11 07:00 | 0.9607 | 0.8698 | | Bybit | USDT | 0.8840 @ 03-11 07:00 | 0.9611 | 0.8500 | @@ -58,7 +58,7 @@ trough: the Fed's FEDS Note (2025-12-17) says "at its trough, USDC traded at | Bitstamp | 2021-01-01 | 0 | — (API silent before 2021; listing date n/v) | | Chainlink | 2021-02-17 | 1 (of 2 026 days) | 3 phase aggregators, round ids non-contiguous; 3 240 rounds total | | Bybit | 2021-10-22 | 0 | — | -| Kraken OHLC | 2024-09-14 | 0 | endpoint caps at 720 candles; older history only via trades | +| third-party OHLC | 2024-09-14 | 0 | endpoint caps at 720 candles; older history only via trades | Our own series starts 2021-01-25 ([[0247]]). So the **only** key-less USD sources that reach our start are Coin Metrics (daily, non-commercial licence) @@ -72,9 +72,9 @@ From `data/dispersion_daily.csv` (calm = every common day outside 2023-03-08..18 | pair | common days | median calm | p95 calm | max calm | median stress | max stress | |---|---|---|---|---|---|---| | Chainlink – Bitstamp (USD–USD) | 2025 | **0.8** | 6.8 | 36 | **4.0** | 33 | -| Chainlink – Kraken (USD–USD) | 721 | **0.6** | 1.7 | 2.8 | — | — | +| Chainlink – third-party (USD–USD) | 721 | **0.6** | 1.7 | 2.8 | — | — | | Chainlink – Coin Metrics (USD–USD) | 2024 | 0.9 | 5.0 | 83 | 2.1 | 25 | -| Bitstamp – Kraken (USD–USD) | 721 | 0.8 | 2.3 | 6.3 | — | — | +| Bitstamp – third-party (USD–USD) | 721 | 0.8 | 2.3 | 6.3 | — | — | | Bitstamp – Coin Metrics (USD–USD) | 2072 | 0.9 | 6.5 | 105 | 2.4 | 33 | | Binance – Bybit (USDT–USDT) | 1614 | **1.0** | 3.0 | 48 | 1.5 | 19 | | Binance – OKX (USDT–USDT) | 2274 | 1.0 | 5.0 | 610 | 1.0 | 15 | @@ -128,7 +128,7 @@ times, nothing verified. [docs] cost hundreds, history from 2021-02-17. Cost: a few thousand `eth_call`s once, then one call per new round; no key needed today. 2. **Fallback and cross-check: Bitstamp USDC/USD** (from 2021-01-01, real USD - book, free, hourly), with Kraken trades as the second USD venue where the + book, free, hourly), with third-party trades as the second USD venue where the two need arbitration. 3. **Reject as primary:** every USDT-quoted venue (cluster offset above), Binance specifically (gap on the event), Coin Metrics (licence), CoinGecko diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-composition-rule.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-composition-rule.md index 40a48071..b7644a09 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-composition-rule.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-composition-rule.md @@ -26,9 +26,9 @@ history: | step | rule | why (measured) | |---|---|---| -| **primary** | Chainlink USDC/USD rounds, Ethereum mainnet proxy `0x8fFf…18f6` | real USD, verifiable by anyone with an RPC, no licence, 0.25 % deviation + 23 h heartbeat, from 2021-02-17; agrees with Bitstamp to 0.8 bps median calm / 4 bps stress and with Kraken to 0.6 bps | +| **primary** | Chainlink USDC/USD rounds, Ethereum mainnet proxy `0x8fFf…18f6` | real USD, verifiable by anyone with an RPC, no licence, 0.25 % deviation + 23 h heartbeat, from 2021-02-17; agrees with Bitstamp to 0.8 bps median calm / 4 bps stress and with third-party to 0.6 bps | | **fallback** | Bitstamp USDC/USD, used only where the primary has **no observation in the bucket** | the only key-less real-USD book reaching 2021-01-01; same cluster as the primary | -| **cross-check** | Kraken (OHLC last 720 d, trades for 2023-03) and Bitstamp | independent USD venues; USDT-quoted venues are excluded from arbitration because their cluster sits 3–4 bps off in calm and 20–30 bps off in stress (that is USDT/USD, not USDC/USD) | +| **cross-check** | third-party (OHLC last 720 d, trades for 2023-03) and Bitstamp | independent USD venues; USDT-quoted venues are excluded from arbitration because their cluster sits 3–4 bps off in calm and 20–30 bps off in stress (that is USDT/USD, not USDC/USD) | | **disagreement** | if a cross-check differs from the primary by **> 25 bps** on the bucket close, keep the primary value and flag `measured-disputed` | 25 bps is 3.5× the calm p95 between USD sources (≤ 7 bps) and below the stress median between them (4 bps median, 33 bps max). No averaging: a median of venues is a number nobody printed, and the reviewer's question "where did 0.9681 come from" must have a one-word answer | | **gaps** | a bucket with no source publishes **no price** and `quality = missing` | forward-fill is the defect being removed. The only carry allowed is *inside the primary's own heartbeat* on the hourly grain: an hour with no round means "no 0.25 % move since the last round", and the row says `n_obs = 0` | @@ -65,7 +65,7 @@ Against our series on the 2 042 common days: median |diff| 0.8 bps, p99 max/min, close = last; `n_obs` = number of rounds. A day with one round is honestly flat (68 % of Chainlink days), which is why flatness is *not* a synthetic signal in phase 4. -- **Candle from trades (Kraken):** standard OHLC over trade prints, +- **Candle from trades (third-party):** standard OHLC over trade prints, `n_obs` = trades. - **Venues that were closed** (Binance 2022-09-26 → 2023-03-11 14:00) are simply absent for those buckets; they are never interpolated. diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-guardrails.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-guardrails.md index 180df1b1..38b9c4d7 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-guardrails.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/S-guardrails.md @@ -12,7 +12,7 @@ history: status: seed who: akot note: > - Thresholds measured over Chainlink USDC/USDT/DAI, Bitstamp, Kraken and + Thresholds measured over Chainlink USDC/USDT/DAI, Bitstamp, third-party and OKX daily bars; tests run green on 2026-09-04 (`python3 analysis/guardrails.py`). --- @@ -27,9 +27,9 @@ legitimate twin on real data: | signal | why it false-positives alone [data] | |---|---| | O = H = L = C | 68 % of Chainlink USDC days (one heartbeat round per day), 88 % of USDT days; 20 thin Stellar assets with a single trade a day | -| close exactly 1.0 | 16 % of Kraken days (4-decimal prints), up to 67 % of a 30-day Kraken window; 8–10 % on every other venue | +| close exactly 1.0 | 16 % of third-party days (4-decimal prints), up to 67 % of a 30-day third-party window; 8–10 % on every other venue | | zero realised vol | never on real data, but a 30-day window can be as quiet as 0.23 bps | -| long constant run | Kraken 18 days, OKX 10 days | +| long constant run | third-party 18 days, OKX 10 days | So the invariants are joined with `trade_count`, and the price-shaped ones carry thresholds sized off the *noisiest real window*, not off intuition. @@ -40,9 +40,9 @@ carry thresholds sized off the *noisiest real window*, not off intuition. |---|---|---|---| | `zero_trades_share_30d` | ≥ 0.9 → **reject** | 0.00 on every venue | 1.00 | | `flat_and_no_trades_share_30d` | ≥ 0.9 → reject | 0.00 | 1.00 | -| `exact_peg_share_30d` | ≥ 0.9 → alert | 0.67 (Kraken) | 1.00 | -| `constant_close_run_days` | > 30 → alert | 18 (Kraken) | 1 864 | -| `min_rvol_30d_bps` | < 0.1 bps → alert | 0.23 (Bitstamp), 0.33 (Chainlink), 0.58 (Kraken), 0.85 (OKX); 0 never | 0.00 | +| `exact_peg_share_30d` | ≥ 0.9 → alert | 0.67 (third-party) | 1.00 | +| `constant_close_run_days` | > 30 → alert | 18 (third-party) | 1 864 | +| `min_rvol_30d_bps` | < 0.1 bps → alert | 0.23 (Bitstamp), 0.33 (Chainlink), 0.58 (third-party), 0.85 (OKX); 0 never | 0.00 | | `zero_volume_share_30d` | ≥ 0.9 → alert | 0.00 | 1.00 | "Reject" means the series fails CI (a fixture test over `/ohlcv` for every @@ -56,7 +56,7 @@ production series, one evaluation per day, which fits the [[0125]] dashboard |---|---| | `test_our_usdc_series_fails` | our `/ohlcv` USDC series trips zero-trades, exact-peg, constant-run and rvol | | `test_our_native_control_passes_price_checks` | XLM, a non-stablecoin, trips nothing | -| `test_real_usd_sources_pass` | Chainlink USDC/USDT/DAI, Bitstamp, Kraken, OKX and the composed series trip nothing over their full history | +| `test_real_usd_sources_pass` | Chainlink USDC/USDT/DAI, Bitstamp, third-party, OKX and the composed series trip nothing over their full history | | `test_stress_window_does_not_trigger` | 2023-02 → 2023-04 on Chainlink trips nothing (a depeg is not an anomaly of the *series*) | | `test_a_quiet_real_quarter_does_not_trigger` | the quietest 30 days ever printed on Bitstamp (0.23 bps) pass | | `test_synthetic_injection_is_caught` | a 45-day splice of exactly 1.0 into Bitstamp trips the constant-run check | @@ -80,5 +80,5 @@ production series, one evaluation per day, which fits the [[0125]] dashboard `data/peer_corr.csv`. Thresholds must be re-derived if the surface changes grain or precision: -Kraken's 4-decimal prints are what put `exact_peg_share_30d` at 0.9 rather +third-party's 4-decimal prints are what put `exact_peg_share_30d` at 0.9 rather than 0.5. diff --git a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/memo.md b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/memo.md index b65694f0..17f09412 100644 --- a/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/memo.md +++ b/lore/1-tasks/archive/0265_FEATURE_price-usdc-from-measurement-not-the-peg/notes/memo.md @@ -61,7 +61,7 @@ assumed"; a consumer cannot tell it from "USDC unmeasured". low **0.8800** at 07:51:23 UTC. 2. **Fallback: Bitstamp USDC/USD**, only where the primary has no observation (24 days: the 2021-01-25 → 02-16 tail and a few sparse days). - **Cross-check: Kraken and Bitstamp.** A daily close more than 25 bps from + **Cross-check: third-party and Bitstamp.** A daily close more than 25 bps from a cross-check keeps the primary value and is flagged `measured-disputed` (4 days in five years). No averaging: every published number is one venue's print. @@ -86,7 +86,7 @@ assumed"; a consumer cannot tell it from "USDC unmeasured". ## Rationale - **Dispersion decides the source order.** USD sources (Chainlink, Bitstamp, - Kraken, Coin Metrics) agree to < 1 bps median; USDT-quoted venues agree + third-party, Coin Metrics) agree to < 1 bps median; USDT-quoted venues agree with each other to 1 bps but sit **3–4 bps** off the USD cluster in calm and **20–30 bps** off in stress — that offset is USDT/USD. So USDT-quoted candles are cross-checks after correction, never primaries. Binance diff --git a/lore/1-tasks/archive/0267_FEATURE_usdc-measured-history-from-chainlink-anchor.md b/lore/1-tasks/archive/0267_FEATURE_usdc-measured-history-from-chainlink-anchor.md index 46cac714..47548518 100644 --- a/lore/1-tasks/archive/0267_FEATURE_usdc-measured-history-from-chainlink-anchor.md +++ b/lore/1-tasks/archive/0267_FEATURE_usdc-measured-history-from-chainlink-anchor.md @@ -74,7 +74,7 @@ The acceptance fixture is the falsifying date: **2023-03-11 must close at - The source decision and its evidence: [[0265]] `notes/memo.md`. Primary Chainlink USDC/USD rounds (on-chain, key-less, 2021-02-17 →), fallback - Bitstamp, cross-check Kraken. USDT-quoted venues and peer stablecoins + Bitstamp, cross-check third-party. USDT-quoted venues and peer stablecoins rejected on measured dispersion and correlation. - The composed daily series already exists (`compose_usdc.py` output, 2 049 rows 2021-01-25 → 2026-03-10: 98.6 % measured, 1.2 % fallback, @@ -218,7 +218,7 @@ What shipped, on `feat/0267_usdc-measured-history-from-chainlink-anchor` **The source, its granularity and its fetch date**, because an imported series with no provenance is not evidence ([[0247]]'s fifth criterion). Primary **Chainlink** USDC/USD on-chain rounds, fallback **Bitstamp**, cross-checked -against Kraken; composed by [[0265]]'s `compose_usdc.py` and versioned in this repo at **two +against third-party; composed by [[0265]]'s `compose_usdc.py` and versioned in this repo at **two grains**: - `lore/1-tasks/archive/0265_.../data/composed_usdc_usd_1d.csv` — **2049 daily From ec3d21bf8d6c9926e7ced072edaf24de14a426d5 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:58:50 +0200 Subject: [PATCH 090/144] fix(lore-0331): roll up wide tiers from priced 1h rows On 4h/1d/1w/1M the step priced the whole bucket's volume at the rate of its last hour, which is far off when XLM moves within a month. These tiers now sum their priced 1h rows (preroll.sql rule), so 1h runs first. Re-runs that would set priced rows back to 0 are guarded: 1m needs --allow-unprice, the coarse tiers a snapshot or --snapshots-verified. The dry run reports rows to write, left unpriced and set back to 0; a month with nothing to write gets no FREEZE. XLM is resolved once, the watermark bind is gone, and the drain loop is shared with the external tier. The /ohlcv text no longer implies XLM is a dollar token. --- packages/coarse-sweep-worker/src/main.rs | 2 +- .../enrichment-worker/examples/run_ch_pass.rs | 2 +- .../src/bin/coarse-repair.rs | 68 ++- packages/enrichment-worker/src/ch_enrich.rs | 453 ++++++++++-------- packages/enrichment-worker/src/main.rs | 2 +- packages/enrichment-worker/src/repair.rs | 46 +- packages/prices-api/src/assets/handlers.rs | 8 +- packages/prices-api/src/assets/queries_ch.rs | 4 +- .../prices-api/src/openapi/descriptions.rs | 4 +- web/portal/public/openapi.json | 4 +- 10 files changed, 335 insertions(+), 258 deletions(-) diff --git a/packages/coarse-sweep-worker/src/main.rs b/packages/coarse-sweep-worker/src/main.rs index 38eeb452..02181537 100644 --- a/packages/coarse-sweep-worker/src/main.rs +++ b/packages/coarse-sweep-worker/src/main.rs @@ -90,7 +90,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { // epoch explicitly, via the `coarse-repair` binary. usd_reset: None, // Opt-in coarse-repair mode only (task 0331). - pre_usdc_xlm_rate: false, + pre_usdc_xlm_rate: None, }; // Cold start, mirroring the sibling workers: build and probe the mTLS client diff --git a/packages/enrichment-worker/examples/run_ch_pass.rs b/packages/enrichment-worker/examples/run_ch_pass.rs index 778df13d..0fc68dc2 100644 --- a/packages/enrichment-worker/examples/run_ch_pass.rs +++ b/packages/enrichment-worker/examples/run_ch_pass.rs @@ -30,7 +30,7 @@ async fn main() -> Result<(), Box> { one_shot: env_parse_or("ENRICHMENT_ONE_SHOT", false), time_window: None, usd_reset: None, - pre_usdc_xlm_rate: false, + pre_usdc_xlm_rate: None, }; let pass = ChEnrichmentPass::new(cfg); diff --git a/packages/enrichment-worker/src/bin/coarse-repair.rs b/packages/enrichment-worker/src/bin/coarse-repair.rs index af73cff9..c98f5794 100644 --- a/packages/enrichment-worker/src/bin/coarse-repair.rs +++ b/packages/enrichment-worker/src/bin/coarse-repair.rs @@ -26,13 +26,16 @@ //! //! `price_ohlcv_1m` is accepted too, for a whole-history drain (task 0286 §7b): //! the hourly Lambda prices at most 200 k rows an hour, which is months for a -//! re-ingested history. Only the fill-the-zeros mode is allowed there — no -//! `--reset-*` — and only with `--skip-snapshot`, see [`check_table`]. +//! re-ingested history. Only the fill-the-zeros mode (and `--pre-usdc-xlm-rate`) +//! is allowed there — no `--reset-*` — and only with `--skip-snapshot`, see +//! [`check_table`]. use std::path::PathBuf; use clap::{Parser, ValueEnum}; -use enrichment_worker::ch_enrich::{ChEnrichConfig, UsdResetSpec, external_window_s}; +use enrichment_worker::ch_enrich::{ + ChEnrichConfig, PreUsdcXlmRate, UsdResetSpec, external_window_s, +}; use enrichment_worker::repair::{CoarseRepairConfig, CoarseRepairDriver}; use prices_clickhouse::USDC_ORACLE_EPOCH_S; use tracing::{info, warn}; @@ -75,7 +78,7 @@ struct Args { mtls_ca_path: Option, /// Table to repair (e.g. `price_ohlcv_1h`). One table per run. - /// `price_ohlcv_1m` takes only the fill-the-zeros mode (see `check_table`). + /// `price_ohlcv_1m` takes no `--reset-*` (see `check_table`). #[arg(long)] table: String, @@ -122,16 +125,28 @@ struct Args { /// promoted `method = 'external'` XLM/USD rows, and dollar-token-quoted ones /// through that token's own XLM market within `--pivot-window-s`. Writes only /// rows whose values differ, so a re-run is idempotent. Refused until the - /// series is promoted. On `price_ohlcv_1m` it needs `--skip-snapshot`. + /// series is promoted. + /// + /// Run `price_ohlcv_1h` (and 1m/15m) first: 4h, 1d, 1w and 1M are rolled up + /// from the priced 1h rows (Σ USD volume, last priced hour's ratio), so + /// `--pivot-window-s` matters only up to 1h, and a wide tier run before its + /// 1h writes nothing. A re-run can set priced rows back to 0: on + /// `price_ohlcv_1m` that needs `--allow-unprice`, elsewhere a snapshot. #[arg(long, conflicts_with = "reset_quote_asset_id")] pre_usdc_xlm_rate: bool, + /// With `--pre-usdc-xlm-rate` on `price_ohlcv_1m`: accept writing priced + /// rows back to 0 (the dry run's `reset` column), with no snapshot. Without + /// it a real run stops at the first such month; earlier months stay written. + #[arg(long, requires = "pre_usdc_xlm_rate")] + allow_unprice: bool, + /// Re-open already-written USD values for this quote `asset_id` so the /// corrected tiers recompute them (task 0182). Requires /// `--reset-not-before`. /// - /// ⚠️ This is the only flag here that DISCARDS a computed value. Everything - /// else in this tool is purely additive — it fills zeros. Use it only to + /// ⚠️ It DISCARDS a computed value, like the 0228 mode and + /// `--pre-usdc-xlm-rate`; the default mode only fills zeros. Use it only to /// correct a *pricing* defect, where the stored number is wrong rather than /// missing, and only against a FREEZE-snapshotted partition. /// @@ -286,7 +301,9 @@ fn validate_month(label: &str, m: u32) -> Result<(), String> { /// nothing, so it can run beside the Lambda (task 0286 §7b). A reset there /// would discard prices every other tier is rolled from, so it is refused. /// FREEZE is refused too: `prices_writer` cannot take it, and a 1m snapshot -/// protects nothing in a mode that only fills zeros. +/// protects nothing in a mode that only fills zeros. `--pre-usdc-xlm-rate` is +/// allowed there too: it can discard a price, so the pass refuses a 1m run +/// that would unless `--allow-unprice` is passed. fn check_table( table: &str, resetting: bool, @@ -311,7 +328,8 @@ fn check_table( if !skip_snapshot && !dry_run { return Err( "price_ohlcv_1m needs --skip-snapshot: prices_writer cannot FREEZE, and \ - the fill-the-zeros mode discards nothing a snapshot would protect." + the fill-the-zeros mode discards nothing a snapshot would protect \ + (--pre-usdc-xlm-rate refuses to without --allow-unprice)." .into(), ); } @@ -345,9 +363,10 @@ async fn main() -> Result<(), Box> { args.dry_run, )?; - // Reset mode is the only path here that discards data, so its two footguns - // are refused rather than warned about. - if args.reset_quote_asset_id.is_some() && !args.dry_run { + // Reset mode discards data, and so can a --pre-usdc-xlm-rate re-run (on 1m + // the pass guards that instead), so their footguns are refused. + let pre_usdc_coarse = args.pre_usdc_xlm_rate && args.table != "price_ohlcv_1m"; + if (args.reset_quote_asset_id.is_some() || pre_usdc_coarse) && !args.dry_run { // 1. Rollback for a bad reset IS `ATTACH PARTITION` from the shadow copy, // so the snapshots must exist. They usually will not have been taken by // this tool: `prices_writer` cannot FREEZE and cannot be granted it, so @@ -356,8 +375,9 @@ async fn main() -> Result<(), Box> { // outright would make reset mode unusable on the only cluster it exists // for. So require the operator to say which case they are in. if args.skip_snapshot && !args.snapshots_verified { - return Err("--skip-snapshot with --reset-quote-asset-id requires \ - --snapshots-verified: a reset discards stored values, and its only \ + return Err("--skip-snapshot with --reset-quote-asset-id or \ + --pre-usdc-xlm-rate requires --snapshots-verified: both can discard \ + stored values, and their only \ rollback is ATTACH PARTITION from the frozen copy.\n\ If the CH admin already froze this span (runbook Step 3b — the \ normal prod path, since prices_writer cannot FREEZE), confirm with \ @@ -484,7 +504,9 @@ async fn main() -> Result<(), Box> { // clap's `requires` makes the mixed cases unreachable. _ => None, }, - pre_usdc_xlm_rate: args.pre_usdc_xlm_rate, + pre_usdc_xlm_rate: args.pre_usdc_xlm_rate.then_some(PreUsdcXlmRate { + allow_unprice: args.allow_unprice, + }), }; let repair_cfg = CoarseRepairConfig { @@ -539,7 +561,10 @@ async fn main() -> Result<(), Box> { driver.preflight().await?; if args.skip_snapshot && !args.dry_run && args.table == "price_ohlcv_1m" { - info!("snapshot disabled on price_ohlcv_1m: fill-the-zeros mode, nothing is discarded"); + info!( + allow_unprice = args.allow_unprice, + "snapshot disabled on price_ohlcv_1m: nothing is discarded unless --allow-unprice" + ); } else if args.skip_snapshot && !args.dry_run { warn!( "snapshot DISABLED (--skip-snapshot): partitions will be repaired with no FREEZE \ @@ -602,14 +627,19 @@ async fn main() -> Result<(), Box> { if args.pre_usdc_xlm_rate { println!( "--pre-usdc-xlm-rate: zeros_before = rows to write, enriched = rows written, \ - zeros_after = rows still differing (> 0 after a capped run: re-run)" + zeros_after = rows left unpriced, reset = priced rows set back to 0" ); } println!( - "{} month(s): {} enriched, {} left at the no_reference floor{}", + "{} month(s): {} enriched, {} {}{}", summary.months.len(), summary.total_enriched(), summary.total_remaining(), + if args.pre_usdc_xlm_rate { + "left unpriced" + } else { + "left at the no_reference floor" + }, if args.dry_run { " [DRY RUN — nothing written]" } else { @@ -621,7 +651,7 @@ async fn main() -> Result<(), Box> { // arithmetic. `rows_reset` > `rows_enriched` means values were discarded and // not recomputed — the one outcome worse than the defect being repaired. let (reset, enriched) = (summary.total_reset(), summary.total_enriched()); - if reset > 0 { + if reset > 0 && !args.pre_usdc_xlm_rate { println!("{reset} row(s) re-opened by the USD reset, {enriched} recomputed"); println!( "Run the damage check (runbook Appendix A, 'Extra verification') on \ diff --git a/packages/enrichment-worker/src/ch_enrich.rs b/packages/enrichment-worker/src/ch_enrich.rs index 6ea66209..7f633b24 100644 --- a/packages/enrichment-worker/src/ch_enrich.rs +++ b/packages/enrichment-worker/src/ch_enrich.rs @@ -1002,9 +1002,25 @@ pub struct ChEnrichConfig { /// computed value if an operator explicitly names the quote leg and epoch. /// See [`UsdResetSpec`]. pub usd_reset: Option, - /// Opt-in coarse-repair mode (task 0331), `false` on every scheduled path. - /// When `true` the pass runs only the pre-2021 XLM/USD step. - pub pre_usdc_xlm_rate: bool, + /// Opt-in coarse-repair mode (task 0331), `None` on every scheduled path. + /// When set the pass runs only the pre-2021 XLM/USD step. + pub pre_usdc_xlm_rate: Option, +} + +/// Options of the pre-2021 XLM/USD step (task 0331). +#[derive(Debug, Clone, Copy, Default)] +pub struct PreUsdcXlmRate { + /// Let a `price_ohlcv_1m` run set priced rows back to 0 (1m has no snapshot). + pub allow_unprice: bool, +} + +/// The pre-2021 step's counts: rows a run writes, priced rows it sets back to +/// 0, and rows left unpriced after it. +#[derive(Debug, Clone, Copy, Default, clickhouse::Row, Deserialize)] +pub struct PreUsdcCounts { + pub pending: u64, + pub unprice: u64, + pub unpriced: u64, } impl Default for ChEnrichConfig { @@ -1022,7 +1038,7 @@ impl Default for ChEnrichConfig { one_shot: false, time_window: None, usd_reset: None, - pre_usdc_xlm_rate: false, + pre_usdc_xlm_rate: None, } } } @@ -1515,8 +1531,11 @@ impl ChEnrichmentPass { /// The last step (task 0208) is `assert_reset_epoch_is_covered`: a /// pivot leg's `not_before` below the table's first priced reference /// candle is refused, because nothing could refill the rows in between. + /// + /// With [`ChEnrichConfig::pre_usdc_xlm_rate`] set it runs only that mode's + /// refusals ([`Self::assert_pre_usdc_admissible`]). pub async fn assert_reset_is_admissible(&self) -> Result<(), ChEnrichError> { - if self.cfg.pre_usdc_xlm_rate { + if self.cfg.pre_usdc_xlm_rate.is_some() { return self.assert_pre_usdc_admissible().await.map(|_| ()); } let Some(spec) = self.cfg.usd_reset.as_ref() else { @@ -2039,8 +2058,8 @@ impl ChEnrichmentPass { } /// The pre-2021 step's refusals; returns native XLM's id. The driver calls - /// this through [`Self::assert_reset_is_admissible`], so a dry run refuses too. - async fn assert_pre_usdc_admissible(&self) -> Result { + /// this once up front, so a dry run refuses too. + pub async fn assert_pre_usdc_admissible(&self) -> Result { let refuse = |m: &str| Err(ChEnrichError::PreUsdcXlmRateRefused(m.to_string())); if self.cfg.usd_reset.is_some() { return refuse("it cannot be combined with a --reset-* spec"); @@ -2063,69 +2082,80 @@ impl ChEnrichmentPass { } fn pre_usdc_statements(&self, xlm: u64) -> (String, String) { - let pw = self.cfg.pivot_window_s; - // Bound the token-market scan to the month plus its look-back. - let ref_window = match self.cfg.time_window { - Some((s, e)) => format!( - " AND timestamp >= toDateTime({}) AND timestamp < toDateTime({e})", - s.saturating_sub(pw) - ), - None => String::new(), - }; - let db = &self.cfg.database; - let (cand, tbl) = (self.window_pred("timestamp"), &self.cfg.table); - pre_usdc_sql(db, tbl, xlm, pw, &cand, &ref_window) + let (db, tbl) = (&self.cfg.database, &self.cfg.table); + pre_usdc_sql(db, tbl, xlm, self.cfg.pivot_window_s, self.cfg.time_window) } - /// Rows a run with this config would write now (the dry-run figure). - pub async fn pre_usdc_pending(&self) -> Result { - let xlm = self.assert_pre_usdc_admissible().await?; - let watermark = self.watermark().await?; + /// [`PreUsdcCounts`] now; `pending` is the dry-run figure. + pub async fn pre_usdc_counts(&self, xlm: u64) -> Result { let (_, count) = self.pre_usdc_statements(xlm); - Ok(self - .client - .query(&count) - .bind(watermark) - .fetch_one::() - .await?) + Ok(self.client.query(&count).fetch_one().await?) } - /// The pre-2021 XLM/USD step (task 0331), looped to a fixed point like - /// [`Self::run_external_tier`], counting with its own pending query. - async fn run_pre_usdc(&self, watermark: u32) -> Result { + /// The pre-2021 XLM/USD step (task 0331), drained to a fixed point. In its + /// stats `candidates_after` is the rows left unpriced and `rows_reset` the + /// priced rows set back to 0. + pub async fn run_pre_usdc(&self, xlm: u64) -> Result { let start = std::time::Instant::now(); - let xlm = self.assert_pre_usdc_admissible().await?; - let (insert, count) = self.pre_usdc_statements(xlm); - let pending = || self.client.query(&count).bind(watermark).fetch_one::(); - let before = pending().await?; - let (mut remaining, mut batches) = (before, 0u32); + let before = self.pre_usdc_counts(xlm).await?; + let allow = self.cfg.pre_usdc_xlm_rate.is_some_and(|m| m.allow_unprice); + if before.unprice > 0 && self.cfg.table == "price_ohlcv_1m" && !allow { + return Err(ChEnrichError::PreUsdcXlmRateRefused(format!( + "it would set {} priced price_ohlcv_1m row(s) back to 0, and 1m has no \ + snapshot to roll back to (pass --allow-unprice to accept)", + before.unprice + ))); + } + let (insert, _) = self.pre_usdc_statements(xlm); + let insert = self.client.query(&insert).bind(self.cfg.batch_size); + let last = std::cell::Cell::new(before); + let (remaining, batches) = self + .drain( + before.pending, + async || Ok(insert.clone().execute().await?), + async || { + last.set(self.pre_usdc_counts(xlm).await?); + Ok(last.get().pending) + }, + "pre-2021 XLM/USD step stalled — rows still differ", + ) + .await?; + Ok(ChPassStats { + batches, + candidates_before: before.pending, + candidates_after: last.get().unpriced, + rows_enriched: before.pending.saturating_sub(remaining), + rows_reset: before.unprice, + duration_ms: start.elapsed().as_millis() as u64, + ..Default::default() + }) + } + + /// Runs `insert` until `pending` reaches 0 or stops falling (logged as + /// `stalled`), at most `effective_max_batches` times. Returns + /// `(remaining, batches)`. + async fn drain( + &self, + mut remaining: u64, + insert: impl AsyncFn() -> Result<(), ChEnrichError>, + pending: impl AsyncFn() -> Result, + stalled: &str, + ) -> Result<(u64, u32), ChEnrichError> { + let mut batches = 0; for _ in 0..self.effective_max_batches() { if remaining == 0 { break; } - self.client - .query(&insert) - .bind(watermark) - .bind(self.cfg.batch_size) - .execute() - .await?; + insert().await?; let after = pending().await?; batches += 1; if after >= remaining { - remaining = after; - break; + info!(remaining = after, table = %self.cfg.table, "{stalled}"); + return Ok((after, batches)); } remaining = after; } - info!(before, remaining, batches, table = %self.cfg.table, "pre-2021 XLM/USD step done"); - Ok(ChPassStats { - batches, - candidates_before: before, - candidates_after: remaining, - rows_enriched: before.saturating_sub(remaining), - duration_ms: start.elapsed().as_millis() as u64, - ..Default::default() - }) + Ok((remaining, batches)) } /// Tier 2 — the peg-pivot deep-history backbone, over the fixed `watermark` @@ -2166,8 +2196,8 @@ impl ChEnrichmentPass { &self, usdc_id: u64, watermark: u32, - mut remaining: u64, - mut batches: u32, + remaining: u64, + batches: u32, ) -> Result<(u64, u32), ChEnrichError> { let sql = external_sql( &self.cfg.database, @@ -2176,31 +2206,17 @@ impl ChEnrichmentPass { &self.window_pred("timestamp"), ); let stale = external_window_s(&self.cfg.table); - for _ in 0..self.effective_max_batches() { - if remaining == 0 { - break; - } - self.client - .query(&sql) - .bind(watermark) - .bind(stale) - .bind(self.cfg.batch_size) - .execute() - .await?; - let after = self.count_candidates(watermark).await?; - batches += 1; - if after >= remaining { - info!( - remaining = after, - table = %self.cfg.table, - "external tier drained — remaining candles have no imported USD rate for their bucket" - ); - remaining = after; - break; - } - remaining = after; - } - Ok((remaining, batches)) + let insert = self.client.query(&sql).bind(watermark).bind(stale); + let insert = insert.bind(self.cfg.batch_size); + let (remaining, n) = self + .drain( + remaining, + async || Ok(insert.clone().execute().await?), + async || self.count_candidates(watermark).await, + "external tier drained — remaining candles have no imported USD rate for their bucket", + ) + .await?; + Ok((remaining, batches + n)) } /// Returns the updated `(remaining, batches)`. @@ -2274,8 +2290,10 @@ impl ChEnrichmentPass { /// pin the boundary and assert that candles newer than the snapshot are /// deferred, not enriched, in the current pass. pub async fn run_through(&self, watermark: u32) -> Result { - if self.cfg.pre_usdc_xlm_rate { - return self.run_pre_usdc(watermark).await; + if self.cfg.pre_usdc_xlm_rate.is_some() { + return self + .run_pre_usdc(self.assert_pre_usdc_admissible().await?) + .await; } let start = std::time::Instant::now(); @@ -2889,94 +2907,108 @@ pub fn pre_usdc_scope_pred(db: &str) -> String { ) } -/// Pre-2021 step (task 0331): `close_usd = close × f × rate`, where `rate` is -/// the newest `external` XLM/USD row before the bucket end (within -/// [`external_window_s`]) and `f` is 1 on an XLM quote, else the dollar -/// token's XLM price from its own XLM markets (both orientations, pf columns) -/// at or before the bucket start, within `pivot_window_s`. +/// Pre-2021 step (task 0331). Up to `_1h`, `close_usd = close × f × rate`: +/// `rate` is the newest `external` XLM/USD row before the bucket end (within +/// [`external_window_s`]); `f` is 1 on an XLM quote, else the dollar token's +/// XLM price from its own XLM markets (both orientations, pf columns) at or +/// before the bucket start, within `pivot_window_s`. Wider tiers roll up their +/// `price_ohlcv_1h` rows as `preroll.sql` does (Σ USD volume, `close` × the +/// last priced hour's `close_usd / close`), so `_1h` must be priced first. /// /// Writes only rows whose computed pair differs from the stored one, so the /// batch loop terminates, a re-run writes nothing, and a newer rate or a /// narrower window re-prices (an unreachable row computes 0). /// -/// Returns `(insert, pending_count)` from one body. Binds: the watermark, then -/// (insert only) the `LIMIT`. +/// Returns `(insert, counts)` over one body; the insert binds the `LIMIT`. fn pre_usdc_sql( db: &str, tbl: &str, xlm: u64, pivot_window_s: u32, - cand_window: &str, - ref_window: &str, + window: Option<(u32, u32)>, ) -> (String, String) { let end = prices_clickhouse::PRE_USDC_END_S; let bend = bucket_end_expr(tbl, "timestamp"); - let stale = external_window_s(tbl); - let scope = pre_usdc_scope_pred(db); - let series = usd_rate_series_sql(db, XLM_RATE_IDENTITY); - let dollar = format!( - "SELECT asset_id FROM {db}.assets FINAL WHERE {}", - dollar_token_asset_pred() - ); - // Decimal256 sums: a Decimal(38, 14) sum wraps silently, a Float64 one - // depends on merge order and would make the differs filter flap. - let amt = |a: &str, b: &str| { - format!("toFloat64(sum(toDecimal256(if(asset_id = {xlm}, {a}, {b}), 14)))") + // `[s - back, upper(e))` of the driver's month window `[s, e)`, if any. + let span = |back: u32, upper: &dyn Fn(String) -> String| { + window.map_or(String::new(), |(s, e)| { + let (s, e) = (s.saturating_sub(back), upper(format!("toDateTime({e})"))); + format!(" AND timestamp >= toDateTime({s}) AND timestamp < {e}") + }) }; - let txlm = format!( - "{} / {}", - amt("pf_volume", "pf_price_volume"), - amt("pf_price_volume", "pf_volume") - ); - let xr = format!( - "if(r.usd > 0 AND (toUInt32(pt.bend) - toUInt32(r.rts)) <= {stale}, toFloat64(r.usd), 0.)" - ); + let scope = pre_usdc_scope_pred(db); + let parents = format!("FROM {db}.{tbl} FINAL WHERE {scope}{}", span(0, &|e| e)); + let cols = prices_clickhouse::CANDLE_COLUMNS; + let list = |f: &dyn Fn(&str) -> String| cols.iter().map(|c| f(c)).collect::>(); + let all = cols.join(", "); + let from = |a: &str| list(&|c| format!("{a}.{c} AS {c}")).join(", "); // Float64, then a NULL-on-overflow parse: a Decimal product wraps silently // and an out-of-range CAST aborts the batch. - let usd = |col: &str| { + let dec = + |f: String| format!("ifNull(toDecimal128OrNull(toString({f}), 14), toDecimal128(0, 14))"); + let inner = if bucket_width_s(tbl) > 3_600 { + let tiny = "toDecimal128('0.000000000001', 14)"; format!( - "ifNull(toDecimal128OrNull(toString(toFloat64(pt.{col}) * {xr} * pt.xf), 14), \ - toDecimal128(0, 14))" + "SELECT {p}, {cu} AS new_cu, c.vqu AS new_vqu \ + FROM (SELECT {all} {parents}) AS p \ + LEFT JOIN ( \ + SELECT b.asset_id AS a, b.quote_asset_id AS q, b.source AS s, b.timestamp AS ts, \ + sum(h.volume_quote_usd) AS vqu, \ + argMaxIf(toFloat64(h.close_usd) / toFloat64(h.close), h.timestamp, \ + h.close_usd >= {tiny} AND h.close >= {tiny}) AS ratio \ + FROM ( \ + SELECT asset_id, quote_asset_id, source, timestamp, close, close_usd, \ + volume_quote_usd \ + FROM {db}.price_ohlcv_1h FINAL \ + WHERE timestamp < toDateTime({end}){hours} \ + ) AS h \ + ASOF JOIN ( \ + SELECT asset_id, quote_asset_id, source, timestamp, {bend} AS bend {parents} \ + ) AS b \ + ON h.asset_id = b.asset_id AND h.quote_asset_id = b.quote_asset_id \ + AND h.source = b.source AND h.timestamp >= b.timestamp \ + WHERE h.timestamp < b.bend \ + GROUP BY a, q, s, ts \ + ) AS c \ + ON c.a = p.asset_id AND c.q = p.quote_asset_id AND c.s = p.source \ + AND c.ts = p.timestamp", + p = from("p"), + cu = dec("toFloat64(p.close) * c.ratio".into()), + hours = span(0, &|e| bucket_end_expr(tbl, &e)), ) - }; - let (new_cu, new_vqu) = (usd("close"), usd("volume_quote")); - let cols = prices_clickhouse::CANDLE_COLUMNS; - let from = |a: &str| { - cols.iter() - .map(|c| format!("{a}.{c} AS {c}")) - .collect::>() - .join(", ") - }; - let (p_cols, pt_cols, x_cols) = (from("p"), from("pt"), cols.join(", ")); - let out = cols - .iter() - .map(|c| match *c { - "close_usd" => "x.new_cu AS close_usd".to_string(), - "volume_quote_usd" => "x.new_vqu AS volume_quote_usd".to_string(), - "version" => "x.version + 1 AS version".to_string(), - c => format!("x.{c}"), - }) - .collect::>() - .join(", "); - let body = format!( - "SELECT {out} \ - FROM ( \ - SELECT {pt_cols}, {new_cu} AS new_cu, {new_vqu} AS new_vqu \ + } else { + let dollar = format!( + "SELECT asset_id FROM {db}.assets FINAL WHERE {}", + dollar_token_asset_pred() + ); + // Decimal256 sums: a Decimal(38, 14) sum wraps silently, a Float64 one + // depends on merge order and would make the differs filter flap. + let amt = |a: &str, b: &str| { + format!("toFloat64(sum(toDecimal256(if(asset_id = {xlm}, {a}, {b}), 14)))") + }; + let txlm = format!( + "{} / {}", + amt("pf_volume", "pf_price_volume"), + amt("pf_price_volume", "pf_volume") + ); + let xr = format!( + "if(r.usd > 0 AND (toUInt32(pt.bend) - toUInt32(r.rts)) <= {}, toFloat64(r.usd), 0.)", + external_window_s(tbl) + ); + let usd = |col: &str| dec(format!("toFloat64(pt.{col}) * {xr} * pt.xf")); + format!( + "SELECT {pt}, {cu} AS new_cu, {vqu} AS new_vqu \ FROM ( \ - SELECT {p_cols}, p.k AS k, p.bend AS bend, \ + SELECT {p}, p.k AS k, p.bend AS bend, \ if(p.quote_asset_id = {xlm}, 1., if(t.txlm > 0 \ AND (toUInt32(p.timestamp) - toUInt32(t.tts)) <= {pivot_window_s}, \ t.txlm, 0.)) AS xf \ - FROM ( \ - SELECT {x_cols}, 1 AS k, {bend} AS bend \ - FROM {db}.{tbl} FINAL \ - WHERE {scope} AND timestamp <= toDateTime(?){cand_window} \ - ) AS p \ + FROM (SELECT {all}, 1 AS k, {bend} AS bend {parents}) AS p \ ASOF LEFT JOIN ( \ SELECT if(asset_id = {xlm}, quote_asset_id, asset_id) AS tok, \ timestamp AS tts, {txlm} AS txlm \ FROM {db}.{tbl} FINAL \ - WHERE timestamp < toDateTime({end}){ref_window} \ + WHERE timestamp < toDateTime({end}){refs} \ AND pf_trade_count > 0 AND pf_volume > 0 AND pf_price_volume > 0 \ AND ((quote_asset_id = {xlm} AND asset_id IN ({dollar})) \ OR (asset_id = {xlm} AND quote_asset_id IN ({dollar}))) \ @@ -2985,16 +3017,35 @@ fn pre_usdc_sql( ON t.tok = p.quote_asset_id AND t.tts <= p.timestamp \ ) AS pt \ ASOF LEFT JOIN ( {series} ) AS r \ - ON r.k = pt.k AND r.rts < pt.bend \ - ) AS x \ - WHERE x.close_usd != x.new_cu OR x.volume_quote_usd != x.new_vqu" - ); + ON r.k = pt.k AND r.rts < pt.bend", + pt = from("pt"), + p = from("p"), + cu = usd("close"), + vqu = usd("volume_quote"), + // The token-market scan: the month plus its look-back. + refs = span(pivot_window_s, &|e| e), + series = usd_rate_series_sql(db, XLM_RATE_IDENTITY), + ) + }; + let differs = "x.close_usd != x.new_cu OR x.volume_quote_usd != x.new_vqu"; + let out = list(&|c| match c { + "close_usd" => "x.new_cu AS close_usd".into(), + "volume_quote_usd" => "x.new_vqu AS volume_quote_usd".into(), + "version" => "x.version + 1 AS version".into(), + c => format!("x.{c}"), + }) + .join(", "); ( format!( - "INSERT INTO {db}.{tbl} ({}) {body} ORDER BY x.timestamp LIMIT ?", + "INSERT INTO {db}.{tbl} ({}) SELECT {out} FROM ({inner}) AS x \ + WHERE {differs} ORDER BY x.timestamp LIMIT ?", insert_columns() ), - format!("SELECT count() FROM ({body})"), + format!( + "SELECT countIf({differs}) AS pending, \ + countIf(x.close_usd > 0 AND x.new_cu = 0) AS unprice, \ + countIf(x.new_cu = 0 AND x.close > 0) AS unpriced FROM ({inner}) AS x" + ), ) } @@ -4441,23 +4492,21 @@ mod tests { /// `_1M` fixtures in `ch_enrich_it.rs`. Task file, Issues 10. #[test] fn every_timezone_sensitive_expression_pins_utc() { - let tables = ["price_ohlcv_1d", "price_ohlcv_1w", "price_ohlcv_1M"]; - for (t, sql) in tables.iter().flat_map(|t| { - [ + for t in ["price_ohlcv_1d", "price_ohlcv_1w", "price_ohlcv_1M"] { + for sql in [ external_sql("prices", t, 3, ""), - pre_usdc_sql("prices", t, 9, 86_400, "", "").0, - ] - .map(|sql| (t, sql)) - }) { - for f in ["addDays(", "addWeeks(", "addMonths(", "toDate("] { - for (i, _) in sql.match_indices(f) { - let tail = &sql[i..]; - let close = tail.find(')').unwrap(); - assert!( - tail[..close].ends_with("'UTC'"), - "{t}: {f} without an explicit 'UTC': {}", - &tail[..close + 1] - ); + pre_usdc_sql("prices", t, 9, 86_400, None).0, + ] { + for f in ["addDays(", "addWeeks(", "addMonths(", "toDate("] { + for (i, _) in sql.match_indices(f) { + let tail = &sql[i..]; + let close = tail.find(')').unwrap(); + assert!( + tail[..close].ends_with("'UTC'"), + "{t}: {f} without an explicit 'UTC': {}", + &tail[..close + 1] + ); + } } } } @@ -4969,7 +5018,11 @@ mod tests { ("pivot_sql", pivot_sql("prices", "price_ohlcv_1m", 5, 3, "")), ( "pre_usdc_sql", - pre_usdc_sql("prices", "price_ohlcv_1m", 9, 86_400, "", "").0, + pre_usdc_sql("prices", "price_ohlcv_1m", 9, 86_400, None).0, + ), + ( + "pre_usdc_sql rollup", + pre_usdc_sql("prices", "price_ohlcv_1d", 9, 86_400, None).0, ), ] } @@ -5658,48 +5711,30 @@ mod tests { #[test] fn pre_usdc_sql_renders_its_contract() { - let end = prices_clickhouse::PRE_USDC_END_S; - let win = |s| format!(" AND timestamp >= toDateTime({s}) AND timestamp < toDateTime(9)"); - for (t, (cw, rw)) in GRAINS - .iter() - .flat_map(|t| [(t, (String::new(), String::new())), (t, (win(5), win(1)))]) - { - let (ins, cnt) = pre_usdc_sql("prices", t, 9, 86_400, &cw, &rw); - let must = [ - "asset_kind = 'native' AND asset_code = 'XLM' AND issuer_address = '' \ - AND contract_address = '' AND method = 'external' AND usd_rate > 0" - .to_string(), - "r.rts < pt.bend".into(), - format!("{} AS bend", bucket_end_expr(t, "timestamp")), - format!("<= {}, toFloat64(r.usd)", external_window_s(t)), - "<= 86400, t.txlm".into(), - "if(asset_id = 9, pf_volume, pf_price_volume)".into(), - "if(asset_id = 9, pf_price_volume, pf_volume)".into(), - "pf_trade_count > 0".into(), - dollar_token_asset_pred(), - "x.close_usd != x.new_cu OR x.volume_quote_usd != x.new_vqu".into(), - "toDecimal128OrNull(toString(".into(), - "toDecimal256(".into(), - format!("{cw} ) AS p"), - format!("{end}){rw} AND pf_trade_count"), - ]; - for needle in must { - assert!(ins.contains(&needle), "{t}: `{needle}` missing in {ins}"); - } - for needle in ["LIKE", "startsWith", "candidate", &win(1)] { - assert_eq!(ins.contains(needle), needle == rw, "{t}: `{needle}`"); + let end = format!( + "timestamp < toDateTime({})", + prices_clickhouse::PRE_USDC_END_S + ); + let month = " AND timestamp >= toDateTime(5) AND timestamp < toDateTime(9)"; + for t in GRAINS { + for w in [None, Some((5, 9))] { + let (ins, cnt) = pre_usdc_sql("prices", t, 9, 86_400, w); + let rollup = bucket_width_s(t) > 3_600; + assert_eq!( + ins.matches("method = ").count(), + usize::from(!rollup), + "{ins}" + ); + assert_eq!(ins.contains("argMaxIf("), rollup, "{t}"); + for needle in ["LIKE", "startsWith", "candidate"] { + assert!(!ins.contains(needle), "{t}: {needle}"); + } + assert!(ins.matches(&end).count() >= 2, "{ins}"); + assert_eq!(ins.contains(month), w.is_some(), "{t}"); + assert_eq!((ins.matches('?').count(), cnt.matches('?').count()), (1, 0)); + let inner = &cnt[cnt.find("AS unpriced FROM (").unwrap() + 18..cnt.len() - 6]; + assert!(ins.contains(&format!("FROM ({inner}) AS x WHERE")), "{t}"); } - assert_eq!(ins.matches("method = ").count(), 1, "{ins}"); - assert!( - ins.matches(&format!("timestamp < toDateTime({end})")) - .count() - >= 2 - ); - assert_eq!((ins.matches('?').count(), cnt.matches('?').count()), (2, 1)); - let body = &cnt["SELECT count() FROM (".len()..cnt.len() - 1]; - let head = format!("INSERT INTO prices.{t} ({}) ", insert_columns()); - assert_eq!(ins, format!("{head}{body} ORDER BY x.timestamp LIMIT ?")); - assert!(cnt.starts_with("SELECT count() FROM (")); } } } diff --git a/packages/enrichment-worker/src/main.rs b/packages/enrichment-worker/src/main.rs index 25689095..8fc46ac8 100644 --- a/packages/enrichment-worker/src/main.rs +++ b/packages/enrichment-worker/src/main.rs @@ -70,7 +70,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { // env-driven — there is no ENRICHMENT_USD_RESET. usd_reset: None, // Opt-in coarse-repair mode only (task 0331). - pre_usdc_xlm_rate: false, + pre_usdc_xlm_rate: None, }; tracing::info!( diff --git a/packages/enrichment-worker/src/repair.rs b/packages/enrichment-worker/src/repair.rs index 3c8df7d4..c9461c7d 100644 --- a/packages/enrichment-worker/src/repair.rs +++ b/packages/enrichment-worker/src/repair.rs @@ -209,7 +209,7 @@ impl CoarseRepairDriver { /// bound. Both cast to `UInt32` to match [`MonthWindow`]. async fn months_with_zeros(&self) -> Result, ChEnrichError> { let e = &self.cfg.enrich; - let pred = if e.pre_usdc_xlm_rate { + let pred = if e.pre_usdc_xlm_rate.is_some() { pre_usdc_scope_pred(&e.database) } else { repair_target_pred(&e.database, &e.table, e.usd_reset.as_ref()) @@ -276,9 +276,12 @@ impl CoarseRepairDriver { // green having touched nothing. Found by the 0228 prove run and its two // review rounds. `reset_step` calls the SAME method — one list, not two // that must be kept in step by hand. - ChEnrichmentPass::with_client(self.client.clone(), self.cfg.enrich.clone()) - .assert_reset_is_admissible() - .await?; + let pass = ChEnrichmentPass::with_client(self.client.clone(), self.cfg.enrich.clone()); + // Task 0331: its refusals, and native XLM resolved once for every month. + let xlm = match self.cfg.enrich.pre_usdc_xlm_rate { + Some(_) => Some(pass.assert_pre_usdc_admissible().await?), + None => pass.assert_reset_is_admissible().await.map(|()| None)?, + }; let months = self.months_with_zeros().await?; info!( count = months.len(), @@ -312,26 +315,30 @@ impl CoarseRepairDriver { let mut ecfg = self.cfg.enrich.clone(); ecfg.time_window = Some((mw.start_ts, mw.end_ts)); ecfg.one_shot = self.cfg.one_shot; - let pre_usdc = ecfg.pre_usdc_xlm_rate; let pass = ChEnrichmentPass::with_client(self.client.clone(), ecfg); + // Task 0331: rows to write, rows left unpriced, priced rows set to 0. + let counts = match xlm { + Some(x) => Some(pass.pre_usdc_counts(x).await?), + None => None, + }; + let (zeros, after, reset) = counts.map_or((mw.zeros, mw.zeros, 0), |c| { + (c.pending, c.unpriced, c.unprice) + }); - if self.cfg.dry_run { - // Task 0331: report the rows the real run writes, not scope rows. - let zeros = if pre_usdc { - pass.pre_usdc_pending().await? + // A 0331 month with nothing to write gets no FREEZE either. + if self.cfg.dry_run || zeros == 0 { + let what = if zeros == 0 { + "nothing to write" } else { - mw.zeros + "DRY RUN — would repair" }; - info!( - month = mw.month, - zeros, "coarse repair: DRY RUN — would repair" - ); + info!(month = mw.month, zeros, "coarse repair: {what}"); summary.months.push(MonthRepair { month: mw.month, zeros_before: zeros, - zeros_after: zeros, + zeros_after: after, rows_enriched: 0, - rows_reset: 0, + rows_reset: reset, snapshot_name: None, }); continue; @@ -352,7 +359,10 @@ impl CoarseRepairDriver { None }; - let stats = pass.run().await?; + let stats = match xlm { + Some(x) => pass.run_pre_usdc(x).await?, + None => pass.run().await?, + }; info!( month = mw.month, @@ -373,7 +383,7 @@ impl CoarseRepairDriver { // Defensive cross-check: the enumeration count and the pass's own // pre-count should agree (both are the month's zero population). // In the 0331 mode the pass counts rows to write, not scope rows. - if !pre_usdc && mw.zeros != stats.candidates_before { + if xlm.is_none() && mw.zeros != stats.candidates_before { warn!( month = mw.month, enumerated = mw.zeros, diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index 5a783961..35c66db9 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -438,10 +438,10 @@ async fn resolve_usd_refs( volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` \ and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a \ price in the bucket, and from every market otherwise. If\nsuch a market has no USD value \ - yet, those fields are `null`. Before 2021-02-01, when Stellar had no USDC, candles \ - quoted in\nXLM or in another dollar token are converted with an imported, measured \ - XLM/USD rate\n(`method` is `external`); a dollar token is converted through its own XLM \ - market and\nstays unpriced without one.\n\nThe window is `timeframe`, ending \ + yet, those fields are `null`. Before 2021-02-01, candles\nquoted in XLM or in a \ + dollar token are converted with an imported, measured\nXLM/USD rate (`method` is \ + `external`); a dollar token is converted through its own XLM\nmarket and stays unpriced \ + without one.\n\nThe window is `timeframe`, ending \ now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is \ rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the \ historical backfill is still running.", diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index 44010648..3293b47a 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -1697,7 +1697,9 @@ pub(crate) fn usd_method_expr(usdc: u64, pivots: &[u64], granularity: Granularit timestamp < toDateTime({pre}), 'external', \ {traded_arm}\ '') AS meth", - // Task 0331: a priced pre-2021 non-USDC row took the XLM/USD rate. + // Task 0331: keys on the timestamp alone because only `coarse-repair + // --pre-usdc-xlm-rate` writes a non-USDC close_usd below this boundary. + // Another pre-2021 pricer needs its own arm above this one. pre = prices_clickhouse::PRE_USDC_END_S, ) } diff --git a/packages/prices-api/src/openapi/descriptions.rs b/packages/prices-api/src/openapi/descriptions.rs index 5544c651..5e529cf8 100644 --- a/packages/prices-api/src/openapi/descriptions.rs +++ b/packages/prices-api/src/openapi/descriptions.rs @@ -392,8 +392,8 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was \ measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* \ `external` — an imported, measured USD series supplied the rate: USDC/USD on a \ - USDC-quoted candle; before 2021-02-01, when Stellar had no USDC, XLM/USD on a \ - candle quoted in XLM or in another dollar token (USD, USDT or a USDC other than \ + USDC-quoted candle; before 2021-02-01, XLM/USD on a \ + candle quoted in XLM or in a dollar token (USD, USDT or a USDC other than \ Circle's), the latter converted through that token's own XLM market.\n* `oracle` — \ a measured Reflector reading supplied the rate.\n* `traded` — priced through a \ reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series \ diff --git a/web/portal/public/openapi.json b/web/portal/public/openapi.json index a6f3986f..2a14160f 100644 --- a/web/portal/public/openapi.json +++ b/web/portal/public/openapi.json @@ -319,7 +319,7 @@ "get": { "tags": ["prices"], "summary": "`GET /assets/{asset_identifier}/ohlcv` — candlestick history.", - "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a price in the bucket, and from every market otherwise. If\nsuch a market has no USD value yet, those fields are `null`. Before 2021-02-01, when Stellar had no USDC, candles quoted in\nXLM or in another dollar token are converted with an imported, measured XLM/USD rate\n(`method` is `external`); a dollar token is converted through its own XLM market and\nstays unpriced without one.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", + "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a price in the bucket, and from every market otherwise. If\nsuch a market has no USD value yet, those fields are `null`. Before 2021-02-01, candles\nquoted in XLM or in a dollar token are converted with an imported, measured\nXLM/USD rate (`method` is `external`); a dollar token is converted through its own XLM\nmarket and stays unpriced without one.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", "operationId": "get_ohlcv", "parameters": [ { @@ -1165,7 +1165,7 @@ }, "method": { "type": ["string", "null"], - "description": "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* `external` — an imported, measured USD series supplied the rate: USDC/USD on a USDC-quoted candle; before 2021-02-01, when Stellar had no USDC, XLM/USD on a candle quoted in XLM or in another dollar token (USD, USDT or a USDC other than Circle's), the latter converted through that token's own XLM market.\n* `oracle` — a measured Reflector reading supplied the rate.\n* `traded` — priced through a reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`): no measured USDC/USD observation covered the bucket, so the $1 fallback was rendered. Never appears on a quote leg — there the same situation is `assumed-par`.\n\nEach value names the INPUT the rate came from, so `assumed-par` and `external` are never interchangeable: one is an assumption, the other a measurement that may sit percent off par.\n\nOn the USDC self-series a bucket that holds both a Reflector reading and an imported one reports `oracle`: a measured poll outranks an imported rate outright, whichever was observed first. Observation time only breaks ties between rows of the same kind. The imported series is loaded at HOURLY grain, so an hourly request over an imported day reports `external` — and that hour's own measured rate — for each of its twenty-four hours; on 2023-03-11 the 07:00 bucket reports the trough rather than the day's close. Where only a daily row exists for a day, that one row prices every bucket of its UTC day at every `granularity`, so an imported day mixes the two only where the oracle epoch falls inside it (2026-03-11): a bucket that extends past 14:00 UTC that day is not priced from the imported series, which ends at 13:00. The self-series carries the rate row's own `method`, so a measured rate that reads exactly 1.0 is `external` there.\n\nOn a quote leg the label is reconstructed at read time — the candle rows carry no provenance column — so one case is deliberately not separated: a bucket reading exactly 1.0 reports `assumed-par` whether the dollar was assumed or a measured rate happened to land on it. The two leave an identical row and carry the same number, so the label is the conservative one; `external` is reported only for a bucket whose value was actually scaled by an imported rate.\n\n`null` in three cases: when the price fields are `null`; always for `base_currency=XLM`, where nothing is converted; and for a USDC-quoted bucket below the oracle epoch that carries a converted rate no imported series covers — nothing can attribute it, and a label would be a claim.\n\nIn USD mode the label follows the markets that set the prices: those quoted in USDC or XLM when one of them formed a price in the bucket, otherwise every market. A bucket that traded mostly against USDT can therefore report the `assumed-par` of its USDC market.", + "description": "Where the USD rate behind this bucket came from:\n\n* `assumed-par` — nothing was measured; the literal 1.0 supplied the value, i.e. USDC was taken at 1 USD.\n* `external` — an imported, measured USD series supplied the rate: USDC/USD on a USDC-quoted candle; before 2021-02-01, XLM/USD on a candle quoted in XLM or in a dollar token (USD, USDT or a USDC other than Circle's), the latter converted through that token's own XLM market.\n* `oracle` — a measured Reflector reading supplied the rate.\n* `traded` — priced through a reference asset's own trades.\n* `peg` — only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`): no measured USDC/USD observation covered the bucket, so the $1 fallback was rendered. Never appears on a quote leg — there the same situation is `assumed-par`.\n\nEach value names the INPUT the rate came from, so `assumed-par` and `external` are never interchangeable: one is an assumption, the other a measurement that may sit percent off par.\n\nOn the USDC self-series a bucket that holds both a Reflector reading and an imported one reports `oracle`: a measured poll outranks an imported rate outright, whichever was observed first. Observation time only breaks ties between rows of the same kind. The imported series is loaded at HOURLY grain, so an hourly request over an imported day reports `external` — and that hour's own measured rate — for each of its twenty-four hours; on 2023-03-11 the 07:00 bucket reports the trough rather than the day's close. Where only a daily row exists for a day, that one row prices every bucket of its UTC day at every `granularity`, so an imported day mixes the two only where the oracle epoch falls inside it (2026-03-11): a bucket that extends past 14:00 UTC that day is not priced from the imported series, which ends at 13:00. The self-series carries the rate row's own `method`, so a measured rate that reads exactly 1.0 is `external` there.\n\nOn a quote leg the label is reconstructed at read time — the candle rows carry no provenance column — so one case is deliberately not separated: a bucket reading exactly 1.0 reports `assumed-par` whether the dollar was assumed or a measured rate happened to land on it. The two leave an identical row and carry the same number, so the label is the conservative one; `external` is reported only for a bucket whose value was actually scaled by an imported rate.\n\n`null` in three cases: when the price fields are `null`; always for `base_currency=XLM`, where nothing is converted; and for a USDC-quoted bucket below the oracle epoch that carries a converted rate no imported series covers — nothing can attribute it, and a label would be a claim.\n\nIn USD mode the label follows the markets that set the prices: those quoted in USDC or XLM when one of them formed a price in the bucket, otherwise every market. A bucket that traded mostly against USDT can therefore report the `assumed-par` of its USDC market.", "example": "oracle" }, "derived": { From fa0a14fdedc4909f135809e4eabe33a61719f962 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:59:52 +0200 Subject: [PATCH 091/144] test(lore-0331): pin the roll-up, boundaries and non-par legs Fixtures no longer sit at par, so a close_usd = close implementation fails. New cases: 1w ending at 2021-02-01, 1M 2021-01, a 1w starting at the boundary (untouched), wide tiers from 1h hours at different rates, an unpriced last hour, both orientations pooled, XLM's own XLM/USDT leg, the native-XLM refusal, and no FREEZE for a month with nothing to write. --- .../enrichment-worker/tests/ch_enrich_it.rs | 281 +++++++++++------- 1 file changed, 176 insertions(+), 105 deletions(-) diff --git a/packages/enrichment-worker/tests/ch_enrich_it.rs b/packages/enrichment-worker/tests/ch_enrich_it.rs index 6650109b..31442799 100644 --- a/packages/enrichment-worker/tests/ch_enrich_it.rs +++ b/packages/enrichment-worker/tests/ch_enrich_it.rs @@ -9,9 +9,11 @@ //! at the end, so they never touch `prices` and can run in parallel. use clickhouse::Client; -use enrichment_worker::ch_enrich::{ChEnrichConfig, ChEnrichError, ChEnrichmentPass, UsdResetSpec}; +use enrichment_worker::ch_enrich::{ + ChEnrichConfig, ChEnrichError, ChEnrichmentPass, PreUsdcXlmRate, UsdResetSpec, +}; use enrichment_worker::repair::{ - CoarseRepairConfig, CoarseRepairDriver, CoarseSweepConfig, run_coarse_sweep, + CoarseRepairConfig, CoarseRepairDriver, CoarseSweepConfig, MonthRepair, run_coarse_sweep, }; use prices_clickhouse::asset_id::fixture::{AssetFixture, assets_insert, fetch_ids}; use prices_clickhouse::{PRE_USDC_END_S, USDC_ISSUER, USDC_ORACLE_EPOCH_S, USDT_ISSUER}; @@ -159,7 +161,7 @@ fn cfg(db: &str) -> ChEnrichConfig { one_shot: false, time_window: None, usd_reset: None, - pre_usdc_xlm_rate: false, + pre_usdc_xlm_rate: None, } } @@ -2215,13 +2217,20 @@ fn utc_day_start(ts: u32) -> u32 { /// test can stage BOTH series for the same bucket and prove which one wins. /// [`seed_external_rate`] is the `'external'` case and carries the day-start /// convention note. -async fn seed_usd_rate(client: &Client, db: &str, ts: u32, rate: f64, method: &str) { +/// One `usd_rate` row of identity `(asset_kind, asset_code, issuer_address)`. +async fn seed_usd_rate( + client: &Client, + db: &str, + (kind, code, issuer): (&str, &str, &str), + (ts, rate, method): (u32, f64, &str), + version: u64, +) { client .query(&format!( "INSERT INTO {db}.usd_rate \ (asset_kind, asset_code, issuer_address, contract_address, \ timestamp, usd_rate, method, reference_asset, hops, version) VALUES \ - ('credit', 'USDC', '{USDC_ISSUER}', '', {ts}, {rate}, '{method}', '', 0, 1)" + ('{kind}', '{code}', '{issuer}', '', {ts}, {rate}, '{method}', '', 0, {version})" )) .execute() .await @@ -3186,7 +3195,8 @@ async fn a_pivot_leg_prefers_the_oracle_rate_over_the_external_rate() { let db = "it_enrich_0228_oracle_wins"; let client = setup_0228_pivot(db, "price_ohlcv_1d", DEPEG_DAY).await; // The oracle says 0.95 at day start; the import says 0.9681 an hour LATER. - seed_usd_rate(&client, db, DEPEG_DAY, 0.95, "oracle").await; + let usdc = ("credit", "USDC", USDC_ISSUER); + seed_usd_rate(&client, db, usdc, (DEPEG_DAY, 0.95, "oracle"), 1).await; seed_external_rate(&client, db, DEPEG_DAY + 3_600, DEPEG_RATE).await; let mut c = cfg(db); @@ -5713,71 +5723,72 @@ async fn the_epoch_is_measured_on_the_pass_s_own_table() { // --- Task 0331: pre-2021 candles priced from the hourly XLM/USD rate --- const USDX: AssetFixture = AssetFixture::new("USD", "classic", "GUSDX", ""); -/// 2020-06-01 00:00 UTC. +/// 2020-06-01 00:00 UTC, a Monday. const D0331: u32 = 1_590_969_600; const H1: &str = "price_ohlcv_1h"; +const WIDE: [&str; 4] = [ + "price_ohlcv_4h", + "price_ohlcv_1d", + "price_ohlcv_1w", + "price_ohlcv_1M", +]; +const END: u32 = PRE_USDC_END_S; const fn h(n: u32) -> u32 { D0331 + n * 3600 } -/// (ts, rate, method) of `('native','XLM','','')` rows in `usd_rate`. +/// (ts, rate, method) of `('native','XLM','')` rows in `usd_rate`. type Rate = (u32, f64, &'static str); /// (label, table, ts, asset, quote, close, volume_base, volume_quote, close_usd, volume_quote_usd) #[rustfmt::skip] type Leg = (&'static str, &'static str, u32, AssetFixture<'static>, AssetFixture<'static>, f64, f64, f64, f64, f64); +const XLM_RATE: (&str, &str, &str) = ("native", "XLM", ""); const RATES_0331: &[Rate] = &[ (h(10), 0.25, "external"), (h(23), 0.20, "external"), (h(24), 0.50, "external"), + (h(29), 0.40, "external"), (h(11), 9.0, "external-candidate"), (h(81), 0.25, "external"), - (PRE_USDC_END_S - 3600, 0.30, "external"), + (END - 3600, 0.30, "external"), ]; +// No dollar-quoted expectation equals par: a token is 5 XLM via USDT/XLM, 3 via +// XLM/USDX and 4.5 where both USDT orientations pool at h(13). Wide tiers are +// Σ of their hours' USD volume and close × the last priced hour's ratio. #[rustfmt::skip] const LEGS_0331: &[Leg] = &[ ("FOO/XLM 1h", H1, h(10), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), - ("FOO/XLM 1d takes 23:00, not the next midnight", "price_ohlcv_1d", D0331, FOO, XLM, 10.0, 3.0, 30.0, 2.0, 6.0), - ("USDT/XLM", H1, h(10), USDT, XLM, 4.0, 10.0, 40.0, 1.0, 10.0), - ("XLM/USDX", H1, h(10), XLM, USDX, 0.25, 40.0, 10.0, 0.25, 10.0), - ("FOO/USDT via USDT/XLM", H1, h(10), FOO, USDT, 2.0, 3.0, 6.0, 2.0, 6.0), - ("FOO/USDX via XLM/USDX", H1, h(10), FOO, USDX, 2.0, 3.0, 6.0, 2.0, 6.0), ("FOO/XLM ignores the candidate's 9.0", H1, h(11), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), - ("FOO/XLM, newest rate 30 h old", H1, h(53), FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), + ("FOO/XLM 23:00 takes 23:00, not midnight", H1, h(23), FOO, XLM, 10.0, 3.0, 30.0, 2.0, 6.0), + ("FOO/XLM, rate 20 h old", H1, h(48), FOO, XLM, 10.0, 3.0, 30.0, 4.0, 12.0), + ("FOO/XLM, newest rate 25 h old", H1, h(53), FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), ("FOO/XLM, rate 2 h old", H1, h(82), FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), - ("FOO/USDT, market 72 h old", H1, h(82), FOO, USDT, 2.0, 3.0, 6.0, 0.0, 0.0), - ("FOO/USDT, market 2 h old", H1, h(12), FOO, USDT, 2.0, 3.0, 6.0, 2.0, 6.0), - ("FOO/XLM at 2021-02-01", H1, PRE_USDC_END_S, FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), + ("USDT/XLM", H1, h(10), USDT, XLM, 5.0, 10.0, 50.0, 1.25, 12.5), + ("XLM/USDX", H1, h(10), XLM, USDX, 0.4, 30.0, 10.0, 0.3, 7.5), + ("FOO/USDT via USDT/XLM", H1, h(10), FOO, USDT, 2.0, 3.0, 6.0, 2.5, 7.5), + ("FOO/USDX via XLM/USDX", H1, h(10), FOO, USDX, 2.0, 3.0, 6.0, 1.5, 4.5), + ("FOO/USDT, market 2 h old", H1, h(12), FOO, USDT, 2.0, 3.0, 6.0, 2.5, 7.5), + ("USDT/XLM pooled", H1, h(13), USDT, XLM, 5.0, 10.0, 50.0, 1.25, 12.5), + ("XLM/USDT pooled: XLM's own USD candle", H1, h(13), XLM, USDT, 0.24, 40.0, 10.0, 0.27, 11.25), + ("FOO/USDT via the pooled market", H1, h(13), FOO, USDT, 2.0, 3.0, 6.0, 2.25, 6.75), + ("FOO/USDT, market 69 h old", H1, h(82), FOO, USDT, 2.0, 3.0, 6.0, 0.0, 0.0), + ("FOO/XLM 2021-01-31 23:00", H1, END - 3600, FOO, XLM, 10.0, 3.0, 30.0, 3.0, 9.0), + ("FOO/XLM at 2021-02-01", H1, END, FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), ("XLM/USDC (canonical)", H1, h(10), XLM, USDC, 0.25, 40.0, 10.0, 0.0, 0.0), ("FOO/USDC (canonical)", H1, h(10), FOO, USDC, 4.0, 1.0, 4.0, 0.0, 0.0), + ("4h FOO/XLM", WIDE[0], h(8), FOO, XLM, 10.0, 6.0, 60.0, 2.5, 15.0), + ("1d FOO/XLM, hours at 0.25 and 0.20", WIDE[1], D0331, FOO, XLM, 10.0, 9.0, 90.0, 2.0, 21.0), + ("1d FOO/XLM, last hour unpriced", WIDE[1], h(48), FOO, XLM, 10.0, 6.0, 60.0, 4.0, 12.0), + ("1d FOO/USDT", WIDE[1], D0331, FOO, USDT, 2.0, 9.0, 18.0, 2.25, 21.75), + ("1w FOO/XLM, hours at three rates", WIDE[2], D0331, FOO, XLM, 10.0, 18.0, 180.0, 2.5, 40.5), + ("1w FOO/XLM ending at 2021-02-01", WIDE[2], END - 7 * 86_400, FOO, XLM, 10.0, 3.0, 30.0, 3.0, 9.0), + ("1M FOO/XLM 2021-01", WIDE[3], 1_609_459_200, FOO, XLM, 10.0, 3.0, 30.0, 3.0, 9.0), + ("1w FOO/XLM at 2021-02-01", WIDE[2], END, FOO, XLM, 10.0, 3.0, 30.0, 0.0, 0.0), ]; -/// The 1h legs whose bucket end resolves to the h(10) rate. -const ON_H10_RATE: &[&str] = &[ - "FOO/XLM 1h", - "USDT/XLM", - "XLM/USDX", - "FOO/USDT via USDT/XLM", - "FOO/USDX via XLM/USDX", - "FOO/XLM ignores the candidate's 9.0", - "FOO/USDT, market 2 h old", -]; - -async fn seed_xlm_rate(client: &Client, db: &str, (ts, rate, method): Rate, version: u64) { - client - .query(&format!( - "INSERT INTO {db}.usd_rate \ - (asset_kind, asset_code, issuer_address, contract_address, \ - timestamp, usd_rate, method, reference_asset, hops, version) VALUES \ - ('native', 'XLM', '', '', {ts}, {rate}, '{method}', '', 0, {version})" - )) - .execute() - .await - .unwrap(); -} - async fn seed_leg(client: &Client, db: &str, leg: &Leg) { let (_, table, ts, asset, quote, close, vb, vq, _, _) = *leg; client @@ -5815,7 +5826,7 @@ async fn setup_0331(db: &str, rates: &[Rate], legs: &[Leg]) -> Client { .await .unwrap(); for r in rates { - seed_xlm_rate(&client, db, *r, 1).await; + seed_usd_rate(&client, db, XLM_RATE, *r, 1).await; } for l in legs { seed_leg(&client, db, l).await; @@ -5827,7 +5838,7 @@ fn cfg_0331(db: &str, table: &str) -> ChEnrichConfig { ChEnrichConfig { table: table.to_string(), one_shot: true, - pre_usdc_xlm_rate: true, + pre_usdc_xlm_rate: Some(PreUsdcXlmRate::default()), ..cfg(db) } } @@ -5839,9 +5850,14 @@ fn written_in(table: &str) -> u64 { .count() as u64 } -async fn assert_legs(client: &Client, db: &str, scale: impl Fn(&str) -> f64) -> Vec { +async fn assert_legs<'a>( + client: &Client, + db: &str, + legs: impl IntoIterator, + scale: impl Fn(&str) -> f64, +) -> Vec { let mut versions = Vec::new(); - for leg in LEGS_0331 { + for leg in legs { let (cu, vqu, v) = usd_of(client, db, leg).await; let (want_cu, want_vqu) = (leg.8 * scale(leg.0), leg.9 * scale(leg.0)); assert!( @@ -5854,34 +5870,35 @@ async fn assert_legs(client: &Client, db: &str, scale: impl Fn(&str) -> f64) -> versions } -async fn drop_db(client: &Client, db: &str) { - client - .query(&format!("DROP DATABASE {db}")) - .execute() - .await - .unwrap(); -} - #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn pre_usdc_xlm_rate_prices_xlm_and_dollar_quoted_legs() { let db = "test_ch_enrich_pre_usdc_prices"; let client = setup_0331(db, RATES_0331, LEGS_0331).await; - for table in [H1, "price_ohlcv_1d"] { - let stats = ChEnrichmentPass::new(cfg_0331(db, table)) - .run() - .await - .unwrap(); - assert_eq!(stats.rows_enriched, written_in(table), "{table}: {stats:?}"); + let run = async |table: &str| { + let pass = ChEnrichmentPass::new(cfg_0331(db, table)); + pass.run().await.unwrap().rows_enriched + }; + assert_eq!( + run(WIDE[1]).await, + 0, + "a wide tier before its 1h writes nothing" + ); + for table in std::iter::once(H1).chain(WIDE) { + assert_eq!(run(table).await, written_in(table), "{table}"); } - let versions = assert_legs(&client, db, |_| 1.0).await; + let versions = assert_legs(&client, db, LEGS_0331, |_| 1.0).await; // Only written rows bump their version; unreachable and out-of-scope ones stay at 1. let want: Vec = LEGS_0331 .iter() .map(|l| 1 + u64::from(l.8 != 0.0)) .collect(); assert_eq!(versions, want); - drop_db(&client, db).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); } #[tokio::test] @@ -5889,36 +5906,41 @@ async fn pre_usdc_xlm_rate_prices_xlm_and_dollar_quoted_legs() { async fn pre_usdc_xlm_rate_rewrites_only_values_that_differ() { let db = "test_ch_enrich_pre_usdc_rewrite"; let client = setup_0331(db, RATES_0331, LEGS_0331).await; - let run = |pivot_window_s| { + let run = async |pivot_window_s| { let pass = ChEnrichmentPass::new(ChEnrichConfig { pivot_window_s, ..cfg_0331(db, H1) }); - async move { pass.run().await.unwrap().rows_enriched } + let stats = pass.run().await.unwrap(); + (stats.rows_enriched, stats.rows_reset) }; - assert_eq!(run(86_400).await, written_in(H1)); - let d1 = ChEnrichmentPass::new(cfg_0331(db, "price_ohlcv_1d")); - d1.run().await.unwrap(); - let first = assert_legs(&client, db, |_| 1.0).await; + let h1 = || LEGS_0331.iter().filter(|l| l.1 == H1); + assert_eq!(run(86_400).await, (written_in(H1), 0)); + let first = assert_legs(&client, db, h1(), |_| 1.0).await; assert_eq!( run(86_400).await, - 0, + (0, 0), "a re-run with the same rate writes nothing" ); assert_eq!( - assert_legs(&client, db, |_| 1.0).await, + assert_legs(&client, db, h1(), |_| 1.0).await, first, "and bumps no version" ); - seed_xlm_rate(&client, db, (h(10), 0.5, "external"), 2).await; - assert_eq!(run(86_400).await, ON_H10_RATE.len() as u64); - let doubled = |l: &str| if ON_H10_RATE.contains(&l) { 2.0 } else { 1.0 }; - assert_legs(&client, db, doubled).await; + // A corrected h(10) rate re-prices exactly the priced 1h legs of h(10)..h(13). + seed_usd_rate(&client, db, XLM_RATE, (h(10), 0.5, "external"), 2).await; + let on_h10: Vec<&str> = h1() + .filter(|l| (h(10)..=h(13)).contains(&l.2) && l.8 != 0.0) + .map(|l| l.0) + .collect(); + assert_eq!(run(86_400).await, (on_h10.len() as u64, 0)); + let doubled = |l: &str| if on_h10.contains(&l) { 2.0 } else { 1.0 }; + assert_legs(&client, db, h1(), doubled).await; // A 1 h window can no longer reach the 2 h old USDT market: written back to 0. - assert_eq!(run(3600).await, 1); + assert_eq!(run(3600).await, (1, 1)); let narrowed = |l: &str| { if l == "FOO/USDT, market 2 h old" { 0.0 @@ -5926,36 +5948,38 @@ async fn pre_usdc_xlm_rate_rewrites_only_values_that_differ() { doubled(l) } }; - assert_legs(&client, db, narrowed).await; - drop_db(&client, db).await; + assert_legs(&client, db, h1(), narrowed).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); } #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn pre_usdc_xlm_rate_repair_refuses_unpromoted_rates_and_previews_exactly() { - let driver = |client: &Client, db: &str, dry_run| { + let driver = |client: &Client, db: &str, dry_run, snapshot| { CoarseRepairDriver::with_client( client.clone(), CoarseRepairConfig { enrich: cfg_0331(db, H1), start_month: 202_001, end_month: 202_103, - snapshot: false, + snapshot, dry_run, one_shot: true, deadline: None, }, ) }; + let refused = |res: &Result<_, ChEnrichError>, why: &str| matches!(res, Err(ChEnrichError::PreUsdcXlmRateRefused(m)) if m.contains(why)); // (i) Only a staged row: refused before any write, in a dry run too. let db = "test_ch_enrich_pre_usdc_refuse"; let client = setup_0331(db, &[(h(10), 0.25, "external-candidate")], &LEGS_0331[..1]).await; for dry_run in [true, false] { - let res = driver(&client, db, dry_run).run().await; - assert!( - matches!(res, Err(ChEnrichError::PreUsdcXlmRateRefused(_))), - "{res:?}" - ); + let res = driver(&client, db, dry_run, false).run().await.map(|_| ()); + assert!(refused(&res, "promote"), "{res:?}"); } assert_eq!(usd_of(&client, db, &LEGS_0331[0]).await.2, 1); // (ii) Never combined with a reset. @@ -5972,33 +5996,80 @@ async fn pre_usdc_xlm_rate_repair_refuses_unpromoted_rates_and_previews_exactly( }) .assert_reset_is_admissible() .await; - assert!( - matches!(res, Err(ChEnrichError::PreUsdcXlmRateRefused(_))), - "{res:?}" - ); - drop_db(&client, db).await; + assert!(refused(&res, "--reset-*"), "{res:?}"); + // (iii) Native XLM does not resolve. + client + .query(&format!("TRUNCATE TABLE {db}.assets")) + .execute() + .await + .unwrap(); + let res = driver(&client, db, true, false).run().await.map(|_| ()); + assert!(refused(&res, "native XLM"), "{res:?}"); + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); - // (iii) The dry run reports exactly what the real run then writes. + // (iv) The dry run reports exactly what the real run then writes; a month + // with nothing to write is skipped, FREEZE included. let db = "test_ch_enrich_pre_usdc_preview"; let client = setup_0331(db, RATES_0331, LEGS_0331).await; - let preview = driver(&client, db, true).run().await.unwrap(); - let months: Vec<(u32, u64)> = preview - .months - .iter() - .map(|m| (m.month, m.zeros_before)) - .collect(); - assert_eq!(months, [(202_006, written_in(H1))]); + // (month, to write / written, left unpriced, set back to 0, frozen) + let months = async |dry_run, snapshot| { + let summary = driver(&client, db, dry_run, snapshot).run().await.unwrap(); + let n = |m: &MonthRepair| { + if dry_run { + m.zeros_before + } else { + m.rows_enriched + } + }; + let row = |m: &MonthRepair| { + let frozen = m.snapshot_name.is_some(); + (m.month, n(m), m.zeros_after, m.rows_reset, frozen) + }; + summary.months.iter().map(row).collect::>() + }; + let want = [(202_006, 13, 2, 0, false), (202_101, 1, 0, 0, false)]; + assert_eq!(months(true, false).await, want); assert_eq!( usd_of(&client, db, &LEGS_0331[0]).await.2, 1, "the dry run wrote nothing" ); - let real = driver(&client, db, false).run().await.unwrap(); - let months: Vec<(u32, u64, u64)> = real - .months - .iter() - .map(|m| (m.month, m.rows_enriched, m.zeros_after)) - .collect(); - assert_eq!(months, [(202_006, written_in(H1), 0)]); - drop_db(&client, db).await; + assert_eq!(months(false, false).await, want); + let rerun = [(202_006, 0, 2, 0, false), (202_101, 0, 0, 0, false)]; + assert_eq!(months(false, true).await, rerun); + + // (v) On 1m a run that would set a priced row back to 0 needs allow_unprice. + #[rustfmt::skip] + let legs: &[Leg] = &[ + ("USDT/XLM", "price_ohlcv_1m", h(10), USDT, XLM, 5.0, 10.0, 50.0, 1.25, 12.5), + ("FOO/USDT, market 2 h old", "price_ohlcv_1m", h(12), FOO, USDT, 2.0, 3.0, 6.0, 2.5, 7.5), + ]; + for l in legs { + seed_leg(&client, db, l).await; + } + let run = async |pivot_window_s, allow_unprice| { + ChEnrichmentPass::new(ChEnrichConfig { + pivot_window_s, + pre_usdc_xlm_rate: Some(PreUsdcXlmRate { allow_unprice }), + ..cfg_0331(db, "price_ohlcv_1m") + }) + .run() + .await + .map(|s| (s.rows_enriched, s.rows_reset)) + }; + assert_eq!(run(86_400, false).await.unwrap(), (2, 0)); + let res = run(3600, false).await.map(|_| ()); + assert!(refused(&res, "--allow-unprice"), "{res:?}"); + assert_legs(&client, db, legs, |_| 1.0).await; + assert_eq!(run(3600, true).await.unwrap(), (1, 1)); + assert_legs(&client, db, &legs[..1], |_| 1.0).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); } From 69b8454108ae1aa97a914e05fa188ae47cc14f05 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:15:04 +0200 Subject: [PATCH 092/144] docs(lore-0333): set the XLM/USD range back to 2018-01-01 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Priced history starts at the beginning of 2018; 2017 is out of scope. The source stays the exchange's OHLCVT files, with a second exchange's XLM/BTC × BTC/USDT as a cross-check. --- ...RE_xlm-usd-for-2017-01-17-to-2018-02-16.md | 98 ------------------- ...RE_xlm-usd-for-2018-01-01-to-2018-02-16.md | 95 ++++++++++++++++++ 2 files changed, 95 insertions(+), 98 deletions(-) delete mode 100644 lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2017-01-17-to-2018-02-16.md create mode 100644 lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md diff --git a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2017-01-17-to-2018-02-16.md b/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2017-01-17-to-2018-02-16.md deleted file mode 100644 index 15ea20f0..00000000 --- a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2017-01-17-to-2018-02-16.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -id: "0333" -title: "XLM/USD for 2017-01-17 → 2018-02-16" -type: FEATURE -status: backlog -related_adr: [] -related_tasks: ["0331"] -tags: [layer-backend, priority-medium, effort-small, enrichment, clickhouse, usd-reference] -links: - - "../active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md" -history: - - date: "2026-10-15" - status: backlog - who: akot - note: > - Scope widened from 2018-01-01 → 02-16 to 2017-01-17 → 2018-02-16, the - start of the exchange's XLM/USD market. The downloadable OHLCVT files - are the chosen source. The 68 in-scope 1h rows before 2017-01-17 stay - unpriced. - - date: "2026-10-14" - status: backlog - who: akot - note: > - Spawned from 0331. The hourly XLM/USD series prepared for 0331 starts - 2018-02-16 23:00 UTC; the 47 days before it have no rate and stay - unpriced until this lands. Source not decided yet. ---- - -# XLM/USD for 2017-01-17 → 2018-02-16 - -## Summary - -[[0331]] prices pre-2021 candles from an hourly XLM/USD series loaded into -`prices.usd_rate`. That series starts at 2018-02-16 23:00 UTC, so XLM- and -dollar-quoted candles before it stay at `close_usd = 0`. This task extends the -series back to 2017-01-17, when the same exchange opened its XLM/USD market, -and loads it in the same shape. That prices almost all of the history before -2018-02-16. - -## Context - -- **Why the 0331 series stops at 2018-02-16.** The exchange's trades API - cannot reach trades before 2018-02-16 23:03 when paged. A time-based `since` - anywhere from 2017-12 to 2018-02-16 lands on that first reachable trade, and - paging from the start jumps to it after the first 1,000 trades. Probed - 2026-10-14. -- **When the exchange's markets start** (public trades API, `since=0`, probed - 2026-10-15): XLM/USD on 2017-01-17 20:17 UTC (first price $0.00225), and - XLM/BTC on 2016-07-19. -- **What is left to price** (prod, 1h rows with volume quoted in XLM or a - dollar token, all at `close_usd = 0`, 2026-10-15): - - | Period | 1h rows | Quote volume (M XLM) | - |---|---|---| - | 2015-11-18 → 2016-07-19 | 32 | 0.6 | - | 2016-07-19 → 2017-01-17 | 36 | 0.2 | - | 2017-01-17 → 2017-12-31 | 12,751 | 383.6 | - | 2018-01-01 → 2018-02-16 | 6,432 | 119.2 | - - XLM/USD from 2017-01-17 covers 19,183 of these 19,251 rows (99.6 %). -- **Thin market early on.** The first three XLM/USD trades are spread over - three days. 0331's rate window on tiers ≤ 1h is 24 h, so a candle with no - trade on the exchange in the preceding 24 h stays unpriced. - -## Decision (2026-10-15, Adam) - -- **Source:** the exchange's downloadable historical OHLCVT files, which - cover each pair "from the beginning of each market". This is option 1 of - the original task. They have to be fetched by hand. -- **Range:** 2017-01-17 → 2018-02-16 22:00 UTC. -- **Left unpriced:** the 68 in-scope 1h rows before 2017-01-17. Pricing them - would need XLM/BTC × BTC/USD or another exchange, which is not worth it for - that volume. - -## Implementation Plan - -1. Fetch the OHLCVT archive and record its provenance (file, sha256, date) - and the terms of use. -2. Build the hourly XLM/USD series for 2017-01-17 → 2018-02-16 22:00 UTC from - the 60-minute XLM/USD file, using the same rate definition as the 0331 rows. -3. Check the seam: compare it with the 0331 series where both have a value - (from 2018-02-16 23:00), and compare the 0331 range itself against the file. -4. Insert in the same shape as 0331's rows (`('native','XLM','','')`, staged - `external-candidate` first, same verify queries), then promote together with - them or on top of them. -5. Re-run 0331's `coarse-repair --pre-usdc-xlm-rate` for 201701–201802: 1h - first, then the wider tiers, which 0331 builds from 1h. - -## Acceptance Criteria - -- [ ] Source recorded with provenance and terms of use. -- [ ] Hourly XLM/USD rows for every hour of 2017-01-17 → 2018-02-16 that has a - trade on the exchange. -- [ ] Agreement with the 0331 series on the overlap recorded (median and max - deviation). -- [ ] XLM 1d candles from 2017-01-18 on return a USD price wherever the - exchange traded within the window; the remaining days counted and - explained, including the 68 rows before 2017-01-17. diff --git a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md b/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md new file mode 100644 index 00000000..92ac4e15 --- /dev/null +++ b/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md @@ -0,0 +1,95 @@ +--- +id: "0333" +title: "XLM/USD for 2018-01-01 → 2018-02-16" +type: FEATURE +status: backlog +related_adr: [] +related_tasks: ["0331"] +tags: [layer-backend, priority-medium, effort-small, enrichment, clickhouse, usd-reference] +links: + - "../active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md" +history: + - date: "2026-10-15" + status: backlog + who: akot + note: > + Scope set back to 2018-01-01 → 2018-02-16: priced history starts at the + beginning of 2018. 2017 is out of scope. The source stays the exchange's + downloadable OHLCVT files. + - date: "2026-10-15" + status: backlog + who: akot + note: > + Scope widened from 2018-01-01 → 02-16 to 2017-01-17 → 2018-02-16, the + start of the exchange's XLM/USD market. The downloadable OHLCVT files + are the chosen source. The 68 in-scope 1h rows before 2017-01-17 stay + unpriced. + - date: "2026-10-14" + status: backlog + who: akot + note: > + Spawned from 0331. The hourly XLM/USD series prepared for 0331 starts + 2018-02-16 23:00 UTC; the 47 days before it have no rate and stay + unpriced until this lands. Source not decided yet. +--- + +# XLM/USD for 2018-01-01 → 2018-02-16 + +## Summary + +[[0331]] prices pre-2021 candles from an hourly XLM/USD series loaded into +`prices.usd_rate`. That series starts at 2018-02-16 23:00 UTC. The priced +history should start at the beginning of 2018, so this task fills the 47 days +before it, from the same exchange, in the same shape. + +## Context + +- **Why the 0331 series stops at 2018-02-16.** The exchange's trades API + cannot reach trades before 2018-02-16 23:03 when paged. A time-based `since` + anywhere from 2017-12 to 2018-02-16 lands on that first reachable trade, and + paging from the start jumps to it after the first 1,000 trades. Probed + 2026-10-14. +- **The market did trade then.** Its XLM/USD market opened on 2017-01-17 and + had about 1,000 trades a day around 2018-01-01. +- **What is left to price** (prod, 2026-10-15): 6,432 1h rows with volume + quoted in XLM or a dollar token, worth 119.2 M XLM of quote volume, all at + `close_usd = 0`. +- **Out of scope:** everything before 2018-01-01. There are 12,819 such 1h + rows (2015-11 → 2017-12), and they stay unpriced. +- **A cross-check is available.** A second exchange's public hourly XLM/BTC × + BTC/USDT tracks the 0331 series at a median deviation of 0.43 % per day + (0.66 % for 2018-02-17 → 04-30). This was measured on 2026-10-15 for the + synthetic XLM series lab. + +## Decision (2026-10-15, Adam) + +- **Range:** 2018-01-01 00:00 → 2018-02-16 22:00 UTC. +- **Source:** the exchange's downloadable historical OHLCVT files. They cover + each pair from the start of its market and have to be fetched by hand. Same + source as the 0331 series. + +## Implementation Plan + +1. Fetch the OHLCVT archive and record its provenance (file, sha256, date) + and its terms of use. +2. Build the hourly XLM/USD series for the range from the 60-minute XLM/USD + file, using the same rate definition as the 0331 rows (hourly VWAP stamped + at the hour start). +3. Check the seam against the 0331 series where both have a value (from + 2018-02-16 23:00), and the range against the XLM/BTC × BTC/USDT + cross-check. +4. Insert in the same shape as the 0331 rows: `('native','XLM','','')`, + `reference_asset = 'third-party'`, staged `external-candidate` first, then + promote. +5. Re-run 0331's `coarse-repair --pre-usdc-xlm-rate` for 201801–201802, first + 1h, then the wider tiers. + +## Acceptance Criteria + +- [ ] Source recorded with provenance and terms of use. +- [ ] Hourly XLM/USD rows for every hour of 2018-01-01 → 2018-02-16 that has a + trade on the exchange. +- [ ] Agreement recorded with the 0331 series on the overlap and with the + cross-check (median and max deviation). +- [ ] XLM daily USD prices from 2018-01-01 on, wherever the exchange traded + within the window; any remaining days counted and explained. From c3fb5e29ecdc595324a1c1a5d87f7c2cbb3f3bd6 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:35:45 +0200 Subject: [PATCH 093/144] feat(lore-0331): synthesize pre-2021 XLM/USD candles on /ohlcv Before 2021-02-01 XLM's own candles exist only where its XLM/USDT market traded (21, 63 and 257 days in 2018-2020). In USD mode, /assets/native/ohlcv now builds them from the promoted third-party hourly XLM/USD rate on every grain: open/close are the first and last hourly rate, high/low their extremes. Volume is XLM traded against dollar tokens, minus four off-peg ones, priced at the rate within 24 h. Buckets without a rate fall back to today's candles; nothing changes from 2021-02-01 on. The dollar-token predicate moves to prices-clickhouse so coarse-repair and the API share it; coarse-repair renders the same text as before. --- packages/enrichment-worker/src/ch_enrich.rs | 17 +- packages/prices-api/src/assets/dto.rs | 7 +- packages/prices-api/src/assets/handlers.rs | 19 +- packages/prices-api/src/assets/queries_ch.rs | 271 ++++++++++++++++++ .../prices-api/src/openapi/descriptions.rs | 47 +-- packages/prices-clickhouse/src/lib.rs | 14 + web/portal/public/openapi.json | 22 +- 7 files changed, 362 insertions(+), 35 deletions(-) diff --git a/packages/enrichment-worker/src/ch_enrich.rs b/packages/enrichment-worker/src/ch_enrich.rs index 7f633b24..ccfece09 100644 --- a/packages/enrichment-worker/src/ch_enrich.rs +++ b/packages/enrichment-worker/src/ch_enrich.rs @@ -110,7 +110,7 @@ use tracing::{info, warn}; /// Re-exported from `prices-clickhouse` (the single source of truth, also used by /// `sdex-backfill`) so the `asset_id` the backfill interns under these issuers /// matches the `quote_asset_id` enriched here — no hand-synced literal to drift. -use prices_clickhouse::{USDC_ISSUER, USDT_ISSUER}; +use prices_clickhouse::{DOLLAR_TOKEN_PRED, USDC_ISSUER, USDT_ISSUER}; #[derive(Debug, thiserror::Error)] pub enum ChEnrichError { @@ -2885,9 +2885,7 @@ fn external_sql(db: &str, tbl: &str, usdc_id: u64, window: &str) -> String { /// Keyed on code/issuer/contract, never `asset_type` (fixtures disagree on it). fn dollar_token_asset_pred() -> String { format!( - "contract_address = '' AND issuer_address != '' \ - AND asset_code IN ('USD', 'USDT', 'USDC') \ - AND NOT (asset_code = 'USDC' AND issuer_address = '{USDC_ISSUER}')" + "{DOLLAR_TOKEN_PRED} AND NOT (asset_code = 'USDC' AND issuer_address = '{USDC_ISSUER}')" ) } @@ -3374,6 +3372,17 @@ fn pivot_sql(db: &str, tbl: &str, ref_id: u64, usdc_id: u64, window: &str) -> St mod tests { use super::*; + /// lore-0331 moved the code list into `DOLLAR_TOKEN_PRED`; the text is unchanged. + #[test] + fn dollar_token_asset_pred_renders_the_pre_0331_text() { + let want = format!( + "contract_address = '' AND issuer_address != '' \ + AND asset_code IN ('USD', 'USDT', 'USDC') \ + AND NOT (asset_code = 'USDC' AND issuer_address = '{USDC_ISSUER}')" + ); + assert_eq!(dollar_token_asset_pred(), want); + } + #[test] fn peg_sql_is_none_without_stablecoins() { assert!(peg_sql("prices", "price_ohlcv_1m", &[], "").is_none()); diff --git a/packages/prices-api/src/assets/dto.rs b/packages/prices-api/src/assets/dto.rs index a0608e8f..2247f22e 100644 --- a/packages/prices-api/src/assets/dto.rs +++ b/packages/prices-api/src/assets/dto.rs @@ -582,7 +582,8 @@ pub struct Candle { /// `chainlink` or `bitstamp` (task 0267, task 0265's composed history). /// /// ⚠️ **Populated only on canonical USDC's OWN synthesized series** (§6, the - /// `ohlcv_peg_series` path). `None` on every other asset — not because those + /// `ohlcv_peg_series` path), and on native XLM's own series before 2021-02-01, as + /// `third-party` (`ohlcv_native_usd`). `None` on every other asset — not because those /// candles have no provenance, but because the candle TABLES carry no /// provenance column to report it from. That gap is task 0268's Issue 9 and /// is deliberately out of scope here; a value invented for those rows would @@ -613,8 +614,8 @@ pub struct Candle { /// better than the $1 assumption it replaces, but a different instrument /// on a different venue. /// - /// Same scope as [`Candle::source`]: non-null only on USDC's own synthesized - /// series, `None` everywhere else and on the peg fallback. + /// Same scope as [`Candle::source`]: non-null only on the synthesized USDC and + /// pre-2021 XLM series, `None` everywhere else and on the peg fallback. pub quality: Option, } diff --git a/packages/prices-api/src/assets/handlers.rs b/packages/prices-api/src/assets/handlers.rs index 35c66db9..6184c383 100644 --- a/packages/prices-api/src/assets/handlers.rs +++ b/packages/prices-api/src/assets/handlers.rs @@ -441,7 +441,17 @@ async fn resolve_usd_refs( yet, those fields are `null`. Before 2021-02-01, candles\nquoted in XLM or in a \ dollar token are converted with an imported, measured\nXLM/USD rate (`method` is \ `external`); a dollar token is converted through its own XLM\nmarket and stays unpriced \ - without one.\n\nThe window is `timeframe`, ending \ + without one.\n\nFor native XLM itself (`GET /assets/native/ohlcv`) before 2021-02-01, a bucket \ + covered by\nthe imported hourly XLM/USD rate is built from that rate instead: `method` is \ + `external`,\n`derived` is `true` and `source` is `third-party`. `open` and `close` are the \ + bucket's\nfirst and last hourly rate, and `high` and `low` are the extremes of the hourly \ + averages,\nnot of individual trades. The volumes and trade count are those of XLM traded \ + against\ndollar tokens (USD, USDT and USDC of any issuer, except four that lost their \ + peg) in\nevery hour of the bucket; an hour without a rate of its own is valued in USD at \ + the\nnewest rate of the previous 24 hours. From 2021-02-01 the volumes are those of \ + XLM's own\nmarkets against canonical USDC and USDT only. At\n`15m` and `1m` a bucket appears only where XLM traded, and takes the rate of its \ + hour. A\nbucket the rate does not cover is built as above.\n\n\ + The window is `timeframe`, ending \ now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is \ rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the \ historical backfill is still running.", @@ -676,6 +686,8 @@ pub async fn get_ohlcv( .eq_ignore_ascii_case(&usdc_identifier().to_canonical()) }); + // Native XLM in USD: its pre-2021 buckets come from the imported rate. + let native_xlm = queries_ch::native_xlm_usd(&args.denomination, asset_id); let mut data = if is_peg_asset { // Both references anchor the series: USDC is the rate's identity, XLM is // the market the buckets come from (and, in XLM mode, the denominator). @@ -700,6 +712,11 @@ pub async fn get_ohlcv( Ok(d) => d, Err(e) => return errors::db_error(&e, "ohlcv peg series"), } + } else if let Some(xlm) = native_xlm { + match queries_ch::ohlcv_native_usd(state.ch(), args, xlm).await { + Ok(d) => d, + Err(e) => return errors::db_error(&e, "ohlcv native usd"), + } } else { match queries_ch::ohlcv(state.ch(), args).await { Ok(d) => d, diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index 3293b47a..db9c6fb0 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -6,6 +6,7 @@ //! field order MUST match the `SELECT` column order. use clickhouse::Client; +use prices_clickhouse::{DOLLAR_TOKEN_PRED, OFF_PEG_DOLLAR_TOKENS, PRE_USDC_END_S}; use crate::assets::dto::Candle; use crate::common::cursor::Cursor; @@ -1911,6 +1912,184 @@ pub async fn ohlcv(ch: &Client, args: OhlcvArgs) -> Result, clickhou q.fetch_all::().await } +/// The aggregates of [`native_usd_series_sql`] over its rows +/// `(bkt, hk, hts, hr, hs, hq, hx, htc, hxr, hxp)`, in `OUTER_ALIASES` order +/// minus `ts`. Prices and provenance come from the rate rows (`hk = 1`), volume +/// from every row; `hxr`/`hxp` are the priced XLM amount × rate, and amount. +fn native_usd_aggregates() -> String { + // Float64, parsed back: a Decimal × Decimal product wraps silently. + let dec = |f: &str| format!("toDecimal128OrNull(toString({f}), 14)"); + // The worst quality of the bucket wins; an unknown label ranks worst. + let rank = "multiIf(hq = '', 0, hq = 'measured', 1, hq = 'measured-disputed', 2, \ + hq = 'fallback', 3, 4)"; + format!( + "toNullable(toString(argMinIf(hr, hts, hk))) AS o, \ + toNullable(toString(maxIf(hr, hk))) AS h, toNullable(toString(minIf(hr, hk))) AS l, \ + toNullable(toString(argMaxIf(hr, hts, hk))) AS c, \ + toString(sum(hx)) AS vb, toString(ifNull({vqu}, toDecimal128(0, 14))) AS vqu, \ + toNullable(toString(least(greatest(ifNull({vw}, argMaxIf(hr, hts, hk)), \ + minIf(hr, hk)), maxIf(hr, hk)))) AS vw, \ + toUInt64(sum(htc)) AS tc, toNullable('external') AS meth, \ + toNullable(toUInt8(1)) AS drv, {PF_NULL_TAIL}, \ + if(uniqExactIf(hs, hk) = 1, nullIf(anyIf(hs, hk), ''), NULL) AS src, \ + nullIf(argMaxIf(hq, {rank}, hk), '') AS qual", + // NULL (→ 0) only on overflow, ≥ 1e24 USD: unreachable before 2021. + vqu = dec("sum(hxr)"), + vw = dec("if(sum(hxp) > 0, sum(hxr) / sum(hxp), avgIf(toFloat64(hr), hk))"), + ) +} + +/// XLM's markets against a dollar token other than the off-peg ones, in either +/// orientation; each leg starts on a key prefix (see [`ohlcv_peg_series`]). +fn dollar_leg_pred(xlm: u64) -> String { + let off_peg = OFF_PEG_DOLLAR_TOKENS.map(|(code, issuer)| format!("('{code}', '{issuer}')")); + let d = format!( + "SELECT asset_id FROM assets FINAL WHERE {DOLLAR_TOKEN_PRED} \ + AND (asset_code, issuer_address) NOT IN ({})", + off_peg.join(", ") + ); + format!( + "((asset_id = {xlm} AND quote_asset_id IN ({d})) \ + OR (asset_id IN ({d}) AND quote_asset_id = {xlm}))" + ) +} + +/// Native XLM's own USD candles before [`PRE_USDC_END_S`], built from the +/// promoted hourly XLM/USD rate (task 0331). Only typed integers are +/// interpolated, so nothing binds. +fn native_usd_series_sql(args: &OhlcvArgs, xlm: u64) -> String { + let (pre, iv) = (PRE_USDC_END_S, args.granularity.interval_sql()); + // ` AND {lhs}toDateTime(t){rhs}` when the window bound `t` is set. + let w = |t: Option, lhs: &str, rhs: &str| { + t.map_or(String::new(), |t| format!(" AND {lhs}toDateTime({t}){rhs}")) + }; + let (amt, legs) = ( + format!("if(asset_id = {xlm}, volume_base, volume_quote)"), + dollar_leg_pred(xlm), + ); + let since = w(args.start, "timestamp >= ", ""); + // `reference_asset` by equality, so `source` is `third-party` by construction. + let rates = |cols: &str, window: String| { + format!( + "SELECT 1 AS k, timestamp AS rts, usd_rate AS rate, \ + CAST(reference_asset AS String) AS rsrc, CAST(quality AS String) AS rq{cols} \ + FROM usd_rate FINAL \ + WHERE asset_kind = 'native' AND asset_code = 'XLM' AND issuer_address = '' \ + AND contract_address = '' AND method = 'external' \ + AND reference_asset = 'third-party' AND usd_rate > 0 \ + AND timestamp < toDateTime({pre}){window}" + ) + }; + let hours = if args.granularity.seconds() < 3600 { + // 1m/15m: a bucket only where XLM traded, at its hour's rate. `quote_asset_id` + // is not a key prefix; read-only on prod (2026-10-15) a pre-2021 month holds + // ≤ 302,183 rows on 1m and ≤ 94,724 on 15m, so the 5000-bucket worst case + // (3 months) reads under 1 M rows. + let until = w(args.end, "timestamp <= ", ""); + format!( + "SELECT t.bkt AS bkt, 1 AS hk, r.rts AS hts, r.rate AS hr, r.rsrc AS hs, \ + r.rq AS hq, t.tx AS hx, t.ttc AS htc, \ + toFloat64(t.tx) * toFloat64(r.rate) AS hxr, toFloat64(t.tx) AS hxp \ + FROM ( SELECT timestamp AS bkt, toStartOfHour(timestamp, 'UTC') AS th, \ + sumIf({amt}, {legs}) AS tx, sumIf(trade_count, {legs}) AS ttc \ + FROM price_ohlcv_{g} FINAL \ + WHERE (asset_id = {xlm} OR quote_asset_id = {xlm}) \ + AND timestamp < toDateTime({pre}){since}{until} \ + GROUP BY timestamp ) AS t \ + INNER JOIN ( {r} ) AS r ON r.rts = t.th", + g = args.granularity.as_str(), + r = rates( + "", + w(args.start, "timestamp >= toStartOfHour(", ", 'UTC')") + &until + ), + ) + } else { + // ≥1h: the bucket set and prices from the rate rows (`hk = 1`); volume from + // every dollar-leg hour, in USD at the newest rate at or before it within 24 h + // (`ch_enrich::external_sql` at 1h), else 0. + let bkt = + |c: &str| format!("toDateTime(toStartOfInterval({c}, INTERVAL {iv}, 'UTC'), 'UTC')"); + let ok = "ifNull(r.rate > 0 AND toUInt32(v.vend) - toUInt32(r.rts) <= 86400, 0)"; + format!( + "SELECT bkt, 1 AS hk, rts AS hts, rate AS hr, rsrc AS hs, rq AS hq, \ + toDecimal128(0, 14) AS hx, toUInt64(0) AS htc, 0. AS hxr, 0. AS hxp \ + FROM ( {r} ) \ + UNION ALL \ + SELECT {vb} AS bkt, 0 AS hk, v.vh AS hts, toDecimal128(0, 14) AS hr, '' AS hs, \ + '' AS hq, v.vx AS hx, v.vtc AS htc, \ + if({ok}, toFloat64(v.vx) * ifNull(toFloat64(r.rate), 0.), 0.) AS hxr, \ + if({ok}, toFloat64(v.vx), 0.) AS hxp \ + FROM ( SELECT 1 AS k, timestamp AS vh, timestamp + INTERVAL 1 HOUR AS vend, \ + sum({amt}) AS vx, toUInt64(sum(trade_count)) AS vtc \ + FROM price_ohlcv_1h FINAL \ + WHERE {legs} AND timestamp < toDateTime({pre}){since}{until} \ + GROUP BY timestamp ) AS v \ + ASOF LEFT JOIN ( {back} ) AS r ON r.k = v.k AND r.rts < v.vend", + r = rates( + &format!(", {} AS bkt", bkt("timestamp")), + w(args.start, "bkt >= ", "") + &w(args.end, "bkt <= ", ""), + ), + vb = bkt("v.vh"), + back = rates("", w(args.start, "timestamp >= ", " - INTERVAL 1 DAY")), + until = w(args.end, "timestamp < ", &format!(" + INTERVAL {iv}")), + ) + }; + format!( + "SELECT {OUTER_ALIASES} FROM ( \ + SELECT formatDateTime(bkt, '%Y-%m-%dT%H:%i:%SZ', 'UTC') AS ts, {agg}, bkt \ + FROM ( {hours} ) \ + GROUP BY bkt HAVING sum(hk) > 0 ORDER BY bkt DESC LIMIT {limit} \ + ) ORDER BY ts ASC", + agg = native_usd_aggregates(), + limit = args.limit, + ) +} + +/// `own` minus every bucket `synth` covers, plus `synth`: ascending, the last +/// `limit` kept, as [`ohlcv`] keeps the most recent. The dedup compares strings: +/// `synth` prints UTC, [`ohlcv`] the server's timezone, which must be UTC (as its +/// `Z` suffix already assumes). +fn merge_native_usd(mut synth: Vec, own: Vec, limit: usize) -> Vec { + let have: std::collections::HashSet = + synth.iter().map(|c| c.timestamp.clone()).collect(); + synth.extend(own.into_iter().filter(|c| !have.contains(&c.timestamp))); + synth.sort_by(|a, b| a.timestamp.cmp(&b.timestamp)); + synth.split_off(synth.len().saturating_sub(limit)) +} + +/// XLM's id when `asset_id` is native XLM in USD, the one case +/// [`ohlcv_native_usd`] serves; `None` in XLM mode and for every other asset. +pub fn native_xlm_usd(denomination: &Denomination, asset_id: u64) -> Option { + match denomination { + Denomination::Usd(refs) if refs.xlm == Some(asset_id) => refs.xlm, + _ => None, + } +} + +/// Whether a window starting at `start` can hold a pre-2021 bucket. +fn reaches_pre_usdc(start: Option) -> bool { + start.is_none_or(|s| s < i64::from(PRE_USDC_END_S)) +} + +/// Native XLM in USD: the synthesized pre-2021 series over [`ohlcv`] unchanged, +/// which still serves every bucket no promoted rate covers. +pub async fn ohlcv_native_usd( + ch: &Client, + args: OhlcvArgs, + xlm: u64, +) -> Result, clickhouse::error::Error> { + let limit = usize::try_from(args.limit).unwrap_or(usize::MAX); + let synth_sql = reaches_pre_usdc(args.start).then(|| native_usd_series_sql(&args, xlm)); + let synth = async { + let Some(sql) = &synth_sql else { + return Ok(Vec::new()); + }; + ch.query(sql).fetch_all::().await + }; + let (synth, own) = tokio::try_join!(synth, ohlcv(ch, args))?; + Ok(merge_native_usd(synth, own, limit)) +} + /// How far a bucket's `close` may sit from its price-forming mean before the /// response says so: one percent. /// @@ -3437,4 +3616,96 @@ mod tests { quality: None, } } + + #[test] + fn native_usd_series_sql_renders_every_grain() { + let want: Vec<&str> = OUTER_ALIASES.split(", ").skip(1).collect(); + assert_eq!(aliases_of(&native_usd_aggregates()), want); + use Granularity::*; + for g in [M1, M15, H1, H4, D1, W1, Mo1] { + let args = OhlcvArgs { + asset_id: 7, + denomination: Denomination::QuoteLeg(7), + granularity: g, + start: Some(1_600_000_000), + end: Some(1_600_086_400), + limit: 5000, + }; + let sql = native_usd_series_sql(&args, 7); + let fine = g.seconds() < 3600; + assert!( + sql.starts_with(&format!("SELECT {OUTER_ALIASES} FROM (")), + "{sql}" + ); + // One rate read on 1m/15m; ≥1h adds the 24 h look-back for volume. + let reads = if fine { 1 } else { 2 }; + let rate = "method = 'external' AND reference_asset = 'third-party' AND"; + assert_eq!(sql.matches("method = ").count(), reads, "{g:?}"); + assert_eq!(sql.matches(rate).count(), reads, "{g:?}"); + for bad in ["LIKE", "startsWith", "candidate", "SETTINGS", "?"] { + assert!(!sql.contains(bad), "{g:?} {bad}: {sql}"); + } + let pre = format!("toDateTime({PRE_USDC_END_S})"); + assert_eq!(sql.matches(&pre).count(), reads + 1, "{g:?}: {sql}"); + let issuers = OFF_PEG_DOLLAR_TOKENS.map(|(_, i)| i); + for needle in [pre.as_str(), "'UTC'", DOLLAR_TOKEN_PRED] + .iter() + .chain(&issuers) + { + assert!(sql.contains(needle), "{g:?} {needle}: {sql}"); + } + // ≥1h reads volume from 1h on key prefixes only; 1m/15m scan their + // own grain with XLM on either side. + let table = if fine { g.as_str() } else { "1h" }; + assert!( + sql.contains(&format!("price_ohlcv_{table} FINAL")), + "{g:?}: {sql}" + ); + assert_eq!(sql.matches("price_ohlcv_").count(), 1, "{g:?}: {sql}"); + let either = sql.contains("(asset_id = 7 OR quote_asset_id = 7)"); + assert_eq!(either, fine, "{g:?}: {sql}"); + let interval = format!("INTERVAL {}", g.interval_sql()); + assert!(fine || sql.contains(&interval), "{g:?}: {sql}"); + } + } + + #[test] + fn native_usd_routing_and_skip() { + let usd = Denomination::Usd(usd_refs()); + assert_eq!(native_xlm_usd(&usd, 1), Some(1)); + assert_eq!(native_xlm_usd(&usd, 2), None); + assert_eq!(native_xlm_usd(&Denomination::QuoteLeg(1), 1), None); + let pre = i64::from(PRE_USDC_END_S); + let got = [None, Some(pre - 1), Some(pre), Some(pre + 1)].map(reaches_pre_usdc); + assert_eq!(got, [true, true, false, false]); + } + + #[test] + fn merge_native_usd_replaces_orders_and_caps() { + let at = |d: &&str, synth: bool| Candle { + timestamp: format!("2021-{d}"), + source: synth.then(|| "third-party".to_string()), + ..candle_with(None, None) + }; + let tag = + |c: &Candle| c.timestamp[5..].to_string() + ["", "*"][c.source.is_some() as usize]; + // (synthetic days, limit, merged days, `*` = synthetic) + let cases: [(&[&str], usize, &str); 3] = [ + (&["01-30", "01-31"], 5000, "01-29 01-30* 01-31* 02-01"), + (&["01-30", "01-31"], 2, "01-31* 02-01"), + (&[], 5000, "01-29 01-31 02-01"), + ]; + for (synth, limit, want) in cases { + let synth = synth.iter().map(|d| at(d, true)).collect(); + let own = ["01-29", "01-31", "02-01"] + .iter() + .map(|d| at(d, false)) + .collect(); + let got: Vec = merge_native_usd(synth, own, limit) + .iter() + .map(tag) + .collect(); + assert_eq!(got.join(" "), want, "limit {limit}"); + } + } } diff --git a/packages/prices-api/src/openapi/descriptions.rs b/packages/prices-api/src/openapi/descriptions.rs index 5e529cf8..6b7eb3b4 100644 --- a/packages/prices-api/src/openapi/descriptions.rs +++ b/packages/prices-api/src/openapi/descriptions.rs @@ -59,7 +59,10 @@ pub(super) const SCHEMAS: &[(&str, &str)] = &[ that has no USD value: either the conversion has not caught up with the newest \ buckets yet, or the bucket traded only against a quote asset with no USD reference. \ `volume_base`, `volume_quote_usd` and `trade_count` are always present, so such a \ - bucket still shows its activity.", + bucket still shows its activity.\n\nThe exception is native XLM before 2021-02-01 \ + (`GET /assets/native/ohlcv`): a bucket the imported hourly XLM/USD rate covers takes \ + its prices from that rate and its volumes from XLM's trades against dollar tokens \ + (`method` `external`, `derived` `true`).", ), ( "ErrorEnvelope", @@ -347,7 +350,9 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ this bucket, never one carried over from an earlier one. `null` when the bucket has \ no price — see `pf_trade_count`. Exact in USD mode — the other price fields are \ scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when \ - one of them formed a price in the bucket; otherwise every market does.", + one of them formed a price in the bucket; otherwise every market does. On native \ + XLM's synthesized buckets before 2021-02-01 it is the bucket's last hourly XLM/USD \ + rate.", ), ( "Candle", @@ -367,7 +372,8 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ measured. Normally `close` is exact and the other price fields are scaled from the \ quote-asset values with one rate per bucket, so the true USD high may have occurred \ at a different moment than the quote-asset high. For USDC, which has no trades of \ - its own, every price field comes from the rate and `derived` is `true`.\n\n`null` \ + its own, every price field comes from the rate and `derived` is `true`, as it does for \ + native XLM before 2021-02-01 where the hourly XLM/USD rate covers the bucket.\n\n`null` \ when the price fields are `null`, and always `null` for `base_currency=XLM`.", ), ( @@ -451,8 +457,8 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ traded, its `volume_base` and `trade_count` are reported, and it has NO price — \ every price field is `null` rather than carrying a dust print. Always at most \ `trade_count`; `null` only on the synthesized USDC self-series \ - (`GET /assets/USDC:/ohlcv`), which is built from rate observations and has \ - no trades to count.", + (`GET /assets/USDC:/ohlcv`) and on native XLM's synthesized buckets before \ + 2021-02-01, which are built from rate observations.", ), ( "Candle", @@ -462,15 +468,17 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ compares `close` against.\n\nDenominated exactly like `close` and bounded by the \ published `low` and `high`, so the response stays self-consistent. In USD mode it \ weighs the same markets that set those (see `/ohlcv`). `null` when the \ - bucket has no price-forming volume, and on the synthesized USDC self-series. Never \ - `0`.", + bucket has no price-forming volume, on the synthesized USDC self-series and on \ + native XLM's synthesized buckets before 2021-02-01. Never `0`.", ), ( "Candle", "quality", - "How confident the imported series is in that day's observation. Present only \ + "How confident the imported series is in its observation; on a bucket built from \ + several, the least confident of them. Present only \ alongside `source`, i.e. only on the synthesized USDC self-series \ - (`GET /assets/USDC:/ohlcv`) for buckets whose `method` is `external`; \ + (`GET /assets/USDC:/ohlcv`) and on native XLM before 2021-02-01 \ + (`GET /assets/native/ohlcv`), for buckets whose `method` is `external`; \ `null` on every other asset, on the `peg` fallback, and whenever the price fields \ are `null`.\n\n* `measured` — a real observation from the primary feed.\n* \ `measured-disputed` — observed, but a cross-check against a second independent \ @@ -484,9 +492,10 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ ( "Candle", "source", - "Which outside USD series the imported rate came from: `chainlink` or `bitstamp`.\n\n\ - Present only on the synthesized USDC self-series \ - (`GET /assets/USDC:/ohlcv`), and there only for buckets whose `method` is \ + "Which outside USD series the imported rate came from: `chainlink` or `bitstamp` for \ + USDC, `third-party` for XLM.\n\nPresent only on the synthesized USDC self-series \ + (`GET /assets/USDC:/ohlcv`) and on native XLM before 2021-02-01 \ + (`GET /assets/native/ohlcv`), and there only for buckets whose `method` is \ `external`. It is `null` on every other asset — not because those candles lack \ provenance, but because the stored candles carry no column to report it from — \ `null` on an `oracle` bucket (a poll has no outside series, even where an outranked \ @@ -502,14 +511,18 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ ( "Candle", "trade_count", - "Number of trades in the bucket. The documented maximum, 2^53 − 1, is the largest \ - integer JSON carries exactly — a transport limit, not a domain one.", + "Number of trades in the bucket; on native XLM's synthesized buckets before \ + 2021-02-01, only XLM's trades against dollar tokens, so possibly `0`. The documented \ + maximum, 2^53 − 1, is the largest integer JSON carries exactly — a transport limit, \ + not a domain one.", ), ( "Candle", "volume_base", "Volume in units of the asset — the base side of each trade, over EVERY trade in \ - the bucket including the ones too small to form a price. It is therefore a \ + the bucket including the ones too small to form a price (on native XLM's \ + synthesized buckets before 2021-02-01, its trades against dollar tokens only, so \ + possibly `0`). It is therefore a \ complete activity figure and not a price-quality signal: `pf_trade_count` is the \ one to read for that, and a bucket that traded only in such amounts reports its \ volume here with no price at all.\n\nVolume is undistorted by those trades — \ @@ -528,7 +541,9 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "vwap", "Volume-weighted average price of the bucket, in `base_currency`, over the trades it \ holds — including the ones too small to form a price. `pf_vwap` is the same mean \ - with those left out. `null` when the bucket has no price.\n\nWHICH trades it \ + with those left out. `null` when the bucket has no price. On native XLM's synthesized \ + buckets before 2021-02-01 it is the hourly rates weighted by its dollar-token volume, \ + or their mean without any.\n\nWHICH trades it \ weighs depends on the denomination, so the two can differ on a bucket that holds a \ source with no usable price. With `base_currency=XLM` it is every row of the \ bucket, as stored. With `USD` it is every row that has a usable rate to convert it \ diff --git a/packages/prices-clickhouse/src/lib.rs b/packages/prices-clickhouse/src/lib.rs index fe623cd3..58abd372 100644 --- a/packages/prices-clickhouse/src/lib.rs +++ b/packages/prices-clickhouse/src/lib.rs @@ -252,6 +252,20 @@ pub const USDC_ORACLE_EPOCH_S: u32 = 1_773_237_600; /// `timestamp <` it means the whole bucket lies before it. pub const PRE_USDC_END_S: u32 = 1_612_137_600; +/// `prices.assets` rows of a classic dollar token of any issuer, canonical +/// USDC/USDT included; keyed on code/issuer/contract. +pub const DOLLAR_TOKEN_PRED: &str = + "contract_address = '' AND issuer_address != '' AND asset_code IN ('USD', 'USDT', 'USDC')"; + +/// Dollar tokens that lost their peg, excluded by identity (task 0331). +#[rustfmt::skip] +pub const OFF_PEG_DOLLAR_TOKENS: [(&str, &str); 4] = [ + ("USDT", "GC2PJ6J37QM3DOQ63DVGCXTRC5QOKAJUBQO2XKBXOJCWTCOCP75KWUDX"), + ("USD", "GD3LHBHY6NYJXETU6YO5OYI37F3X6L6C46ZWKBFBT35RBRKACDVMRF5K"), + ("USDT", "GC3YRTJQUXVQMG5STS7MSWVF3OF3Z5MVF6NEWZLKNQAJZX5UUFLMCFMA"), + ("USD", "GA5O7NTUIKYDPZSV3MNMIRFO2LCV5RJBNO7TNN7Y3MDO2IXRBSWDEO3T"), +]; + /// ClickHouse client configuration, sourced from environment with local-dev /// defaults. #[derive(Debug, Clone)] diff --git a/web/portal/public/openapi.json b/web/portal/public/openapi.json index 2a14160f..057c82ea 100644 --- a/web/portal/public/openapi.json +++ b/web/portal/public/openapi.json @@ -319,7 +319,7 @@ "get": { "tags": ["prices"], "summary": "`GET /assets/{asset_identifier}/ohlcv` — candlestick history.", - "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a price in the bucket, and from every market otherwise. If\nsuch a market has no USD value yet, those fields are `null`. Before 2021-02-01, candles\nquoted in XLM or in a dollar token are converted with an imported, measured\nXLM/USD rate (`method` is `external`); a dollar token is converted through its own XLM\nmarket and stays unpriced without one.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", + "description": "Candlestick history for one asset. `base_currency` sets the currency the candles are\nexpressed in, not which market they come from. With `USD` (the default) every candle is\nconverted to USD whatever it traded against, so an asset with no USD market still has a\nhistory: `close` is exact, while `open`, `high`, `low` and `vwap` are scaled from the\nquote-asset values with one rate per bucket and flagged with `derived`. With `XLM` only\ntrades against the native asset are returned, as stored.\n\nA bucket that traded but has no USD value yet is returned with its price fields `null`\nrather than omitted, so a chart keeps its time axis. Trades from every venue — and, in\nUSD mode, against every quote asset — are merged per bucket, in ascending time order,\nand all of them count toward the volumes and trade counts. In USD mode `open`, `high`,\n`low`, `close`, `method`, `vwap` and `pf_vwap` come only from markets quoted in USDC or\nXLM when one of them formed a price in the bucket, and from every market otherwise. If\nsuch a market has no USD value yet, those fields are `null`. Before 2021-02-01, candles\nquoted in XLM or in a dollar token are converted with an imported, measured\nXLM/USD rate (`method` is `external`); a dollar token is converted through its own XLM\nmarket and stays unpriced without one.\n\nFor native XLM itself (`GET /assets/native/ohlcv`) before 2021-02-01, a bucket covered by\nthe imported hourly XLM/USD rate is built from that rate instead: `method` is `external`,\n`derived` is `true` and `source` is `third-party`. `open` and `close` are the bucket's\nfirst and last hourly rate, and `high` and `low` are the extremes of the hourly averages,\nnot of individual trades. The volumes and trade count are those of XLM traded against\ndollar tokens (USD, USDT and USDC of any issuer, except four that lost their peg) in\nevery hour of the bucket; an hour without a rate of its own is valued in USD at the\nnewest rate of the previous 24 hours. From 2021-02-01 the volumes are those of XLM's own\nmarkets against canonical USDC and USDT only. At\n`15m` and `1m` a bucket appears only where XLM traded, and takes the rate of its hour. A\nbucket the rate does not cover is built as above.\n\nThe window is `timeframe`, ending now, or `start`/`end`. A window that would exceed 5000\ncandles at the chosen granularity is rejected with a 400. `backfill_note` is present only\nfor `timeframe=all` while the historical backfill is still running.", "operationId": "get_ohlcv", "parameters": [ { @@ -1107,7 +1107,7 @@ }, "Candle": { "type": "object", - "description": "One OHLCV candle, expressed in `base_currency`.\n\nThe prices come only from the bucket's own price-forming trades: `open` and `close` are the first and last of them, `high` and `low` their extremes. Nothing is carried over from a neighbouring bucket, so a period that traded only in amounts too small to carry a price has no price at all — see `pf_trade_count`.\n\nThe price fields — `open`, `high`, `low`, `close`, `vwap` and `pf_vwap` — are `null`, not omitted, on such a bucket and on one that has no USD value: either the conversion has not caught up with the newest buckets yet, or the bucket traded only against a quote asset with no USD reference. `volume_base`, `volume_quote_usd` and `trade_count` are always present, so such a bucket still shows its activity.", + "description": "One OHLCV candle, expressed in `base_currency`.\n\nThe prices come only from the bucket's own price-forming trades: `open` and `close` are the first and last of them, `high` and `low` their extremes. Nothing is carried over from a neighbouring bucket, so a period that traded only in amounts too small to carry a price has no price at all — see `pf_trade_count`.\n\nThe price fields — `open`, `high`, `low`, `close`, `vwap` and `pf_vwap` — are `null`, not omitted, on such a bucket and on one that has no USD value: either the conversion has not caught up with the newest buckets yet, or the bucket traded only against a quote asset with no USD reference. `volume_base`, `volume_quote_usd` and `trade_count` are always present, so such a bucket still shows its activity.\n\nThe exception is native XLM before 2021-02-01 (`GET /assets/native/ohlcv`): a bucket the imported hourly XLM/USD rate covers takes its prices from that rate and its volumes from XLM's trades against dollar tokens (`method` `external`, `derived` `true`).", "required": [ "timestamp", "volume_base", @@ -1137,12 +1137,12 @@ }, "close": { "type": ["string", "null"], - "description": "Closing price of the bucket, in `base_currency`: the LAST price-forming trade of this bucket, never one carried over from an earlier one. `null` when the bucket has no price — see `pf_trade_count`. Exact in USD mode — the other price fields are scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does.", + "description": "Closing price of the bucket, in `base_currency`: the LAST price-forming trade of this bucket, never one carried over from an earlier one. `null` when the bucket has no price — see `pf_trade_count`. Exact in USD mode — the other price fields are scaled, see `derived`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does. On native XLM's synthesized buckets before 2021-02-01 it is the bucket's last hourly XLM/USD rate.", "example": "0.2104557806765" }, "volume_base": { "type": "string", - "description": "Volume in units of the asset — the base side of each trade, over EVERY trade in the bucket including the ones too small to form a price. It is therefore a complete activity figure and not a price-quality signal: `pf_trade_count` is the one to read for that, and a bucket that traded only in such amounts reports its volume here with no price at all.\n\nVolume is undistorted by those trades — they carry almost none — which is why it stays unfiltered where the price fields do not.", + "description": "Volume in units of the asset — the base side of each trade, over EVERY trade in the bucket including the ones too small to form a price (on native XLM's synthesized buckets before 2021-02-01, its trades against dollar tokens only, so possibly `0`). It is therefore a complete activity figure and not a price-quality signal: `pf_trade_count` is the one to read for that, and a bucket that traded only in such amounts reports its volume here with no price at all.\n\nVolume is undistorted by those trades — they carry almost none — which is why it stays unfiltered where the price fields do not.", "example": "256744.7651102" }, "volume_quote_usd": { @@ -1152,13 +1152,13 @@ }, "vwap": { "type": ["string", "null"], - "description": "Volume-weighted average price of the bucket, in `base_currency`, over the trades it holds — including the ones too small to form a price. `pf_vwap` is the same mean with those left out. `null` when the bucket has no price.\n\nWHICH trades it weighs depends on the denomination, so the two can differ on a bucket that holds a source with no usable price. With `base_currency=XLM` it is every row of the bucket, as stored. With `USD` it is every row that has a usable rate to convert it with, on the markets that set the prices (see `/ohlcv`) — a row whose own price is below the precision floor, or that has no USD value yet, cannot be restated in USD and is left out of the mean rather than counted at zero.\n\nBounded by the published `low` and `high` so the response stays self-consistent. Those come from the price-forming trades only, so on a bucket where the trades that formed no price carry most of the volume the raw mean falls outside that range and the value reported is the nearer bound. Compare it with `pf_vwap` when the difference matters.", + "description": "Volume-weighted average price of the bucket, in `base_currency`, over the trades it holds — including the ones too small to form a price. `pf_vwap` is the same mean with those left out. `null` when the bucket has no price. On native XLM's synthesized buckets before 2021-02-01 it is the hourly rates weighted by its dollar-token volume, or their mean without any.\n\nWHICH trades it weighs depends on the denomination, so the two can differ on a bucket that holds a source with no usable price. With `base_currency=XLM` it is every row of the bucket, as stored. With `USD` it is every row that has a usable rate to convert it with, on the markets that set the prices (see `/ohlcv`) — a row whose own price is below the precision floor, or that has no USD value yet, cannot be restated in USD and is left out of the mean rather than counted at zero.\n\nBounded by the published `low` and `high` so the response stays self-consistent. Those come from the price-forming trades only, so on a bucket where the trades that formed no price carry most of the volume the raw mean falls outside that range and the value reported is the nearer bound. Compare it with `pf_vwap` when the difference matters.", "example": "0.2109568287069" }, "trade_count": { "type": "integer", "format": "int64", - "description": "Number of trades in the bucket. The documented maximum, 2^53 − 1, is the largest integer JSON carries exactly — a transport limit, not a domain one.", + "description": "Number of trades in the bucket; on native XLM's synthesized buckets before 2021-02-01, only XLM's trades against dollar tokens, so possibly `0`. The documented maximum, 2^53 − 1, is the largest integer JSON carries exactly — a transport limit, not a domain one.", "example": 666, "maximum": 9007199254740991, "minimum": 0 @@ -1170,19 +1170,19 @@ }, "derived": { "type": ["boolean", "null"], - "description": "Whether `open`, `high`, `low` and `vwap` were derived by scaling rather than measured. Normally `close` is exact and the other price fields are scaled from the quote-asset values with one rate per bucket, so the true USD high may have occurred at a different moment than the quote-asset high. For USDC, which has no trades of its own, every price field comes from the rate and `derived` is `true`.\n\n`null` when the price fields are `null`, and always `null` for `base_currency=XLM`.", + "description": "Whether `open`, `high`, `low` and `vwap` were derived by scaling rather than measured. Normally `close` is exact and the other price fields are scaled from the quote-asset values with one rate per bucket, so the true USD high may have occurred at a different moment than the quote-asset high. For USDC, which has no trades of its own, every price field comes from the rate and `derived` is `true`, as it does for native XLM before 2021-02-01 where the hourly XLM/USD rate covers the bucket.\n\n`null` when the price fields are `null`, and always `null` for `base_currency=XLM`.", "example": true }, "pf_trade_count": { "type": ["integer", "null"], "format": "int64", - "description": "How many of the bucket's trades formed its price.\n\nA trade's price is the ratio of the two integer amounts exchanged, so a trade of a few of the smallest representable units prints an exact small fraction — 1/17, 5/34 — that is arithmetically correct and can sit hundreds of percent off the market. Those trades are not price-forming, and `open`, `high`, `low` and `close` are taken only from the ones that are. Neither is a trade whose price falls below 0.000000000001 — one hundred times the smallest value the price fields can hold, below which a stored price is rounding noise rather than a measurement. The same floor applies at every `interval`.\n\n`0` is a real answer and the one worth acting on: the bucket traded, its `volume_base` and `trade_count` are reported, and it has NO price — every price field is `null` rather than carrying a dust print. Always at most `trade_count`; `null` only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`), which is built from rate observations and has no trades to count.", + "description": "How many of the bucket's trades formed its price.\n\nA trade's price is the ratio of the two integer amounts exchanged, so a trade of a few of the smallest representable units prints an exact small fraction — 1/17, 5/34 — that is arithmetically correct and can sit hundreds of percent off the market. Those trades are not price-forming, and `open`, `high`, `low` and `close` are taken only from the ones that are. Neither is a trade whose price falls below 0.000000000001 — one hundred times the smallest value the price fields can hold, below which a stored price is rounding noise rather than a measurement. The same floor applies at every `interval`.\n\n`0` is a real answer and the one worth acting on: the bucket traded, its `volume_base` and `trade_count` are reported, and it has NO price — every price field is `null` rather than carrying a dust print. Always at most `trade_count`; `null` only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`) and on native XLM's synthesized buckets before 2021-02-01, which are built from rate observations.", "example": 666, "minimum": 0 }, "pf_vwap": { "type": ["string", "null"], - "description": "Volume-weighted average price of the bucket's price-forming trades, in `base_currency` — `vwap` with the dust left out, and the value `close_divergent` compares `close` against.\n\nDenominated exactly like `close` and bounded by the published `low` and `high`, so the response stays self-consistent. In USD mode it weighs the same markets that set those (see `/ohlcv`). `null` when the bucket has no price-forming volume, and on the synthesized USDC self-series. Never `0`.", + "description": "Volume-weighted average price of the bucket's price-forming trades, in `base_currency` — `vwap` with the dust left out, and the value `close_divergent` compares `close` against.\n\nDenominated exactly like `close` and bounded by the published `low` and `high`, so the response stays self-consistent. In USD mode it weighs the same markets that set those (see `/ohlcv`). `null` when the bucket has no price-forming volume, on the synthesized USDC self-series and on native XLM's synthesized buckets before 2021-02-01. Never `0`.", "example": "0.21095682870692" }, "close_divergent": { @@ -1192,12 +1192,12 @@ }, "source": { "type": ["string", "null"], - "description": "Which outside USD series the imported rate came from: `chainlink` or `bitstamp`.\n\nPresent only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`), and there only for buckets whose `method` is `external`. It is `null` on every other asset — not because those candles lack provenance, but because the stored candles carry no column to report it from — `null` on an `oracle` bucket (a poll has no outside series, even where an outranked import shares the bucket), `null` on the `peg` fallback, where no series was consulted at all, and whenever the price fields are `null`. Never the empty string.", + "description": "Which outside USD series the imported rate came from: `chainlink` or `bitstamp` for USDC, `third-party` for XLM.\n\nPresent only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`) and on native XLM before 2021-02-01 (`GET /assets/native/ohlcv`), and there only for buckets whose `method` is `external`. It is `null` on every other asset — not because those candles lack provenance, but because the stored candles carry no column to report it from — `null` on an `oracle` bucket (a poll has no outside series, even where an outranked import shares the bucket), `null` on the `peg` fallback, where no series was consulted at all, and whenever the price fields are `null`. Never the empty string.", "example": null }, "quality": { "type": ["string", "null"], - "description": "How confident the imported series is in that day's observation. Present only alongside `source`, i.e. only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`) for buckets whose `method` is `external`; `null` on every other asset, on the `peg` fallback, and whenever the price fields are `null`.\n\n* `measured` — a real observation from the primary feed.\n* `measured-disputed` — observed, but a cross-check against a second independent source disagreed by more than the composer's spread tolerance. The number is real; treat it as less certain than a plain `measured` day and prefer not to alert on it alone.\n* `fallback` — the primary feed had nothing for that day, so the secondary source supplied it. Still an observation, and far better than assuming $1, but a different instrument on a different venue.\n\nA day with no quality is not a day of unknown quality — the field simply does not apply outside the imported series.", + "description": "How confident the imported series is in its observation; on a bucket built from several, the least confident of them. Present only alongside `source`, i.e. only on the synthesized USDC self-series (`GET /assets/USDC:/ohlcv`) and on native XLM before 2021-02-01 (`GET /assets/native/ohlcv`), for buckets whose `method` is `external`; `null` on every other asset, on the `peg` fallback, and whenever the price fields are `null`.\n\n* `measured` — a real observation from the primary feed.\n* `measured-disputed` — observed, but a cross-check against a second independent source disagreed by more than the composer's spread tolerance. The number is real; treat it as less certain than a plain `measured` day and prefer not to alert on it alone.\n* `fallback` — the primary feed had nothing for that day, so the secondary source supplied it. Still an observation, and far better than assuming $1, but a different instrument on a different venue.\n\nA day with no quality is not a day of unknown quality — the field simply does not apply outside the imported series.", "example": null } } From 737226cfcc79802910b7bf3fc9dc1adf9074492f Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:36:41 +0200 Subject: [PATCH 094/144] test(lore-0331): pin the synthetic XLM series Seeded hourly rates and dollar legs: OHLC from the hours on 1h, 4h, 1d, 1w and 1M with no duplicate bucket; 15m only where XLM traded; volume excludes off-peg and non-dollar legs; the 24 h carry; worst quality per bucket; a non-third-party rate row ignored; no synthetic bucket from 2021-02-01; before promotion the response equals today's. --- packages/prices-api/tests/ohlcv_it.rs | 224 ++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/packages/prices-api/tests/ohlcv_it.rs b/packages/prices-api/tests/ohlcv_it.rs index bf19c68e..c50a5e32 100644 --- a/packages/prices-api/tests/ohlcv_it.rs +++ b/packages/prices-api/tests/ohlcv_it.rs @@ -3273,3 +3273,227 @@ async fn ohlcv_labels_pre_usdc_candles_external() { } teardown(db).await; } + +// A dollar token that is not off-peg, and one that is (lore-0331). +const USD_X: AssetFixture = AssetFixture::new("USD", "credit", USDT_ISSUER, ""); +const OFF: (&str, &str) = prices_clickhouse::OFF_PEG_DOLLAR_TOKENS[0]; +const OFF_PEG: AssetFixture = AssetFixture::new(OFF.0, "credit", OFF.1, ""); + +/// XLM rates as `rate_method`: 24 hourly on 2021-01-31 (0.33 at 00:00, 0.31 at +/// 05:00, 0.34 at 22:00, 0.35 at 23:00; max 0.36, min 0.30), three on 01-27/28, +/// three after the seam; 9.0 decoys staged or from another source. Plus XLM's +/// dollar, off-peg and non-dollar markets. +async fn seed_native_usd(db: &str, admin: &Client, m: &str) { + exec(admin, &assets_insert(db, &[USDT, USD_X, OFF_PEG])).await; + exec( + admin, + &format!( + "INSERT INTO {db}.usd_rate (asset_kind, asset_code, issuer_address, contract_address, \ + timestamp, usd_rate, method, reference_asset, quality, hops, version) \ + SELECT 'native', 'XLM', '', '', toDateTime('2021-01-31 00:00:00') + number * 3600, \ + toDecimal128(30 + (number + 3) % 7, 14) / 100, '{m}', 'third-party', \ + 'measured', 0, 1 FROM numbers(24) \ + UNION ALL SELECT 'native', 'XLM', '', '', toDateTime(t), toDecimal128(r, 14), \ + m, s, q, 0, 1 FROM values('t String, r String, m String, s String, q String', \ + ('2021-01-27 06:00:00', '0.20', '{m}', 'third-party', 'measured'), \ + ('2021-01-28 12:00:00', '0.25', '{m}', 'third-party', 'fallback'), \ + ('2021-01-28 13:00:00', '0.27', '{m}', 'third-party', 'measured-disputed'), \ + ('2021-02-01 00:00:00', '0.5', '{m}', 'third-party', 'measured'), \ + ('2021-02-01 01:00:00', '0.5', '{m}', 'third-party', 'measured'), \ + ('2021-02-01 02:00:00', '0.5', '{m}', 'third-party', 'measured'), \ + ('2021-01-31 05:00:00', '9', 'external-candidate', 'third-party', 'measured'), \ + ('2021-01-30 12:00:00', '9', 'external-candidate', 'third-party', 'measured'), \ + ('2021-01-30 06:00:00', '9', 'external', 'other-source', 'measured'))" + ), + ) + .await; + // (grain, timestamp, base, quote, close, close_usd, volume_base, volume_quote, trades). + // close_usd != close: an exact-par dollar row is dropped as a peg signature. + #[rustfmt::skip] + let candles = [ + ("1h", "2021-01-31 05:00", XLM, USDT, 0.5, 0.49, 100.0, 50.0, 2), + ("1h", "2021-01-31 05:00", XLM, USDC, 0.48, 0.4801, 10.0, 4.8, 1), + ("1h", "2021-01-31 05:00", USD_X, XLM, 2.0, 0.0, 25.0, 50.0, 3), + ("1h", "2021-01-31 05:00", OFF_PEG, XLM, 2.0, 0.0, 500.0, 1000.0, 9), + ("1h", "2021-01-31 05:00", FOO, XLM, 1.0, 0.0, 777.0, 777.0, 11), + ("1h", "2021-01-31 22:00", USD_X, XLM, 2.0, 0.0, 20.0, 40.0, 1), + // 01-28: 01:00 carries 01-27 06:00's rate (20 h), 06:00 has none within + // 24 h, 20:00 carries 13:00's. + ("1h", "2021-01-28 01:00", XLM, USDT, 0.5, 0.49, 10.0, 5.0, 1), + ("1h", "2021-01-28 06:00", XLM, USDT, 0.5, 0.49, 20.0, 10.0, 1), + ("1h", "2021-01-28 20:00", USD_X, XLM, 2.0, 0.0, 15.0, 30.0, 1), + ("1h", "2021-02-01 00:00", XLM, USDC, 0.3215, 0.3216, 50.0, 16.075, 1), + ("4h", "2021-01-31 04:00", XLM, USDT, 0.5, 0.49, 100.0, 50.0, 2), + ("1d", "2021-01-30 00:00", XLM, USDT, 0.2905, 0.29, 5.0, 1.4525, 1), + ("1d", "2021-01-31 00:00", XLM, USDT, 0.5, 0.49, 999.0, 499.5, 1), + ("1d", "2021-02-01 00:00", XLM, USDC, 0.3221, 0.3222, 7.0, 2.2547, 1), + ("1w", "2021-01-25 00:00", XLM, USDT, 0.5, 0.49, 999.0, 499.5, 1), + ("1w", "2021-02-01 00:00", XLM, USDC, 0.3221, 0.3222, 7.0, 2.2547, 1), + ("1M", "2021-01-01 00:00", XLM, USDT, 0.5, 0.49, 999.0, 499.5, 1), + ("1M", "2021-02-01 00:00", XLM, USDC, 0.3221, 0.3222, 7.0, 2.2547, 1), + ("15m", "2021-01-31 05:15", FOO, XLM, 1.0, 0.0, 5.0, 5.0, 1), + ("15m", "2021-01-31 05:30", XLM, USDT, 0.5, 0.49, 100.0, 50.0, 2), + ("15m", "2021-01-31 05:30", OFF_PEG, XLM, 2.0, 0.0, 500.0, 1000.0, 9), + ]; + for (g, ts, base, quote, p, cu, vb, vq, tc) in candles { + exec( + admin, + &format!( + "INSERT INTO {db}.price_ohlcv_{g} \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + VALUES ('{ts}:00', {base}, {quote}, 'sdex', {p}, {p}, {p}, {p}, \ + {vb}, {vq}, 0, {cu}, {p}, {tc}, 1)" + ), + ) + .await; + } +} + +/// One expected candle: (timestamp, close, and for a synthesized one +/// [open, high, low, volume_base, trade_count, volume_quote_usd, vwap] and quality). +type NativeRow = (&'static str, f64, Option<([f64; 7], &'static str)>); + +/// Seeds `db` with `rate_method` rates, then checks per case (query, candles, +/// synthesized candles, expected rows). Ascending timestamps: no duplicate bucket. +async fn check_native_usd( + db: &str, + rate_method: &str, + cases: &[(&str, usize, usize, &[NativeRow])], +) { + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + seed_native_usd(db, &admin, rate_method).await; + for (query, n, synth_n, rows) in cases { + let uri = format!("/v1/assets/native/ohlcv?{query}&base_currency=USD"); + let (status, json) = get(client.clone(), &uri).await; + assert_eq!(status, StatusCode::OK, "{query}: {json}"); + let data = json["data"].as_array().unwrap(); + assert_eq!(data.len(), *n, "{query}: {json}"); + let ts: Vec<&Value> = data.iter().map(|c| &c["timestamp"]).collect(); + assert!( + ts.windows(2).all(|w| w[0].as_str() < w[1].as_str()), + "{query}: {ts:?}" + ); + let synth = data.iter().filter(|c| c["source"] == "third-party").count(); + assert_eq!(synth, *synth_n, "{query}: {json}"); + for (at, close, s) in rows.iter() { + let c = data.iter().find(|c| c["timestamp"] == *at).expect(at); + approx(&c["close"], *close); + let Some(([o, h, l, vb, tc, vqu, vw], quality)) = s else { + assert!(c["source"].is_null(), "{query} {at}: {c}"); + continue; + }; + let fields = [ + "open", + "high", + "low", + "volume_base", + "volume_quote_usd", + "vwap", + ]; + for (f, v) in fields.iter().zip([o, h, l, vb, vqu, vw]) { + approx(&c[f], *v); + } + assert_eq!(c["trade_count"].as_f64(), Some(*tc), "{query} {at}: {c}"); + let tail = [ + &c["method"], + &c["derived"], + &c["quality"], + &c["pf_trade_count"], + ]; + let want = [ + "external".into(), + true.into(), + (*quality).into(), + Value::Null, + ]; + assert!(tail.into_iter().eq(&want), "{query} {at}: {c}"); + assert!( + c["pf_vwap"].is_null() && c["close_divergent"].is_null(), + "{c}" + ); + } + } + teardown(db).await; +} + +const NATIVE_DAY: &str = "granularity=1d&start=2021-01-30T00:00:00Z&end=2021-02-01T00:00:00Z"; +const NATIVE_HOUR: &str = "granularity=1h&start=2021-01-31T00:00:00Z&end=2021-01-31T23:00:00Z"; +const NATIVE_15M: &str = "granularity=15m&start=2021-01-31T05:00:00Z&end=2021-01-31T06:45:00Z"; +const NATIVE_WEEK: &str = "granularity=1w&start=2021-01-18T00:00:00Z&end=2021-02-01T00:00:00Z"; + +/// Native XLM's pre-2021 USD buckets come from the promoted hourly rate; the +/// rest is today's series (lore-0331). XLM mode never routes here +/// (`native_usd_routing_and_skip`). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_native_usd_synthesizes_pre_2021_from_the_rate() { + // The 01-27 → 01-31 rates and 01-28 + 01-31 volume, worst quality `fallback`. + let jan = Some(( + [0.20, 0.36, 0.20, 260.0, 10.0, 73.3, 73.3 / 240.0], + "fallback", + )); + #[rustfmt::skip] + let cases: [(&str, usize, usize, &[NativeRow]); 8] = [ + (NATIVE_DAY, 3, 1, &[ + // Only a staged or other-source rate covers 01-30: today's candle. + ("2021-01-30T00:00:00Z", 0.29, None), + // 05:00 legs 100 + 10 + 50 and 22:00's 40; off-peg, FOO and the + // stored 1d volume left out. + ("2021-01-31T00:00:00Z", 0.35, Some(([0.33, 0.36, 0.30, 200.0, 7.0, 63.2, 0.316], "measured"))), + // Rates after the seam are ignored. + ("2021-02-01T00:00:00Z", 0.3222, None), + ]), + // Volume from every hour; USD 10 × 0.20 + 30 × 0.27, the 06:00 hour unpriced. + ("granularity=1d&start=2021-01-28T00:00:00Z&end=2021-01-29T00:00:00Z", 1, 1, &[ + ("2021-01-28T00:00:00Z", 0.27, Some(([0.25, 0.27, 0.25, 60.0, 3.0, 10.1, 0.2525], "fallback"))), + ]), + // Every hour from the rate; the own 05:00 candle replaced. + (NATIVE_HOUR, 24, 24, &[ + ("2021-01-31T05:00:00Z", 0.31, Some(([0.31, 0.31, 0.31, 160.0, 6.0, 49.6, 0.31], "measured"))), + ("2021-01-31T22:00:00Z", 0.34, Some(([0.34, 0.34, 0.34, 40.0, 1.0, 13.6, 0.34], "measured"))), + ("2021-01-31T00:00:00Z", 0.33, Some(([0.33, 0.33, 0.33, 0.0, 0.0, 0.0, 0.33], "measured"))), + ]), + ("granularity=1h&start=2021-01-31T22:00:00Z&end=2021-02-01T02:00:00Z", 3, 2, &[ + ("2021-02-01T00:00:00Z", 0.3216, None), + ]), + // Only where XLM traded: FOO/XLM alone at 05:15, off-peg left out at 05:30. + (NATIVE_15M, 2, 2, &[ + ("2021-01-31T05:15:00Z", 0.31, Some(([0.31, 0.31, 0.31, 0.0, 0.0, 0.0, 0.31], "measured"))), + ("2021-01-31T05:30:00Z", 0.31, Some(([0.31, 0.31, 0.31, 100.0, 2.0, 31.0, 0.31], "measured"))), + ]), + // Each bucket must land on the stored one's timestamp, or both show. + ("granularity=4h&start=2021-01-31T00:00:00Z&end=2021-01-31T20:00:00Z", 6, 6, &[ + ("2021-01-31T04:00:00Z", 0.33, Some(([0.30, 0.33, 0.30, 160.0, 6.0, 49.6, 0.31], "measured"))), + ("2021-01-31T20:00:00Z", 0.35, Some(([0.32, 0.35, 0.32, 40.0, 1.0, 13.6, 0.34], "measured"))), + ]), + (NATIVE_WEEK, 2, 1, &[ + ("2021-01-25T00:00:00Z", 0.35, jan), + ("2021-02-01T00:00:00Z", 0.3222, None), + ]), + ("granularity=1M&start=2021-01-01T00:00:00Z&end=2021-02-01T00:00:00Z", 2, 1, &[ + ("2021-01-01T00:00:00Z", 0.35, jan), + ("2021-02-01T00:00:00Z", 0.3222, None), + ]), + ]; + check_native_usd("it_ohlcv_native_usd_0331", "external", &cases).await; +} + +/// Before promotion (only `external-candidate` rates) the series is today's. +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_native_usd_before_promotion_equals_today() { + #[rustfmt::skip] + let cases: [(&str, usize, usize, &[NativeRow]); 4] = [ + (NATIVE_DAY, 3, 0, &[ + ("2021-01-30T00:00:00Z", 0.29, None), + ("2021-01-31T00:00:00Z", 0.49, None), + ("2021-02-01T00:00:00Z", 0.3222, None), + ]), + (NATIVE_HOUR, 1, 0, &[("2021-01-31T05:00:00Z", 0.4801, None)]), + (NATIVE_15M, 1, 0, &[("2021-01-31T05:30:00Z", 0.49, None)]), + (NATIVE_WEEK, 2, 0, &[("2021-01-25T00:00:00Z", 0.49, None)]), + ]; + check_native_usd("it_ohlcv_native_usd_pre_0331", "external-candidate", &cases).await; +} From 64773aa94b6cd77f63794a0c88445273deb1f17c Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:04:48 +0200 Subject: [PATCH 095/144] fix(lore-0331): apply review findings on the pre-2021 rate - coarse-repair reads only the third-party XLM/USD rows, as the API does (shared XLM_RATE_SOURCE), in the pass and its admissibility check. - The native XLM series pins its window arithmetic to UTC. - The snapshot guard for --pre-usdc-xlm-rate moves to check_snapshots, with a table-driven test. - A dry run says where a real 1m run would stop, and how many priced rows a coarse-tier run would set back to 0. - open/high/low/volume_quote_usd describe the synthesized XLM buckets. --- .../src/bin/coarse-repair.rs | 115 ++++++++++++++---- packages/enrichment-worker/src/ch_enrich.rs | 30 +++-- packages/prices-api/src/assets/queries_ch.rs | 76 +++++++++--- .../prices-api/src/openapi/descriptions.rs | 16 ++- packages/prices-clickhouse/src/lib.rs | 4 + web/portal/public/openapi.json | 8 +- 6 files changed, 188 insertions(+), 61 deletions(-) diff --git a/packages/enrichment-worker/src/bin/coarse-repair.rs b/packages/enrichment-worker/src/bin/coarse-repair.rs index c98f5794..bc02f71e 100644 --- a/packages/enrichment-worker/src/bin/coarse-repair.rs +++ b/packages/enrichment-worker/src/bin/coarse-repair.rs @@ -336,6 +336,40 @@ fn check_table( Ok(()) } +/// The snapshot guard of a real run that can discard stored values: reset +/// mode, and `--pre-usdc-xlm-rate` off `price_ohlcv_1m` (on 1m the pass guards +/// that instead, see [`check_table`]). +/// +/// Rollback for a bad reset IS `ATTACH PARTITION` from the shadow copy, so the +/// snapshots must exist. They usually will not have been taken by this tool: +/// `prices_writer` cannot FREEZE and cannot be granted it, so on prod the admin +/// takes them out of band and `--skip-snapshot` is the *correct* flag rather +/// than a shortcut. Refusing that combination outright would make reset mode +/// unusable on the only cluster it exists for. So the operator must say which +/// case they are in. +fn check_snapshots( + table: &str, + resetting: bool, + pre_usdc: bool, + (skip_snapshot, snapshots_verified): (bool, bool), + dry_run: bool, +) -> Result<(), String> { + let discards = resetting || (pre_usdc && table != "price_ohlcv_1m"); + if discards && !dry_run && skip_snapshot && !snapshots_verified { + return Err("--skip-snapshot with --reset-quote-asset-id or \ + --pre-usdc-xlm-rate requires --snapshots-verified: both can discard \ + stored values, and their only \ + rollback is ATTACH PARTITION from the frozen copy.\n\ + If the CH admin already froze this span (runbook Step 3b — the \ + normal prod path, since prices_writer cannot FREEZE), confirm with \ + `ls /var/lib/clickhouse/shadow/ | grep repair_` plus a non-trivial \ + `du`, then pass --snapshots-verified.\n\ + If nobody has, stop: there is no rollback point." + .into()); + } + Ok(()) +} + #[tokio::main] async fn main() -> Result<(), Box> { tracing_subscriber::fmt() @@ -363,30 +397,13 @@ async fn main() -> Result<(), Box> { args.dry_run, )?; - // Reset mode discards data, and so can a --pre-usdc-xlm-rate re-run (on 1m - // the pass guards that instead), so their footguns are refused. - let pre_usdc_coarse = args.pre_usdc_xlm_rate && args.table != "price_ohlcv_1m"; - if (args.reset_quote_asset_id.is_some() || pre_usdc_coarse) && !args.dry_run { - // 1. Rollback for a bad reset IS `ATTACH PARTITION` from the shadow copy, - // so the snapshots must exist. They usually will not have been taken by - // this tool: `prices_writer` cannot FREEZE and cannot be granted it, so - // on prod the admin takes them out of band and `--skip-snapshot` is the - // *correct* flag rather than a shortcut. Refusing that combination - // outright would make reset mode unusable on the only cluster it exists - // for. So require the operator to say which case they are in. - if args.skip_snapshot && !args.snapshots_verified { - return Err("--skip-snapshot with --reset-quote-asset-id or \ - --pre-usdc-xlm-rate requires --snapshots-verified: both can discard \ - stored values, and their only \ - rollback is ATTACH PARTITION from the frozen copy.\n\ - If the CH admin already froze this span (runbook Step 3b — the \ - normal prod path, since prices_writer cannot FREEZE), confirm with \ - `ls /var/lib/clickhouse/shadow/ | grep repair_` plus a non-trivial \ - `du`, then pass --snapshots-verified.\n\ - If nobody has, stop: there is no rollback point." - .into()); - } - } + check_snapshots( + &args.table, + args.reset_quote_asset_id.is_some(), + args.pre_usdc_xlm_rate, + (args.skip_snapshot, args.snapshots_verified), + args.dry_run, + )?; // The window guards below run in dry-run mode TOO (task 0228 review WR-01): // the runbooks say "dry run first", and a dry run that accepts a window the // real run then refuses is a rehearsal of nothing. Only the snapshot guard @@ -647,6 +664,29 @@ async fn main() -> Result<(), Box> { } ); + // A real 1m run refuses the first month that would set priced rows to 0; + // the dry run reports the count, so say what the real run would do with it. + let unprice = summary.total_reset(); + let first = summary.months.iter().find(|m| m.rows_reset > 0); + if let (true, true, Some(m)) = (args.pre_usdc_xlm_rate, args.dry_run, first) { + if args.table == "price_ohlcv_1m" { + if !args.allow_unprice { + println!( + "A real run would stop at month {} ({unprice} priced row(s) in all \ + would be set back to 0); months before it would be written. \ + Pass --allow-unprice to accept.", + m.month + ); + } + } else { + println!( + "A real run would set {unprice} priced row(s) back to 0, from month {} \ + on; only the snapshot can restore them.", + m.month + ); + } + } + // The reset's own safety check, computed rather than left to the operator's // arithmetic. `rows_reset` > `rows_enriched` means values were discarded and // not recomputed — the one outcome worse than the defect being repaired. @@ -698,7 +738,32 @@ async fn main() -> Result<(), Box> { #[cfg(test)] mod tests { - use super::check_table; + use super::{check_snapshots, check_table}; + + /// (table, resetting, pre_usdc, skip_snapshot, snapshots_verified, dry_run, refused) + #[rustfmt::skip] + const SNAPSHOT_CASES: &[(&str, bool, bool, bool, bool, bool, bool)] = &[ + ("price_ohlcv_1h", true, false, true, false, false, true), + ("price_ohlcv_1h", false, true, true, false, false, true), + ("price_ohlcv_1M", false, true, true, false, false, true), + ("price_ohlcv_1h", false, true, true, true, false, false), + ("price_ohlcv_1h", false, true, false, false, false, false), + ("price_ohlcv_1h", false, true, true, false, true, false), + ("price_ohlcv_1m", false, true, true, false, false, false), + ("price_ohlcv_1h", false, false, true, false, false, false), + ]; + + #[test] + fn a_real_run_that_can_discard_needs_a_snapshot_or_its_confirmation() { + for &(t, reset, pre, skip, verified, dry, refused) in SNAPSHOT_CASES { + let got = check_snapshots(t, reset, pre, (skip, verified), dry); + assert_eq!( + got.is_err(), + refused, + "{t} reset={reset} pre={pre} skip={skip} verified={verified} dry={dry}" + ); + } + } #[test] fn coarse_tables_take_every_mode() { diff --git a/packages/enrichment-worker/src/ch_enrich.rs b/packages/enrichment-worker/src/ch_enrich.rs index ccfece09..d921980d 100644 --- a/packages/enrichment-worker/src/ch_enrich.rs +++ b/packages/enrichment-worker/src/ch_enrich.rs @@ -2074,8 +2074,8 @@ impl ChEnrichmentPass { ); if self.client.query(&sql).fetch_one::().await? == 0 { return refuse( - "usd_rate holds no method = 'external' XLM row before 2021-02-01 \ - (promote the staged series first)", + "usd_rate holds no method = 'external' third-party XLM row before \ + 2021-02-01 (promote the staged series first)", ); } Ok(xlm) @@ -2718,21 +2718,27 @@ pub fn external_window_s(table: &str) -> u32 { bucket_width_s(table).max(86_400) } -/// `usd_rate` identity `(asset_kind, asset_code, issuer_address, contract_address)`. -const USDC_RATE_IDENTITY: [&str; 4] = ["credit", "USDC", USDC_ISSUER, ""]; -const XLM_RATE_IDENTITY: [&str; 4] = ["native", "XLM", "", ""]; +/// `usd_rate` identity `(asset_kind, asset_code, issuer_address, contract_address)` +/// and the `reference_asset` to require, `""` for any. +const USDC_RATE_IDENTITY: [&str; 5] = ["credit", "USDC", USDC_ISSUER, "", ""]; +const XLM_RATE_IDENTITY: [&str; 5] = ["native", "XLM", "", "", prices_clickhouse::XLM_RATE_SOURCE]; /// The imported (`method = 'external'`) rate series of one `usd_rate` identity, /// as an ASOF right side keyed `k = 1`. Equality on `method`: the staged /// `external-candidate` rows share its prefix (`external_rate.rs`). -fn usd_rate_series_sql(db: &str, id: [&str; 4]) -> String { - let [kind, code, issuer, contract] = id; +fn usd_rate_series_sql(db: &str, id: [&str; 5]) -> String { + let [kind, code, issuer, contract, source] = id; + let source = if source.is_empty() { + String::new() + } else { + format!(" AND reference_asset = '{source}'") + }; format!( "SELECT 1 AS k, timestamp AS rts, usd_rate AS usd \ FROM {db}.usd_rate FINAL \ WHERE asset_kind = '{kind}' AND asset_code = '{code}' \ AND issuer_address = '{issuer}' AND contract_address = '{contract}' \ - AND method = 'external' AND usd_rate > 0" + AND method = 'external'{source} AND usd_rate > 0" ) } @@ -5729,11 +5735,9 @@ mod tests { for w in [None, Some((5, 9))] { let (ins, cnt) = pre_usdc_sql("prices", t, 9, 86_400, w); let rollup = bucket_width_s(t) > 3_600; - assert_eq!( - ins.matches("method = ").count(), - usize::from(!rollup), - "{ins}" - ); + for needle in ["method = ", "reference_asset = 'third-party'"] { + assert_eq!(ins.matches(needle).count(), usize::from(!rollup), "{ins}"); + } assert_eq!(ins.contains("argMaxIf("), rollup, "{t}"); for needle in ["LIKE", "startsWith", "candidate"] { assert!(!ins.contains(needle), "{t}: {needle}"); diff --git a/packages/prices-api/src/assets/queries_ch.rs b/packages/prices-api/src/assets/queries_ch.rs index db9c6fb0..1df3cf0f 100644 --- a/packages/prices-api/src/assets/queries_ch.rs +++ b/packages/prices-api/src/assets/queries_ch.rs @@ -6,7 +6,9 @@ //! field order MUST match the `SELECT` column order. use clickhouse::Client; -use prices_clickhouse::{DOLLAR_TOKEN_PRED, OFF_PEG_DOLLAR_TOKENS, PRE_USDC_END_S}; +use prices_clickhouse::{ + DOLLAR_TOKEN_PRED, OFF_PEG_DOLLAR_TOKENS, PRE_USDC_END_S, XLM_RATE_SOURCE, +}; use crate::assets::dto::Candle; use crate::common::cursor::Cursor; @@ -1624,12 +1626,15 @@ pub async fn ohlcv_peg_series( /// USDC leg at or after the epoch was priced from a measured reading. /// 4. `quote_asset_id = usdc` -> **`''`**. Scaled, pre-epoch, and no imported /// rate covers the bucket — no tier produces this. Null is the honest answer -/// to "which input priced this"; it also keeps USDC off arm 5 if it is ever -/// tracked as a pivot. -/// 5. `quote_asset_id IN (pivots)` -> **`traded`** (ADR 0011 §4; see [`ohlcv`]). +/// to "which input priced this"; it also keeps USDC off the arms below, e.g. if +/// it is ever tracked as a pivot. +/// 5. `timestamp < PRE_USDC_END_S` -> **`external`** (task 0331): before +/// 2021-02-01 only `coarse-repair --pre-usdc-xlm-rate` writes a non-USDC +/// `close_usd`, from the imported XLM/USD rate. +/// 6. `quote_asset_id IN (pivots)` -> **`traded`** (ADR 0011 §4; see [`ohlcv`]). /// Omitted ENTIRELY when no pivot reference is tracked — an empty `IN ()` is /// a ClickHouse syntax error, and the label is optional. -/// 6. `''` -> the fallback, which [`ohlcv`]'s `nullIf` turns into a JSON `null`. +/// 7. `''` -> the fallback, which [`ohlcv`]'s `nullIf` turns into a JSON `null`. /// A `multiIf` with no else is an error, so this arm always closes the list. /// /// ⚠️ The bare column names below (`close_usd`, `close`, `timestamp`) resolve to @@ -1663,12 +1668,7 @@ pub(crate) fn usd_method_expr(usdc: u64, pivots: &[u64], granularity: Granularit // the plain `+ INTERVAL` form slides the coverage window's upper day by the // server's offset — and by a whole hour across a DST boundary. The worker // guards this at length; the read side must not reintroduce it. - let bucket_end = match granularity { - Granularity::D1 => "addDays(timestamp, 1, 'UTC')".to_string(), - Granularity::W1 => "addWeeks(timestamp, 1, 'UTC')".to_string(), - Granularity::Mo1 => "addMonths(timestamp, 1, 'UTC')".to_string(), - g => format!("timestamp + {}", g.seconds()), - }; + let bucket_end = bucket_end_sql(granularity, "timestamp"); // The imported days, as ONE uncorrelated set built per query (ClickHouse // renders this as a single `CreatingSet` node, not per row). let imported_days = format!( @@ -1705,6 +1705,16 @@ pub(crate) fn usd_method_expr(usdc: u64, pivots: &[u64], granularity: Granularit ) } +/// `col` plus one bucket of `g`, pinned to UTC (see [`usd_method_expr`]). +fn bucket_end_sql(g: Granularity, col: &str) -> String { + match g { + Granularity::D1 => format!("addDays({col}, 1, 'UTC')"), + Granularity::W1 => format!("addWeeks({col}, 1, 'UTC')"), + Granularity::Mo1 => format!("addMonths({col}, 1, 'UTC')"), + g => format!("{col} + {}", g.seconds()), + } +} + /// Read merged candles for one asset at the chosen grain, denominated per /// [ADR 0011]. /// @@ -1968,7 +1978,8 @@ fn native_usd_series_sql(args: &OhlcvArgs, xlm: u64) -> String { dollar_leg_pred(xlm), ); let since = w(args.start, "timestamp >= ", ""); - // `reference_asset` by equality, so `source` is `third-party` by construction. + // `reference_asset` by equality, as `coarse-repair` reads it, so `source` is + // `third-party` by construction. let rates = |cols: &str, window: String| { format!( "SELECT 1 AS k, timestamp AS rts, usd_rate AS rate, \ @@ -1976,7 +1987,7 @@ fn native_usd_series_sql(args: &OhlcvArgs, xlm: u64) -> String { FROM usd_rate FINAL \ WHERE asset_kind = 'native' AND asset_code = 'XLM' AND issuer_address = '' \ AND contract_address = '' AND method = 'external' \ - AND reference_asset = 'third-party' AND usd_rate > 0 \ + AND reference_asset = '{XLM_RATE_SOURCE}' AND usd_rate > 0 \ AND timestamp < toDateTime({pre}){window}" ) }; @@ -2030,8 +2041,11 @@ fn native_usd_series_sql(args: &OhlcvArgs, xlm: u64) -> String { w(args.start, "bkt >= ", "") + &w(args.end, "bkt <= ", ""), ), vb = bkt("v.vh"), - back = rates("", w(args.start, "timestamp >= ", " - INTERVAL 1 DAY")), - until = w(args.end, "timestamp < ", &format!(" + INTERVAL {iv}")), + back = rates("", w(args.start, "timestamp >= ", " - 86400")), + until = args.end.map_or(String::new(), |t| { + let end = bucket_end_sql(args.granularity, &format!("toDateTime({t}, 'UTC')")); + format!(" AND timestamp < {end}") + }), ) }; format!( @@ -3666,6 +3680,38 @@ mod tests { assert_eq!(either, fine, "{g:?}: {sql}"); let interval = format!("INTERVAL {}", g.interval_sql()); assert!(fine || sql.contains(&interval), "{g:?}: {sql}"); + // Calendar arithmetic is pinned to UTC, as in `usd_method_expr`. + let call = |at: usize| { + let mut depth = 0; + for (j, ch) in sql[at..].char_indices() { + match ch { + '(' => depth += 1, + ')' if depth == 1 => return &sql[at..at + j], + ')' => depth -= 1, + _ => {} + } + } + panic!("unbalanced at {at}: {sql}") + }; + for f in [ + "addDays(", + "addWeeks(", + "addMonths(", + "toStartOf", + "formatDateTime(", + ] { + for (i, _) in sql.match_indices(f) { + assert!(call(i).ends_with("'UTC'"), "{g:?}: {}", call(i)); + } + } + for iv in [ + "- INTERVAL", + "+ INTERVAL 1 DAY", + "+ INTERVAL 1 WEEK", + "+ INTERVAL 1 MONTH", + ] { + assert!(!sql.contains(iv), "{g:?} {iv}: {sql}"); + } } } diff --git a/packages/prices-api/src/openapi/descriptions.rs b/packages/prices-api/src/openapi/descriptions.rs index 6b7eb3b4..316d19b2 100644 --- a/packages/prices-api/src/openapi/descriptions.rs +++ b/packages/prices-api/src/openapi/descriptions.rs @@ -382,7 +382,9 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Highest price among this bucket's price-forming trades, in `base_currency`; `null` \ when the bucket has no price — see `pf_trade_count`. In USD mode only markets \ quoted in USDC or XLM set it when one of them formed a price in the bucket; \ - otherwise every market does.", + otherwise every market does. On native XLM's synthesized buckets before 2021-02-01 \ + it is the highest hourly XLM/USD rate of the bucket, an hourly average rather than \ + a trade.", ), ( "Candle", @@ -390,7 +392,9 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` \ when the bucket has no price — see `pf_trade_count`. In USD mode only markets \ quoted in USDC or XLM set it when one of them formed a price in the bucket; \ - otherwise every market does.", + otherwise every market does. On native XLM's synthesized buckets before 2021-02-01 \ + it is the lowest hourly XLM/USD rate of the bucket, an hourly average rather than \ + a trade.", ), ( "Candle", @@ -440,7 +444,8 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of \ this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD \ mode only markets quoted in USDC or XLM set it when one of them formed a price in \ - the bucket; otherwise every market does.", + the bucket; otherwise every market does. On native XLM's synthesized buckets \ + before 2021-02-01 it is the bucket's first hourly XLM/USD rate.", ), ( "Candle", @@ -534,7 +539,10 @@ pub(super) const FIELDS: &[(&str, &str, &str)] = &[ "volume_quote_usd", "USD volume of the bucket, summed over the trades that have a USD value. A trade not \ yet priced counts in `volume_base` and `trade_count` but not here, so this is the \ - USD volume that can be accounted for, not the bucket's total restated in USD.", + USD volume that can be accounted for, not the bucket's total restated in USD. On \ + native XLM's synthesized buckets before 2021-02-01 it covers XLM's trades against \ + dollar tokens only, each hour valued at the newest XLM/USD rate of the previous \ + 24 hours.", ), ( "Candle", diff --git a/packages/prices-clickhouse/src/lib.rs b/packages/prices-clickhouse/src/lib.rs index 58abd372..e0f8d1d3 100644 --- a/packages/prices-clickhouse/src/lib.rs +++ b/packages/prices-clickhouse/src/lib.rs @@ -252,6 +252,10 @@ pub const USDC_ORACLE_EPOCH_S: u32 = 1_773_237_600; /// `timestamp <` it means the whole bucket lies before it. pub const PRE_USDC_END_S: u32 = 1_612_137_600; +/// `usd_rate.reference_asset` of the imported pre-2021 XLM/USD series (task +/// 0331). Both its readers filter on it, so they read the same rows. +pub const XLM_RATE_SOURCE: &str = "third-party"; + /// `prices.assets` rows of a classic dollar token of any issuer, canonical /// USDC/USDT included; keyed on code/issuer/contract. pub const DOLLAR_TOKEN_PRED: &str = diff --git a/web/portal/public/openapi.json b/web/portal/public/openapi.json index 057c82ea..028a534b 100644 --- a/web/portal/public/openapi.json +++ b/web/portal/public/openapi.json @@ -1122,17 +1122,17 @@ }, "open": { "type": ["string", "null"], - "description": "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does.", + "description": "Opening price of the bucket, in `base_currency`: the FIRST price-forming trade of this bucket. `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does. On native XLM's synthesized buckets before 2021-02-01 it is the bucket's first hourly XLM/USD rate.", "example": "0.21155612364244" }, "high": { "type": ["string", "null"], - "description": "Highest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does.", + "description": "Highest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does. On native XLM's synthesized buckets before 2021-02-01 it is the highest hourly XLM/USD rate of the bucket, an hourly average rather than a trade.", "example": "0.21336358305927" }, "low": { "type": ["string", "null"], - "description": "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does.", + "description": "Lowest price among this bucket's price-forming trades, in `base_currency`; `null` when the bucket has no price — see `pf_trade_count`. In USD mode only markets quoted in USDC or XLM set it when one of them formed a price in the bucket; otherwise every market does. On native XLM's synthesized buckets before 2021-02-01 it is the lowest hourly XLM/USD rate of the bucket, an hourly average rather than a trade.", "example": "0.21019557835832" }, "close": { @@ -1147,7 +1147,7 @@ }, "volume_quote_usd": { "type": "string", - "description": "USD volume of the bucket, summed over the trades that have a USD value. A trade not yet priced counts in `volume_base` and `trade_count` but not here, so this is the USD volume that can be accounted for, not the bucket's total restated in USD.", + "description": "USD volume of the bucket, summed over the trades that have a USD value. A trade not yet priced counts in `volume_base` and `trade_count` but not here, so this is the USD volume that can be accounted for, not the bucket's total restated in USD. On native XLM's synthesized buckets before 2021-02-01 it covers XLM's trades against dollar tokens only, each hour valued at the newest XLM/USD rate of the previous 24 hours.", "example": "54161.26730887106528" }, "vwap": { From 2e243ab02743fee10172519cd10f7dd90779f4cf Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:05:42 +0200 Subject: [PATCH 096/144] test(lore-0331): pin month edges, rate source and XLM mode - A driver run prices a dollar-quoted 1h row from the previous month's token market, and a week that crosses a month edge rolls up both months' hours. - An external XLM row from another source is never read. - base_currency=XLM on native ignores the promoted rate. --- .../enrichment-worker/tests/ch_enrich_it.rs | 70 ++++++++++++++++--- packages/prices-api/tests/ohlcv_it.rs | 30 +++++++- 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/packages/enrichment-worker/tests/ch_enrich_it.rs b/packages/enrichment-worker/tests/ch_enrich_it.rs index 31442799..572d8228 100644 --- a/packages/enrichment-worker/tests/ch_enrich_it.rs +++ b/packages/enrichment-worker/tests/ch_enrich_it.rs @@ -16,7 +16,9 @@ use enrichment_worker::repair::{ CoarseRepairConfig, CoarseRepairDriver, CoarseSweepConfig, MonthRepair, run_coarse_sweep, }; use prices_clickhouse::asset_id::fixture::{AssetFixture, assets_insert, fetch_ids}; -use prices_clickhouse::{PRE_USDC_END_S, USDC_ISSUER, USDC_ORACLE_EPOCH_S, USDT_ISSUER}; +use prices_clickhouse::{ + PRE_USDC_END_S, USDC_ISSUER, USDC_ORACLE_EPOCH_S, USDT_ISSUER, XLM_RATE_SOURCE, +}; use std::fmt::Display; // The fixture assets. Each displays as its id, derived from its identity @@ -2213,15 +2215,15 @@ fn utc_day_start(ts: u32) -> u32 { ts - ts % 86_400 } -/// Seed one `usd_rate` row for canonical USDC under an explicit `method`, so a -/// test can stage BOTH series for the same bucket and prove which one wins. -/// [`seed_external_rate`] is the `'external'` case and carries the day-start +/// Seed one `usd_rate` row of identity `(asset_kind, asset_code, issuer_address)` +/// and `reference_asset` under an explicit `method`, so a test can stage BOTH +/// series for the same bucket and prove which one wins. [`seed_external_rate`] +/// is the canonical USDC `'external'` case and carries the day-start /// convention note. -/// One `usd_rate` row of identity `(asset_kind, asset_code, issuer_address)`. async fn seed_usd_rate( client: &Client, db: &str, - (kind, code, issuer): (&str, &str, &str), + (kind, code, issuer, source): (&str, &str, &str, &str), (ts, rate, method): (u32, f64, &str), version: u64, ) { @@ -2230,7 +2232,7 @@ async fn seed_usd_rate( "INSERT INTO {db}.usd_rate \ (asset_kind, asset_code, issuer_address, contract_address, \ timestamp, usd_rate, method, reference_asset, hops, version) VALUES \ - ('{kind}', '{code}', '{issuer}', '', {ts}, {rate}, '{method}', '', 0, {version})" + ('{kind}', '{code}', '{issuer}', '', {ts}, {rate}, '{method}', '{source}', 0, {version})" )) .execute() .await @@ -3195,7 +3197,7 @@ async fn a_pivot_leg_prefers_the_oracle_rate_over_the_external_rate() { let db = "it_enrich_0228_oracle_wins"; let client = setup_0228_pivot(db, "price_ohlcv_1d", DEPEG_DAY).await; // The oracle says 0.95 at day start; the import says 0.9681 an hour LATER. - let usdc = ("credit", "USDC", USDC_ISSUER); + let usdc = ("credit", "USDC", USDC_ISSUER, ""); seed_usd_rate(&client, db, usdc, (DEPEG_DAY, 0.95, "oracle"), 1).await; seed_external_rate(&client, db, DEPEG_DAY + 3_600, DEPEG_RATE).await; @@ -5744,7 +5746,10 @@ type Rate = (u32, f64, &'static str); #[rustfmt::skip] type Leg = (&'static str, &'static str, u32, AssetFixture<'static>, AssetFixture<'static>, f64, f64, f64, f64, f64); -const XLM_RATE: (&str, &str, &str) = ("native", "XLM", ""); +const XLM_RATE: (&str, &str, &str, &str) = ("native", "XLM", "", XLM_RATE_SOURCE); +/// An `external` XLM row from another source, which no 0331 reader may take: +/// at h(30) it would price the h(48) and h(53) legs. +const OTHER_XLM_RATE: (&str, &str, &str, &str) = ("native", "XLM", "", "other-source"); const RATES_0331: &[Rate] = &[ (h(10), 0.25, "external"), (h(23), 0.20, "external"), @@ -5828,6 +5833,7 @@ async fn setup_0331(db: &str, rates: &[Rate], legs: &[Leg]) -> Client { for r in rates { seed_usd_rate(&client, db, XLM_RATE, *r, 1).await; } + seed_usd_rate(&client, db, OTHER_XLM_RATE, (h(30), 9.0, "external"), 1).await; for l in legs { seed_leg(&client, db, l).await; } @@ -6073,3 +6079,49 @@ async fn pre_usdc_xlm_rate_repair_refuses_unpromoted_rates_and_previews_exactly( .await .unwrap(); } + +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_repair_reaches_across_a_month_edge() { + let db = "test_ch_enrich_pre_usdc_month_edge"; + let day = |n: u32| D0331 + n * 86_400; + // 2020-06-29 is a Monday: its week runs into July. + let (week, jun30, jul1, jul2) = (day(28), day(29), day(30), day(31)); + let rates: &[Rate] = &[ + (jun30 + 10 * 3600, 0.25, "external"), + (jul1, 0.30, "external"), + (jul2 + 10 * 3600, 0.20, "external"), + ]; + #[rustfmt::skip] + let legs: &[Leg] = &[ + ("USDT/XLM 06-30 20:00", H1, jun30 + 20 * 3600, USDT, XLM, 5.0, 10.0, 50.0, 1.25, 12.5), + ("FOO/USDT 07-01 via June's market", H1, jul1, FOO, USDT, 2.0, 3.0, 6.0, 3.0, 9.0), + ("FOO/XLM 06-30 10:00", H1, jun30 + 10 * 3600, FOO, XLM, 10.0, 3.0, 30.0, 2.5, 7.5), + ("FOO/XLM 07-02 10:00", H1, jul2 + 10 * 3600, FOO, XLM, 10.0, 3.0, 30.0, 2.0, 6.0), + ("1w FOO/XLM from 06-29 rolls up July's hour", WIDE[2], week, FOO, XLM, 10.0, 6.0, 60.0, 2.0, 13.5), + ]; + let client = setup_0331(db, rates, legs).await; + for table in [H1, WIDE[2]] { + CoarseRepairDriver::with_client( + client.clone(), + CoarseRepairConfig { + enrich: cfg_0331(db, table), + start_month: 202_006, + end_month: 202_007, + snapshot: false, + dry_run: false, + one_shot: true, + deadline: None, + }, + ) + .run() + .await + .unwrap(); + } + assert_legs(&client, db, legs, |_| 1.0).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); +} diff --git a/packages/prices-api/tests/ohlcv_it.rs b/packages/prices-api/tests/ohlcv_it.rs index c50a5e32..9612b196 100644 --- a/packages/prices-api/tests/ohlcv_it.rs +++ b/packages/prices-api/tests/ohlcv_it.rs @@ -3425,7 +3425,7 @@ const NATIVE_WEEK: &str = "granularity=1w&start=2021-01-18T00:00:00Z&end=2021-02 /// Native XLM's pre-2021 USD buckets come from the promoted hourly rate; the /// rest is today's series (lore-0331). XLM mode never routes here -/// (`native_usd_routing_and_skip`). +/// (`ohlcv_native_xlm_mode_ignores_the_promoted_rate`). #[tokio::test] #[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] async fn ohlcv_native_usd_synthesizes_pre_2021_from_the_rate() { @@ -3497,3 +3497,31 @@ async fn ohlcv_native_usd_before_promotion_equals_today() { ]; check_native_usd("it_ohlcv_native_usd_pre_0331", "external-candidate", &cases).await; } + +/// `base_currency=XLM` never takes the synthesized series: promoting the rate +/// changes nothing in XLM mode (lore-0331). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn ohlcv_native_xlm_mode_ignores_the_promoted_rate() { + let mut got = Vec::new(); + for (db, m) in [ + ("it_ohlcv_native_xlm_0331", "external"), + ("it_ohlcv_native_xlm_pre_0331", "external-candidate"), + ] { + let client = setup(db).await; + let admin = Client::default().with_url(ch_url()).with_database(db); + seed_native_usd(db, &admin, m).await; + let mut data = Vec::new(); + for q in [NATIVE_DAY, NATIVE_HOUR, NATIVE_15M, NATIVE_WEEK] { + let uri = format!("/v1/assets/native/ohlcv?{q}&base_currency=XLM"); + let (status, json) = get(client.clone(), &uri).await; + assert_eq!(status, StatusCode::OK, "{q}: {json}"); + let candles = json["data"].as_array().unwrap(); + assert!(candles.iter().all(|c| c["source"].is_null()), "{q}: {json}"); + data.push(json["data"].clone()); + } + got.push(data); + teardown(db).await; + } + assert_eq!(got[0], got[1]); +} From f230398a412d969473f37d67672d7359a67e3ac3 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:33:23 +0200 Subject: [PATCH 097/144] fix(lore-0331): refuse a stalled run and guard the roll-up sum - A one-shot pre-2021 run that stops with rows still differing now fails (PreUsdcXlmRateStalled) instead of reporting success, and rows_reset counts the priced rows actually set back to 0. - The wide-tier USD volume sums in Decimal256 and parses back, as the narrow branch does: a Decimal(38, 14) sum wraps negative on overflow. --- packages/enrichment-worker/src/ch_enrich.rs | 86 +++++++++++++++++++-- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/packages/enrichment-worker/src/ch_enrich.rs b/packages/enrichment-worker/src/ch_enrich.rs index d921980d..147985c5 100644 --- a/packages/enrichment-worker/src/ch_enrich.rs +++ b/packages/enrichment-worker/src/ch_enrich.rs @@ -450,6 +450,12 @@ pub enum ChEnrichError { #[error("pre-2021 XLM/USD rate mode refused: {0}")] PreUsdcXlmRateRefused(String), + + #[error( + "pre-2021 XLM/USD step stalled: {remaining} row(s) still differ after the last \ + batch, and {reset} priced row(s) were set back to 0 before it stopped" + )] + PreUsdcXlmRateStalled { remaining: u64, reset: u64 }, } /// Opt-in reset of **already-written** USD columns, so a corrected pricing tier @@ -2120,14 +2126,10 @@ impl ChEnrichmentPass { "pre-2021 XLM/USD step stalled — rows still differ", ) .await?; + let stats = pre_usdc_stats(before, last.get(), remaining, batches, self.cfg.one_shot)?; Ok(ChPassStats { - batches, - candidates_before: before.pending, - candidates_after: last.get().unpriced, - rows_enriched: before.pending.saturating_sub(remaining), - rows_reset: before.unprice, duration_ms: start.elapsed().as_millis() as u64, - ..Default::default() + ..stats }) } @@ -2718,6 +2720,31 @@ pub fn external_window_s(table: &str) -> u32 { bucket_width_s(table).max(86_400) } +/// The pre-2021 step's stats from its counts before and after the drain: +/// `rows_reset` is the priced rows actually set back to 0. A one-shot run that +/// leaves rows still differing is an error, because the mode can discard +/// values and a partial run must not read as a finished one. +fn pre_usdc_stats( + before: PreUsdcCounts, + after: PreUsdcCounts, + remaining: u64, + batches: u32, + one_shot: bool, +) -> Result { + let reset = before.unprice.saturating_sub(after.unprice); + if one_shot && remaining > 0 { + return Err(ChEnrichError::PreUsdcXlmRateStalled { remaining, reset }); + } + Ok(ChPassStats { + batches, + candidates_before: before.pending, + candidates_after: after.unpriced, + rows_enriched: before.pending.saturating_sub(remaining), + rows_reset: reset, + ..Default::default() + }) +} + /// `usd_rate` identity `(asset_kind, asset_code, issuer_address, contract_address)` /// and the `reference_asset` to require, `""` for any. const USDC_RATE_IDENTITY: [&str; 5] = ["credit", "USDC", USDC_ISSUER, "", ""]; @@ -2953,11 +2980,11 @@ fn pre_usdc_sql( let inner = if bucket_width_s(tbl) > 3_600 { let tiny = "toDecimal128('0.000000000001', 14)"; format!( - "SELECT {p}, {cu} AS new_cu, c.vqu AS new_vqu \ + "SELECT {p}, {cu} AS new_cu, {vqu} AS new_vqu \ FROM (SELECT {all} {parents}) AS p \ LEFT JOIN ( \ SELECT b.asset_id AS a, b.quote_asset_id AS q, b.source AS s, b.timestamp AS ts, \ - sum(h.volume_quote_usd) AS vqu, \ + sum(toDecimal256(h.volume_quote_usd, 14)) AS vqu, \ argMaxIf(toFloat64(h.close_usd) / toFloat64(h.close), h.timestamp, \ h.close_usd >= {tiny} AND h.close >= {tiny}) AS ratio \ FROM ( \ @@ -2978,6 +3005,8 @@ fn pre_usdc_sql( AND c.ts = p.timestamp", p = from("p"), cu = dec("toFloat64(p.close) * c.ratio".into()), + // Σ in Decimal256, parsed back: a Decimal(38, 14) sum wraps silently. + vqu = dec("c.vqu".into()), hours = span(0, &|e| bucket_end_expr(tbl, &e)), ) } else { @@ -5724,6 +5753,47 @@ mod tests { assert!(external_sql("prices", "price_ohlcv_1h", 3, "").contains(&want)); } + /// (remaining, one_shot, before unprice, after unprice, Ok(rows_reset) or Err(reset)). + type StatsCase = (u64, bool, u64, u64, Result); + + #[test] + fn pre_usdc_stats_reports_resets_written_and_refuses_a_partial_run() { + let counts = |unprice| PreUsdcCounts { + pending: 10, + unprice, + unpriced: 4, + }; + #[rustfmt::skip] + let cases: [StatsCase; 4] = [ + (0, true, 3, 0, Ok(3)), + (0, true, 3, 3, Ok(0)), + (2, false, 3, 1, Ok(2)), + (2, true, 5, 1, Err(4)), + ]; + for (remaining, one_shot, b, a, want) in cases { + let got = pre_usdc_stats(counts(b), counts(a), remaining, 1, one_shot); + match (got, want) { + (Ok(s), Ok(reset)) => { + assert_eq!(s.rows_reset, reset); + assert_eq!(s.rows_enriched, 10 - remaining); + assert_eq!((s.candidates_before, s.candidates_after), (10, 4)); + } + ( + Err(ChEnrichError::PreUsdcXlmRateStalled { + remaining: r, + reset, + }), + Err(want), + ) => { + assert_eq!((r, reset), (remaining, want)); + } + (got, want) => { + panic!("remaining={remaining} one_shot={one_shot}: {got:?} vs {want:?}") + } + } + } + } + #[test] fn pre_usdc_sql_renders_its_contract() { let end = format!( From be0f482a76cee22a187996b9b5b76434133c8200 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:33:26 +0200 Subject: [PATCH 098/144] test(lore-0331): pin the roll-up volume overflow A 1d bucket over three hours of 6e23 USD is written with volume 0, not the wrapped negative sum. --- .../enrichment-worker/tests/ch_enrich_it.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/enrichment-worker/tests/ch_enrich_it.rs b/packages/enrichment-worker/tests/ch_enrich_it.rs index 572d8228..94d24c97 100644 --- a/packages/enrichment-worker/tests/ch_enrich_it.rs +++ b/packages/enrichment-worker/tests/ch_enrich_it.rs @@ -6125,3 +6125,35 @@ async fn pre_usdc_xlm_rate_repair_reaches_across_a_month_edge() { .await .unwrap(); } + +/// A wide tier's USD volume is a Σ of its hours; past Decimal(38, 14) it is +/// written as 0, never as the sum wrapped negative (three hours of 6e23 do). +#[tokio::test] +#[ignore = "requires ClickHouse — run via tools/scripts/ignored-tests.sh (CI runs it)"] +async fn pre_usdc_xlm_rate_rollup_volume_never_wraps() { + let db = "test_ch_enrich_pre_usdc_rollup_overflow"; + #[rustfmt::skip] + let day: Leg = ("1d FOO/XLM over three huge hours", WIDE[1], D0331, FOO, XLM, 10.0, 9.0, 90.0, 2.5, 0.0); + let client = setup_0331(db, RATES_0331, &[day]).await; + for ts in [h(10), h(11), h(12)] { + client + .query(&format!( + "INSERT INTO {db}.{H1} \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + VALUES ({ts}, {FOO}, {XLM}, 'sdex', 10, 10, 10, 10, 3, 30, \ + '600000000000000000000000', 2.5, 10, 1, 1)" + )) + .execute() + .await + .unwrap(); + } + let pass = ChEnrichmentPass::new(cfg_0331(db, WIDE[1])); + assert_eq!(pass.run().await.unwrap().rows_enriched, 1); + assert_legs(&client, db, &[day], |_| 1.0).await; + client + .query(&format!("DROP DATABASE {db}")) + .execute() + .await + .unwrap(); +} From c9276cf7d70717e0cc1c4002881a136184301fb7 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:33:54 +0200 Subject: [PATCH 099/144] docs(lore-0331): record the third-party rate decisions Decisions 5-11 (2026-10-14/15) replace par and the on-chain reference with the imported hourly XLM/USD rate, convert dollar-token quotes through their own XLM market, keep the existing `external` label and widen the scope to native XLM's series in the API. Open questions, plan and acceptance criteria follow, with the pre-deploy measurement. --- .../README.md | 123 +++++++++++++----- 1 file changed, 92 insertions(+), 31 deletions(-) diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md index 773d5a23..15a109e7 100644 --- a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -1,6 +1,6 @@ --- id: "0331" -title: "No USD price before 2021-02 — price that history by counting every dollar token at par ($1)" +title: "No USD price before 2021-02 — price that history from an imported hourly XLM/USD rate" type: FEATURE status: active assignee: akot @@ -13,6 +13,17 @@ links: - "../../../../packages/prices-clickhouse/schema/current.sql" - "../../../../docs/scf/milestone-3-evidence.md" history: + - date: "2026-10-15" + status: active + who: akot + note: > + Decisions amended (5–11): a third-party hourly XLM/USD series replaces + par and the on-chain reference; dollar-token quotes are converted + through their own XLM market; the API builds native XLM's pre-2021 USD + series from the rate at request time. Implemented in PR #403 (PR #401 + closed). Measured read-only on prod before deploy: native XLM 1d + priced on 1081/1081 days, close vs Binance median 0.43 %; 794,672 of + 814,269 in-scope 1h rows priced, 0 set back to 0. - date: "2026-10-14" status: active who: akot @@ -45,16 +56,18 @@ history: as $1, on-chain, with the safeguards below. Pre-run report goes to notes/. --- -# No USD price before 2021-02 — price that history by counting every dollar token at par +# No USD price before 2021-02 — price that history from an imported hourly XLM/USD rate ## Summary Every USD value we publish is anchored on canonical USDC (`GA5Z…`), priced with a measured USDC/USD rate. USDC trades on Stellar from 2021-02-01, so nothing earlier has a USD reference: candles from 2015-11 to 2021-01 carry volumes and no USD price, on -every tier. This task prices that period by counting dollar tokens at $1 before -2021-02, both where they are the quote and as the source of the XLM/USD rate that -everything quoted in XLM is converted through. +every tier. This task prices that period from an imported, measured hourly XLM/USD +rate: candles quoted in XLM directly, candles quoted in a dollar token through that +token's own XLM market, and native XLM's own USD series in the API. The first plan, +dollar tokens at $1 (Decision 2026-10-13, PR #401), was replaced on 2026-10-14 +(Decisions 5–11). ## Context @@ -73,7 +86,47 @@ everything quoted in XLM is converted through. 2018–2025; asked for priced 2018 and 2022 candles. 2022 is repaired; 2018 is this task. -## Decision (2026-10-13) +## Decision (amended 2026-10-14 and 2026-10-15) + +Decided by Adam. Supersedes Decisions 1, 2 and 4 and Safeguards 1, 2 and 4 below; +Decision 3 is widened by Decision 9. + +5. **A third-party hourly XLM/USD series replaces par and the on-chain reference.** + 25,551 hourly rows, 2018-02-16 23:00 → 2021-01-31 23:00 UTC, in `prices.usd_rate` + as `('native','XLM','','')`, staged `external-candidate` with + `reference_asset = 'third-party'`. Code reads only `method = 'external'` rows of + that source, so nothing changes until the rows are promoted. The source is not + named in project documents. It is not Bitstamp, so Safeguard 5 stays an + independent check. PR #401 (par) is closed. +6. **Dollar-token quotes are measured, not assumed.** A candle quoted in a dollar + token (`USD`, `USDT`, `USDC` of any issuer, classic, except canonical USDC) is + converted through that token's own XLM market within `--pivot-window-s` (24 h), + then through the rate. A token that lost its peg is priced at what it traded for, + so the write side has no exclusion list. No market in the window: the row stays + unpriced. +7. **Wide tiers roll up from 1h.** 4h, 1d, 1w and 1M take the sum of their priced + hours' USD volume and the last priced hour's `close_usd / close`, as `preroll.sql` + does. `price_ohlcv_1h` is repaired first. +8. **Label: the existing `external`.** No new word, so Safeguard 4 is dropped. On a + quote leg the API labels every priced non-USDC row before 2021-02-01 `external`. + That holds while `coarse-repair --pre-usdc-xlm-rate` is the only writer of such + rows; on 2026-10-15 production had none from any other source. +9. **Scope: history plus native XLM's series in the API.** In USD mode + `GET /assets/native/ohlcv` builds pre-2021 buckets from the promoted rate at + request time: `open`/`close` are the first and last hourly rate, `high`/`low` the + extremes of the hourly averages, volume is XLM traded against dollar tokens minus + the four off-peg ones (`OFF_PEG_DOLLAR_TOKENS`), and `method = external`, + `derived = true`, `source = third-party`. It goes live when the rate is promoted, + independently of the `coarse-repair` run. This changes the `/ohlcv` contract and + `openapi.json`. Nothing from 2021-02-01 on changes. +10. **XLM's own pre-2021 candles stay priced** through their XLM/USDT legs, as the + M3 reviewer asked. +11. **The 1m/15m native series reads its partitions outside the table key**, because + a bucket appears wherever XLM traded and most XLM markets have XLM as the quote. + Accepted in review on 2026-10-15: at most 1.15 M rows and 0.53 s per request on + production, and pre-2021 partitions no longer grow. + +## Decision (2026-10-13, superseded in part) 1. Before 2021-02-01, **every dollar token** counts as $1. Not a list of issuers. 2. **On-chain only**, chosen over an external XLM/USD series. The trade-off is accepted: @@ -92,42 +145,50 @@ everything quoted in XLM is converted through. Anyone can issue a token called `USD`, so "every dollar token" needs these: -1. The XLM/USD reference is **volume-weighted** across all dollar-token markets in the - bucket, as the USDC reference already is. -2. A market whose implied XLM/USD deviates from the bucket's median by more than a - threshold is left out. Threshold to be set from the report. +1. ~~The XLM/USD reference is volume-weighted across all dollar-token markets in the + bucket.~~ Superseded by Decision 5: the rate is imported. +2. ~~A market whose implied XLM/USD deviates from the bucket's median by more than a + threshold is left out.~~ Superseded by Decision 5. 3. **Pre-run report** (`notes/`): every dollar token traded before 2021-02, with - volume, implied XLM/USD and deviation. -4. A distinct `method` on these candles, so a client can tell an assumed par from a - measured rate. + volume, implied XLM/USD and deviation. Done. +4. ~~A distinct `method` on these candles.~~ Dropped by Decision 8. 5. A spot-check against Bitstamp XLM/USD where Bitstamp has data. ## Open questions -- Which codes make a dollar token: `USD`, `USDT`, `USDC`, others? -- `method` label: the existing `assumed-par`, or a new one? -- ~~The outlier threshold~~ and ~~per-token exclusion~~: decided, Decision 4, from the - [pre-run report](notes/R-pre-run-report-dollar-tokens-before-2021.md). -- Coverage: before 2018 there is no usable reference (one token, $31 in two years). - Priced history starts in August 2018 (AnchorUSD) and is daily from 2019. -- Coverage ceiling: `canonical.rs` ranks quotes USDC > USDT > XLM, so XLM's own daily - series before 2021 is the XLM/USDT market. It has candles on 21 days of 2018, 63 of - 2019 and 257 of 2020 (API, 2026-10-13). XLM's own series cannot exceed that. +- ~~Which codes make a dollar token~~: `USD`, `USDT` and `USDC`, classic, any issuer + (`DOLLAR_TOKEN_PRED` in `prices-clickhouse`). +- ~~`method` label~~: the existing `external`, Decision 8. +- ~~The outlier threshold~~ and ~~per-token exclusion~~: no longer apply (Decision 5). + The four off-peg tokens are still left out of native XLM's volume (Decision 9). +- ~~Coverage~~: priced history starts where the rate does, 2018-02-16 23:00. Before + that nothing is priced (task 0333). +- ~~Coverage ceiling~~: native XLM's series is built from the rate (Decision 9), so it + is no longer bound by the 21, 63 and 257 days of its XLM/USDT candles. +- The tolerance for the Bitstamp spot-check (Safeguard 5), to agree with Marek. ## Plan 1. Pre-run report → `notes/` (done 2026-10-13, data in `data/`). -2. Decide codes, threshold and label; record them here. -3. Implement the par rule for buckets before 2021-02 in the repricing pass, with tests. -4. Dry run, then run over 2015-11 → 2021-01 on all tiers. After 0286 §7b (~2026-10-16): - one pass at a time on the cluster shared with the block explorer. +2. Decide codes, source and label; record them here (done, Decisions 5–11). +3. Implement `coarse-repair --pre-usdc-xlm-rate` and native XLM's series, with tests + (PR #403). +4. Deploy per PR #403: API, promotion of the rate, then after the 0286 drain and A6 a + dry run and a real run over 2018-02 → 2021-01, tier by tier (1h first). One pass at + a time on the cluster shared with the block explorer. 5. Spot-check against Bitstamp. Then the M3 reply and evidence. ## Acceptance Criteria -- [ ] Pre-run report in `notes/`, read by Marek or Adam -- [ ] Codes, threshold and label decided and recorded here -- [ ] XLM daily candles before 2021-02 priced wherever a dollar-token market traded - within the window; the remaining days explained -- [ ] Spot-check against Bitstamp XLM/USD within an agreed tolerance +- [x] Pre-run report in `notes/`, read by Marek or Adam +- [x] Codes, source and label decided and recorded here (Decisions 5–11) +- [ ] Native XLM's daily USD candles priced on every day the rate covers + (2018-02-16 → 2021-01-31). Candles quoted in XLM priced wherever the rate + covers them; candles quoted in a dollar token also need the token's XLM market + within 24 h. The remaining rows explained. Measured on prod before deploy + (2026-10-15): 1081 of 1081 days; 794,672 of 814,269 in-scope 1h rows, the rest + before the rate (task 0333) or 358 dollar-quoted rows with no token market +- [ ] Spot-check against Bitstamp XLM/USD within an agreed tolerance (measured before + deploy: median 0.30 %, p95 1.76 %, without 2020-05-28 → 07-15 where Bitstamp + printed a flat stale price) - [ ] Rows from 2021-02-01 on unchanged From b4cf4a06e9823beb563e193486567c5a8986b5a5 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:23:24 +0200 Subject: [PATCH 100/144] chore(lore-0333): activate task --- .../0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) rename lore/1-tasks/{backlog => active}/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md (94%) diff --git a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md b/lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md similarity index 94% rename from lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md rename to lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md index 92ac4e15..4a5cc97e 100644 --- a/lore/1-tasks/backlog/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md +++ b/lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md @@ -2,13 +2,19 @@ id: "0333" title: "XLM/USD for 2018-01-01 → 2018-02-16" type: FEATURE -status: backlog +status: active related_adr: [] related_tasks: ["0331"] tags: [layer-backend, priority-medium, effort-small, enrichment, clickhouse, usd-reference] links: - "../active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md" history: + - date: "2026-10-15" + status: active + who: akot + note: > + Activated. First step: fetch the OHLCVT archive and check whether it + carries enough to build the 0331 rate definition (hourly VWAP). - date: "2026-10-15" status: backlog who: akot From 9cf6523052ee0dc117293299befbbe3a136d6760 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:18:37 +0200 Subject: [PATCH 101/144] docs(lore-0286): record the phase-3 after-checks The 1m drain finished 2026-10-15 14:27 UTC. Measured read-only on 2026-10-16: 1,802 reachable 1m zeros, all USDT pivot-window residue; post_run_0228_it green; no USDT peg; no carried product on five tiers; the seven dust days within 5 % of Bitstamp. Two round-price buckets on 15m/1h await a decision; the frontier gauge is a recorded deviation. --- ...uilt-from-dust-fills-in-the-wrong-order.md | 123 +++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md b/lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md index a7e268df..1185ad71 100644 --- a/lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md +++ b/lore/1-tasks/active/0286_BUG_candles-are-built-from-dust-fills-in-the-wrong-order.md @@ -197,6 +197,16 @@ history: finished" need two readings, because the frontier gauge skips months still marked exhausted from before the re-ingest. The criterion above now matches. + - date: "2026-10-16" + status: active + who: akot + note: > + 1m drain on fishuser-hero finished 2026-10-15 14:27 UTC (202608). + After-checks run read-only: 1 802 reachable 1m zeros, all USDT + pivot-window residue; post_run_0228_it 2/2; no USDT peg (0212); + no carried product on 5 tiers (0207); seven dust days within 5 % + of Bitstamp; 2 round-price XLM/USDC buckets on 15m/1h, not dust, + awaiting decision. The frontier gauge (49) is a recorded deviation. --- # Candles are built from dust fills in the wrong order @@ -911,6 +921,101 @@ stays DISABLED until M3 completes (decided 2026-09-14). **Rough end:** history rebuilt by mid-October, provided 0300 lands before the loop reaches 202405. +### Phase 3 — after-checks (2026-10-16) + +Runbook §7b-2 (a)(b), §7c, §7e, §7e-2, §7f, §8.3 and §8.4, run read-only on +prod on 2026-10-16 ~08:00–09:30 UTC. Asset ids by identity: XLM +`5209214538714742248`, USDC `1368471977071604712`, USDT +`11430142929638344312`. + +**The 1m drain on fishuser-hero is finished.** Its last batch ran on +2026-10-15 at 14:27:10 UTC, on 202608. That month is `END`, two months +back; 202609 on belongs to the live sweep. `system.query_log` shows +~415 M rows written from 10-15 02:00 with 0 exceptions. Since 14:27 only +the live ingest and the hourly Lambda write to `price_ohlcv_1m`. + +**§7b-2 (a)/(b) cannot pass as written, so the drain was measured +directly.** + +- `EnrichmentFrontierMonthsPending` reads 49. It went 52 → 49 over 48 h, + not one month per hour. +- 45 months, 202212–202608, have no row in `prices.enrichment_frontier`. + 202103 and 202209–202211 are `pending`. +- The Lambda takes 202103 first every hour and leaves it `pending`, with + 81 112 zeros, none of them reachable (below). Why is not investigated. + +In its place, runbook ③'s `coarse_zeros` columns were computed on +`price_ohlcv_1m FINAL`, one partition per query, over 201511–202608 +(130 months): + +| zero_usd | no_volume | no_price | other_quote | before_reference | reachable_left | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 245 112 743 | 0 | 27 658 212 | 212 555 564 | 4 897 165 | **1 802** | + +Every one of the 1 802 is USDT-quoted. Each sits more than 24 h after the +last USDT/USDC candle, which is the pivot-window residue of §7b-2 ⑤: + +| Month | Rows | When | USDT/USDC gap | +| --- | ---: | --- | --- | +| 202102 | 1 787 | 02-08 → 02-14 | silent from 02-09 15:35 to 02-12 01:09 | +| 202304 / 202305 / 202306 | 6 / 1 / 8 | one day each | 24 h 3 min to ~25 h | + +**§7c — `post_run_0228_it`: green, 2 of 2.** Run with +`POST_RUN_0228_VERSION_BEFORE_1W=45340987003` and `_1M=1407435801128`, +both captured on fishuser-hero on 2026-10-13 before `finish`. + +**§7e — USDT peg: none.** The runbook's `peg_written` (rate ≥ 0.9) cannot +tell the peg from the pivot before the depeg: USDT/USDC traded at +0.98–1.00 from 2021-02 to 2022-05. Peg was therefore counted as +`close_usd = close` exactly, per year: + +| Tier | Years | `peg_exact` | Rate < 0.5 | +| --- | --- | ---: | --- | +| 1m | 2023–2026 | 0 | every priced row (2023: 77 000 of 77 000) | +| 1m | 2021–2022 | 6 872 | — | +| 1h | 2023–2026 | 0 | every priced row (2023: 30 685 of 30 685) | +| 1h | 2021–2022 | 988 | — | + +All 988 on 1h fall in 2021-03 … 2022-05. In 960 of them, USDT/USDC's 1h +close in the same hour is exactly 1.0, so they are measured, not pegged. +Priced USDT-quoted rows start at 2021-02-07 19:42, the first USDT/USDC +candle. + +**§7e-2 — carried product: no rows** on 1h, 4h, 1d, 1w or 1M, XLM and USDT +legs. Before the re-ingest there were 1 757. + +**§7f — nothing to start.** All six `mv_reconcile_*` read `Scheduled` and +refreshed at 08:00 UTC. + +**§8.3 — the seven dust days are within 5 % of Bitstamp XLM/USD (1d +close).** + +| Day | Source | Close | Bitstamp | Deviation | +| --- | --- | ---: | ---: | ---: | +| 2021-09-05 | sdex | 0.39600 | 0.39803 | −0.51 % | +| 2021-12-28 | sdex | 0.27520 | 0.27500 | +0.07 % | +| 2022-01-11 | sdex | 0.26252 | 0.26314 | −0.23 % | +| 2023-03-11 | sdex | 0.08220 | 0.07936 | +3.58 % | +| 2023-03-15 | sdex | 0.08382 | 0.08399 | −0.21 % | +| 2026-02-09 | four sources | 0.15956–0.15987 | 0.1595 | +0.03 to +0.23 % | +| 2026-04-02 | five sources | 0.16254–0.16331 | 0.16295 | −0.25 to +0.22 % | + +**§8.4 — two buckets, not zero; neither is quantisation.** The check +applies 0278's definition to XLM/USDC: a close that is an exact `n/d` +with `d ≤ 200`, more than 5 % off the bucket VWAP. 4h, 1d and 1w hold 0; +15m and 1h hold one each: + +- **1h 2023-07-13 17:00, sdex: close 0.125 (1/8).** VWAP 0.1188. The bucket + holds 4 544 price-forming fills and 4.4 M XLM, and Bitstamp's range that + hour was 0.1137–0.1425. It is a real price in a fast hour that happens to + be round. +- **15m 2026-03-09 10:00, aquarius: close 0.128 (16/125).** VWAP 0.1499. The + bucket holds 11 fills and 48 k XLM. Bitstamp was 0.1496–0.1501, so this + is an off-market pool price, 14.6 % below, not stroop dust. Whether + anything was priced from it is not checked. + +Whether these two count as explained is Adam's decision. + ## Out of scope - D9, the cross-source rule on the read path (median instead of largest @@ -1019,6 +1124,9 @@ Phase 3: 85 699 / 30 064 / 10 938 on 15m / 1h / 4h / 1d / 1w) is zero after the re-ingest and pre-roll. [[0266]] closed into this criterion (2026-09-24, [[0314]]) — it is that task's only remaining check. + 2026-10-16: the seven days are within 5 % (worst +3.58 %). Quantised + buckets: 1 on 15m, 1 on 1h, 0 on 4h/1d/1w. Neither is dust; awaiting + Adam's decision ("Phase 3 — after-checks"). - [ ] The whole history re-enriched (`close_usd > 0` wherever a reference exists) and `post_run_0228_it` green on the repaired reference; 0228's reset-mode campaign recorded as superseded, not run. @@ -1032,6 +1140,11 @@ Phase 3: non-`exhausted` or swept before `T7` (the last §4e re-ingest's end). The gauge alone reads 0 for a re-ingested month still marked `exhausted`. + 2026-10-16: the drain finished 10-15 14:27 UTC, after the coarse + re-price, which the runbook allows. Neither reading passes: the + gauge is 49 and 45 months have no frontier row. A direct count on + 1m leaves 1 802 reachable zeros, all explained ("Phase 3 — + after-checks"). Recorded as a known deviation. - [ ] Dry run recorded for all six coarse tables. Every table lists months up to END (two months back), and none reports 0 months. - [ ] `coarse-repair` run in plain mode (no `--reset-*`) on `price_ohlcv_15m`, @@ -1042,18 +1155,22 @@ Phase 3: `reachable_left` ~0 (any remainder explained through the drill-down); `before_reference`, `no_price` and `no_volume` unchanged; `other_quote` not higher than the baseline. - - [ ] `post_run_0228_it` green after this step. -- [ ] No USDT-quoted `price_ohlcv_1m` row carries the $1 peg after the + - [x] `post_run_0228_it` green after this step. ✅ 2026-10-16, 2 of 2. +- [x] No USDT-quoted `price_ohlcv_1m` row carries the $1 peg after the re-enrichment ([[0212]]'s query: `peg_written = 0`, `pivot_written > 0`, measured on 1m and on one coarse tier) — the re-ingest replaces the 1.56 M rows 0172/0182 never reached, so 0212 closes here (re-ingest runbook §7e). 0212 is archived pending this check ([[0314]]). -- [ ] No coarse row whose `close_usd` contradicts its own `close` after §7a + ✅ 2026-10-16: `close_usd = close` on 0 rows from 2023 on, on 1m + and 1h. The 988 at ×1.0 on 1h (2021-03 … 2022-05) are measured + USDT/USDC closes of exactly 1.0 before the depeg. +- [x] No coarse row whose `close_usd` contradicts its own `close` after §7a and §7b-2 ([[0207]]'s query, re-ingest runbook §7e-2: no row on 1h, 4h, 1d, 1w, 1M more than 10x off its quote leg's median rate in the bucket, XLM and USDT legs). Before: 1 757 rows, all XLM-quoted, 2022-01 … 2022-04 — the pre-0286 carried product, which the rate form cannot write. 0207 is archived pending this check. + ✅ 2026-10-16: no rows on any of the five tiers. ## Implementation Notes From b2065c18ce016d0861af65c08ed288d415813942 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:18:41 +0200 Subject: [PATCH 102/144] docs(lore-0331): record the history run Backup of 201802-202101 on six tiers, then coarse-repair --pre-usdc-xlm-rate on seven tiers: every tier wrote what its dry run listed, reset 0, nothing at or after 202102. What stays unpriced is explained; the Bitstamp tolerance is still open. --- .../README.md | 93 ++++++++++++++++++- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md index 15a109e7..68696858 100644 --- a/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md +++ b/lore/1-tasks/active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md @@ -13,6 +13,17 @@ links: - "../../../../packages/prices-clickhouse/schema/current.sql" - "../../../../docs/scf/milestone-3-evidence.md" history: + - date: "2026-10-16" + status: active + who: akot + note: > + History run done (Plan step 4). Backup repair_0331_bak_* of + 201802–202101, source = backup on six tiers. coarse-repair + --pre-usdc-xlm-rate over seven tiers wrote exactly what each dry + run listed, with reset 0 everywhere (1h 794,672; 1m 4,971,301; 1d + 114,117). No insert into 202102 or later. 97 % of what stays + unpriced is quoted in other assets with no two-hop path. The + Bitstamp tolerance is still open. - date: "2026-10-15" status: active who: akot @@ -178,17 +189,93 @@ Anyone can issue a token called `USD`, so "every dollar token" needs these: a time on the cluster shared with the block explorer. 5. Spot-check against Bitstamp. Then the M3 reply and evidence. +Steps 1–4 are done. The API went out and the rate was promoted on 2026-10-15 +(version 1791990002, 25,551 `external` rows). The history run was 2026-10-16. + +## Rollout of the history run (2026-10-16) + +**Order and preconditions.** The 0286 1m drain finished on 2026-10-15 at +14:27 UTC, and 0286's after-checks (A6) passed on 2026-10-16, both recorded +on 0286. + +**Backup.** Partitions 201802–202101 of 15m, 1h, 4h, 1d, 1w and 1M were +copied into `prices.repair_0331_bak_` as `dev_shared`, with +`ATTACH PARTITION … FROM`. Each table holds 36 partitions, with source +equal to backup on every tier: 2,302,390 / 1,043,707 / 454,131 / 143,771 / +36,674 / 13,563 rows. Rollback is +`REPLACE PARTITION

FROM prices.repair_0331_bak_`. + +**Run.** `coarse-repair --pre-usdc-xlm-rate --start-month 201802 +--end-month 202101`, from Adam's laptop as `prices_writer`, 08:34–~09:05 +UTC. Each tier had a dry run first, and a gate stopped the chain on any +`reset` > 0 or any month ≥ 202102. Tier order: 1h → 15m → 1m (`--skip-snapshot`) +→ 4h → 1d → 1w → 1M (`--skip-snapshot --snapshots-verified`). + +| Tier | Written | Left unpriced | Reset | +| --- | ---: | ---: | ---: | +| 1h | 794,672 | 3,351 | 0 | +| 15m | 1,655,500 | 8,545 | 0 | +| 1m | 4,971,301 | 25,553 | 0 | +| 4h | 357,495 | 1,284 | 0 | +| 1d | 114,117 | 383 | 0 | +| 1w | 27,770 | 55 | 0 | +| 1M | 9,303 | 16 | 0 | + +- Every tier wrote exactly what its dry run listed. +- On 1h, 3,005 of the unpriced rows are 201802 hours before the rate starts + (task 0333). The other 346 are spread over 2018–2021. +- `system.part_log` shows new parts only in partitions 201802–202101, with + row counts equal to the table above, and none in 202102–202508. + +**What is still unpriced, and why.** Measured on 1d rows with volume and a +price, 2018-01-01 → 2021-01-31, after the run. + +| Year | Rows priced | +| --- | ---: | +| 2018 | 74.8 % | +| 2019 | 77.7 % | +| 2020 | 81.9 % | +| 2021-01 | 81.8 % | + +Of the 30,054 rows still unpriced: + +| Cause | Rows | +| --- | ---: | +| Quoted in an asset other than XLM or a dollar token (330 quotes, 275 bases) | 29,328 | +| XLM-quoted before the rate (task 0333) | 579 | +| Dollar-token quote with no XLM market within 24 h | 147 | + +- **No two-hop pricing.** The largest group needs a second step, quote → + XLM → USD, which the pipeline does not have on any date. The same pairs + are unpriced after 2021 too. +- **The top quotes have XLM markets.** Those quotes are XCN, HKDT, CMS, + NGNT, LTC, ETH, BTC, XRP and CNY. Most of them have an XLM market on + 450–1,070 of the 1,080 days. + +**XLM-quoted rows, implied XLM/USD.** Median per year 0.229 / 0.083 / 0.073 +(2018 / 2019 / 2020). The 2020 minimum is 0.031, in March 2020. + +**Bitstamp, after the run.** The OHLC endpoint returned data for one of +five dates tried; the other four returned 404. That date was 2020-11-24, +where XLM/USDT 1d `close_usd` is 0.19589 and Bitstamp's close is 0.19589. +Agreement over the whole range was measured on the rate before deploy (AC +below). + ## Acceptance Criteria - [x] Pre-run report in `notes/`, read by Marek or Adam - [x] Codes, source and label decided and recorded here (Decisions 5–11) -- [ ] Native XLM's daily USD candles priced on every day the rate covers +- [x] Native XLM's daily USD candles priced on every day the rate covers (2018-02-16 → 2021-01-31). Candles quoted in XLM priced wherever the rate covers them; candles quoted in a dollar token also need the token's XLM market within 24 h. The remaining rows explained. Measured on prod before deploy (2026-10-15): 1081 of 1081 days; 794,672 of 814,269 in-scope 1h rows, the rest before the rate (task 0333) or 358 dollar-quoted rows with no token market + ✅ 2026-10-16. The native series has been live since 2026-10-15 on 1081 of + 1081 days. The history run priced 794,672 1h rows, as measured. The rows + left are explained in "Rollout of the history run". - [ ] Spot-check against Bitstamp XLM/USD within an agreed tolerance (measured before deploy: median 0.30 %, p95 1.76 %, without 2020-05-28 → 07-15 where Bitstamp - printed a flat stale price) -- [ ] Rows from 2021-02-01 on unchanged + printed a flat stale price). The tolerance is still to agree with Marek. +- [x] Rows from 2021-02-01 on unchanged. ✅ 2026-10-16: the run listed no month + ≥ 202102, and `system.part_log` shows no insert into 202102–202508. From 00027b8c2ca18495b131f9bcef8088b8a9bc511e Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:48:20 +0200 Subject: [PATCH 103/144] docs(lore-0333): source the rate as a cross rate The third-party exchange had no XLM/USD market from 2017-08-22 to 2018-02-16, so the rate is its XLM/BTC x BTC/USD hourly VWAP. Record the measurements, the 2026-10-16 decisions and assign the task. --- ...RE_xlm-usd-for-2018-01-01-to-2018-02-16.md | 97 ++++++++++++------- 1 file changed, 63 insertions(+), 34 deletions(-) diff --git a/lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md b/lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md index 4a5cc97e..af1ae343 100644 --- a/lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md +++ b/lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md @@ -3,12 +3,23 @@ id: "0333" title: "XLM/USD for 2018-01-01 → 2018-02-16" type: FEATURE status: active +assignee: akot related_adr: [] related_tasks: ["0331"] tags: [layer-backend, priority-medium, effort-small, enrichment, clickhouse, usd-reference] links: - "../active/0331_FEATURE_price-pre-2021-history-with-dollar-tokens-at-par/README.md" history: + - date: "2026-10-16" + status: active + who: akot + note: > + Source changed. The exchange had no XLM/USD market from 2017-08-22 to + 2018-02-16 (consecutive trade ids 225906 → 225907), so the OHLCVT files + are empty for the range. The rate is built as the exchange's XLM/BTC × + BTC/USD hourly VWAP from its public trades API. The 2018-01-11 → 01-13 + outage stays without a rate. The 0331 row at 2018-02-16 23:00 is + replaced with the cross rate. - date: "2026-10-15" status: active who: akot @@ -46,56 +57,74 @@ history: [[0331]] prices pre-2021 candles from an hourly XLM/USD series loaded into `prices.usd_rate`. That series starts at 2018-02-16 23:00 UTC. The priced history should start at the beginning of 2018, so this task fills the 47 days -before it, from the same exchange, in the same shape. +before it from the same exchange, in the same shape. The exchange had no +XLM/USD market in that period, so the rate is its XLM/BTC × BTC/USD. ## Context -- **Why the 0331 series stops at 2018-02-16.** The exchange's trades API - cannot reach trades before 2018-02-16 23:03 when paged. A time-based `since` - anywhere from 2017-12 to 2018-02-16 lands on that first reachable trade, and - paging from the start jumps to it after the first 1,000 trades. Probed - 2026-10-14. -- **The market did trade then.** Its XLM/USD market opened on 2017-01-17 and - had about 1,000 trades a day around 2018-01-01. +- **Why the 0331 series stops at 2018-02-16.** The exchange delisted XLM/USD + and XLM/EUR on 2017-08-22 and relisted them on 2018-02-16. Its trade ids run + without a gap across that date: XLM/USD 225906 is at 2017-08-22 06:05 and + 225907 at 2018-02-16 23:43; XLM/EUR does the same. There was no trade to + fetch, so the OHLCVT files would be empty for the range too. Measured + 2026-10-16. Supersedes the 2026-10-14 reading that the trades API could not + reach older trades. +- **XLM/BTC and BTC/USD traded throughout** on the same exchange and page from + its public trades API: 472,990 XLM/BTC and ~2.3 M BTC/USD trades for + 2017-12-31 → 2018-02-28. XLM/BTC has a trade in 1,074 of the range's 1,127 + hours, median 296 trades an hour. +- **The exchange was down 2018-01-11 07:00 → 01-13 14:00** (53 hours without + an XLM/BTC trade, in four runs). A second exchange's XLM/BTC × BTC/USDT + covers them, but mixes sources and takes USDT at $1, so it is not used. - **What is left to price** (prod, 2026-10-15): 6,432 1h rows with volume quoted in XLM or a dollar token, worth 119.2 M XLM of quote volume, all at `close_usd = 0`. - **Out of scope:** everything before 2018-01-01. There are 12,819 such 1h rows (2015-11 → 2017-12), and they stay unpriced. -- **A cross-check is available.** A second exchange's public hourly XLM/BTC × - BTC/USDT tracks the 0331 series at a median deviation of 0.43 % per day - (0.66 % for 2018-02-17 → 04-30). This was measured on 2026-10-15 for the - synthetic XLM series lab. -## Decision (2026-10-15, Adam) +## Measurements (2026-10-16, hourly VWAP of each leg) + +| Cross rate compared with | Hourly median / p95 / max | Daily median / max | +|---|---|---| +| The exchange's direct XLM/USD (the 0331 definition), 2018-02-17 12:00 → 02-28 | 0.44 % / 1.87 % / 6.40 % | 0.21 % / 0.64 % | +| Second exchange's XLM/BTC × the exchange's BTC/USD, in range | 0.41 % / 4.19 % / 21.0 % | 0.33 % / 5.87 % | +| Second exchange's XLM/BTC × BTC/USDT (lab cross-check), in range | 0.98 % / 4.67 % / 20.3 % | 0.95 % / 8.06 % | + +The direct pair's first hours after relisting were off the market: at +2018-02-16 23:00 it printed 0.4056 against 0.4505 for the cross rate (11.1 %), +and 3–4 % for the next six hours. The 23:00 hour is already in the 0331 +series as `external`. + +## Decision (amended 2026-10-16, Adam) + +Supersedes the 2026-10-15 source decision. - **Range:** 2018-01-01 00:00 → 2018-02-16 22:00 UTC. -- **Source:** the exchange's downloadable historical OHLCVT files. They cover - each pair from the start of its market and have to be fetched by hand. Same - source as the 0331 series. +- **Source:** the same exchange, as a cross rate: hourly VWAP of XLM/BTC × + hourly VWAP of BTC/USD, both from its public trades API, stamped at the hour + start. An hour gets a rate only when both legs traded in it. +- **The outage hours stay without a rate.** The pass carries a rate for up to + 24 h, so only the candles more than 24 h past the last rate stay unpriced. + XLM's daily candles are priced on every day of the range. +- **The 0331 row at 2018-02-16 23:00 is replaced** with the cross rate. +- **Tag:** `reference_asset = 'third-party'`, as for 0331. ## Implementation Plan -1. Fetch the OHLCVT archive and record its provenance (file, sha256, date) - and its terms of use. -2. Build the hourly XLM/USD series for the range from the 60-minute XLM/USD - file, using the same rate definition as the 0331 rows (hourly VWAP stamped - at the hour start). -3. Check the seam against the 0331 series where both have a value (from - 2018-02-16 23:00), and the range against the XLM/BTC × BTC/USDT - cross-check. -4. Insert in the same shape as the 0331 rows: `('native','XLM','','')`, - `reference_asset = 'third-party'`, staged `external-candidate` first, then - promote. -5. Re-run 0331's `coarse-repair --pre-usdc-xlm-rate` for 201801–201802, first +1. Build the hourly series for the range plus the 2018-02-16 23:00 hour. +2. Insert as `('native','XLM','','')`, staged `external-candidate`, then + promote; the 23:00 row by a newer version. +3. Re-run 0331's `coarse-repair --pre-usdc-xlm-rate` for 201801–201802, first 1h, then the wider tiers. +4. Count the candles left unpriced around the outage. ## Acceptance Criteria -- [ ] Source recorded with provenance and terms of use. -- [ ] Hourly XLM/USD rows for every hour of 2018-01-01 → 2018-02-16 that has a - trade on the exchange. -- [ ] Agreement recorded with the 0331 series on the overlap and with the +- [x] Source recorded as third-party. +- [ ] Hourly XLM/USD rows for every hour of 2018-01-01 → 2018-02-16 in which + both legs traded on the exchange. +- [x] Agreement recorded with the direct pair on the overlap and with the cross-check (median and max deviation). -- [ ] XLM daily USD prices from 2018-01-01 on, wherever the exchange traded - within the window; any remaining days counted and explained. +- [ ] The 2018-02-16 23:00 row replaced with the cross rate. +- [ ] XLM daily USD prices from 2018-01-01 on; candles left unpriced by the + 2018-01-11 → 01-13 outage counted. From a95b4b51ceb9a262c89a3e4579915e92f92a8165 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:14:49 +0200 Subject: [PATCH 104/144] docs(lore-0334): open task for the Blend v2.1 Comet pool --- ...lend-v2-1-comet-pool-trades-are-dropped.md | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 lore/1-tasks/active/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md diff --git a/lore/1-tasks/active/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md b/lore/1-tasks/active/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md new file mode 100644 index 00000000..f4d2326b --- /dev/null +++ b/lore/1-tasks/active/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md @@ -0,0 +1,90 @@ +--- +id: "0334" +title: "Blend v2.1's Comet BLND/USDC pool is missing from pool_registry — every swap since 2026-10-07 is dropped and both pool alarms fire" +type: BUG +status: active +assignee: akot +related_adr: [] +related_tasks: ["0300", "0291", "0100", "0323", "0329"] +tags: [layer-backend, priority-high, effort-small, ingest, soroban, comet, observability] +links: + - "../../../packages/prices-ingest-core/src/static_pools.rs" + - "../../../packages/coverage-sweep-probe/allowlist.toml" + - "../../../docs/runbooks/seed-pool-registry.md" + - "https://github.com/blend-capital/docs-v2/blob/main/mainnet-deployments.md" + - "https://github.com/CometDEX/comet-contracts-v1/releases/tag/v1.1.0_contracts_pkg1.0.0_cli25.1.0" +history: + - date: "2026-10-16" + status: active + who: akot + note: "Task created from the unregistered-pool and coverage-sweep alarms; pool verified as Blend v2.1's" +--- + +# Blend v2.1's Comet BLND/USDC pool is missing from pool_registry + +## Summary + +Blend v2.1 moved its backstop to a new Comet BLND/USDC pool, +`CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE`. Comet pools have no +factory event, so only `STATIC_POOLS` registers them, and it holds only the old +pool `CAS3FL6T…` ([[0300]]). The live processor drops every swap of the new pool. +It is in `prices-production-ledger-processor-unregistered-pool` (flapping since +2026-10-07, ~12 transitions a day) and makes 393 of the 404 events behind +`prices-production-coverage-sweep-unclassified` (ALARM since 2026-10-12). + +## Verification (2026-10-16) + +- **Official.** `blend-capital/docs-v2` `mainnet-deployments.md` (commit + ef36127c, 2026-10-07) lists it as v2.1's "Comet v1.1 Contract (BLND:USDC + liquidity pool)". The v2.1 backstop `CCS4AZ5O…` is listed there too and in + blend-ui's `.env.production` as `NEXT_PUBLIC_BACKSTOP_V2_1`. It holds ~50% of + the pool's LP supply. +- **Code.** Wasm `d735c339…` is byte-identical (sha256) to CometDEX release + `v1.1.0_contracts_pkg1.0.0_cli25.1.0` (2026-09-11). That release adds + `token_in != token_out` (`ErrSameToken`) to both swap entry points, which closes + the 2026-08-25 self-swap exploit. +- **Payload.** Same `POOL/swap` topic and data map as the old pool (`caller`, + `token_amount_in/out`, `token_in/out`). All 520 lifetime swaps are BLND SAC + `CD25MNVT…` ↔ USDC SAC `CCW67TSZ…`. 730 events in prod ClickHouse match + StellarExpert's count. +- **Price.** Hourly VWAP vs the same hour elsewhere: Soroswap median −0.09% + (p5/p95 −1.8%/+0.8%), SDEX median −0.59% (−3.3%/+1.5%). The only outliers + (≤ −34%) are the launch hours of 2026-10-07, when SDEX and Soroswap disagreed + with each other too. +- **Range.** Pool created 2026-09-30, first swap 2026-10-07 00:16 UTC (ledger + 64,809,271), the same minute as the first WARN. Nothing was traded before. + ~1.55 M BLND / ~9.5 k USDC to 2026-10-16. 80% of swaps come from one arb-like + contract (`CBNVK5PE…`). + +The sweep's two other emitters are not priceable: + +- `CDU47AIU…` (wasm `f3dd3edf…`, 10 events): securities-financing ledger (repo, + stock loan, FX forward; PSG Digital Labs). `trade_opened/closed` carry notional + and `rate_bps`, no token price. Allow-list it. +- `CD35HSTR…` (1 event): an instance of the allow-listed order-book wasm + `4d6abb0d…`, matched only by contract id today. Allow-list this instance. + +## Implementation Plan + +1. Add `CASYAKK3…` (venue `comet`) to `STATIC_POOLS`, with a test on a real swap + of this pool. +2. Allow-list `CDU47AIU…` and `CD35HSTR…` in `coverage-sweep-probe`. +3. Deploy the ledger-processor; its cold start loads the pool. +4. `events-backfill --discover-pools` WRITE (runbook "Factory-less pools"). +5. Reprice from ledger 64,809,271 to the deploy. + +## Acceptance Criteria + +- [ ] `STATIC_POOLS` holds `CASYAKK3…`; a test decodes one of its real swaps +- [ ] Both sweep emitters allow-listed +- [ ] Ledger-processor deployed; no `unregistered_pool_events` WARN after it +- [ ] `prices.pool_registry FINAL` has a `comet` row for `CASYAKK3…` +- [ ] Minutes from 2026-10-07 00:16 UTC repriced; BLND `comet` 1h candles + exist for those hours +- [ ] Next coverage sweep clean; `coverage-sweep-unclassified` back to OK + +## Notes + +- The old pool `CAS3FL6T…`, called frozen after the exploit, still emitted 1,084 + swaps from 2026-09-30 to ~2026-10-11 and is indexed. Its prices were not + checked here. From 1d19f12e63c40730084c5827171d76a8847a4317 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:19:29 +0200 Subject: [PATCH 105/144] fix(lore-0334): register the Blend v2.1 Comet pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blend v2.1 moved its backstop to a new Comet BLND/USDC pool, CASYAKK3…, listed in blend-capital/docs-v2 mainnet-deployments.md. Comet pools have no factory event, so only STATIC_POOLS registers them; without an entry the live processor dropped every swap since 2026-10-07 and fired the unregistered-pool alarm. Its wasm is CometDEX release v1.1.0, which rejects self-swaps, and its POOL/swap payload is the one comet_extractor already decodes. --- docs/runbooks/seed-pool-registry.md | 17 ++++-- packages/events-backfill/src/discover.rs | 28 ++++++--- packages/events-backfill/src/run.rs | 4 +- .../src/soroban/comet_tests.rs | 51 ++++++++++++++++ .../prices-ingest-core/src/static_pools.rs | 60 ++++++++++++------- 5 files changed, 124 insertions(+), 36 deletions(-) diff --git a/docs/runbooks/seed-pool-registry.md b/docs/runbooks/seed-pool-registry.md index ed75991d..5b75a9d7 100644 --- a/docs/runbooks/seed-pool-registry.md +++ b/docs/runbooks/seed-pool-registry.md @@ -234,9 +234,13 @@ Confirm with the same `FINAL` count below, then drop `--dry-run` to write. Some pools have no factory event, so neither `learn_factory` nor this tool can find them. They are registered by a committed list, `STATIC_POOLS` in -`packages/prices-ingest-core/src/static_pools.rs`. Today it holds one pool: -Comet BLND/USDC `CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM` -(venue `comet`). +`packages/prices-ingest-core/src/static_pools.rs`. Today it holds two pools, +both venue `comet`: + +- Comet BLND/USDC `CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM`, + Blend v1/v2's backstop pool (task 0300); +- Comet v1.1 BLND/USDC `CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE`, + Blend v2.1's backstop pool, first swap 2026-10-07 (task 0334). - The live processor and the events-backfill reprice route it from the list, with or without a table row. @@ -249,9 +253,10 @@ Comet BLND/USDC `CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM` --clickhouse-url http://localhost:8123 --dry-run ``` - Expected until written: a `change="new"` line for `CAS3FL6T…` with - `venue="comet"`, then `to_write=1 per_venue={"comet": 1}` (plus any - factory pools created in that ledger). + Expected until written: a `change="new"` line per missing pool with + `venue="comet"`, then `to_write=N per_venue={"comet": N}` (plus any + factory pools created in that ledger). On production after task 0334 that + is `CASYAKK3…` alone, `N = 1`: `CAS3FL6T…` has been in the table since 0300. - sdex-backfill's end-of-run registry write also persists it — the identical row, idempotent under the table's `ReplacingMergeTree`. diff --git a/packages/events-backfill/src/discover.rs b/packages/events-backfill/src/discover.rs index 7b5f5939..172d3999 100644 --- a/packages/events-backfill/src/discover.rs +++ b/packages/events-backfill/src/discover.rs @@ -291,10 +291,11 @@ mod tests { } const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; + const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; - fn comet_row() -> PoolRegistryRow { + fn comet_row(id: &str) -> PoolRegistryRow { PoolRegistryRow { - contract_id: COMET.to_string(), + contract_id: id.to_string(), venue: "comet".to_string(), token0: String::new(), token1: String::new(), @@ -303,19 +304,31 @@ mod tests { } } - /// Task 0300 D3b: over an EMPTY table the factory-less Comet pool is the - /// one row to write — whatever the range, since it has no factory event. + /// Task 0300 D3b: over an EMPTY table the factory-less Comet pools are the + /// rows to write — whatever the range, since they have no factory event. #[test] fn a_static_pool_missing_from_the_table_is_written_once() { let (reg, persisted) = preload(Registries::new()); - assert_eq!(reg.pool_rows_unpersisted(&persisted), vec![comet_row()]); + assert_eq!( + reg.pool_rows_unpersisted(&persisted), + vec![comet_row(COMET), comet_row(COMET_V1_1)] + ); } - /// ...and once the table holds it, a re-run writes nothing. + /// ...a table holding only the first writes only the second (task 0334, + /// production's state), and a table holding both writes nothing. #[test] fn a_static_pool_already_in_the_table_is_not_rewritten() { let mut loaded = Registries::new(); - loaded.load_pool_rows(&[comet_row()]); + loaded.load_pool_rows(&[comet_row(COMET)]); + let (reg, persisted) = preload(loaded); + assert_eq!( + reg.pool_rows_unpersisted(&persisted), + vec![comet_row(COMET_V1_1)] + ); + + let mut loaded = Registries::new(); + loaded.load_pool_rows(&[comet_row(COMET), comet_row(COMET_V1_1)]); let (reg, persisted) = preload(loaded); assert!(reg.pool_rows_unpersisted(&persisted).is_empty()); } @@ -337,6 +350,7 @@ mod tests { written, vec![ (COMET, "comet"), + (COMET_V1_1, "comet"), ( "CAZ4Z273BBAAFL5NYNQJKEMZDQBRCPKAS4GOXDUFXPSE56M4ONBJUOVD", "soroswap" diff --git a/packages/events-backfill/src/run.rs b/packages/events-backfill/src/run.rs index 4c3af165..7ac7d1fa 100644 --- a/packages/events-backfill/src/run.rs +++ b/packages/events-backfill/src/run.rs @@ -853,6 +853,7 @@ mod tests { } const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; + const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; /// Task 0300 D3a: the static pools never mask an unseeded `pool_registry` /// — the guard is about the table, and still fires on an empty one. @@ -873,9 +874,10 @@ mod tests { loaded.venue.insert(AQUA.to_string(), Venue::Aquarius); let reg = reprice_registry(loaded).expect("a seeded table reprices"); - assert_eq!(reg.venue.len(), 2); + assert_eq!(reg.venue.len(), 3); assert_eq!(reg.venue.get(AQUA), Some(&Venue::Aquarius)); assert_eq!(reg.venue.get(COMET), Some(&Venue::Comet)); + assert_eq!(reg.venue.get(COMET_V1_1), Some(&Venue::Comet)); } // ---- task 0139: only new identities, and before their candles ---------- diff --git a/packages/prices-ingest-core/src/soroban/comet_tests.rs b/packages/prices-ingest-core/src/soroban/comet_tests.rs index 814e587d..6ef9938a 100644 --- a/packages/prices-ingest-core/src/soroban/comet_tests.rs +++ b/packages/prices-ingest-core/src/soroban/comet_tests.rs @@ -572,3 +572,54 @@ fn the_static_list_routes_comet_from_an_empty_registry() { assert_eq!((&tick.base, &tick.quote), (&blnd(), &usdc())); assert!(out.unregistered_pool_events.is_empty()); } + +/// Blend v2.1's Comet pool (task 0334), wasm `d735c339…` (CometDEX v1.1.0). +const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; + +/// `v1_1_blnd_to_usdc` — pool `CASYAKK3…`, ledger 64,875,167, tx idx 199, +/// op 0, event 8. BLND → USDC, 16805005302 in / 94368304 out. +fn v1_1_blnd_to_usdc() -> RawSorobanEvent { + RawSorobanEvent { + contract_id: COMET_V1_1.to_string(), + ..comet_event( + 64_875_167, + 199, + 8, + json!([{"type": "sym", "value": "POOL"}, {"type": "sym", "value": "swap"}]), + json!({"type": "map", "value": [{"key": {"type": "sym", "value": "caller"}, "value": {"type": "address", "value": "GDCRZPZYBZ24RHRO3WBPJGFDL7NDFKUQBS3ZDB6YGBJB3TGKMFYBQ3LD"}}, {"key": {"type": "sym", "value": "token_amount_in"}, "value": {"type": "i128", "value": "16805005302"}}, {"key": {"type": "sym", "value": "token_amount_out"}, "value": {"type": "i128", "value": "94368304"}}, {"key": {"type": "sym", "value": "token_in"}, "value": {"type": "address", "value": "CD25MNVTZDL4Y3XBCPCJXGXATV5WUHHOWMYFF4YBEGU5FCPGMYTVG5JY"}}, {"key": {"type": "sym", "value": "token_out"}, "value": {"type": "address", "value": "CCW67TSZV3SSS2HXMBQ5JFGCKJNXKZM7UQUWUZPUTHXSTZLEO7SJMI75"}}]}), + ) + } +} + +/// Task 0334: the v1.1 pool's real swap, unregistered before and routed by +/// the static list after, from an empty registry. +#[test] +fn the_static_list_routes_the_blend_v2_1_comet_pool() { + let mut assets = seeded_assets(); + let out = run( + 64_875_167, + &[v1_1_blnd_to_usdc()], + &mut Registries::new(), + &mut assets, + ); + assert!(out.amm_ticks.is_empty()); + assert_eq!(out.unregistered_pool_events, vec![("comet", 1)]); + assert_eq!( + out.unregistered_pool_contracts, + vec![COMET_V1_1.to_string()] + ); + + let mut reg = Registries::new(); + reg.merge_static_pools(); + let out = run(64_875_167, &[v1_1_blnd_to_usdc()], &mut reg, &mut assets); + assert_eq!(out.amm_ticks.len(), 1); + let (source, tick) = &out.amm_ticks[0]; + assert_eq!(*source, "comet"); + assert_eq!((&tick.base, &tick.quote), (&blnd(), &usdc())); + // BLND → USDC: price = USDC out / BLND in (≈ 0.005616). + assert_eq!(tick.price, d(94_368_304) / d(16_805_005_302)); + assert_eq!(tick.volume_base, d(16_805_005_302)); + assert!(tick.price_forming); + assert!(out.unregistered_pool_events.is_empty()); + assert!(out.dispatch_errors.is_empty()); +} diff --git a/packages/prices-ingest-core/src/static_pools.rs b/packages/prices-ingest-core/src/static_pools.rs index bb4549b0..ed731ab8 100644 --- a/packages/prices-ingest-core/src/static_pools.rs +++ b/packages/prices-ingest-core/src/static_pools.rs @@ -18,12 +18,23 @@ use crate::soroban::Registries; /// issuer, with wasm `8abc28913035c07411ed5d134e6bfeab4723d97ddd4d1a22a0605d35c94d1a36`. /// It has no factory and emits no init / new-pool event (task 0300). /// -/// Its sister `CB3A6LUP…` (same wasm) is deliberately absent: it has emitted -/// no event, ever. -pub const STATIC_POOLS: &[(&str, Venue)] = &[( - "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM", - Venue::Comet, -)]; +/// - `CASYAKK3…UMIHE` — Blend v2.1's Comet BLND/USDC pool, listed in +/// `blend-capital/docs-v2` `mainnet-deployments.md`. Deployed 2026-09-30 +/// with wasm `d735c3395f59510172cf5cf838823a1389b97cfec6bf24e580e9bd77d2b3e687`, +/// CometDEX release v1.1.0, which rejects self-swaps (task 0334). +/// +/// Its sister `CB3A6LUP…` (same wasm as `CAS3FL6T…`) is deliberately absent: +/// it has emitted no event, ever. +pub const STATIC_POOLS: &[(&str, Venue)] = &[ + ( + "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM", + Venue::Comet, + ), + ( + "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE", + Venue::Comet, + ), +]; impl Registries { /// Route every [`STATIC_POOLS`] entry to its venue. The committed list @@ -59,10 +70,14 @@ mod tests { use crate::registry_io::PoolRegistryRow; const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; + const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; #[test] - fn the_list_holds_exactly_the_comet_pool() { - assert_eq!(STATIC_POOLS, &[(COMET, Venue::Comet)]); + fn the_list_holds_exactly_the_two_comet_pools() { + assert_eq!( + STATIC_POOLS, + &[(COMET, Venue::Comet), (COMET_V1_1, Venue::Comet)] + ); } #[test] @@ -79,28 +94,29 @@ mod tests { fn merging_routes_the_pools_and_touches_nothing_else() { let mut reg = Registries::new(); reg.merge_static_pools(); - assert_eq!(reg.venue.len(), 1); + assert_eq!(reg.venue.len(), 2); assert_eq!(reg.venue.get(COMET), Some(&Venue::Comet)); + assert_eq!(reg.venue.get(COMET_V1_1), Some(&Venue::Comet)); assert_eq!(reg.pool_count(), 0, "no pair or phoenix entry"); reg.merge_static_pools(); - assert_eq!(reg.venue.len(), 1, "idempotent"); + assert_eq!(reg.venue.len(), 2, "idempotent"); } #[test] - fn a_merged_pool_flattens_to_one_venue_only_row() { + fn each_merged_pool_flattens_to_one_venue_only_row() { let mut reg = Registries::new(); reg.merge_static_pools(); - assert_eq!( - reg.to_pool_rows(), - vec![PoolRegistryRow { - contract_id: COMET.to_string(), - venue: "comet".to_string(), - token0: String::new(), - token1: String::new(), - pool_type: 0, - wasm_hash: String::new(), - }] - ); + let mut rows = reg.to_pool_rows(); + rows.sort_by(|a, b| a.contract_id.cmp(&b.contract_id)); + let row = |id: &str| PoolRegistryRow { + contract_id: id.to_string(), + venue: "comet".to_string(), + token0: String::new(), + token1: String::new(), + pool_type: 0, + wasm_hash: String::new(), + }; + assert_eq!(rows, vec![row(COMET), row(COMET_V1_1)]); } } From e9b89212c520adf0bcae627b61085248e4beb7ae Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:19:38 +0200 Subject: [PATCH 106/144] chore(lore-0334): allow-list two non-priceable emitters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-10-12 sweep reported two besides the Comet pool: CDU47AIU…, a securities-financing ledger whose trade events carry notional and rate_bps but no token price, and CD35HSTR…, a further instance of the already allow-listed order-book wasm. --- packages/coverage-sweep-probe/allowlist.toml | 12 ++++++++++++ packages/coverage-sweep-probe/src/allowlist.rs | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/coverage-sweep-probe/allowlist.toml b/packages/coverage-sweep-probe/allowlist.toml index 9fa70823..a2536471 100644 --- a/packages/coverage-sweep-probe/allowlist.toml +++ b/packages/coverage-sweep-probe/allowlist.toml @@ -182,6 +182,18 @@ wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" reason = "on-chain order book of tokenised instruments (NewSellInitiatedTrade); instruments are u64 ids with no token addresses, not priceable here" task = "0100" +[[contract]] +id = "CD35HSTRTNL4O3BFXQL75XGAI6AOCWFFAEPR5OCBS7A7HVUDUZOVLCAB" +wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" +reason = "on-chain order book of tokenised instruments (NewBuyInitiatedTrade), a further instance of the wasm above; instruments are u64 ids with no token addresses, not priceable here" +task = "0334" + +[[contract]] +id = "CDU47AIU3BKKRTJBTPFHFXLVJG4BGNOXKDYUBDTKRIWAFP2RLWFKL4HV" +wasm = "f3dd3edfd637a9654054181a9c47391ddafa723697b982698b3c9662a9df974e" +reason = "securities-financing trade ledger (repo, stock loan, FX forward); trade_opened/trade_closed carry notional and rate_bps, no token price" +task = "0334" + [[contract]] id = "CBP76I2FRMUKYKIYYBKN3DH7TSWMFAJF2WTDC7Q2OHQFBIVWP7CAAI5Q" wasm = "76e60d025ee6c0322cb19f4846b9587ab172870802251de903e93545371678b6" diff --git a/packages/coverage-sweep-probe/src/allowlist.rs b/packages/coverage-sweep-probe/src/allowlist.rs index 08f0d1bc..e93a5586 100644 --- a/packages/coverage-sweep-probe/src/allowlist.rs +++ b/packages/coverage-sweep-probe/src/allowlist.rs @@ -179,7 +179,7 @@ mod tests { #[test] fn embedded_list_parses_with_the_seeded_entries() { let list = AllowList::embedded().expect("embedded allow-list parses"); - assert_eq!(list.contract.len(), 27, "{list:?}"); + assert_eq!(list.contract.len(), 29, "{list:?}"); assert_eq!(list.wasm.len(), 3, "{list:?}"); // Every family-wide entry is temporary and names the task that ends it: // SushiSwap V3 (0290) and the Comet BLND/USDC pool (0300). From 12a65fd586524b160278373798ee59432735af21 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:27:24 +0200 Subject: [PATCH 107/144] docs(lore-0335): open task for the stellar-xdr 30 bump Protocol 30 (CAP-0088) makes the millisecond StellarValueExt arms mandatory from the first ledger after the vote; stellar-xdr 28 cannot decode them. BE's xdr-parser on develop now requires 30 (BE #673). --- ...FEATURE_stellar-xdr-30-protocol-30-bump.md | 135 ++++++++++++++++++ ...RE_xlm-usd-for-2018-01-01-to-2018-02-16.md | 0 2 files changed, 135 insertions(+) create mode 100644 lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md rename lore/1-tasks/{active => archive}/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md (100%) diff --git a/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md b/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md new file mode 100644 index 00000000..6c7c064b --- /dev/null +++ b/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md @@ -0,0 +1,135 @@ +--- +id: "0335" +title: "Bump stellar-xdr 28→30 before the protocol 30 vote — every ledger after it carries a millisecond StellarValue arm that 28 cannot decode" +type: FEATURE +status: active +related_adr: [] +related_tasks: ["0277", "0330", "0325", "0098", "0091"] +tags: [layer-backend, priority-high, effort-small, phase-live, resilience, ingestion, deployment] +links: + - "../archive/0277_FEATURE_stellar-xdr-28-protocol-28-bump.md" + - "../../../docs/runbooks/xdr-protocol-watch.md" + - "https://github.com/rumblefishdev/soroban-block-explorer/pull/673" + - "https://github.com/stellar/rs-stellar-xdr/releases/tag/v30.0.0" +history: + - date: "2026-10-19" + status: active + who: akot + note: > + Created after BE announced their stellar-xdr 30 bump (BE #673, lore-0649, + merged; BE develop head d1251f30). Our lock still pins xdr-parser at + 31be5f7, so nothing is broken today, but the next `cargo update -p + xdr-parser` fails to build unless stellar-xdr moves to 30 in the same + change. No vote date announced yet. + - date: "2026-10-19" + status: active + who: akot + note: > + Bump done on feat/0335_stellar-xdr-30-protocol-30-bump, not yet + committed. The lock resolves one stellar-xdr (30.0.0) and xdr-parser is + at d1251f30. The only compile break was BE's event API rework + (ADR 0059), not XDR. On 485 real mainnet ledgers the adapted extraction + gives the same 360,847 events as develop. Workspace tests: 1448 passed, + 0 failed. +--- + +# stellar-xdr 28 → 30 for protocol 30 + +## Summary + +Protocol 30 (CAP-0088) makes the millisecond `StellarValueExt` arms +(`SignedMs`, `EmptyTxSetMs`) mandatory from the first ledger after the vote. +Our workspace pins `stellar-xdr = "=28.0.0"` (`Cargo.toml:30`), and the +ledger-processor decodes every Galexie object with it. On 28 the first +protocol-30 ledger fails to decode, the batch errors, and the cursor stops +moving. That is the protocol-27 freeze (0091) all over again. The bump has to +ship and be **deployed** before the vote. + +## Context + +- **BE side is done.** `xdr-parser` on BE `develop` now requires + `stellar-xdr` 30 (BE `Cargo.toml:41`, `version = "30"`). We track it with + `branch = "develop"`, and our lock pins rev `31be5f7` (the proto-28 rev from + 0277). Bumping one without the other puts two XDR majors in the graph and + `decode_object` stops type-checking. The two move together. +- **The compiler will not prove readiness** (0277's correction). CAP-0088 + changes `StellarValueExt`, and we only read `scp_value.close_time`, a plain + field. So a clean `cargo check` says nothing. Proof has to be a decode of a + ledger carrying the new arm through our own path. +- **`closed_at` stays in seconds.** CAP-0088 keeps + `closeTime == closeTimeMs / 1000`, and BE asserts the same in + `crates/xdr-parser/tests/protocol_30_arms.rs`. +- **CAP-0084** adds `ScAddress::MuxedContract` (`W…`). We only construct + `ScAddress::Contract` and never match `ScAddress` exhaustively, so no compile + break is expected. It matters only if a pool swap names a muxed contract. +- **The watch** (`xdr-protocol-watch`, 0098/0330) goes LAGGING once Horizon + reports `core_supported_protocol_version = 30`, because crate 30.0.0 is + already published. That alarm is a backstop, not the plan. +- **Out of our hands:** BE's Galexie on captive core 30 (BE lore-0650). + Protocol 29 stalled on Galexie, not on XDR (0325). + +## Implementation Plan + +1. `Cargo.toml`: `stellar-xdr = "=28.0.0"` → `"=30.0.0"`, keeping the exact pin. +2. `cargo update -p xdr-parser -p stellar-xdr`. The lock must resolve exactly + one `stellar-xdr`, and `xdr-parser` lands on BE `develop`. +3. Fix whatever the compiler reports. +4. Decode test: a ledger with `StellarValueExt::SignedMs` and one with + `EmptyTxSetMs`, packed as Galexie writes them (zstd `LedgerCloseMetaBatch`), + through `decode_object` → `ledger_sequence` / `ledger_close_time` in seconds. +5. Workspace tests, plus the ClickHouse `#[ignore]` suite. +6. Deploy every binary that links `stellar-xdr`. That is the ledger-processor + first (it decodes ledgers), then the EventBridge Lambdas. Confirm + `CodeSha256` changed (0091: a merged bump with a stale binary changes + nothing). +7. After the vote: measure the candle frontier crossing the activation ledger, + as 0277 did. + +## Findings 2026-10-19 + +**The break was BE's event API, not XDR.** Between `31be5f7` and `d1251f30`, BE +replaced `xdr_parser::extract_events` with `LedgerEvents::new(..).extract(tx, +hash)` (ADR 0059: stellar-rpc event ids). `EventSource` is gone, and +`ExtractedEvent` now carries `body: EventBody { event_type, contract_id, +topics, data }`. The new `TxEvents` returns consensus events in `.events` +(tx-level first, then per-op) and diagnostic events separately in +`.diagnostic`. The old function returned the same consensus list with the +diagnostic events appended after it. So an event's position in `.events` is +exactly the old per-tx `event_index`. `prices-ingest-core/src/soroban.rs` is +the only consumer that needed changes. `decompress_zstd`, `deserialize_batch`, +`ParseError` and `scval.rs` are byte-identical between the two revs. + +**Equivalence measured, not assumed.** A scratch dump ran on both `develop` +and the branch over 485 mainnet ledgers from the public data lake (`65,021,700+` +on protocol 29 with V4 meta, `57,000,000+` and `60,000,000+` older). It printed +every consensus event (tx, index, type, contract, topics, data) plus the full +`process_ledger` output. All 360,847 events match, and so do all `amm_ticks` +and `oracle`. The one diff line is the order of two `unresolved` rows in ledger +65,021,722. They come from iterating the `amm_groups` `HashMap`, whose order is +random per process on `develop` too. + +**XDR 30 gates nothing.** There is no `cap_*` feature in `stellar-xdr` +30.0.0, so `SignedMs`/`EmptyTxSetMs` are ungated. 28 rejects both +(`stellar_value_ext.rs:135`, `_ => Err(Invalid)`). `stellar-xdr` 30 also +pulls in `stellar-strkey` 1.0.0 beside our 0.0.13. No types cross between +them. + +**Deploy scope.** Only the ledger-processor decodes ledgers now +(`reconcile.rs:292`). `asset-discovery` no longer calls `decode_object`, which +was 0277's scope correction. The EventBridge Lambdas still link `stellar-xdr`, +so they get rebuilt too. + +**Pre-existing, not ours:** `cargo clippy --workspace --all-targets -D +warnings` fails only on `prices-clickhouse/tests/current_mv_it.rs:1187` +(`too_many_arguments`), a file this branch does not touch. Excluding that +crate, clippy is clean. + +## Acceptance Criteria + +- [x] Workspace pin `=30.0.0`; `xdr-parser` on BE `develop` at `d1251f30`; one `stellar-xdr` in the lock +- [ ] `cargo test --workspace` (1448 passed / 0 failed locally) and the CH ignored suite green (CI runs it on the PR) +- [x] A test proves a `SignedMs` / `EmptyTxSetMs` ledger decodes through `decode_object` with `close_time` in seconds (`decode.rs`, table-driven) +- [x] The `xdr-parser` event API change preserves extraction: 360,847 events identical on 485 real ledgers +- [ ] Ledger-processor deployed on the 30 binary, `CodeSha256` before/after recorded +- [ ] EventBridge Lambdas linking `stellar-xdr` deployed on the 30 binary +- [ ] Candle frontier measured across the protocol-30 activation ledger (after the vote) diff --git a/lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md b/lore/1-tasks/archive/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md similarity index 100% rename from lore/1-tasks/active/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md rename to lore/1-tasks/archive/0333_FEATURE_xlm-usd-for-2018-01-01-to-2018-02-16.md From e0301f91dcf05092cc9571d275f62a369d3399f7 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:28:06 +0200 Subject: [PATCH 108/144] feat(lore-0335): bump stellar-xdr to 30 for protocol 30 Protocol 30 (CAP-0088) puts a millisecond StellarValueExt arm on every ledger after the vote. stellar-xdr 28 rejects it, so decode_object would fail and the ledger-processor cursor would stop, as in protocol 27. xdr-parser moves to BE develop d1251f30 in the same change; it requires stellar-xdr 30. Its event API changed (ADR 0059): extract_events is replaced by LedgerEvents, and process_ledger now reads the consensus events from it. Over 485 mainnet ledgers the 360,847 extracted events and the process_ledger output match develop. --- Cargo.lock | 75 +++++++++++++++++--- Cargo.toml | 2 +- packages/prices-ingest-core/Cargo.toml | 2 + packages/prices-ingest-core/src/decode.rs | 82 ++++++++++++++++++++++ packages/prices-ingest-core/src/soroban.rs | 41 ++++++----- 5 files changed, 169 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f6a9512..160549de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,7 +165,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "stellar-strkey", + "stellar-strkey 0.0.13", "stellar-xdr", "thiserror 2.0.18", "tokio", @@ -1021,6 +1021,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" @@ -1259,7 +1265,7 @@ dependencies = [ "prices-clickhouse", "serde", "serde_json", - "stellar-strkey", + "stellar-strkey 0.0.13", "thiserror 2.0.18", "tokio", "toml", @@ -1490,7 +1496,7 @@ dependencies = [ [[package]] name = "domain" version = "0.1.0" -source = "git+https://github.com/rumblefishdev/soroban-block-explorer.git?branch=develop#31be5f747abadb2e0cc0d2ae4f59702604519bd5" +source = "git+https://github.com/rumblefishdev/soroban-block-explorer.git?branch=develop#d1251f30d1a9258bbe6643f8f5e35feafc7f6c4a" dependencies = [ "chrono", "serde", @@ -1882,6 +1888,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1908,6 +1923,17 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32", + "stable_deref_trait", + "zeroize", +] + [[package]] name = "heck" version = "0.5.0" @@ -2694,7 +2720,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "stellar-strkey", + "stellar-strkey 0.0.13", "stellar-xdr", "thiserror 2.0.18", "tokio", @@ -2880,7 +2906,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "stellar-strkey", + "stellar-strkey 0.0.13", "thiserror 2.0.18", "tokio", "tower 0.5.3", @@ -2931,12 +2957,13 @@ dependencies = [ "serde_json", "sha2 0.10.9", "soroswap-extractor", - "stellar-strkey", + "stellar-strkey 0.0.13", "stellar-xdr", "thiserror 2.0.18", "tokio", "tracing", "xdr-parser", + "zstd", ] [[package]] @@ -3575,7 +3602,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "stellar-strkey", + "stellar-strkey 0.0.13", "stellar-xdr", "thiserror 2.0.18", "tokio", @@ -3930,18 +3957,30 @@ dependencies = [ "data-encoding", ] +[[package]] +name = "stellar-strkey" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5038ed2bf4e11842210d118f65791d8ca4747622daddd222338b3d06b6051bc1" +dependencies = [ + "crate-git-revision 0.0.9", + "data-encoding", + "heapless", + "zeroize", +] + [[package]] name = "stellar-xdr" -version = "28.0.0" +version = "30.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93d09ff8b9f919b084f664003c4c546ac66a76affd5429460dbe29f4b326f8e" +checksum = "56f2fb2008b42fc746c0f6b46e375600068dd91f07748e9f1737c6341a2255f2" dependencies = [ "crate-git-revision 0.0.9", "escape-bytes", "ethnum", "hex", "sha2 0.10.9", - "stellar-strkey", + "stellar-strkey 1.0.0", ] [[package]] @@ -4840,7 +4879,7 @@ dependencies = [ [[package]] name = "xdr-parser" version = "0.1.0" -source = "git+https://github.com/rumblefishdev/soroban-block-explorer.git?branch=develop#31be5f747abadb2e0cc0d2ae4f59702604519bd5" +source = "git+https://github.com/rumblefishdev/soroban-block-explorer.git?branch=develop#d1251f30d1a9258bbe6643f8f5e35feafc7f6c4a" dependencies = [ "base64", "domain", @@ -4928,6 +4967,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index d3f7ac9c..558cfaca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ members = [ ] [workspace.dependencies] -stellar-xdr = { version = "=28.0.0" } +stellar-xdr = { version = "=30.0.0" } stellar-strkey = "0.0.13" sha2 = "0.10" xdr-parser = { git = "https://github.com/rumblefishdev/soroban-block-explorer.git", branch = "develop" } diff --git a/packages/prices-ingest-core/Cargo.toml b/packages/prices-ingest-core/Cargo.toml index 547ea808..54b7c795 100644 --- a/packages/prices-ingest-core/Cargo.toml +++ b/packages/prices-ingest-core/Cargo.toml @@ -42,3 +42,5 @@ ledger-processor = { path = "../ledger-processor" } [dev-dependencies] # task 0078: reuse the shared Phoenix xyk swap fixture in the classify-seam test. phoenix-extractor = { path = "../phoenix-extractor", features = ["test-fixtures"] } +# task 0335: `decode.rs` tests pack a ledger the way Galexie writes it. +zstd = "0.13" diff --git a/packages/prices-ingest-core/src/decode.rs b/packages/prices-ingest-core/src/decode.rs index 8ce07c19..0db3a186 100644 --- a/packages/prices-ingest-core/src/decode.rs +++ b/packages/prices-ingest-core/src/decode.rs @@ -43,3 +43,85 @@ pub fn ledger_close_time(lcm: &LedgerCloseMeta) -> i64 { LedgerCloseMeta::V2(v) => v.ledger_header.header.scp_value.close_time.0 as i64, } } + +#[cfg(test)] +mod tests { + use stellar_xdr::{ + Hash, LedgerCloseMetaBatch, LedgerCloseMetaV2, LedgerCloseValueSignature, LedgerHeader, + LedgerHeaderHistoryEntry, Limits, NodeId, PublicKey, Signature, StellarValue, + StellarValueExt, StellarValueProposedMsValue, StellarValueSignedMsValue, TimePoint, + TimePointMs, Uint256, WriteXdr, + }; + + use super::*; + + const SEQ: u32 = 70_000_000; + const CLOSE_TIME: u64 = 1_793_000_000; + + fn signature() -> LedgerCloseValueSignature { + LedgerCloseValueSignature { + node_id: NodeId(PublicKey::PublicKeyTypeEd25519(Uint256([1; 32]))), + signature: Signature(vec![2; 64].try_into().unwrap()), + } + } + + /// One ledger with `ext`, packed as Galexie writes it: a zstd + /// `LedgerCloseMetaBatch`. + fn object(ext: StellarValueExt) -> Vec { + let lcm = LedgerCloseMeta::V2(LedgerCloseMetaV2 { + ledger_header: LedgerHeaderHistoryEntry { + header: LedgerHeader { + ledger_seq: SEQ, + scp_value: StellarValue { + close_time: TimePoint(CLOSE_TIME), + ext, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }); + let batch = LedgerCloseMetaBatch { + start_sequence: SEQ, + end_sequence: SEQ, + ledger_close_metas: vec![lcm].try_into().unwrap(), + }; + let xdr = batch.to_xdr(Limits::none()).unwrap(); + zstd::encode_all(&xdr[..], 0).unwrap() + } + + /// Protocol 30 (CAP-0088) puts one of the two millisecond arms on every + /// ledger after the vote; `stellar-xdr` 28 rejects both. `close_time` + /// stays whole seconds (`closeTime == closeTimeMs / 1000`). + #[test] + fn protocol_30_millisecond_arms_decode_with_close_time_in_seconds() { + let close_time_ms = TimePointMs(CLOSE_TIME * 1000 + 250); + let cases = [ + ( + "SignedMs", + StellarValueExt::SignedMs(StellarValueSignedMsValue { + close_time_ms: close_time_ms.clone(), + lc_value_signature: signature(), + }), + ), + ( + "EmptyTxSetMs", + StellarValueExt::EmptyTxSetMs(StellarValueProposedMsValue { + close_time_ms, + tx_set_hash: Hash([3; 32]), + previous_ledger_hash: Hash([4; 32]), + previous_ledger_version: 29, + lc_value_signature: signature(), + }), + ), + ]; + for (arm, ext) in cases { + let lcms = decode_object(&object(ext)).unwrap_or_else(|e| panic!("{arm}: {e}")); + assert_eq!(lcms.len(), 1, "{arm}"); + assert_eq!(ledger_sequence(&lcms[0]), SEQ, "{arm}"); + assert_eq!(ledger_close_time(&lcms[0]), CLOSE_TIME as i64, "{arm}"); + } + } +} diff --git a/packages/prices-ingest-core/src/soroban.rs b/packages/prices-ingest-core/src/soroban.rs index 1d31771e..fee415fb 100644 --- a/packages/prices-ingest-core/src/soroban.rs +++ b/packages/prices-ingest-core/src/soroban.rs @@ -21,8 +21,8 @@ use extractors_core::{SorobanEventRow, TaggedValue, Venue, VenueRegistry}; use ledger_processor::dispatch::{PairRegistries, dispatch}; use phoenix_extractor::PhoenixPoolRegistry; use soroswap_extractor::SoroswapPoolRegistry; -use xdr_parser::extract_events; -use xdr_parser::types::EventSource; +use xdr_parser::LedgerEvents; +use xdr_parser::types::ExtractedEvent; // USDT_ISSUER is deliberately NOT imported here: task 0172 removed the `USDT` // arm from `reflector_key_to_identity`, so this module must not resolve that @@ -314,37 +314,38 @@ pub fn process_ledger( ) -> LedgerSoroban { let (ledger_seq, closed_at) = ledger_header(lcm); let tx_metas = collect_tx_metas(lcm); + let ledger_events = LedgerEvents::new(ledger_seq, closed_at, &tx_metas); let mut out = LedgerSoroban::default(); - for (tx_index, tx_meta) in tx_metas.iter().enumerate() { + for tx_index in 0..tx_metas.len() { let tx_id = format!("{ledger_seq}:{tx_index}"); - let events = extract_events(tx_meta, &tx_id, ledger_seq, closed_at); + // Consensus events only; the diagnostic mirrors come back in + // `.diagnostic`, unread. An event's position here is its per-tx + // `event_index`: tx-level events first, then per-op. + let events = ledger_events.extract(tx_index, &tx_id).events; // Grouped AMM event rows by emitting contract, order preserved. let mut amm_groups: HashMap> = HashMap::new(); - for ev in &events { - if ev.source == EventSource::Diagnostic { - continue; // drop diagnostic mirrors - } - let Some(contract_id) = ev.contract_id.clone() else { + for (event_index, ev) in events.iter().enumerate() { + let Some(contract_id) = ev.body.contract_id.clone() else { continue; }; - let sig = signature(&ev.topics); + let sig = signature(&ev.body.topics); match sig { Some("REFLECTOR") => decode_reflector(ev, assets, &mut out), Some("REDSTONE") => decode_redstone(ev, &mut out), _ => { // Factory events grow the registry; pool events are queued. - learn_factory(&ev.topics, &ev.data, reg); + learn_factory(&ev.body.topics, &ev.body.data, reg); let row = SorobanEventRow { contract_id: contract_id.clone(), transaction_id: tx_id.clone(), ledger_sequence: ledger_seq as u64, - event_index: ev.event_index, - topics: topics_to_tagged(&ev.topics), - data: json_to_tagged(&ev.data), + event_index: event_index as u32, + topics: topics_to_tagged(&ev.body.topics), + data: json_to_tagged(&ev.body.data), }; amm_groups.entry(contract_id).or_default().push(row); } @@ -931,12 +932,9 @@ fn amm_trade_to_tick( /// Decode a REFLECTOR `update` event: data.update_data is a vec of /// [asset_key, price_i128@1e14] pairs; topic[2] is the u64 ms timestamp. -fn decode_reflector( - ev: &xdr_parser::types::ExtractedEvent, - assets: &mut AssetRegistry, - out: &mut LedgerSoroban, -) { +fn decode_reflector(ev: &ExtractedEvent, assets: &mut AssetRegistry, out: &mut LedgerSoroban) { let ts_ms = ev + .body .topics .as_array() .and_then(|t| t.get(2)) @@ -946,7 +944,7 @@ fn decode_reflector( .unwrap_or((ev.created_at.max(0) as u64) * 1000); let timestamp = (ts_ms / 1000) as u32; - let TaggedValue::Map(m) = json_to_tagged(&ev.data) else { + let TaggedValue::Map(m) = json_to_tagged(&ev.body.data) else { return; }; let Some((_, update_data)) = m.iter().find(|(k, _)| k.as_str() == Some("update_data")) else { @@ -998,8 +996,9 @@ fn decode_reflector( /// row. That id is functionally dead anyway (price_usd = 0, oracle_name = /// 'redstone'; never read by the `reflector` ASOF join), and the raw payload is /// preserved for the byte-footprint measurement. -fn decode_redstone(ev: &xdr_parser::types::ExtractedEvent, out: &mut LedgerSoroban) { +fn decode_redstone(ev: &ExtractedEvent, out: &mut LedgerSoroban) { let raw = ev + .body .data .as_object() .and_then(|o| o.get("value")) From 0705243275d3ed26901ed8b18f06187bd9559cfd Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:14:40 +0200 Subject: [PATCH 109/144] docs(lore-0335): record the compute and eventbridge deploys --- ...FEATURE_stellar-xdr-30-protocol-30-bump.md | 67 ++++++++++++++++++- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md b/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md index 6c7c064b..b90289c9 100644 --- a/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md +++ b/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md @@ -31,6 +31,22 @@ history: (ADR 0059), not XDR. On 485 real mainnet ledgers the adapted extraction gives the same 360,847 events as develop. Workspace tests: 1448 passed, 0 failed. + - date: "2026-10-20" + status: active + who: akot + note: > + PR #406 merged to develop as af504c37 (approved). CI green, including + 328/328 ClickHouse integration tests. Not deployed yet: ledger-processor + and the EventBridge Lambdas are next, then the post-vote crossing + measurement. + - date: "2026-10-20" + status: active + who: akot + note: > + Deployed on the XDR 30 binary: Compute (ledger-processor) at 08:31 UTC + and EventBridge (asset-discovery and the rest) at 09:02 UTC. CodeSha256 + changed on both, with 0 errors after. Stays active until the candle + frontier is measured across the protocol-30 activation ledger. --- # stellar-xdr 28 → 30 for protocol 30 @@ -124,12 +140,57 @@ warnings` fails only on `prices-clickhouse/tests/current_mv_it.rs:1187` (`too_many_arguments`), a file this branch does not touch. Excluding that crate, clippy is clean. +## Deploy 2026-10-20 + +Preflight, read 2026-10-20 at about 08:00 UTC: +`npm run xdr:verify-protocol-gap` reports pinned 30, mainnet current 29, and +**core supports 30**. Validators already run core 30, so the vote can come at +any time. + +| Lambda | `CodeSha256` before | `LastModified` before | +| --- | --- | --- | +| `prices-production-ledger-processor` | `ZFuumECCwyZNKDt1B6lH48YeFxgwH6aHT+1t36FsQzg=` | 2026-10-19 08:26:51 UTC | +| `prices-production-asset-discovery` | `Zaeo0F6RslW+kdc2e5nNxScnikNbjNGkjGIks0ESIos=` | 2026-10-19 08:59:03 UTC | + +Candle frontier before: sdex and aquarius 88 s behind now (07:54 UTC candle). + +**Compute deployed** (`make deploy-production-compute`, stage cache flushed): + +| Lambda | `CodeSha256` after | `LastModified` after | +| --- | --- | --- | +| `prices-production-ledger-processor` | `WCokytgM7mqlxS/+KiQk+QTf665qHwB7a+ssUGTD26g=` | 2026-10-20 08:31:01 UTC | + +Read at 09:00 UTC, about 30 minutes later: Lambda `Errors` = 0 in every +10-minute bucket from 07:30 to 09:10 UTC. sdex and aquarius were 86 s behind, +with 29 and 28 one-minute candles since the deploy. soroswap (5) and sushiswap +(2) also wrote candles since the deploy, so the Soroban event path works on the +new `LedgerEvents` extraction. comet (1.7 h behind) and phoenix (2.7 h) had no +candle since the deploy. That is normal for them: in the previous 24 h they +wrote 12 and 8 candles, and phoenix's longest gap was 16 h. + +**EventBridge deployed** (`make deploy-production-eventbridge`), every Lambda +in the stack at 2026-10-20 09:02:43–44 UTC: + +| Lambda | `CodeSha256` after | +| --- | --- | +| `prices-production-asset-discovery` | `m7eceW0ZGn+gykUhHb3yhn6/haa1q86CFj+LsiH2u18=` (before `Zaeo0F6R…`) | +| `prices-production-oracle` | `+D6AZ+rytOpspELZ37v4iRBiIgj342g6NACnEvrgnY0=` | +| `prices-production-enrichment` | `M6YAUmixrmN6FlTdZkVfr60l2+yX1k97Oe9zYtdcMpc=` | +| `prices-production-supply` | `q8UWEMWZcI0gdsgRSg/VjqaH+91FoUV5w2/h38YTmc0=` | +| `prices-production-coarse-sweep` | `VLMFBYOXdBTXNDEBxS5t7ykJmCnpPCgYnYnm3B2iPX8=` | + +The probes and `cleanup` were rebuilt too. The `prices-production-cleanup` rule +is still `DISABLED` (0200). By 09:13 UTC, oracle (2 runs) and the backfill and +rollup freshness probes (1 run each) had run with 0 errors, and the +ledger-processor had run 119 times with 0 errors. asset-discovery, enrichment +and supply run `rate(1 hour)` and had not run since the deploy yet. + ## Acceptance Criteria - [x] Workspace pin `=30.0.0`; `xdr-parser` on BE `develop` at `d1251f30`; one `stellar-xdr` in the lock -- [ ] `cargo test --workspace` (1448 passed / 0 failed locally) and the CH ignored suite green (CI runs it on the PR) +- [x] `cargo test --workspace` (1448 passed / 0 failed locally) and the CH ignored suite green (CI on #406: 328/328) - [x] A test proves a `SignedMs` / `EmptyTxSetMs` ledger decodes through `decode_object` with `close_time` in seconds (`decode.rs`, table-driven) - [x] The `xdr-parser` event API change preserves extraction: 360,847 events identical on 485 real ledgers -- [ ] Ledger-processor deployed on the 30 binary, `CodeSha256` before/after recorded -- [ ] EventBridge Lambdas linking `stellar-xdr` deployed on the 30 binary +- [x] Ledger-processor deployed on the 30 binary, `CodeSha256` before/after recorded (2026-10-20 08:31 UTC) +- [x] EventBridge Lambdas linking `stellar-xdr` deployed on the 30 binary (2026-10-20 09:02 UTC) - [ ] Candle frontier measured across the protocol-30 activation ledger (after the vote) From 9bd70df81d0a333f88d80a6612d69e7f1e03ca7a Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:19:03 +0200 Subject: [PATCH 110/144] docs(lore-0335): close the stellar-xdr 30 bump --- ...FEATURE_stellar-xdr-30-protocol-30-bump.md | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) rename lore/1-tasks/{active => archive}/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md (84%) diff --git a/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md b/lore/1-tasks/archive/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md similarity index 84% rename from lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md rename to lore/1-tasks/archive/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md index b90289c9..406de56e 100644 --- a/lore/1-tasks/active/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md +++ b/lore/1-tasks/archive/0335_FEATURE_stellar-xdr-30-protocol-30-bump.md @@ -2,7 +2,7 @@ id: "0335" title: "Bump stellar-xdr 28→30 before the protocol 30 vote — every ledger after it carries a millisecond StellarValue arm that 28 cannot decode" type: FEATURE -status: active +status: completed related_adr: [] related_tasks: ["0277", "0330", "0325", "0098", "0091"] tags: [layer-backend, priority-high, effort-small, phase-live, resilience, ingestion, deployment] @@ -47,6 +47,15 @@ history: and EventBridge (asset-discovery and the rest) at 09:02 UTC. CodeSha256 changed on both, with 0 errors after. Stays active until the candle frontier is measured across the protocol-30 activation ledger. + - date: "2026-10-20" + status: completed + who: akot + note: > + Closed as done before the vote. Pin, lock, event API adaptation, decode + test and both deploys are complete (6 of 7 AC). The post-vote crossing + cannot be measured yet: on 2026-10-20 mainnet, testnet and futurenet + are all on protocol 29. It is not tracked as a task; xdr-protocol-watch + is the backstop. --- # stellar-xdr 28 → 30 for protocol 30 @@ -193,4 +202,36 @@ and supply run `rate(1 hour)` and had not run since the deploy yet. - [x] The `xdr-parser` event API change preserves extraction: 360,847 events identical on 485 real ledgers - [x] Ledger-processor deployed on the 30 binary, `CodeSha256` before/after recorded (2026-10-20 08:31 UTC) - [x] EventBridge Lambdas linking `stellar-xdr` deployed on the 30 binary (2026-10-20 09:02 UTC) -- [ ] Candle frontier measured across the protocol-30 activation ledger (after the vote) +- [ ] Candle frontier measured across the protocol-30 activation ledger (after the vote). Not done: on 2026-10-20 no public network had voted on 30 (Horizon: mainnet, testnet and futurenet all `current_protocol_version` 29). Closed without it. + +## Design Decisions + +### From Plan + +1. **Exact pin `=30.0.0`**, moved together with `xdr-parser` so the lock holds + one XDR major. +2. **Readiness proved by decode, not compile**: the `decode.rs` test packs + `SignedMs` / `EmptyTxSetMs` ledgers as Galexie does and goes through + `decode_object`. + +### Emerged + +3. **Adapted to BE's `LedgerEvents` API instead of pinning the old rev.** The + rev that requires XDR 30 also removed `extract_events`. Staying on the old + API was not possible without forking BE. +4. **`event_index` = position in `TxEvents.events`.** Diagnostic events are + now separate and unread. Equivalence was measured on 485 real ledgers + rather than argued. +5. **Whole EventBridge stack redeployed**, not just asset-discovery, since + every Lambda in it links `stellar-xdr`. +6. **Closed before the vote** at Adam's call. The crossing measurement is + left out, with no follow-up task. + +## Issues Encountered + +- **The compile break was BE's event API (ADR 0059), not XDR.** See Findings. +- **The runbook is stale** (`docs/runbooks/deploy-ledger-processor.md`): it + says asset-discovery decodes ledgers and uses the `soroban-admin` profile. + Not fixed here. +- **The task-opening commit 12a65fd5 also carried an unrelated staged rename + of 0333** (active → archive), done by mistake. From 8c1006bc23825526d3694fc774c6696a607f0c87 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:23:15 +0200 Subject: [PATCH 111/144] docs(lore-0336): open task for the securities-ledger wasm family --- ...ties-ledger-wasm-family-pages-the-sweep.md | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 lore/1-tasks/active/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md diff --git a/lore/1-tasks/active/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md b/lore/1-tasks/active/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md new file mode 100644 index 00000000..e128b89b --- /dev/null +++ b/lore/1-tasks/active/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md @@ -0,0 +1,84 @@ +--- +id: "0336" +title: "coverage-sweep-unclassified stays in ALARM: new instances of the securities-financing ledger wasm page one by one — allow-list the family as not a venue" +type: BUG +status: active +assignee: akot +related_adr: [] +related_tasks: ["0100", "0323", "0334"] +tags: [layer-infra, observability, coverage, priority-high, effort-small] +links: + - "../../../packages/coverage-sweep-probe/allowlist.toml" + - "../../../packages/coverage-sweep-probe/src/allowlist.rs" + - "../../../docs/runbooks/0100-coverage-sweep-triage.md" +history: + - date: "2026-10-23" + status: active + who: akot + note: "Task created from the daily stuck-alarm digest: coverage-sweep-unclassified in ALARM for 11d 4h" +--- + +# New instances of the securities-financing ledger wasm keep the sweep in ALARM + +## Summary + +`prices-production-coverage-sweep-unclassified` has been in ALARM since +2026-10-12 07:18 CEST. [[0334]] fixed the residual it was raised for, yet the +manual run after its deploy still found 4 contracts. All four run wasm +`f3dd3edf…`, the securities-financing ledger that [[0334]] allow-listed by +one contract id (`CDU47AIU…`). The family deploys new instances every few +days, so a `[[contract]]` entry per instance would page again every Monday. + +## Context + +Runs from `/aws/lambda/prices-production-coverage-sweep-probe` +(`coverage sweep complete`): + +| Run (UTC) | unclassified | events | Note | +| --- | --- | --- | --- | +| 2026-10-12 05:17 | 3 | 404 | Comet `CASYAKK3…` (393), `CDU47AIU…`, `CD35HSTR…` ([[0334]]) | +| 2026-10-19 05:17 | 7 | 595 | before PR #405 merged | +| 2026-10-19 09:03 | 4 | 15 | manual, after the 0334 deploy; Comet and both 0334 entries gone | + +The four left on 2026-10-19 09:03, all wasm +`f3dd3edfd637a9654054181a9c47391ddafa723697b982698b3c9662a9df974e`, +`trade_opened`/`trade_closed` with shape `sym|u64|bytes -> map`: + +- `CAKQ6DWL…` 8 events, ledgers 64,908,423–64,941,821 +- `CCLN72NM…` 4 events, 64,958,834–64,962,719 +- `CBZYHS2D…` 2 events, 64,944,905–64,959,086 +- `CB4B4ELL…` 1 event, 64,978,560 + +Since [[0323]] the alarm holds the latest run's state, so it stays in ALARM +until a run finds none: the 2026-10-26 run, or a manual invoke after deploy. + +## Design + +[[0100]] D4 forbids a `[[wasm]]` entry without `until`, because a permanent +family-wide ignore would hide the next pool of a real venue. That reason does +not hold for a family whose code is not a market. Add an explicit exception: + +- `[[wasm]]` takes `not_a_venue = true` in place of `until`. Both together, + or neither, fails validation. +- `f3dd3edf…` moves from `CDU47AIU…`'s `[[contract]]` entry to one such + `[[wasm]]` entry. +- Runbook §3 outcome (c) says when to use it: the wasm itself rules out a + market, not the size of the instances seen so far. + +## Acceptance Criteria + +- [ ] Validator: `not_a_venue` without `until` accepted; with `until`, or + `until` missing without it, refused (unit tests) +- [ ] The five known instances match `wasm:f3dd3edf…` (unit test) +- [ ] Runbook §3 describes `not_a_venue` +- [ ] EventBridge deployed; one manual invoke logs `coverage sweep complete` + with `unclassified=0` +- [ ] `coverage-sweep-unclassified` back to OK + +## Notes + +- The order-book wasm `4d6abb0d…` is allow-listed by two contract ids + (`CBOSXSEZ…`, `CD35HSTR…` from [[0334]]). It is the same pattern and a + candidate for `not_a_venue` if a third instance appears. +- The Comet `[[wasm]]` entry `8abc2891…` (`until = "0300"`) is stale: 0300 + shipped. Not removed here. From e0a1bec7d4d2413e2014980947d441b1e79f5752 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:23:21 +0200 Subject: [PATCH 112/144] fix(lore-0336): allow-list the securities-ledger wasm as not a venue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-10-19 manual sweep after 0334 still found four contracts, all new instances of wasm f3dd3edf…, which 0334 allow-listed by one contract id. A [[wasm]] entry may now carry not_a_venue = true in place of until, for a family whose code cannot be a market; the ledger moves to one such entry. --- docs/runbooks/0100-coverage-sweep-triage.md | 8 +- packages/coverage-sweep-probe/allowlist.toml | 17 +++-- .../coverage-sweep-probe/src/allowlist.rs | 73 ++++++++++++++++--- 3 files changed, 80 insertions(+), 18 deletions(-) diff --git a/docs/runbooks/0100-coverage-sweep-triage.md b/docs/runbooks/0100-coverage-sweep-triage.md index be556602..09db093f 100644 --- a/docs/runbooks/0100-coverage-sweep-triage.md +++ b/docs/runbooks/0100-coverage-sweep-triage.md @@ -128,6 +128,11 @@ three outcomes: permanent family-wide ignore would hide the next pool of a real venue. - **(c) A router, aggregator or other non-AMM contract.** Add a permanent `[[contract]]` entry with `reason` and `task`. Never register it. + If the same non-AMM wasm keeps deploying new instances, each new one pages + on its own. Add one `[[wasm]]` entry with `not_a_venue = true` in place of + `until` instead (task 0336: the securities-financing ledger `f3dd3edf…`, + four new instances in one week). Do this only when the code itself rules out + a market, not because the instances seen so far are small. Also look at: @@ -147,7 +152,8 @@ Also look at: `packages/coverage-sweep-probe/allowlist.toml`: - `cargo test -p coverage-sweep-probe` validates it: reason and task on every - entry, `until` on every `[[wasm]]` entry, valid `C…` strkeys, 64-lowercase-hex + entry, `until` on every `[[wasm]]` entry except a `not_a_venue` one (which + must not have it), valid `C…` strkeys, 64-lowercase-hex hashes, no duplicates, no unknown keys. - The list is compiled into the binary, so an edit **ships only with an EventBridge deploy** (§4.4). diff --git a/packages/coverage-sweep-probe/allowlist.toml b/packages/coverage-sweep-probe/allowlist.toml index a2536471..ed999768 100644 --- a/packages/coverage-sweep-probe/allowlist.toml +++ b/packages/coverage-sweep-probe/allowlist.toml @@ -17,6 +17,9 @@ # the task that registers the family). A permanent family-wide # ignore would hide the next pool of a real venue, so a [[wasm]] # entry without `until` fails validation. +# The one exception is `not_a_venue = true` (task 0336), in +# place of `until`: a family whose code cannot be a market we +# price, read from the wasm, that keeps deploying new instances. # # Every entry needs a non-empty `reason` and `task`. Unknown keys are rejected. # @@ -188,12 +191,6 @@ wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" reason = "on-chain order book of tokenised instruments (NewBuyInitiatedTrade), a further instance of the wasm above; instruments are u64 ids with no token addresses, not priceable here" task = "0334" -[[contract]] -id = "CDU47AIU3BKKRTJBTPFHFXLVJG4BGNOXKDYUBDTKRIWAFP2RLWFKL4HV" -wasm = "f3dd3edfd637a9654054181a9c47391ddafa723697b982698b3c9662a9df974e" -reason = "securities-financing trade ledger (repo, stock loan, FX forward); trade_opened/trade_closed carry notional and rate_bps, no token price" -task = "0334" - [[contract]] id = "CBP76I2FRMUKYKIYYBKN3DH7TSWMFAJF2WTDC7Q2OHQFBIVWP7CAAI5Q" wasm = "76e60d025ee6c0322cb19f4846b9587ab172870802251de903e93545371678b6" @@ -225,3 +222,11 @@ hash = "8abc28913035c07411ed5d134e6bfeab4723d97ddd4d1a22a0605d35c94d1a36" reason = "Comet weighted pool, Blend backstop BLND/USDC (CAS3FL6T…), a venue not yet indexed; 53k swaps since 2024-05-02" task = "0100" until = "0300" + +# --- non-venue families (permanent, keyed by wasm) ---------------------------- + +[[wasm]] +hash = "f3dd3edfd637a9654054181a9c47391ddafa723697b982698b3c9662a9df974e" +reason = "securities-financing trade ledger (repo, stock loan, FX forward); trade_opened/trade_closed carry notional and rate_bps, no token price; new instances keep appearing (CDU47AIU… in 0334, four more by 2026-10-19), so this is keyed by wasm" +task = "0336" +not_a_venue = true diff --git a/packages/coverage-sweep-probe/src/allowlist.rs b/packages/coverage-sweep-probe/src/allowlist.rs index e93a5586..9634eef0 100644 --- a/packages/coverage-sweep-probe/src/allowlist.rs +++ b/packages/coverage-sweep-probe/src/allowlist.rs @@ -31,7 +31,8 @@ pub struct ContractEntry { pub task: String, } -/// A whole wasm family, allow-listed until `until` ships. +/// A whole wasm family, allow-listed until `until` ships, or for good when +/// `not_a_venue` (task 0336). #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct WasmEntry { @@ -39,9 +40,14 @@ pub struct WasmEntry { pub hash: String, pub reason: String, pub task: String, - /// The task whose delivery removes this entry. Required (D4). + /// The task whose delivery removes this entry. Required (D4) unless + /// `not_a_venue`. #[serde(default)] pub until: String, + /// The family's code cannot be a market we price (a securities ledger, + /// say), so no task will ever register it. Takes the place of `until`. + #[serde(default)] + pub not_a_venue: bool, } /// Why an allow-list was refused. @@ -87,7 +93,8 @@ impl AllowList { /// Enforce the rules a list must meet before it may hide anything: /// - every entry has a non-empty `reason` and `task`; /// - every `[[wasm]]` entry has a non-empty `until` (D4: a permanent - /// family-wide ignore would hide the next pool of a real venue); + /// family-wide ignore would hide the next pool of a real venue), or + /// `not_a_venue` and no `until` (task 0336); /// - every contract `id` is a valid `C…` strkey (an account key or a typo /// would silently match nothing); /// - every hash, including a contract entry's informational `wasm`, is @@ -127,7 +134,13 @@ impl AllowList { } require(&who, "reason", &e.reason)?; require(&who, "task", &e.task)?; - require(&who, "until", &e.until)?; + if !e.not_a_venue { + require(&who, "until", &e.until)?; + } else if !e.until.trim().is_empty() { + return Err(AllowListError::Invalid(format!( + "{who}: `not_a_venue` and `until` are exclusive" + ))); + } if !hashes.insert(e.hash.as_str()) { return Err(AllowListError::Invalid(format!("{who}: duplicate hash"))); } @@ -174,23 +187,48 @@ mod tests { use super::*; const SUSHI_V3: &str = "003710b383f9da7d650a7f719a7be479110266427817ebbed61d924505fcd7c7"; + const SECURITIES_LEDGER: &str = + "f3dd3edfd637a9654054181a9c47391ddafa723697b982698b3c9662a9df974e"; const AQUARIUS_ROUTER: &str = "CBQDHNBFBZYE4MKPWBSJOPIYLW4SFSXAXUTSXJN76GNKYVYPCKWC6QUK"; #[test] fn embedded_list_parses_with_the_seeded_entries() { let list = AllowList::embedded().expect("embedded allow-list parses"); - assert_eq!(list.contract.len(), 29, "{list:?}"); - assert_eq!(list.wasm.len(), 3, "{list:?}"); - // Every family-wide entry is temporary and names the task that ends it: - // SushiSwap V3 (0290) and the Comet BLND/USDC pool (0300). + assert_eq!(list.contract.len(), 28, "{list:?}"); + assert_eq!(list.wasm.len(), 4, "{list:?}"); + // Every family-wide entry is temporary and names the task that ends it + // (SushiSwap V3, 0290; the Comet BLND/USDC pool, 0300), except the one + // non-venue family (0336). assert!( - list.wasm - .iter() - .all(|w| w.until == "0290" || w.until == "0300"), + list.wasm.iter().all(|w| if w.not_a_venue { + w.hash == SECURITIES_LEDGER + } else { + w.until == "0290" || w.until == "0300" + }), "{list:?}" ); } + #[test] + fn securities_ledger_family_is_allow_listed_by_wasm() { + // The 2026-10-19 sweep (task 0336): four new instances of the wasm + // 0334 allow-listed by one contract id, each paging on its own. + let list = AllowList::embedded().unwrap(); + for id in [ + "CDU47AIU3BKKRTJBTPFHFXLVJG4BGNOXKDYUBDTKRIWAFP2RLWFKL4HV", + "CAKQ6DWLPTKFDHYON346SBAYJJDTVO5DKTMZ7737HBF6NCGASIMOM4U2", + "CCLN72NMUDZVXK6V6W5RQ4GHGHMX437G2FMGT6YKVNB7BAAH7XFLP5SZ", + "CBZYHS2DVYAIOURIAXVUNTTAGTUXH3I4SWFWKN7FAKFKNOKLQ5LIR5YB", + "CB4B4ELLM4YTF7HLITB52OFCYPGHPT763JZBHVUOD4AN4FJR7RWVZXJ6", + ] { + assert_eq!( + list.match_row(id, Some(SECURITIES_LEDGER)), + Some(format!("wasm:{SECURITIES_LEDGER}")), + "{id}" + ); + } + } + #[test] fn comet_pool_family_is_allow_listed_until_0300() { // Phase-1 classification (task 0100): the Comet BLND/USDC pool is a real @@ -276,6 +314,19 @@ mod tests { refused(&wasm_toml(SUSHI_V3, Some(" "))); } + #[test] + fn not_a_venue_takes_the_place_of_until() { + let entry = |extra: &str| { + format!( + "[[wasm]]\nhash = \"{SECURITIES_LEDGER}\"\nreason = \"r\"\ntask = \"0336\"\n{extra}" + ) + }; + AllowList::parse(&entry("not_a_venue = true\n")).unwrap(); + let e = refused(&entry("not_a_venue = true\nuntil = \"0336\"\n")); + assert!(e.contains("exclusive"), "{e}"); + refused(&entry("not_a_venue = false\n")); + } + #[test] fn duplicates_are_refused() { let e = refused(&(contract_toml(GOOD_ID, "a", "1") + &contract_toml(GOOD_ID, "b", "2"))); From 19eaab2ea4543eb47c0f80f234aa594e19097d13 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:56:42 +0200 Subject: [PATCH 113/144] docs(lore-0336): close the securities-ledger allow-list fix --- .../0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lore/1-tasks/{active => archive}/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md (100%) diff --git a/lore/1-tasks/active/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md b/lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md similarity index 100% rename from lore/1-tasks/active/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md rename to lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md From 4d612847bbf1f476239429d53dfbc7b0cd15cf1f Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:58:04 +0200 Subject: [PATCH 114/144] docs(lore-0336): mark the task completed with the deploy result --- ...ties-ledger-wasm-family-pages-the-sweep.md | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md b/lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md index e128b89b..faae8526 100644 --- a/lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md +++ b/lore/1-tasks/archive/0336_BUG_securities-ledger-wasm-family-pages-the-sweep.md @@ -2,7 +2,7 @@ id: "0336" title: "coverage-sweep-unclassified stays in ALARM: new instances of the securities-financing ledger wasm page one by one — allow-list the family as not a venue" type: BUG -status: active +status: completed assignee: akot related_adr: [] related_tasks: ["0100", "0323", "0334"] @@ -16,6 +16,13 @@ history: status: active who: akot note: "Task created from the daily stuck-alarm digest: coverage-sweep-unclassified in ALARM for 11d 4h" + - date: "2026-10-23" + status: completed + who: akot + note: > + PR #407 merged as 06e788a5. EventBridge deployed at 10:54 UTC (diff: + only the coverage-sweep-probe S3Key). Manual invoke: rows=21, + allowlisted=21, unclassified=0. Alarm OK at 10:55:59 UTC. --- # New instances of the securities-financing ledger wasm keep the sweep in ALARM @@ -67,13 +74,25 @@ not hold for a family whose code is not a market. Add an explicit exception: ## Acceptance Criteria -- [ ] Validator: `not_a_venue` without `until` accepted; with `until`, or +- [x] Validator: `not_a_venue` without `until` accepted; with `until`, or `until` missing without it, refused (unit tests) -- [ ] The five known instances match `wasm:f3dd3edf…` (unit test) -- [ ] Runbook §3 describes `not_a_venue` -- [ ] EventBridge deployed; one manual invoke logs `coverage sweep complete` +- [x] The five known instances match `wasm:f3dd3edf…` (unit test) +- [x] Runbook §3 describes `not_a_venue` +- [x] EventBridge deployed; one manual invoke logs `coverage sweep complete` with `unclassified=0` -- [ ] `coverage-sweep-unclassified` back to OK +- [x] `coverage-sweep-unclassified` back to OK + +## Result (2026-10-23) + +- `cdk diff` for `Prices-production-EventBridge` changed only + `CoverageSweepProbeFunction` `.S3Key`. Deployed with + `make -C infra deploy-production-eventbridge`; probe `LastModified` + 10:54:04 UTC, `CodeSha256` `5QXzXnsLtgTvLGb9/CkJIXjAZq7XQBz8gdk3ikopGvI=`. +- Manual invoke at 10:55 UTC, window 64,872,241–65,093,419: `rows=21`, + `allowlisted=21`, `unclassified=0`, `unresolved=0`. +- `Prices/Coverage` `UnclassifiedSwapEvents` = 0 for the 10:55 UTC hour; + `prices-production-coverage-sweep-unclassified` OK at 10:55:59 UTC, after 11 + days in ALARM. ## Notes From b54ebfe0d30c81fa5c29c8d90ee92a33faa0fe1e Mon Sep 17 00:00:00 2001 From: karczuRF Date: Mon, 26 Oct 2026 09:22:24 +0100 Subject: [PATCH 115/144] docs(lore-0337): open task for new Comet pools in the sweep The 2026-10-26 coverage sweep found three unregistered Comet v1.1 pools (BLNT/USDC, BLNT/XLM, BLND/USDC) and the BLNT backfill contract. --- ...-pools-and-blnt-backfill-page-the-sweep.md | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 lore/1-tasks/active/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md diff --git a/lore/1-tasks/active/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md b/lore/1-tasks/active/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md new file mode 100644 index 00000000..51afa0d3 --- /dev/null +++ b/lore/1-tasks/active/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md @@ -0,0 +1,87 @@ +--- +id: "0337" +title: "Three new Comet v1.1 pools (BLNT/USDC, BLNT/XLM, a small BLND/USDC) are missing from pool_registry and the BLNT backfill contract pages the sweep" +type: BUG +status: active +assignee: okarcz +related_adr: [] +related_tasks: ["0300", "0334", "0336", "0100", "0291"] +tags: [layer-backend, ingest, soroban, comet, observability, coverage, priority-high, effort-small] +links: + - "../../../packages/prices-ingest-core/src/static_pools.rs" + - "../../../packages/coverage-sweep-probe/allowlist.toml" + - "../../../docs/runbooks/0100-coverage-sweep-triage.md" + - "../../../docs/runbooks/seed-pool-registry.md" + - "https://github.com/BLNT-Protocol/blnt-backfill-contract/tree/fe366da68da7f24ef20063c43aaf0b13ea752257" +history: + - date: "2026-10-26" + status: active + who: okarcz + note: "Task created from the coverage-sweep-unclassified alarm of the 2026-10-26 05:17 UTC run" +--- + +# New Comet pools and the BLNT backfill contract page the sweep + +## Summary + +`prices-production-coverage-sweep-unclassified` went to ALARM at +2026-10-26 05:18 UTC. The Monday run found 4 contracts with 25 swap-shaped +events. The previous run (2026-10-23 10:54, after [[0336]]) found 0. + +Three of the four are Comet pools on wasm `d735c339…`, the CometDEX v1.1 +code [[0334]] registered for Blend's `CASYAKK3…`. Comet has no factory, so +only `STATIC_POOLS` registers a pool. The live processor drops their swaps: +`ledger-processor-unregistered-pool` flapped 00:35–01:56 UTC on 10-26, in +step with the BLNT swaps. The fourth contract is not a market. + +## Findings (StellarExpert public API, 2026-10-26) + +| Contract | What | Created | Swaps in window | +| --- | --- | --- | --- | +| `CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3` | Comet v1.1, BLNT/USDC | 2026-10-25 19:18 by `GDEA2WMO…` | 9 | +| `CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC` | Comet v1.1, BLNT/XLM | 2026-10-25 19:19, same creator | 9 | +| `CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ` | Comet v1.1, BLND/USDC | 2026-09-23 by `GCRKRXU2…` | 4 | +| `CANIE64BDFKLHROLOIY2KTQVBWUZNUKZP75DLXC272QM7TIN5N3M4LVA` | BLNT backfill contract (wasm `dcdc0b18…`) | 2026-10-25 19:30 by `GDEA2WMO…` | 3 (`swap_blnd`) | + +- **Tokens.** BLNT SAC `CCBNWP5L…` (classic `BLNT-GAP2ACGH…`, created + 2026-10-25), USDC SAC `CCW67TSZ…`, XLM SAC `CAS3J7GY…`, BLND SAC + `CD25MNVT…`. Payload is the same `POOL/swap` map as [[0334]]. +- **Prices are coherent.** ~0.0048 USDC/BLNT and ~0.036 XLM/BLNT (implies + XLM ≈ $0.135); the BLND pool ~0.0054 USDC/BLND, near the Blend pool. +- **BLNT swaps come from one contract** `CB2DT56G…` routing USDC → BLNT → + XLM, so each tx swaps in both BLNT pools. The BLND pool's 4 swaps + (~$0.10 in total) come from `CDYYYMHP…`. +- **`CANIE64B…` is not a market.** Verified source + (`BLNT-Protocol/blnt-backfill-contract`): it custodies BLNT v3's initial + allocation; `swap_blnd` is a fixed 1:1 BLND → BLNT conversion (burns + BLND, pays pre-funded BLNT). No admin, no upgrade. Pricing it would pin + BLNT to BLND. + +## Decision + +- Register all three pools as `comet`, the [[0334]] pattern. The BLND pool + barely trades, but the rule stays one rule: a Comet pool is registered. +- Allow-list `CANIE64B…` as a permanent `[[contract]]`. +- No wasm-hash auto-registration for now: three pools in one week come from + one launch. Revisit if Comet pools keep appearing. + +## Implementation Plan + +1. Add the three pools to `STATIC_POOLS` with a decode test on a real swap + of each. +2. Allow-list `CANIE64B…` in `coverage-sweep-probe`. +3. Deploy Compute (ledger-processor cold start loads the pools) and + EventBridge (the allow-list is compiled into the probe). +4. `events-backfill --discover-pools` WRITE. +5. Reprice from ledger 65,099,227 (first swap of `CCNDNFA7…`, 2026-10-23) + to the deploy. +6. Invoke the probe in a later clock hour than 05:17 UTC. + +## Acceptance Criteria + +- [ ] `STATIC_POOLS` holds the three pools; a test decodes one real swap of each +- [ ] `CANIE64B…` allow-listed with reason and task +- [ ] Compute and EventBridge deployed; the running Lambdas carry the change +- [ ] No `unregistered_pool_events` WARN for these pools after the deploy +- [ ] Registry rows written; range from ledger 65,099,227 repriced +- [ ] Manual probe run reads `unclassified = 0`; alarm back to OK From 7bc33d5162841e5f674038e03d603ef03a984557 Mon Sep 17 00:00:00 2001 From: karczuRF Date: Mon, 26 Oct 2026 09:25:55 +0100 Subject: [PATCH 116/144] fix(lore-0337): register three new Comet v1.1 pools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-10-26 coverage sweep found three Comet pools on the CometDEX v1.1 wasm (d735c339…) that only STATIC_POOLS can register: BLNT/USDC CDQMIKIH… and BLNT/XLM CA5HX3D4…, deployed for the BLNT launch on 2026-10-25, and a small BLND/USDC pool CCNDNFA7…. The live processor dropped their swaps and the unregistered-pool alarm flapped overnight. Their POOL/swap payload is the one comet_extractor already decodes. --- docs/runbooks/seed-pool-registry.md | 16 +- packages/events-backfill/src/discover.rs | 34 ++++- packages/events-backfill/src/run.rs | 2 +- .../src/soroban/comet_tests.rs | 139 ++++++++++++++++++ .../prices-ingest-core/src/static_pools.rs | 48 ++++-- 5 files changed, 217 insertions(+), 22 deletions(-) diff --git a/docs/runbooks/seed-pool-registry.md b/docs/runbooks/seed-pool-registry.md index 5b75a9d7..6ae9d42b 100644 --- a/docs/runbooks/seed-pool-registry.md +++ b/docs/runbooks/seed-pool-registry.md @@ -234,13 +234,18 @@ Confirm with the same `FINAL` count below, then drop `--dry-run` to write. Some pools have no factory event, so neither `learn_factory` nor this tool can find them. They are registered by a committed list, `STATIC_POOLS` in -`packages/prices-ingest-core/src/static_pools.rs`. Today it holds two pools, -both venue `comet`: +`packages/prices-ingest-core/src/static_pools.rs`. Today it holds five pools, +all venue `comet`: - Comet BLND/USDC `CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM`, Blend v1/v2's backstop pool (task 0300); - Comet v1.1 BLND/USDC `CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE`, - Blend v2.1's backstop pool, first swap 2026-10-07 (task 0334). + Blend v2.1's backstop pool, first swap 2026-10-07 (task 0334); +- Comet v1.1 BLNT/USDC `CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3` + and BLNT/XLM `CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC`, + deployed for the BLNT launch, first swaps 2026-10-25 (task 0337); +- Comet v1.1 BLND/USDC `CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ`, + a small pool, first swap 2026-10-23 (task 0337). - The live processor and the events-backfill reprice route it from the list, with or without a table row. @@ -255,8 +260,9 @@ both venue `comet`: Expected until written: a `change="new"` line per missing pool with `venue="comet"`, then `to_write=N per_venue={"comet": N}` (plus any - factory pools created in that ledger). On production after task 0334 that - is `CASYAKK3…` alone, `N = 1`: `CAS3FL6T…` has been in the table since 0300. + factory pools created in that ledger). On production after task 0337 that + is `CDQMIKIH…`, `CA5HX3D4…` and `CCNDNFA7…`, `N = 3`: the two Blend pools + have been in the table since 0300 and 0334. - sdex-backfill's end-of-run registry write also persists it — the identical row, idempotent under the table's `ReplacingMergeTree`. diff --git a/packages/events-backfill/src/discover.rs b/packages/events-backfill/src/discover.rs index 172d3999..e113e6fd 100644 --- a/packages/events-backfill/src/discover.rs +++ b/packages/events-backfill/src/discover.rs @@ -292,6 +292,9 @@ mod tests { const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; + const BLNT_USDC: &str = "CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3"; + const BLNT_XLM: &str = "CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC"; + const BLND_USDC_SMALL: &str = "CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ"; fn comet_row(id: &str) -> PoolRegistryRow { PoolRegistryRow { @@ -311,24 +314,40 @@ mod tests { let (reg, persisted) = preload(Registries::new()); assert_eq!( reg.pool_rows_unpersisted(&persisted), - vec![comet_row(COMET), comet_row(COMET_V1_1)] + vec![ + comet_row(BLNT_XLM), + comet_row(COMET), + comet_row(COMET_V1_1), + comet_row(BLND_USDC_SMALL), + comet_row(BLNT_USDC), + ] ); } - /// ...a table holding only the first writes only the second (task 0334, - /// production's state), and a table holding both writes nothing. + /// ...a table holding the two Blend pools writes only the three of task + /// 0337 (production's state), and a table holding all five writes nothing. #[test] fn a_static_pool_already_in_the_table_is_not_rewritten() { let mut loaded = Registries::new(); - loaded.load_pool_rows(&[comet_row(COMET)]); + loaded.load_pool_rows(&[comet_row(COMET), comet_row(COMET_V1_1)]); let (reg, persisted) = preload(loaded); assert_eq!( reg.pool_rows_unpersisted(&persisted), - vec![comet_row(COMET_V1_1)] + vec![ + comet_row(BLNT_XLM), + comet_row(BLND_USDC_SMALL), + comet_row(BLNT_USDC), + ] ); let mut loaded = Registries::new(); - loaded.load_pool_rows(&[comet_row(COMET), comet_row(COMET_V1_1)]); + loaded.load_pool_rows(&[ + comet_row(COMET), + comet_row(COMET_V1_1), + comet_row(BLNT_USDC), + comet_row(BLNT_XLM), + comet_row(BLND_USDC_SMALL), + ]); let (reg, persisted) = preload(loaded); assert!(reg.pool_rows_unpersisted(&persisted).is_empty()); } @@ -349,12 +368,15 @@ mod tests { assert_eq!( written, vec![ + (BLNT_XLM, "comet"), (COMET, "comet"), (COMET_V1_1, "comet"), ( "CAZ4Z273BBAAFL5NYNQJKEMZDQBRCPKAS4GOXDUFXPSE56M4ONBJUOVD", "soroswap" ), + (BLND_USDC_SMALL, "comet"), + (BLNT_USDC, "comet"), ] ); } diff --git a/packages/events-backfill/src/run.rs b/packages/events-backfill/src/run.rs index 7ac7d1fa..7c38ccd1 100644 --- a/packages/events-backfill/src/run.rs +++ b/packages/events-backfill/src/run.rs @@ -874,7 +874,7 @@ mod tests { loaded.venue.insert(AQUA.to_string(), Venue::Aquarius); let reg = reprice_registry(loaded).expect("a seeded table reprices"); - assert_eq!(reg.venue.len(), 3); + assert_eq!(reg.venue.len(), 1 + prices_ingest_core::STATIC_POOLS.len()); assert_eq!(reg.venue.get(AQUA), Some(&Venue::Aquarius)); assert_eq!(reg.venue.get(COMET), Some(&Venue::Comet)); assert_eq!(reg.venue.get(COMET_V1_1), Some(&Venue::Comet)); diff --git a/packages/prices-ingest-core/src/soroban/comet_tests.rs b/packages/prices-ingest-core/src/soroban/comet_tests.rs index 6ef9938a..954593df 100644 --- a/packages/prices-ingest-core/src/soroban/comet_tests.rs +++ b/packages/prices-ingest-core/src/soroban/comet_tests.rs @@ -623,3 +623,142 @@ fn the_static_list_routes_the_blend_v2_1_comet_pool() { assert!(out.unregistered_pool_events.is_empty()); assert!(out.dispatch_errors.is_empty()); } + +// ---- task 0337: three more factory-less Comet v1.1 pools ------------------- +// +// Fixtures from StellarExpert's copy of each event (`/contract//events`), +// decoded from its `bodyXdr`; ledger and tx application order are the event +// id's TOID (`ledger << 32 | tx << 12 | op`), event index its suffix. + +const BLNT_USDC: &str = "CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3"; +const BLNT_XLM: &str = "CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC"; +const BLND_USDC_SMALL: &str = "CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ"; +/// BLNT's issuer; its SAC is `CCBNWP5L…`. +const BLNT_ISSUER: &str = "GAP2ACGH3O6TRFAERGM4USJ6XWU3U2BFIK224JPV6NI3CUNEEEGTBTFB"; + +fn blnt() -> AssetIdentity { + AssetIdentity::Credit { + code: "BLNT".to_string(), + issuer: BLNT_ISSUER.to_string(), + } +} + +/// One `POOL/swap` of `pool`, its payload map built from the five fields. +fn v1_1_swap( + pool: &str, + (ledger, tx_index, event_index): (u32, u16, u32), + caller: &str, + (token_in, amount_in): (&str, &str), + (token_out, amount_out): (&str, &str), +) -> RawSorobanEvent { + RawSorobanEvent { + contract_id: pool.to_string(), + ..comet_event( + ledger, + tx_index, + event_index, + json!([{"type": "sym", "value": "POOL"}, {"type": "sym", "value": "swap"}]), + json!({"type": "map", "value": [{"key": {"type": "sym", "value": "caller"}, "value": {"type": "address", "value": caller}}, {"key": {"type": "sym", "value": "token_amount_in"}, "value": {"type": "i128", "value": amount_in}}, {"key": {"type": "sym", "value": "token_amount_out"}, "value": {"type": "i128", "value": amount_out}}, {"key": {"type": "sym", "value": "token_in"}, "value": {"type": "address", "value": token_in}}, {"key": {"type": "sym", "value": "token_out"}, "value": {"type": "address", "value": token_out}}]}), + ) + } +} + +const USDC_SAC: &str = "CCW67TSZV3SSS2HXMBQ5JFGCKJNXKZM7UQUWUZPUTHXSTZLEO7SJMI75"; +const BLND_SAC: &str = "CD25MNVTZDL4Y3XBCPCJXGXATV5WUHHOWMYFF4YBEGU5FCPGMYTVG5JY"; +const BLNT_SAC: &str = "CCBNWP5L7ZSXA3DBWCMFAVKFVRW3ZHWO7B5J557W42O64DDZNPJP7VDS"; +const XLM_SAC: &str = "CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA"; +/// The router behind every BLNT swap so far (USDC → BLNT → XLM in one tx). +const BLNT_ROUTER: &str = "CB2DT56GJPXZG7SXGPJOW3F3WNF6RQ7RT5HZLAD2XW4TJ7NUN2YWUZLW"; + +/// Event `279761757207027713-0001` — `CDQMIKIH…`, ledger 65,137,110, tx 262, +/// event 1. USDC → BLNT, 500000000 in / 103337956008 out. +fn blnt_usdc_swap() -> RawSorobanEvent { + v1_1_swap( + BLNT_USDC, + (65_137_110, 262, 1), + BLNT_ROUTER, + (USDC_SAC, "500000000"), + (BLNT_SAC, "103337956008"), + ) +} + +/// Event `279761757207027713-0005` — `CA5HX3D4…`, the same transaction's +/// second hop. BLNT → XLM, 103337956008 in / 3711330029 out. +fn blnt_xlm_swap() -> RawSorobanEvent { + v1_1_swap( + BLNT_XLM, + (65_137_110, 262, 5), + BLNT_ROUTER, + (BLNT_SAC, "103337956008"), + (XLM_SAC, "3711330029"), + ) +} + +/// Event `279599050962505729-0009` — `CCNDNFA7…`, ledger 65,099,227, tx 641, +/// event 9, the pool's first swap. BLND → USDC, 42983166 in / 231757 out. +fn blnd_usdc_small_swap() -> RawSorobanEvent { + v1_1_swap( + BLND_USDC_SMALL, + (65_099_227, 641, 9), + "CDYYYMHP3RKMPDY7VARR5J65RBLUFBHMARTHZM5NZ774D4CMOV7VHRR2", + (BLND_SAC, "42983166"), + (USDC_SAC, "231757"), + ) +} + +/// Run `event` from an empty registry, then from one with the static list +/// merged: unregistered before, exactly one `"comet"` tick after. +fn routed_by_the_static_list(event: RawSorobanEvent) -> TradeTick { + let mut assets = AssetRegistry::from_existing(vec![usdc(), blnd(), blnt()]); + let ledger = event.ledger_sequence; + let pool = event.contract_id.clone(); + + let out = run( + ledger, + std::slice::from_ref(&event), + &mut Registries::new(), + &mut assets, + ); + assert!(out.amm_ticks.is_empty()); + assert_eq!(out.unregistered_pool_events, vec![("comet", 1)]); + assert_eq!(out.unregistered_pool_contracts, vec![pool]); + + let mut reg = Registries::new(); + reg.merge_static_pools(); + let out = run(ledger, &[event], &mut reg, &mut assets); + assert!(out.unregistered_pool_events.is_empty()); + assert!(out.dispatch_errors.is_empty()); + assert!(out.unresolved.is_empty()); + assert_eq!(out.amm_ticks.len(), 1); + let (source, tick) = out.amm_ticks.into_iter().next().unwrap(); + assert_eq!(source, "comet"); + assert!(tick.price_forming); + tick +} + +#[test] +fn the_static_list_routes_the_blnt_usdc_pool() { + let tick = routed_by_the_static_list(blnt_usdc_swap()); + assert_eq!((&tick.base, &tick.quote), (&blnt(), &usdc())); + // USDC → BLNT: price = USDC in / BLNT out (≈ 0.00484). + assert_eq!(tick.price, d(500_000_000) / d(103_337_956_008)); + assert_eq!(tick.volume_base, d(103_337_956_008)); +} + +#[test] +fn the_static_list_routes_the_blnt_xlm_pool() { + let tick = routed_by_the_static_list(blnt_xlm_swap()); + assert_eq!((&tick.base, &tick.quote), (&blnt(), &AssetIdentity::Native)); + // BLNT → XLM: price = XLM out / BLNT in (≈ 0.0359). + assert_eq!(tick.price, d(3_711_330_029) / d(103_337_956_008)); + assert_eq!(tick.volume_base, d(103_337_956_008)); +} + +#[test] +fn the_static_list_routes_the_small_blnd_usdc_pool() { + let tick = routed_by_the_static_list(blnd_usdc_small_swap()); + assert_eq!((&tick.base, &tick.quote), (&blnd(), &usdc())); + // BLND → USDC: price = USDC out / BLND in (≈ 0.00539). + assert_eq!(tick.price, d(231_757) / d(42_983_166)); + assert_eq!(tick.volume_base, d(42_983_166)); +} diff --git a/packages/prices-ingest-core/src/static_pools.rs b/packages/prices-ingest-core/src/static_pools.rs index ed731ab8..7d856906 100644 --- a/packages/prices-ingest-core/src/static_pools.rs +++ b/packages/prices-ingest-core/src/static_pools.rs @@ -23,6 +23,13 @@ use crate::soroban::Registries; /// with wasm `d735c3395f59510172cf5cf838823a1389b97cfec6bf24e580e9bd77d2b3e687`, /// CometDEX release v1.1.0, which rejects self-swaps (task 0334). /// +/// - `CDQMIKIH…C7Y3` (BLNT/USDC) and `CA5HX3D4…3FLC` (BLNT/XLM) — Comet v1.1 +/// pools deployed 2026-10-25 by `GDEA2WMO…` for the BLNT token launch, same +/// wasm `d735c339…`. First swaps at ledger 65,137,110 (task 0337). +/// +/// - `CCNDNFA7…7QFZ` — a small Comet v1.1 BLND/USDC pool deployed 2026-09-23 +/// by `GCRKRXU2…`, same wasm. First swap at ledger 65,099,227 (task 0337). +/// /// Its sister `CB3A6LUP…` (same wasm as `CAS3FL6T…`) is deliberately absent: /// it has emitted no event, ever. pub const STATIC_POOLS: &[(&str, Venue)] = &[ @@ -34,6 +41,18 @@ pub const STATIC_POOLS: &[(&str, Venue)] = &[ "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE", Venue::Comet, ), + ( + "CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3", + Venue::Comet, + ), + ( + "CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC", + Venue::Comet, + ), + ( + "CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ", + Venue::Comet, + ), ]; impl Registries { @@ -71,13 +90,19 @@ mod tests { const COMET: &str = "CAS3FL6TLZKDGGSISDBWGGPXT3NRR4DYTZD7YOD3HMYO6LTJUVGRVEAM"; const COMET_V1_1: &str = "CASYAKK3SHDN6S2E6IUOBVRXT3KKPVQCVQO4GNBDTVE6XZQQMW4UMIHE"; + const BLNT_USDC: &str = "CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3"; + const BLNT_XLM: &str = "CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC"; + const BLND_USDC_SMALL: &str = "CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ"; + const ALL: [&str; 5] = [COMET, COMET_V1_1, BLNT_USDC, BLNT_XLM, BLND_USDC_SMALL]; #[test] - fn the_list_holds_exactly_the_two_comet_pools() { - assert_eq!( - STATIC_POOLS, - &[(COMET, Venue::Comet), (COMET_V1_1, Venue::Comet)] - ); + fn the_list_holds_exactly_the_five_comet_pools() { + let listed: Vec<(&str, Venue)> = STATIC_POOLS + .iter() + .map(|(id, v)| (*id, v.clone())) + .collect(); + let expected: Vec<(&str, Venue)> = ALL.iter().map(|id| (*id, Venue::Comet)).collect(); + assert_eq!(listed, expected); } #[test] @@ -94,13 +119,14 @@ mod tests { fn merging_routes_the_pools_and_touches_nothing_else() { let mut reg = Registries::new(); reg.merge_static_pools(); - assert_eq!(reg.venue.len(), 2); - assert_eq!(reg.venue.get(COMET), Some(&Venue::Comet)); - assert_eq!(reg.venue.get(COMET_V1_1), Some(&Venue::Comet)); + assert_eq!(reg.venue.len(), ALL.len()); + for id in ALL { + assert_eq!(reg.venue.get(id), Some(&Venue::Comet), "{id}"); + } assert_eq!(reg.pool_count(), 0, "no pair or phoenix entry"); reg.merge_static_pools(); - assert_eq!(reg.venue.len(), 2, "idempotent"); + assert_eq!(reg.venue.len(), ALL.len(), "idempotent"); } #[test] @@ -117,6 +143,8 @@ mod tests { pool_type: 0, wasm_hash: String::new(), }; - assert_eq!(rows, vec![row(COMET), row(COMET_V1_1)]); + let mut expected: Vec = ALL.iter().map(|id| row(id)).collect(); + expected.sort_by(|a, b| a.contract_id.cmp(&b.contract_id)); + assert_eq!(rows, expected); } } From 3c301b6501beafda68d8ee04ea78f4386710a1dc Mon Sep 17 00:00:00 2001 From: karczuRF Date: Mon, 26 Oct 2026 09:25:57 +0100 Subject: [PATCH 117/144] chore(lore-0337): allow-list the BLNT backfill contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CANIE64B… emits swap_blnd, a fixed 1:1 BLND -> BLNT conversion from the BLNT v3 allocation (verified source), not a market. --- packages/coverage-sweep-probe/allowlist.toml | 6 ++++++ packages/coverage-sweep-probe/src/allowlist.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/coverage-sweep-probe/allowlist.toml b/packages/coverage-sweep-probe/allowlist.toml index ed999768..55e7f8bd 100644 --- a/packages/coverage-sweep-probe/allowlist.toml +++ b/packages/coverage-sweep-probe/allowlist.toml @@ -203,6 +203,12 @@ wasm = "91d5ab7e075d533a9404373453a548bc21c3d2893d78f9782785985f396b4454" reason = "bonding-curve token launchpad (pad_created, launched, buy, bump_curve); its trade events price a launched token on its own curve, not a pool; first seen 2026-10-04 as a test launch (ANYCHAIN Test, CCRIN5F5…), 3 trades by its creator" task = "0323" +[[contract]] +id = "CANIE64BDFKLHROLOIY2KTQVBWUZNUKZP75DLXC272QM7TIN5N3M4LVA" +wasm = "dcdc0b1860799629c7c88804955dd7373cb6febaf069e2ba863e98ebbaa95c27" +reason = "BLNT v3 backfill contract (verified source BLNT-Protocol/blnt-backfill-contract); swap_blnd is a fixed 1:1 BLND -> BLNT conversion (burns BLND, pays pre-funded BLNT), not a market; no admin, no upgrade" +task = "0337" + # --- unindexed venue families (temporary, keyed by wasm) ---------------------- [[wasm]] diff --git a/packages/coverage-sweep-probe/src/allowlist.rs b/packages/coverage-sweep-probe/src/allowlist.rs index 9634eef0..cde2b1c5 100644 --- a/packages/coverage-sweep-probe/src/allowlist.rs +++ b/packages/coverage-sweep-probe/src/allowlist.rs @@ -194,7 +194,7 @@ mod tests { #[test] fn embedded_list_parses_with_the_seeded_entries() { let list = AllowList::embedded().expect("embedded allow-list parses"); - assert_eq!(list.contract.len(), 28, "{list:?}"); + assert_eq!(list.contract.len(), 29, "{list:?}"); assert_eq!(list.wasm.len(), 4, "{list:?}"); // Every family-wide entry is temporary and names the task that ends it // (SushiSwap V3, 0290; the Comet BLND/USDC pool, 0300), except the one From a667570fd5625abb1b3ae52238049f14d6ad2516 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:36:42 +0100 Subject: [PATCH 118/144] docs(lore-0338): open task for clearing the pool-coverage alarms --- ...ool-coverage-alarms-after-0334-and-0337.md | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md diff --git a/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md b/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md new file mode 100644 index 00000000..941f65f2 --- /dev/null +++ b/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md @@ -0,0 +1,137 @@ +--- +id: "0338" +title: "coverage-sweep-unclassified stays in ALARM: 0334 and 0337 never finished their registry and reprice steps, and three new emitters are unclassified" +type: BUG +status: active +assignee: akot +related_adr: [] +related_tasks: ["0334", "0337", "0336", "0242", "0100", "0200"] +tags: [layer-backend, ingest, soroban, comet, observability, coverage, priority-high, effort-small] +links: + - "../../../packages/coverage-sweep-probe/allowlist.toml" + - "../../../packages/prices-ingest-core/src/static_pools.rs" + - "../../../docs/runbooks/0100-coverage-sweep-triage.md" + - "../../../docs/runbooks/seed-pool-registry.md" + - "../../../docs/runbooks/events-sourced-amm-reprice.md" +history: + - date: "2026-11-02" + status: active + who: akot + note: "Task created from the coverage-sweep-unclassified alarm of the 2026-11-02 05:17 UTC run; one task for all of it" +--- + +# Clear the pool-coverage alarms after 0334 and 0337 + +## Summary + +`prices-production-coverage-sweep-unclassified` has been in ALARM since +2026-10-26 05:18 UTC. The 2026-11-02 run completed with 6 contracts and 594 +events. Three contracts are the Comet pools of [[0337]]: merged and deployed, +but without a `prices.pool_registry` row, which is the only thing the sweep +reads (`sweep.rs:156`). The other three are new emitters. + +The trades are no longer being dropped: the last `unregistered_pool_events` +WARN is 2026-10-26 01:40 UTC, and comet candles exist from the deploys on. What +is missing is the history. Neither [[0334]] nor [[0337]] has been repriced. + +## Findings (2026-11-02) + +### The sweep + +Window 65,040,990–65,262,168. `with_pool` is over all txs of the contract in +the window. + +| Contract | What | Wasm | Events | `with_pool` | Outcome | +| --- | --- | --- | --- | --- | --- | +| `CDQMIKIHKA5UY3FQJ2UDTLNX3GOPD3GPNZWSPTOBVOHEUMQJQXOPC7Y3` | Comet v1.1, BLNT/USDC | `d735c339…` | 299 | | Registry row | +| `CA5HX3D47PQZWC2E7VWXCKH4534P2NOJEOIXAWU47MI4QV2FQUYY3FLC` | Comet v1.1, BLNT/XLM | `d735c339…` | 281 | | Registry row | +| `CCNDNFA7LEVUJRYWTSWVAA2ESZAKFO6MYOPI6E5AZT3V2UBP2MNV7QFZ` | Comet v1.1, BLND/USDC | `d735c339…` | 6 | | Registry row | +| `CAF5RWQG5GSTEPD5XAKCUH2H77NWA4J6DNFT6TUP2NU6Q6VW3R6HCZYO` | Permissioned order book, grBENJI/USDC | `4d6abb0d…` | 4 | 0/50 | `[[contract]]` | +| `CC5QR5OOQMCPVSL3SJMH7QESCFINWUMX3427CT4NIDBS3FRVDHAM4NAJ` | Tokenised fund aLQD (Alloy Liquid Dollar) | `a85d6d53…` | 1 | 3/8 | `[[contract]]` | +| `CAZE5GVMH5332L6FEGVPPDLEPVYUFGQB7ZOCH2OKKBJCBYZQ5FGKZN7L` | Stableswap "Spreadless USD 3pool" | `ae835560…` | 3 | 0/7 | `[[contract]]` | + +- **Registry.** `prices.pool_registry FINAL` holds 2 `comet` rows, `CAS3FL6T…` + and `CASYAKK3…`. The three pools above are absent. +- **`CC5QR5OO…`.** Unverified source; interface read from the wasm: `issue`, + `redeem`, `nav`, `rebalance`, `trade`, `set_router`. Created 2026-10-26. Its one + `trade` wraps `SoroswapRouter/swap` (allow-listed `CAG5LRYQ…`) and + `SoroswapPair/swap` on registered `CAM7DY53…`. Indexing it would double-count. +- **`CAZE5GVM…`.** Unverified source. An AMM: `get_amp` = 100, `get_reserves`, + `swap_exact_in/out`, `deposit`, `withdraw`, own LP token `SPR-USD3`. USDT0, PYUSD + and USDC, about $29 of reserves, created 2026-10-29 by `GBAXG35E…`. Its three + swaps are 1 unit each. No extractor exists and it has `upgrade()`. +- **`CAF5RWQG…`.** The fourth instance of order-book wasm `4d6abb0d…`. Its + `get_config` returns `base_token` grBENJI (`CBEFW4IK…`) and `quote_token` USDC, + and the book has `whitelist`, `access_control` and admin-set phases and + reference price. Decoded trades are ~75,000 at 0.9999999–1.0000000. +- **The existing reason is wrong.** The three `4d6abb0d…` entries say + "instruments are u64 ids with no token addresses". The u64s are order ids. The + trade event carries no token address, but the book's config does. This also + rules out the `not_a_venue` that [[0336]]'s notes suggest: the code is a market. + +### The history that is missing + +- **[[0334]] was not repriced.** `CASYAKK3…` emitted 174 swaps on 2026-10-12–18 + and `source = 'comet'` has 1 trade on those days. Its 1m candles start at + 2026-10-19 11:02 UTC. BE holds 560 swaps for 2026-10-07–18. Comet candles of + 10-07–11 exist but come from the old pool `CAS3FL6T…`. +- **[[0337]] was not repriced.** The first BLNT candle is 2026-10-26 14:22 UTC; + the 18 BLNT swaps of 10-25 23:35–10-26 01:40 UTC have none. `CCNDNFA7…`'s swaps + from 10-23 18:58 UTC (ledger 65,099,227) are not distinguishable in BLND/USDC + candles and are assumed missing. +- **BLNT has two identities.** The first 10 comet trades (10-26 14:22–16:36 UTC) + sit on `asset_id` 14236029787030528679, `soroban`, contract `CCBNWP5L…`, created + at 14:22:57. From 18:42 UTC, when `sdex` first traded it, the trades sit on + 15676158627979534220, `classic` BLNT-`GAP2ACGH…`, `sac_address` `CCBNWP5L…`. + Same shape as [[0242]]; the cause is not confirmed here. + +### What does not constrain this + +- `prices-production-cleanup` is DISABLED ([[0200]]), so the 1m rows survive. +- `mv_reconcile_*` look back 7 days and BE holds the events, so there is no hard + deadline, but every minute repriced here is outside that window. + +## Decision + +- All of it lives in this task, with no separate venue task. +- **`CAZE5GVM…` is a `[[contract]]`, not a temporary `[[wasm]]`.** A `[[wasm]]` + needs `until` = a task that removes it, and none exists. The contract entry + covers this pool only, so the next pool of the family still pages and reopens + the question. It says plainly that this is a small unindexed AMM. +- `CAF5RWQG…` is a `[[contract]]`, and the reason on the three older order-book + entries is corrected in the same commit. It is no `not_a_venue`. +- `CC5QR5OO…` is a permanent `[[contract]]`. +- Removing the stale `until` entries (`003710b3…`, `95a8e001…`, `8abc2891…`) is + out of scope. + +## Implementation Plan + +1. Allow-list the three contracts and correct the order-book reason; fix what + the count-pinning tests report; `cargo test -p coverage-sweep-probe`. +2. Decide how the BLNT identity split is repaired, against [[0242]]: whether a + reprice writes the 10 trades under the classic id, and what happens to the + orphan rows, before anything below is run. +3. After merge, in this order: + 1. `make -C infra diff-production`, read in full; then + `make -C infra deploy-production-eventbridge`. + 2. A **new** `events-backfill` from develop. An older binary does not hold the + pools of 0337 and reports a false `to_write=0`. + 3. `--discover-pools --dry-run` (expect `to_write=3`, `change="new"`), WRITE, + dry run again (`to_write=0`). + 4. Reprice ledgers 64,809,271–65,150,000 (10-07 to 10-26 ~16:50 UTC), dry run + first. The end covers the orphan BLNT minutes, assuming step 2 keeps it. + 5. `preroll-live-gap.sql` over that range. Never `preroll-amm-reprice.sql`. + 6. Invoke the probe in a later clock hour than the last run; expect + `unclassified = 0` and the alarm in OK. +4. Close [[0334]], [[0337]] and this task. + +## Acceptance Criteria + +- [ ] `CAF5RWQG…`, `CC5QR5OO…` and `CAZE5GVM…` allow-listed with reason and task +- [ ] The reason on the three older `4d6abb0d…` entries is corrected +- [ ] EventBridge deployed; the running probe carries the change +- [ ] `prices.pool_registry FINAL` has `comet` rows for the three 0337 pools +- [ ] `source = 'comet'` candles cover 2026-10-12–18 and 10-25 23:35–10-26 01:40 UTC +- [ ] BLNT comet trades sit under one `asset_id`, or the split is handed to [[0242]] + with the reason recorded here +- [ ] Manual probe run reads `unclassified = 0`; alarm back to OK From 0fa88fe13fbad3271770fe702516ffc734951b06 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:45:39 +0100 Subject: [PATCH 119/144] chore(lore-0338): allow-list three sweep emitters and correct the order-book reason --- packages/coverage-sweep-probe/allowlist.toml | 26 ++++++++++++-- .../coverage-sweep-probe/src/allowlist.rs | 34 ++++++++++++++++++- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/coverage-sweep-probe/allowlist.toml b/packages/coverage-sweep-probe/allowlist.toml index 55e7f8bd..3ab6676c 100644 --- a/packages/coverage-sweep-probe/allowlist.toml +++ b/packages/coverage-sweep-probe/allowlist.toml @@ -176,21 +176,33 @@ task = "0100" [[contract]] id = "CA5YJ5H2HH6KUZBGQ7M7R2TX63LQKQVG57Y3HE23NVC5C6Q7BQFMSOI5" wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" -reason = "on-chain order book of tokenised instruments (NewBuyInitiatedTrade); instruments are u64 ids with no token addresses, not priceable here" +reason = "permissioned order book, grBENJI/USDC (whitelist, admin-set phases and reference price); its NewBuyInitiatedTrade carries order ids, no token address, and has no extractor" task = "0100" [[contract]] id = "CBOSXSEZXU7P66O6F3AQV65URR5XUVKMY6CAQ7FT7C5TQLGYNV7IG5I6" wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" -reason = "on-chain order book of tokenised instruments (NewSellInitiatedTrade); instruments are u64 ids with no token addresses, not priceable here" +reason = "permissioned order book, AMA1/XUSDT (whitelist, admin-set phases and reference price); its NewSellInitiatedTrade carries order ids, no token address, and has no extractor" task = "0100" [[contract]] id = "CD35HSTRTNL4O3BFXQL75XGAI6AOCWFFAEPR5OCBS7A7HVUDUZOVLCAB" wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" -reason = "on-chain order book of tokenised instruments (NewBuyInitiatedTrade), a further instance of the wasm above; instruments are u64 ids with no token addresses, not priceable here" +reason = "permissioned order book, AMA1/XUSDT, a further instance of the wasm above (whitelist, admin-set phases and reference price); its NewBuyInitiatedTrade carries order ids, no token address, and has no extractor" task = "0334" +[[contract]] +id = "CAF5RWQG5GSTEPD5XAKCUH2H77NWA4J6DNFT6TUP2NU6Q6VW3R6HCZYO" +wasm = "4d6abb0d5dfae9b2450c694e123f1f4f2e614bd109c6b383a03d36d49b2b6bf8" +reason = "permissioned order book, grBENJI/USDC, a further instance of the wasm above (whitelist, admin-set phases and reference price); its trade events carry order ids, no token address, and have no extractor; decoded trades ~75,000 at ~1.00; 0/50 txs hold a registered pool's swap" +task = "0338" + +[[contract]] +id = "CC5QR5OOQMCPVSL3SJMH7QESCFINWUMX3427CT4NIDBS3FRVDHAM4NAJ" +wasm = "a85d6d53dfe894ef6483aad0613359584e5a78629c159cf3dbf53d74395a0404" +reason = "tokenised managed fund aLQD (issue, redeem, nav, rebalance), not a market; its trade event wraps a swap through the Soroswap router, and indexing it double-counts; 3/8 txs hold a registered pool's swap, its one trade among them" +task = "0338" + [[contract]] id = "CBP76I2FRMUKYKIYYBKN3DH7TSWMFAJF2WTDC7Q2OHQFBIVWP7CAAI5Q" wasm = "76e60d025ee6c0322cb19f4846b9587ab172870802251de903e93545371678b6" @@ -209,6 +221,14 @@ wasm = "dcdc0b1860799629c7c88804955dd7373cb6febaf069e2ba863e98ebbaa95c27" reason = "BLNT v3 backfill contract (verified source BLNT-Protocol/blnt-backfill-contract); swap_blnd is a fixed 1:1 BLND -> BLNT conversion (burns BLND, pays pre-funded BLNT), not a market; no admin, no upgrade" task = "0337" +# --- unindexed pools (one contract, not their family) ------------------------- + +[[contract]] +id = "CAZE5GVMH5332L6FEGVPPDLEPVYUFGQB7ZOCH2OKKBJCBYZQ5FGKZN7L" +wasm = "ae835560f930e73b3ac8ce0ab2b26543b815eb6bb61b0d87b894f4dc8dc6e8df" +reason = "Spreadless USD 3pool, a real stableswap AMM (USDT0, PYUSD, USDC; amp 100) with no extractor; ~$29 of reserves and three 1-unit swaps at 2026-11-02; 0/7 txs hold a registered pool's swap; this pool only, so the next pool of the wasm still pages" +task = "0338" + # --- unindexed venue families (temporary, keyed by wasm) ---------------------- [[wasm]] diff --git a/packages/coverage-sweep-probe/src/allowlist.rs b/packages/coverage-sweep-probe/src/allowlist.rs index cde2b1c5..5bff2e74 100644 --- a/packages/coverage-sweep-probe/src/allowlist.rs +++ b/packages/coverage-sweep-probe/src/allowlist.rs @@ -194,7 +194,7 @@ mod tests { #[test] fn embedded_list_parses_with_the_seeded_entries() { let list = AllowList::embedded().expect("embedded allow-list parses"); - assert_eq!(list.contract.len(), 29, "{list:?}"); + assert_eq!(list.contract.len(), 32, "{list:?}"); assert_eq!(list.wasm.len(), 4, "{list:?}"); // Every family-wide entry is temporary and names the task that ends it // (SushiSwap V3, 0290; the Comet BLND/USDC pool, 0300), except the one @@ -229,6 +229,38 @@ mod tests { } } + #[test] + fn the_2026_11_02_emitters_are_allow_listed_by_contract() { + // The sweep of 2026-11-02 (task 0338). + let list = AllowList::embedded().unwrap(); + for id in [ + "CAF5RWQG5GSTEPD5XAKCUH2H77NWA4J6DNFT6TUP2NU6Q6VW3R6HCZYO", + "CC5QR5OOQMCPVSL3SJMH7QESCFINWUMX3427CT4NIDBS3FRVDHAM4NAJ", + "CAZE5GVMH5332L6FEGVPPDLEPVYUFGQB7ZOCH2OKKBJCBYZQ5FGKZN7L", + ] { + assert_eq!( + list.match_row(id, None), + Some(format!("contract:{id}")), + "{id}" + ); + } + } + + #[test] + fn the_next_spreadless_pool_still_pages() { + // Task 0338 lists one pool of an AMM we do not index, not its family: a + // new pool has to reach a human. + let list = AllowList::embedded().unwrap(); + let spreadless = "ae835560f930e73b3ac8ce0ab2b26543b815eb6bb61b0d87b894f4dc8dc6e8df"; + assert_eq!( + list.match_row( + "CCCRWH6Q3FNP3I2I57BDLM5AFAT7O6OF6GKQOC6SSJNDAVRZ57SPHGU2", + Some(spreadless) + ), + None + ); + } + #[test] fn comet_pool_family_is_allow_listed_until_0300() { // Phase-1 classification (task 0100): the Comet BLND/USDC pool is a real From 7ab7302bc2d263b20fab6fd69cc63616f37441c3 Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:45:46 +0100 Subject: [PATCH 120/144] docs(lore-0338): record the BLNT identity split and hand it to 0242 --- ...nted-as-a-second-identity-splits-volume.md | 2 ++ ...ool-coverage-alarms-after-0334-and-0337.md | 23 +++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md index 3a199d58..6f8e25d0 100644 --- a/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md +++ b/lore/1-tasks/active/0242_BUG_sac-minted-as-a-second-identity-splits-volume.md @@ -243,6 +243,8 @@ The six latent SACs have other causes. SUSHI and HYPE traded before SushiSwap wa - **2026-10-06, seed (runbook §3).** Stage B finished 202404. The seed ran at 10:29:11 UTC as `dev_shared`, straight from `seed-0242.sql` rather than through the staged variant of §3. There is therefore no `bak_0242_seed_stage`, and `bak_0242_seed` was created afterwards from the 29 rows with that `created_at`. - Check: `29 29 / 29 / 0`, with collisions at 0. No candle under any seeded id on any tier. - Comet's `pool_registry` row is present (1). +- **2026-11-02, a new split (found in [[0338]]).** BLNT's SAC `CCBNWP5L…` traded on Comet from 2026-10-26 14:22 UTC, before SDEX interned its classic at 18:42, so 10 trades sit on the `soroban` id 14236029787030528679. The "no contract row since BURNMKR" of the October window no longer holds. + - The reprice in 0338 rewrites those trades under the classic id, so the rows left on that id are copies. Add it to the residual cleanup. It does not fall under PC9, because the classic exists. ## Acceptance Criteria diff --git a/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md b/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md index 941f65f2..e3567264 100644 --- a/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md +++ b/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md @@ -83,7 +83,14 @@ the window. sit on `asset_id` 14236029787030528679, `soroban`, contract `CCBNWP5L…`, created at 14:22:57. From 18:42 UTC, when `sdex` first traded it, the trades sit on 15676158627979534220, `classic` BLNT-`GAP2ACGH…`, `sac_address` `CCBNWP5L…`. - Same shape as [[0242]]; the cause is not confirmed here. + This is the race in [[0242]]: the SAC traded before anything interned its + classic. PR #390 is still open, so live can do this again for any new SAC. +- **A reprice heals the trades, not the orphan id.** `events-backfill` builds its + registry from `prices.assets` (`run.rs:220`), and the classic row now carries + the SAC, so it writes these minutes under the classic id. The rows under + 14236029787030528679 and its `prices.assets` row stay. +- **The order books.** `CA5YJ5H2…` is grBENJI/USDC like `CAF5RWQG…`; `CBOSXSEZ…` + and `CD35HSTR…` are AMA1/XUSDT. ### What does not constrain this @@ -103,14 +110,16 @@ the window. - `CC5QR5OO…` is a permanent `[[contract]]`. - Removing the stale `until` entries (`003710b3…`, `95a8e001…`, `8abc2891…`) is out of scope. +- **The BLNT orphan id goes to [[0242]].** The reprice ends past 16:36 UTC so the + 10 trades reach the classic id. This task deletes nothing on production: after + the reprice the orphan rows are copies, and they join 0242's residual cleanup, + which is where every other SAC-keyed row is removed. ## Implementation Plan 1. Allow-list the three contracts and correct the order-book reason; fix what the count-pinning tests report; `cargo test -p coverage-sweep-probe`. -2. Decide how the BLNT identity split is repaired, against [[0242]]: whether a - reprice writes the 10 trades under the classic id, and what happens to the - orphan rows, before anything below is run. +2. Add BLNT's orphan id to [[0242]]'s residual list (done in its Heal log). 3. After merge, in this order: 1. `make -C infra diff-production`, read in full; then `make -C infra deploy-production-eventbridge`. @@ -119,7 +128,7 @@ the window. 3. `--discover-pools --dry-run` (expect `to_write=3`, `change="new"`), WRITE, dry run again (`to_write=0`). 4. Reprice ledgers 64,809,271–65,150,000 (10-07 to 10-26 ~16:50 UTC), dry run - first. The end covers the orphan BLNT minutes, assuming step 2 keeps it. + first. The end covers the orphan BLNT minutes. 5. `preroll-live-gap.sql` over that range. Never `preroll-amm-reprice.sql`. 6. Invoke the probe in a later clock hour than the last run; expect `unclassified = 0` and the alarm in OK. @@ -132,6 +141,6 @@ the window. - [ ] EventBridge deployed; the running probe carries the change - [ ] `prices.pool_registry FINAL` has `comet` rows for the three 0337 pools - [ ] `source = 'comet'` candles cover 2026-10-12–18 and 10-25 23:35–10-26 01:40 UTC -- [ ] BLNT comet trades sit under one `asset_id`, or the split is handed to [[0242]] - with the reason recorded here +- [ ] BLNT's 10 comet trades of 10-26 14:22–16:36 UTC exist under the classic id, + and the orphan id is on [[0242]]'s residual list - [ ] Manual probe run reads `unclassified = 0`; alarm back to OK From 7799bb32fce7195d1eb99cbfdc3389d1b8ad2e4c Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 2 Nov 2026 13:11:17 +0100 Subject: [PATCH 121/144] docs(lore-0338): correct the reprice plan to what versioning allows --- ...ool-coverage-alarms-after-0334-and-0337.md | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md b/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md index e3567264..4eb19afe 100644 --- a/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md +++ b/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md @@ -92,11 +92,26 @@ the window. - **The order books.** `CA5YJ5H2…` is grBENJI/USDC like `CAF5RWQG…`; `CBOSXSEZ…` and `CD35HSTR…` are AMA1/XUSDT. -### What does not constrain this +### What a reprice can and cannot recover + +`events-backfill` has no venue filter, and an enriched row sits at `version + 1`, +above what the reprice writes for the same ledger. So a reprice fills an empty +minute, and is discarded in a minute that already has an enriched row. That +protects the other venues' USD values, and it limits what comes back here. + +Of `CASYAKK3…`'s 578 swaps up to 2026-10-19, 302 sit in minutes with no comet row +and come back. Up to 286 sit in minutes that already hold an old-pool row: 282 on +10-07–11 and 4 on 10-19. All of 10-12–18 comes back. + +### Timing - `prices-production-cleanup` is DISABLED ([[0200]]), so the 1m rows survive. -- `mv_reconcile_*` look back 7 days and BE holds the events, so there is no hard - deadline, but every minute repriced here is outside that window. +- `mv_reconcile_*` look back 7 days, and the runbook wants them stopped when a + pre-roll overlaps that window. A range ending 2026-10-27 00:00 UTC stops + overlapping on 2026-11-03 00:00 UTC. +- The pre-roll's 1w and 1M rows for the buckets it cuts short lose on version, so + the reconcile MVs have to rebuild them: week of 10-26 and October are still in + their windows until about 2026-11-08. ## Decision @@ -114,6 +129,11 @@ the window. 10 trades reach the classic id. This task deletes nothing on production: after the reprice the orphan rows are copies, and they join 0242's residual cleanup, which is where every other SAC-keyed row is removed. +- **The reprice is plain, with no `DELETE` of comet rows.** Up to 286 swaps of + 2026-10-07–11 and 10-19 stay undercounted, which is recorded here as known. + Removing that residual needs comet `DELETE`s on `1m` and on the coarse tiers, + which is not worth the risk for one pool in five days. +- **Two ranges, not one**, minute-aligned, so 10-19–23 is not rewritten for nothing. ## Implementation Plan @@ -127,11 +147,15 @@ the window. pools of 0337 and reports a false `to_write=0`. 3. `--discover-pools --dry-run` (expect `to_write=3`, `change="new"`), WRITE, dry run again (`to_write=0`). - 4. Reprice ledgers 64,809,271–65,150,000 (10-07 to 10-26 ~16:50 UTC), dry run - first. The end covers the orphan BLNT minutes. - 5. `preroll-live-gap.sql` over that range. Never `preroll-amm-reprice.sql`. - 6. Invoke the probe in a later clock hour than the last run; expect + 4. Invoke the probe in a later clock hour than the last run; expect `unclassified = 0` and the alarm in OK. + 5. From 2026-11-03 00:15 UTC and before 2026-11-08: reprice ledgers + 64,809,264–65,025,000 and 65,099,220–65,150,000, dry run first. The second + range ends past the orphan BLNT minutes. + 6. `preroll-live-gap.sql` with `start_ts = '2026-10-07 00:00:00'`, + `end_ts = '2026-10-27 00:00:00'`. Never `preroll-amm-reprice.sql`. + 7. Check the repriced 1m rows got `close_usd`; if not, `coarse-repair + --table price_ohlcv_1m --start-month 202610 --end-month 202610 --skip-snapshot`. 4. Close [[0334]], [[0337]] and this task. ## Acceptance Criteria @@ -140,7 +164,8 @@ the window. - [ ] The reason on the three older `4d6abb0d…` entries is corrected - [ ] EventBridge deployed; the running probe carries the change - [ ] `prices.pool_registry FINAL` has `comet` rows for the three 0337 pools -- [ ] `source = 'comet'` candles cover 2026-10-12–18 and 10-25 23:35–10-26 01:40 UTC +- [ ] `source = 'comet'` candles cover 2026-10-12–18 and 10-25 23:35–10-26 01:40 UTC, + on `1m` and on `1h`; the residual of 10-07–11 is recorded with its measured size - [ ] BLNT's 10 comet trades of 10-26 14:22–16:36 UTC exist under the classic id, and the orphan id is on [[0242]]'s residual list - [ ] Manual probe run reads `unclassified = 0`; alarm back to OK From 0696ab96debd241e23400c1545cf53b6541ccf9a Mon Sep 17 00:00:00 2001 From: Adam <65679285+adamkoot@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:38:01 +0100 Subject: [PATCH 122/144] docs(lore-0338): close the pool-coverage work with 0334 and 0337 --- ...lend-v2-1-comet-pool-trades-are-dropped.md | 26 ++++++--- ...-pools-and-blnt-backfill-page-the-sweep.md | 25 ++++++--- ...ool-coverage-alarms-after-0334-and-0337.md | 53 ++++++++++++++++--- 3 files changed, 82 insertions(+), 22 deletions(-) rename lore/1-tasks/{active => archive}/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md (82%) rename lore/1-tasks/{active => archive}/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md (83%) rename lore/1-tasks/{active => archive}/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md (77%) diff --git a/lore/1-tasks/active/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md b/lore/1-tasks/archive/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md similarity index 82% rename from lore/1-tasks/active/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md rename to lore/1-tasks/archive/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md index f4d2326b..49887958 100644 --- a/lore/1-tasks/active/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md +++ b/lore/1-tasks/archive/0334_BUG_blend-v2-1-comet-pool-trades-are-dropped.md @@ -2,7 +2,7 @@ id: "0334" title: "Blend v2.1's Comet BLND/USDC pool is missing from pool_registry — every swap since 2026-10-07 is dropped and both pool alarms fire" type: BUG -status: active +status: completed assignee: akot related_adr: [] related_tasks: ["0300", "0291", "0100", "0323", "0329"] @@ -18,6 +18,11 @@ history: status: active who: akot note: "Task created from the unregistered-pool and coverage-sweep alarms; pool verified as Blend v2.1's" + - date: "2026-11-02" + status: completed + who: akot + note: > + Reprice finished under 0338, which has the result and the residual. --- # Blend v2.1's Comet BLND/USDC pool is missing from pool_registry @@ -75,16 +80,23 @@ The sweep's two other emitters are not priceable: ## Acceptance Criteria -- [ ] `STATIC_POOLS` holds `CASYAKK3…`; a test decodes one of its real swaps -- [ ] Both sweep emitters allow-listed -- [ ] Ledger-processor deployed; no `unregistered_pool_events` WARN after it -- [ ] `prices.pool_registry FINAL` has a `comet` row for `CASYAKK3…` -- [ ] Minutes from 2026-10-07 00:16 UTC repriced; BLND `comet` 1h candles +- [x] `STATIC_POOLS` holds `CASYAKK3…`; a test decodes one of its real swaps +- [x] Both sweep emitters allow-listed +- [x] Ledger-processor deployed; no `unregistered_pool_events` WARN after it +- [x] `prices.pool_registry FINAL` has a `comet` row for `CASYAKK3…` +- [x] Minutes from 2026-10-07 00:16 UTC repriced; BLND `comet` 1h candles exist for those hours -- [ ] Next coverage sweep clean; `coverage-sweep-unclassified` back to OK +- [x] Next coverage sweep clean; `coverage-sweep-unclassified` back to OK ## Notes - The old pool `CAS3FL6T…`, called frozen after the exploit, still emitted 1,084 swaps from 2026-09-30 to ~2026-10-11 and is indexed. Its prices were not checked here. + +## Result (2026-11-02) + +The reprice and pre-roll were done under [[0338]], over ledgers +64,809,264–65,025,000. 2026-10-12–18 matches BE's swap count per day. 93 swaps of +10-07–11 stay undercounted: they share a minute with an enriched row of the old +pool, which outranks the reprice. Comet 1h candles exist for all those hours. diff --git a/lore/1-tasks/active/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md b/lore/1-tasks/archive/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md similarity index 83% rename from lore/1-tasks/active/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md rename to lore/1-tasks/archive/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md index 51afa0d3..46538b7d 100644 --- a/lore/1-tasks/active/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md +++ b/lore/1-tasks/archive/0337_BUG_new-comet-pools-and-blnt-backfill-page-the-sweep.md @@ -2,7 +2,7 @@ id: "0337" title: "Three new Comet v1.1 pools (BLNT/USDC, BLNT/XLM, a small BLND/USDC) are missing from pool_registry and the BLNT backfill contract pages the sweep" type: BUG -status: active +status: completed assignee: okarcz related_adr: [] related_tasks: ["0300", "0334", "0336", "0100", "0291"] @@ -18,6 +18,11 @@ history: status: active who: okarcz note: "Task created from the coverage-sweep-unclassified alarm of the 2026-10-26 05:17 UTC run" + - date: "2026-11-02" + status: completed + who: akot + note: > + Registry rows and the reprice finished under 0338, which has the result. --- # New Comet pools and the BLNT backfill contract page the sweep @@ -79,9 +84,15 @@ step with the BLNT swaps. The fourth contract is not a market. ## Acceptance Criteria -- [ ] `STATIC_POOLS` holds the three pools; a test decodes one real swap of each -- [ ] `CANIE64B…` allow-listed with reason and task -- [ ] Compute and EventBridge deployed; the running Lambdas carry the change -- [ ] No `unregistered_pool_events` WARN for these pools after the deploy -- [ ] Registry rows written; range from ledger 65,099,227 repriced -- [ ] Manual probe run reads `unclassified = 0`; alarm back to OK +- [x] `STATIC_POOLS` holds the three pools; a test decodes one real swap of each +- [x] `CANIE64B…` allow-listed with reason and task +- [x] Compute and EventBridge deployed; the running Lambdas carry the change +- [x] No `unregistered_pool_events` WARN for these pools after the deploy +- [x] Registry rows written; range from ledger 65,099,227 repriced +- [x] Manual probe run reads `unclassified = 0`; alarm back to OK + +## Result (2026-11-02) + +The registry write, reprice, pre-roll and manual probe run were done under +[[0338]]: three rows written, ledgers 65,099,220–65,150,000 repriced, 22 dropped +swaps back in the candles, alarm OK at 12:35:41 UTC. diff --git a/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md b/lore/1-tasks/archive/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md similarity index 77% rename from lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md rename to lore/1-tasks/archive/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md index 4eb19afe..15e198bf 100644 --- a/lore/1-tasks/active/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md +++ b/lore/1-tasks/archive/0338_BUG_clear-pool-coverage-alarms-after-0334-and-0337.md @@ -2,7 +2,7 @@ id: "0338" title: "coverage-sweep-unclassified stays in ALARM: 0334 and 0337 never finished their registry and reprice steps, and three new emitters are unclassified" type: BUG -status: active +status: completed assignee: akot related_adr: [] related_tasks: ["0334", "0337", "0336", "0242", "0100", "0200"] @@ -18,6 +18,12 @@ history: status: active who: akot note: "Task created from the coverage-sweep-unclassified alarm of the 2026-11-02 05:17 UTC run; one task for all of it" + - date: "2026-11-02" + status: completed + who: akot + note: > + PR #409 merged. EventBridge deployed, three registry rows written, 0334 and + 0337 repriced and pre-rolled. Alarm OK at 12:35:41 UTC. Residual recorded. --- # Clear the pool-coverage alarms after 0334 and 0337 @@ -160,12 +166,43 @@ and come back. Up to 286 sit in minutes that already hold an old-pool row: 282 o ## Acceptance Criteria -- [ ] `CAF5RWQG…`, `CC5QR5OO…` and `CAZE5GVM…` allow-listed with reason and task -- [ ] The reason on the three older `4d6abb0d…` entries is corrected -- [ ] EventBridge deployed; the running probe carries the change -- [ ] `prices.pool_registry FINAL` has `comet` rows for the three 0337 pools -- [ ] `source = 'comet'` candles cover 2026-10-12–18 and 10-25 23:35–10-26 01:40 UTC, +- [x] `CAF5RWQG…`, `CC5QR5OO…` and `CAZE5GVM…` allow-listed with reason and task +- [x] The reason on the three older `4d6abb0d…` entries is corrected +- [x] EventBridge deployed; the running probe carries the change +- [x] `prices.pool_registry FINAL` has `comet` rows for the three 0337 pools +- [x] `source = 'comet'` candles cover 2026-10-12–18 and 10-25 23:35–10-26 01:40 UTC, on `1m` and on `1h`; the residual of 10-07–11 is recorded with its measured size -- [ ] BLNT's 10 comet trades of 10-26 14:22–16:36 UTC exist under the classic id, +- [x] BLNT's 10 comet trades of 10-26 14:22–16:36 UTC exist under the classic id, and the orphan id is on [[0242]]'s residual list -- [ ] Manual probe run reads `unclassified = 0`; alarm back to OK +- [x] Manual probe run reads `unclassified = 0`; alarm back to OK + +## Result (2026-11-02) + +- PR #409 merged as `380aed31`. EventBridge deployed; probe `LastModified` + 12:17:53 UTC, and the deployed `bootstrap` holds the three contracts. `cdk diff` + for that stack changed only `CoverageSweepProbeFunction` `.S3Key`. +- `--discover-pools` WRITE at 12:33:22 UTC: `written=3`, then `to_write=0`. + `comet` holds 5 rows. The binary was `3e0e521a…`, built with `cargo lambda + build --target x86_64-unknown-linux-musl`, since the workstation has no musl-gcc. +- Manual invoke, window 65,046,238–65,267,416: `rows=22`, `allowlisted=22`, + `unclassified=0`, `unresolved=0`. Alarm OK at 12:35:41 UTC. +- Reprice of 65,099,220–65,150,000 (18,717 rows), then 64,809,264–65,025,000 + (86,791 rows). Other venues kept their rows, trades and `close_usd` zero counts, + which confirms that an enriched row outranks the reprice. +- Comet gained 512 trades on 1m: 481 in the first range and 31 in the second. + 10-12–18 matches BE's swap count per day. +- **Residual: 93 swaps of `CASYAKK3…` on 10-07–11**, not the 286 estimated above: + a repriced row with a higher version did replace an old-pool row. The minutes + affected remain undercounted, as decided. +- 334 comet rows at `close_usd = 0` were enriched in about 90 s, before the pre-roll. +- `preroll-live-gap.sql` over 10-07 to 10-27 ran on 2026-11-02, **without `SYSTEM + STOP VIEW` and before the 11-03 start above**, which departs from runbook 0142. + Its reason, a half-refilled `1m`, did not apply: no partition was dropped. No + rollup or `mv-refresh` alarm fired. +- After it, comet agrees on 1m, 15m, 1h, 4h and 1d: 1,726 trades, $23,364.42. The + reconcile MVs finished 1w and 1M: each week equals its days, October is 2,630 + trades, $40,006.36. +- The pre-roll put 72 `sdex` rows at `close_usd = 0` (46, 16, 7, 3 on 15m, 1h, 4h, + 1d). The 13:30 UTC coarse sweep restored every count to its value before. +- BLNT's 10 trades now exist under the classic id. The orphan id's rows stay for + [[0242]]'s cleanup. #390 was merged at 13:28 UTC and is not deployed. From 6d7fe79a173c720e359bb4010468b6e3970b23d4 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:42:15 +0100 Subject: [PATCH 123/144] docs(lore-0339): add task to serve the portal at /api/prices-api/ The product portals gather under /api/ on the explorer's host (soroban-block-explorer task 0660, PR #699). The task carries the deploy order: sync, then the explorer, then Compute. --- ...FEATURE_portal-served-at-api-prices-api.md | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 lore/1-tasks/active/0339_FEATURE_portal-served-at-api-prices-api.md diff --git a/lore/1-tasks/active/0339_FEATURE_portal-served-at-api-prices-api.md b/lore/1-tasks/active/0339_FEATURE_portal-served-at-api-prices-api.md new file mode 100644 index 00000000..8a6c540b --- /dev/null +++ b/lore/1-tasks/active/0339_FEATURE_portal-served-at-api-prices-api.md @@ -0,0 +1,126 @@ +--- +id: "0339" +title: "Serve the portal at /api/prices-api/ on the explorer's host" +type: FEATURE +status: active +related_adr: [] +related_tasks: ["0326", "0194"] +tags: ["portal", "hosting", "effort-small"] +links: + - https://github.com/rumblefishdev/soroban-block-explorer/pull/699 +history: + - date: 2026-11-03 + status: active + who: stkrolikiewicz + note: > + Task created. The product portals gather under /api/ on the explorer's + host (team decision): /api/prices-api/ now, /api/x402-facilitator/ and + /api/sorobanscan/ later, /api/ a hub. Explorer side: + soroban-block-explorer task 0660, PR #699. +--- + +# Serve the portal at /api/prices-api/ on the explorer's host + +## Summary + +The portal moves from `https://sorobanscan.rumblefish.dev/prices-api/` to +`/api/prices-api/`, under the hub at `/api/` that will also list the other +product portals. The explorer (task 0660, PR #699) serves +`/api/prices-api(/*)` from the bucket prefix `api/prices-api/` and answers +`/prices-api…`, `/pricing-api…` and every other `/api/…` with one `301` to the +same path under `/api/prices-api`, query string kept. Same kind of move as +[[0326]]. + +## Status: Active + +**Current state:** code and docs on the branch; nothing deployed. Next: the +deploy order below, explorer PR #699 first. + +## Context + +The bundle bakes its prefix in (Vite `base`, router `basename`), the sync puts +it under the same key prefix in the explorer's bucket (CloudFront hands S3 the +viewer path unchanged), and the backend's OAuth callback lands the popup on +`/prices-api/`. All three move. On the API's own host nothing +moves: `/api/auth/*`, `/api/key`, `/api/usage`, `/api/config`, the session +cookie's `Path=/api/`, the Discord `redirect_uri`, `portalWebOrigin`. + +## Implementation Plan + +1. `web/portal/src/base-path.ts` + `vite.config.mts`: `BASE_PATH` → + `/api/prices-api/`; specs that pin bundle paths follow. +2. `packages/prices-api/src/portal/auth/mod.rs`: `PORTAL_HOME` → + `/api/prices-api/`; every test asserting a landing follows. +3. `infra/Makefile` `sync-portal-explorer`: upload to `api/prices-api/`, + invalidate `/api/prices-api/*`. +4. Current-state docs (README, `docs/scf/api-endpoints.md`, overview, + runbook, skill) name the new URL. Dated SCF evidence stays as submitted. + +### Deploy order (production, manual) + +1. Here: `make -C infra sync-portal-explorer` — the new bundle under + `api/prices-api/`. Nothing serves it until step 2; the old `prices-api/` + keeps serving. +2. Explorer PR #699: Delivery (CDK) deploy, then the explorer SPA. +3. Here: Compute with the new `PORTAL_HOME`. **Never before step 2:** until + then `/api/prices-api/?signin=…` would `301` to `/prices-api/prices-api/`. + Until step 3 the old home `/prices-api/?signin=…` reaches the portal + through the `301`. + +Then: + +- The old `prices-api/` bucket prefix is dead: drop it, like explorer task + 0631 does for `api/` and `pricing-api/`. Any delete of the old `api/` + prefix must now exclude `api/prices-api/`, or it takes the live portal. +- A person completes a Discord sign-in round-trip on production. + +## Acceptance Criteria + +- [ ] Portal boots at `/api/prices-api/`, `/api/prices-api/dashboard`, + `/api/prices-api/docs` (hard refresh included) +- [ ] Discord sign-in round-trip lands on `/api/prices-api/?…` (after + Compute; until then via the `301`), done by a person on production +- [ ] `/prices-api/?utm_source=stellar-skill` → one `301` to + `/api/prices-api/?utm_source=stellar-skill` +- [ ] Old `prices-api/` prefix dropped from the bucket once the above holds + +## Design Decisions + +### From Plan + +1. **Backend `/api/` untouched.** Only `PORTAL_HOME`, the landing, moves. + The API host and the explorer host now both have an `/api/`, but they are + different hosts and nothing routes between them; the session cookie is + host-only, so its `Path=/api/` never reaches the explorer's host. ~110 + Location literals in `packages/prices-api/tests/portal_*.rs` follow + `PORTAL_HOME`. + +### Emerged + +2. **`PORTAL_HOME` on the API host is no longer "nothing".** A relative + landing (no `PORTAL_WEB_ORIGIN`) on the API host would now fall into the + backend's own `/api/` namespace instead of the gateway's `404`. Production + always sets the origin; the doc comment says so instead of claiming a 404. +3. **Dev proxy spec pins the real bundle paths.** `dev-proxy.spec.ts` listed + `/api/login`, `/api/@vite/client`… from the 0194 layout; it now lists + `/api/prices-api/…`, the paths Vite actually serves. The enumerated + backend regex is load-bearing again: the bundle shares `/api/`. +4. **Dated SCF evidence left as submitted** (`docs/scf/milestone-*`): the + `301` keeps their links working, as in [[0326]]. +5. **Task created on the branch, not on `develop`.** Asked for explicitly + (nothing pushed straight to `develop`), against the usual rule in + `lore/CLAUDE.md`. + +## Implementation Notes + +Checked locally on 2026-11-03: `cargo test -p prices-api` (lib 271 + every +`portal_*` suite green), `cargo clippy -p prices-api --all-targets -D +warnings`, `cargo fmt --all --check`, `nx run-many -t lint typecheck test` +on `portal` and the CDK project. `make -C infra build-portal-explorer`: every +asset URL in `dist/index.html` is under `/api/prices-api/`; `vite preview` +boots `/api/prices-api/`, `/api/prices-api/docs` and `/api/prices-api/dashboard`. + +## Notes + +- The usage plan names (`pricing-api--`) and the SSM parameter + `/prices//pricing-api-free-plan-id` are AWS resource names, not URLs. From b6fd158fea6b172111d87dcc268be79b0c3a2237 Mon Sep 17 00:00:00 2001 From: stkrolikiewicz <46401055+stkrolikiewicz@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:42:25 +0100 Subject: [PATCH 124/144] feat(lore-0339): serve the portal at /api/prices-api/ The product portals gather under /api/ on the explorer's host: the Prices portal now, x402-facilitator and sorobanscan later, /api/ itself a hub (soroban-block-explorer task 0660, PR #699). Base path, router basename, OAuth landing (PORTAL_HOME) and the sync prefix move to /api/prices-api/. The backend's /api/ on the API host, the session cookie's Path=/api/, the Discord redirect URI and portalWebOrigin stay: that is another host. Deploy only after explorer PR #699 (order in the task); Compute must not go before the explorer's Delivery deploy. --- README.md | 2 +- docs/prices-api-general-overview.md | 2 +- docs/runbooks/portal-oauth-deploy-prep.md | 5 +- docs/scf/api-endpoints.md | 64 +++++++------ infra/Makefile | 10 +-- infra/src/lib/app.ts | 2 +- infra/src/lib/stacks/api-gateway-stack.ts | 23 ++--- infra/src/lib/types.ts | 13 +-- packages/prices-api/src/config.rs | 2 +- packages/prices-api/src/portal/auth/issue.rs | 2 +- packages/prices-api/src/portal/auth/mod.rs | 32 ++++--- .../prices-api/src/portal/auth/state_token.rs | 2 +- packages/prices-api/src/portal/mod.rs | 14 +-- packages/prices-api/tests/portal_auth.rs | 90 ++++++++++--------- packages/prices-api/tests/portal_issue.rs | 70 ++++++++------- packages/prices-api/tests/portal_keys_logs.rs | 2 +- packages/prices-api/tests/portal_lazy_load.rs | 9 +- packages/prices-api/tests/portal_rework.rs | 60 ++++++------- skills/stellar-prices-api/SKILL.md | 4 +- web/portal/index.html | 8 +- web/portal/src/api-origin.ts | 13 +-- web/portal/src/api/portal.ts | 12 +-- web/portal/src/app/app.tsx | 20 ++--- web/portal/src/base-path.spec.ts | 6 +- web/portal/src/base-path.ts | 9 +- web/portal/src/dev-proxy.spec.ts | 22 +++-- web/portal/src/landing/Chrome.tsx | 2 +- web/portal/src/landing/LoginSection.tsx | 2 +- web/portal/src/landing/links.spec.ts | 4 +- web/portal/src/landing/links.ts | 24 ++--- web/portal/src/landing/oauthPopup.ts | 2 +- web/portal/src/main.spec.tsx | 2 +- web/portal/src/main.tsx | 4 +- web/portal/vite.config.mts | 19 ++-- 34 files changed, 299 insertions(+), 258 deletions(-) diff --git a/README.md b/README.md index a1774abb..ec333b7f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ keys. - **API:** `https://prices-api.sorobanscan.rumblefish.dev` (key-gated `/v1/*`; the OpenAPI document is public at `/api-docs-json`) -- **Portal (key sign-up):** `https://sorobanscan.rumblefish.dev/prices-api/` +- **Portal (key sign-up):** `https://sorobanscan.rumblefish.dev/api/prices-api/` - **Endpoint reference:** [`docs/scf/api-endpoints.md`](docs/scf/api-endpoints.md) - **Architecture overview:** [`docs/prices-api-general-overview.md`](docs/prices-api-general-overview.md) diff --git a/docs/prices-api-general-overview.md b/docs/prices-api-general-overview.md index b27e1694..03a749b9 100644 --- a/docs/prices-api-general-overview.md +++ b/docs/prices-api-general-overview.md @@ -170,7 +170,7 @@ to their own infrastructure at any time if needed. | **EventBridge Scheduler** | Scheduled triggers | Cron/rate rules for all periodic Lambda workers | | **Secrets Manager** | Credentials & mTLS material | Per-env client `{cert,key,ca}` for Caddy:443 mTLS (single JSON bundle secret per identity, named by `MTLS_SECRET_NAME`); Soroswap/Aquarius API keys; oracle contract address | | **CloudWatch + X-Ray** | Observability | API latency, error rates, ingestion lag, Lambda duration/concurrency, backfill progress; mTLS cert NotAfter alarm | -| **S3** (API docs) | Documentation hosting | self-service onboarding portal + API reference, served from the block explorer's bucket and CloudFront distribution at `sorobanscan.rumblefish.dev/prices-api/`; the OpenAPI document is served by the API itself | +| **S3** (API docs) | Documentation hosting | self-service onboarding portal + API reference, served from the block explorer's bucket and CloudFront distribution at `sorobanscan.rumblefish.dev/api/prices-api/`; the OpenAPI document is served by the API itself | **Components no longer in the Prices API budget** (eliminated by ADR 0007): diff --git a/docs/runbooks/portal-oauth-deploy-prep.md b/docs/runbooks/portal-oauth-deploy-prep.md index 1b808501..d74ff5ad 100644 --- a/docs/runbooks/portal-oauth-deploy-prep.md +++ b/docs/runbooks/portal-oauth-deploy-prep.md @@ -75,8 +75,9 @@ In the [Discord Developer Portal](https://discord.com/developers/applications): `` is **the host the backend answers on**, not the host the page is served from — since task 0194 those are two hostnames: `prices-api.sorobanscan.rumblefish.dev` runs the callback and sets the - session cookie, `sorobanscan.rumblefish.dev/prices-api/` is the page it sends the - visitor back to (`PORTAL_WEB_ORIGIN`). Register the former. In production: + session cookie, `sorobanscan.rumblefish.dev/api/prices-api/` is the page it + sends the visitor back to (`PORTAL_WEB_ORIGIN`). Register the former. In + production: ``` https://prices-api.sorobanscan.rumblefish.dev/api/auth/callback diff --git a/docs/scf/api-endpoints.md b/docs/scf/api-endpoints.md index 702d9c08..4932165b 100644 --- a/docs/scf/api-endpoints.md +++ b/docs/scf/api-endpoints.md @@ -93,21 +93,23 @@ missing or wrong `x-api-key` is a different answer and stays `403 {"message": ## Portal hosting — two hosts, no distribution of ours (tasks 0194, 0195) -**Portal:** `https://sorobanscan.rumblefish.dev/prices-api/` -**API reference:** `https://sorobanscan.rumblefish.dev/prices-api/docs` +**Portal:** `https://sorobanscan.rumblefish.dev/api/prices-api/` +**API reference:** `https://sorobanscan.rumblefish.dev/api/prices-api/docs` -Until task 0326 (2026-10) the portal lived at `/api/`, and for a few hours -on 2026-10-02 at `/pricing-api/` before the name settled; the explorer -answers every `/api…` and `/pricing-api…` URL with a `301` to the same path -under `/prices-api`, query string kept, so links shared before the move still -land on it. +Until task 0326 (2026-10) the portal lived at `/api/`, for a few hours on +2026-10-02 at `/pricing-api/` before the name settled, then at `/prices-api/` +until task 0339 (2026-11). The explorer (its task 0660) answers every +`/prices-api…`, `/pricing-api…` and other `/api/…` URL with one `301` to the +same path under `/api/prices-api`, query string kept, so links shared before +any of the moves still land on it. `/api/` itself is the explorer's hub of +product portals. The portal is a page on one host that calls an API on another: -| What | Where | Owned by | -| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | -| the page (bundle) | `s3://production-soroban-explorer-api-spa/prices-api/`, behaviour `/prices-api/*` of the block explorer's distribution `EA2TLS5SS5M87` (alias `sorobanscan.rumblefish.dev`) | `soroban-block-explorer` (their stack); this repo syncs the bundle with `make -C infra sync-portal-explorer` | -| the backend + spec | `https://prices-api.sorobanscan.rumblefish.dev` — the custom domain above, no CloudFront in front of it | `ApiGatewayStack` here | +| What | Where | Owned by | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| the page (bundle) | `s3://production-soroban-explorer-api-spa/api/prices-api/`, behaviour `/api/prices-api/*` of the block explorer's distribution `EA2TLS5SS5M87` (alias `sorobanscan.rumblefish.dev`) | `soroban-block-explorer` (their stack); this repo syncs the bundle with `make -C infra sync-portal-explorer` | +| the backend + spec | `https://prices-api.sorobanscan.rumblefish.dev` — the custom domain above, no CloudFront in front of it | `ApiGatewayStack` here | Decided 2026-08-31 (task 0194, "decision A"). The bundle is built with `VITE_PORTAL_API_ORIGIN` set to the API hostname, so every backend call is @@ -129,23 +131,27 @@ now answers `403`. ### The path convention -The bundle lives under **`/prices-api/`** on the explorer's host -(`/prices-api/login`, `/prices-api/docs` are pages); the backend under +The bundle lives under **`/api/prices-api/`** on the explorer's host +(`/api/prices-api/login`, `/api/prices-api/docs` are pages); the backend under **`/api/`** on the API's own host (`/api/auth/login`, `/api/api-docs-json`). -On the explorer's host the root belongs to the block explorer, so nothing of -ours may assume it. +On the explorer's host the root and `/api/` belong to the block explorer, so +nothing of ours may assume them. Both hosts have an `/api/`, and nothing +routes between them: the session cookie is host-only, and the bundle calls +the API host by name. From 2026-08-31 to task 0326 both shared one prefix, `/api/`, which read on the explorer's host as the explorer's own API; the move gave the bundle a -name of its own and left the backend where it was. The shared prefix +name of its own and left the backend where it was. Task 0339 put the bundle +back under `/api/`, now as one named portal among the product portals the +explorer's `/api/` hub lists. The shared prefix replaced task 0161's `/*` + `/api/*` convention on 2026-08-31, which for an app called "api" produced `/api/api/…` (task 0235's record of the three days that layout lived). The rule that decides which side of the split is enumerated: -- **the bundle's paths are a short, fixed list** — `/prices-api/`, - `/prices-api/index.html`, `/prices-api/favicon.ico`, - `/prices-api/assets/*` and the app's routes — and +- **the bundle's paths are a short, fixed list** — `/api/prices-api/`, + `/api/prices-api/index.html`, `/api/prices-api/favicon.ico`, + `/api/prices-api/assets/*` and the app's routes — and - **the backend is the open-ended side** (five slices added routes; none touched infrastructure), so it gets the catch-all. @@ -156,12 +162,13 @@ error in a browser (`web/portal/src/api/portal.ts` names the URL when it happens). On the explorer's host the split needs no routing table at all: their -`/prices-api/*` behaviour is S3, and their viewer-request function rewrites -**every path whose last segment has no `.`** to `/prices-api/index.html`. -That is what makes a hard refresh on `/prices-api/dashboard` — or on -`/prices-api/docs` — boot the portal, and it is also why the API reference is -a **route of the portal** rather than a static `docs/` folder in the bundle: a -folder would have been reachable only as `/prices-api/docs/index.html`. +`/api/prices-api/*` behaviour is S3, and their viewer-request function +rewrites **every path whose last segment has no `.`** to +`/api/prices-api/index.html`. That is what makes a hard refresh on +`/api/prices-api/dashboard` — or on `/api/prices-api/docs` — boot the portal, +and it is also why the API reference is a **route of the portal** rather than +a static `docs/` folder in the bundle: a folder would have been reachable only +as `/api/prices-api/docs/index.html`. Adding a page to the app therefore needs nothing on the hosting side; adding a backend route needs nothing either, because the gateway maps `/api/{proxy+}` and the bundle calls the API host directly. @@ -193,7 +200,7 @@ curl -sS https://prices-api.sorobanscan.rumblefish.dev/api-docs-json \ The same bytes are served at `/api/api-docs-json` (the portal's alias, task 0194), and rendered as the portal's API reference at -`https://sorobanscan.rumblefish.dev/prices-api/docs` (task 0195) — Swagger UI's +`https://sorobanscan.rumblefish.dev/api/prices-api/docs` (task 0195) — Swagger UI's shape (tags, collapsible operations, parameters, responses, schemas) in the portal's own design system, reading the live document on every visit. @@ -250,8 +257,9 @@ npm run openapi:extract # → target/openapi.json, servers stamped from config sends a request until the data routes answer CORS (task 0126). - **Onboarding portal** — open (`PORTAL_ENABLED=true` since task 0194) and public since 2026-09-23 (task 0305) at - `https://sorobanscan.rumblefish.dev/prices-api/` (`/api/` until task - 0326): the block explorer's basic auth is off for it. The move from the + `https://sorobanscan.rumblefish.dev/api/prices-api/` (`/api/` until task + 0326, `/prices-api/` until task 0339): the block explorer's basic auth is + off for it. The move from the test guild to the Stellar guild (task 0179) still precedes advertising the URL. - **CORS on `/v1`** (task 0126) — no browser can call the data routes yet; diff --git a/infra/Makefile b/infra/Makefile index 4c936edd..d9ae22d0 100644 --- a/infra/Makefile +++ b/infra/Makefile @@ -131,11 +131,11 @@ destroy-production-apigateway: build-production npx cdk --app "$(PRODUCTION_APP)" destroy Prices-production-ApiGateway --force # The portal (task 0194; the only deploy path since task 0195). -# `https://sorobanscan.rumblefish.dev/prices-api/` is served by the block explorer's +# `https://sorobanscan.rumblefish.dev/api/prices-api/` is served by the block explorer's # distribution (EA2TLS5SS5M87) from ITS bucket, which nothing in this repo # deploys — this sync is the only way a build reaches it. Built with the API's # own hostname baked in (`VITE_PORTAL_API_ORIGIN`, see -# `web/portal/src/api-origin.ts`): on that host `/prices-api/*` is a static SPA, so +# `web/portal/src/api-origin.ts`): on that host `/api/prices-api/*` is a static SPA, so # the bundle must call the backend on `apiDomain.domainName` rather than # relatively. `--skip-nx-cache` because the variable is not an Nx input — a # cached build would hand back whichever variant ran last, and the relative @@ -193,13 +193,13 @@ verify-portal-guild: cd .. && node tools/scripts/verify-discord-guild.mjs --ssm --env production sync-portal-explorer: build-portal-explorer verify-portal-guild - aws s3 sync ../web/portal/dist s3://$(EXPLORER_PORTAL_BUCKET)/prices-api/ \ + aws s3 sync ../web/portal/dist s3://$(EXPLORER_PORTAL_BUCKET)/api/prices-api/ \ --exclude '*' --include 'assets/*' \ --cache-control 'public, max-age=31536000, immutable' - aws s3 sync ../web/portal/dist s3://$(EXPLORER_PORTAL_BUCKET)/prices-api/ \ + aws s3 sync ../web/portal/dist s3://$(EXPLORER_PORTAL_BUCKET)/api/prices-api/ \ --exclude 'assets/*' \ --cache-control 'public, max-age=0, must-revalidate' - aws cloudfront create-invalidation --distribution-id $(EXPLORER_DISTRIBUTION_ID) --paths '/prices-api/*' + aws cloudfront create-invalidation --distribution-id $(EXPLORER_DISTRIBUTION_ID) --paths '/api/prices-api/*' deploy-production-eventbridge: build-production build-lambdas npx cdk --app "$(PRODUCTION_APP)" deploy Prices-production-EventBridge --exclusively --require-approval broadening diff --git a/infra/src/lib/app.ts b/infra/src/lib/app.ts index f2c9918d..27dca7a7 100644 --- a/infra/src/lib/app.ts +++ b/infra/src/lib/app.ts @@ -45,7 +45,7 @@ export function createApp({ config }: CreateAppOptions): void { // private bucket and a CloudFront distribution fronting both the bundle // and this API — was retired by task 0195 on 2026-09-01: since task 0194 // the page is served from the block explorer's distribution at - // `https://sorobanscan.rumblefish.dev/prices-api/` (its bucket, synced by + // `https://sorobanscan.rumblefish.dev/api/prices-api/` (its bucket, synced by // `make -C infra sync-portal-explorer`) and calls this API on // `config.apiDomain` directly, so the distribution had become a second, // ungated front door to the same portal. Constructing `ApiGatewayStack` is diff --git a/infra/src/lib/stacks/api-gateway-stack.ts b/infra/src/lib/stacks/api-gateway-stack.ts index cc4ca08c..abf9ec0e 100644 --- a/infra/src/lib/stacks/api-gateway-stack.ts +++ b/infra/src/lib/stacks/api-gateway-stack.ts @@ -554,9 +554,9 @@ export class ApiGatewayStack extends cdk.Stack { // below — declared outside the `cacheEnabled` branch, because with the stage // cache off is exactly when an unbounded anonymous route costs the most. // `/api/{proxy+}` directly under the root: the bundle's paths share the - // prefix but never reach the gateway — the page lives on another host - // (task 0194) and only its backend calls come here, on the API's own - // hostname. The OpenAPI alias `/api/api-docs-json` rides on this same + // prefix (`/api/prices-api/`, task 0339) but never reach the gateway — the + // page lives on another host (task 0194) and only its backend calls come + // here, on the API's own hostname. The OpenAPI alias `/api/api-docs-json` rides on this same // proxy. const portalProxy = this.api.root .addResource('api') @@ -568,14 +568,15 @@ export class ApiGatewayStack extends cdk.Stack { // The preflight for the bundle's cross-origin calls (task 0194). // // The bundle is served from `config.portalWebOrigin` — another - // application's distribution, whose `/api/*` behaviour is a static SPA - // that rewrites every extensionless path to its `index.html`. A - // same-origin `fetch('/api/config')` from there gets the bundle back as - // `200 text/html`, so the bundle calls this API on `config.apiDomain` - // instead, and a browser asks before a non-simple request crosses an - // origin: the revoke's marker header (`PORTAL_REQUEST_HEADER`) is one, and - // so is any `DELETE`. Answered here by a MOCK integration rather than by - // the Lambda — no invocation, and the `max-age` has to be set by whoever + // application's distribution, whose `/api/*` behaviour is a `301` into a + // static SPA (`/api/prices-api/*`) that rewrites every extensionless path + // to its `index.html`. A same-origin `fetch('/api/config')` from there + // gets the bundle back as `200 text/html`, so the bundle calls this API on + // `config.apiDomain` instead, and a browser asks before a non-simple + // request crosses an origin: the revoke's marker header + // (`PORTAL_REQUEST_HEADER`) is one, and so is any `DELETE`. Answered here + // by a MOCK integration rather than by the Lambda — no invocation, and + // the `max-age` has to be set by whoever // answers the preflight, which is the gateway — with the handler adding // `Access-Control-Allow-Origin` on the actual responses (`portal::apply` // in the Rust crate names the same one origin). diff --git a/infra/src/lib/types.ts b/infra/src/lib/types.ts index 914bee14..c16ac5c7 100644 --- a/infra/src/lib/types.ts +++ b/infra/src/lib/types.ts @@ -170,13 +170,14 @@ export interface EnvironmentConfig { * not a resource under it. * * Why the portal needs it at all: the bundle is served from another - * application's distribution (`portalWebOrigin`), whose `/prices-api/*` + * application's distribution (`portalWebOrigin`), whose `/api/prices-api/*` * behaviour is a static SPA — every extensionless path under it is rewritten - * to `/prices-api/index.html` at the edge and answered `200 text/html`, and - * `/api/config` there is a `301` into it (task 0326). There is nothing on - * that host for a same-origin call to reach, so the bundle calls this - * hostname directly, cross-origin and same-site — the pattern the explorer's - * own SPA uses for its API. + * to `/api/prices-api/index.html` at the edge and answered `200 text/html`, + * and `/api/config` there is a `301` into it (explorer task 0660; that host's + * `/api/` is not this API's). There is nothing on that host for a + * same-origin call to reach, so the bundle calls this hostname directly, + * cross-origin and same-site — the pattern the explorer's own SPA uses for + * its API. * * `hostedZoneName` must be a suffix of `domainName`, and the zone must live * in this account: the certificate's validation record and the alias records diff --git a/packages/prices-api/src/config.rs b/packages/prices-api/src/config.rs index cc1770ed..5f020b86 100644 --- a/packages/prices-api/src/config.rs +++ b/packages/prices-api/src/config.rs @@ -581,7 +581,7 @@ mod web_origin_tests { Some(" "), Some("/"), Some("sorobanscan.rumblefish.dev"), - Some("https://sorobanscan.rumblefish.dev/prices-api/"), + Some("https://sorobanscan.rumblefish.dev/api/prices-api/"), Some("https://sorobanscan.rumblefish.dev?x=1"), Some("https://"), ] { diff --git a/packages/prices-api/src/portal/auth/issue.rs b/packages/prices-api/src/portal/auth/issue.rs index dfd3ebf1..9f30e890 100644 --- a/packages/prices-api/src/portal/auth/issue.rs +++ b/packages/prices-api/src/portal/auth/issue.rs @@ -696,7 +696,7 @@ mod tests { ]; for (i, a) in fixed.iter().enumerate() { assert!(a.starts_with("?issue=")); - assert!(format!("{PORTAL_HOME}{a}").starts_with("/prices-api/?")); + assert!(format!("{PORTAL_HOME}{a}").starts_with("/api/prices-api/?")); for b in &fixed[i + 1..] { assert_ne!(a, b); } diff --git a/packages/prices-api/src/portal/auth/mod.rs b/packages/prices-api/src/portal/auth/mod.rs index 09484f68..6f094c84 100644 --- a/packages/prices-api/src/portal/auth/mod.rs +++ b/packages/prices-api/src/portal/auth/mod.rs @@ -95,7 +95,7 @@ pub const LOGOUT_PATH: &str = "/api/auth/logout"; /// origin is the one that handles OAuth. The portal has several pages now and /// this did not change: the page it lands on decides where to go next; this /// handler still will not. -const PORTAL_HOME: &str = "/prices-api/"; +const PORTAL_HOME: &str = "/api/prices-api/"; /// Appended to [`PORTAL_HOME`] when the visitor declined at Discord's consent /// screen, so [0185]'s page can say "sign-in cancelled" instead of silently @@ -215,10 +215,13 @@ impl AuthState { /// The callback runs on this backend's host, where the session cookie is /// set; the page the visitor is sent back to lives on /// `AppConfig::portal_web_origin`. A relative `Location` would keep them on - /// the API host, where `/prices-api/` is nothing — the gateway's own - /// `404`. `None` keeps the bare path, which is the same-origin deployment - /// and what every existing test sees. A builder, like [`Self::with_issue`], - /// so every constructor and test stays valid. + /// the API host, where `/api/prices-api/` is not the page: it falls under + /// this backend's own [`PORTAL_API_PREFIX`](super::PORTAL_API_PREFIX) and + /// gets the router's empty `404` (task 0339; the explorer host's `/api/` + /// and this one share a name, not a route). `None` keeps the bare path, + /// which is the same-origin deployment and what every existing test sees. + /// A builder, like [`Self::with_issue`], so every constructor and test + /// stays valid. pub fn with_web_origin(mut self, origin: Option<&str>) -> Self { if let Some(origin) = origin { self.home = format!("{origin}{PORTAL_HOME}").into(); @@ -1232,7 +1235,10 @@ mod tests { vec!["a=1; Path=/".into(), "b=2; Path=/".into()], ); assert_eq!(response.status(), StatusCode::SEE_OTHER); - assert_eq!(response.headers().get(LOCATION).unwrap(), "/prices-api/"); + assert_eq!( + response.headers().get(LOCATION).unwrap(), + "/api/prices-api/" + ); assert_eq!(response.headers().get_all(SET_COOKIE).iter().count(), 2); assert_eq!(response.headers().get("cache-control").unwrap(), "no-store"); } @@ -1323,7 +1329,7 @@ mod tests { assert_ne!(CANCELLED_QUERY, FAILED_QUERY); assert_eq!(ERROR_ACCESS_DENIED, "access_denied"); for query in [CANCELLED_QUERY, FAILED_QUERY] { - assert!(format!("{PORTAL_HOME}{query}").starts_with("/prices-api/?")); + assert!(format!("{PORTAL_HOME}{query}").starts_with("/api/prices-api/?")); } } @@ -1333,7 +1339,7 @@ mod tests { /// delete this to do it. #[test] fn the_only_redirect_targets_are_the_portal_itself() { - assert_eq!(PORTAL_HOME, "/prices-api/"); + assert_eq!(PORTAL_HOME, "/api/prices-api/"); assert!(PORTAL_HOME.starts_with('/')); assert!(!PORTAL_HOME.starts_with("//")); for query in [ @@ -1353,7 +1359,7 @@ mod tests { &issue::too_young_query(173), &issue::capped_query("2026-09-01"), ] { - assert!(format!("{PORTAL_HOME}{query}").starts_with("/prices-api/?")); + assert!(format!("{PORTAL_HOME}{query}").starts_with("/api/prices-api/?")); } } @@ -1363,19 +1369,19 @@ mod tests { #[test] fn the_landing_is_prefixed_with_the_configured_origin_and_nothing_else() { let bare = AuthState::new(None, discord::Endpoints::default()); - assert_eq!(&*bare.home, "/prices-api/"); + assert_eq!(&*bare.home, "/api/prices-api/"); let kept = AuthState::new(None, discord::Endpoints::default()).with_web_origin(None); - assert_eq!(&*kept.home, "/prices-api/"); + assert_eq!(&*kept.home, "/api/prices-api/"); let hosted = AuthState::new(None, discord::Endpoints::default()) .with_web_origin(Some("https://sorobanscan.example")); - assert_eq!(&*hosted.home, "https://sorobanscan.example/prices-api/"); + assert_eq!(&*hosted.home, "https://sorobanscan.example/api/prices-api/"); let response = redirect(&format!("{}{CANCELLED_QUERY}", hosted.home), vec![]); assert_eq!( response.headers()[LOCATION], - "https://sorobanscan.example/prices-api/?signin=cancelled" + "https://sorobanscan.example/api/prices-api/?signin=cancelled" ); } diff --git a/packages/prices-api/src/portal/auth/state_token.rs b/packages/prices-api/src/portal/auth/state_token.rs index b232cb5c..fa144683 100644 --- a/packages/prices-api/src/portal/auth/state_token.rs +++ b/packages/prices-api/src/portal/auth/state_token.rs @@ -277,7 +277,7 @@ pub fn accept( /// /// ⚠️ **This authorises nothing and must never be used to.** It chooses /// between two fixed literals, both failures, both reachable by anyone who -/// types `/prices-api/?issue=failed` into a link — so a forged claim buys an +/// types `/api/prices-api/?issue=failed` into a link — so a forged claim buys an /// attacker nothing they did not already have. Everything a claim could /// matter for goes through [`accept`], which verifies both halves before /// reading a byte of either. `None` on anything that is not a token of ours diff --git a/packages/prices-api/src/portal/mod.rs b/packages/prices-api/src/portal/mod.rs index ba173c6b..157b700f 100644 --- a/packages/prices-api/src/portal/mod.rs +++ b/packages/prices-api/src/portal/mod.rs @@ -69,12 +69,14 @@ use crate::config::AppConfig; /// /// `/api/` is the portal's backend on the API's own hostname. The bundle shared /// the prefix from task 0194 (2026-08-31) until task 0326 moved it to -/// `sorobanscan.rumblefish.dev/prices-api/` (the explorer now answers `301` on -/// `/api…` there). Nothing routes between them either way, because they are on -/// different HOSTS — the bundle is served from the block explorer's -/// distribution, and the page calls this backend cross-origin and same-site. So -/// from in here the prefix is simply ours end to end; a bundle path that -/// arrives (it should not) is a plain `404`, the same as any unrouted path. +/// `sorobanscan.rumblefish.dev/prices-api/`, and shares it again since task +/// 0339 at `sorobanscan.rumblefish.dev/api/prices-api/` (the explorer answers +/// `301` there on every other `/api/…`, `/prices-api…` and `/pricing-api…`). +/// Nothing routes between them either way, because they are on different +/// HOSTS — the bundle is served from the block explorer's distribution, and +/// the page calls this backend cross-origin and same-site. So from in here the +/// prefix is simply ours end to end; a bundle path that arrives (it should +/// not) is a plain `404`, the same as any unrouted path. /// /// Replaces [0161]'s `/*` + `/api/*` convention, which produced /// `/api/api/…` for an app that is itself called "api". The OAuth redirect URI diff --git a/packages/prices-api/tests/portal_auth.rs b/packages/prices-api/tests/portal_auth.rs index 7b685660..295fcc09 100644 --- a/packages/prices-api/tests/portal_auth.rs +++ b/packages/prices-api/tests/portal_auth.rs @@ -462,7 +462,7 @@ async fn login_lands_an_unwired_issue_round_trip_on_failed() { let open = signed_in_app(true); let refused = fetch(&open, &format!("{LOGIN_PATH}?action=issue"), &[]).await; assert_eq!(refused.status, StatusCode::SEE_OTHER); - assert_eq!(refused.location(), "/prices-api/?issue=failed"); + assert_eq!(refused.location(), "/api/prices-api/?issue=failed"); assert!(refused.set_cookies().is_empty()); // And not a body the browser would render as text. assert!(refused.body.is_empty(), "{:?}", refused.body); @@ -493,11 +493,11 @@ async fn neither_arm_503s_on_a_deployment_with_no_credentials() { let signin = fetch(&router, LOGIN_PATH, &[]).await; assert_eq!(signin.status, StatusCode::SEE_OTHER); - assert_eq!(signin.location(), "/prices-api/?signin=not_open"); + assert_eq!(signin.location(), "/api/prices-api/?signin=not_open"); let issue = fetch(&router, &format!("{LOGIN_PATH}?action=issue"), &[]).await; assert_eq!(issue.status, StatusCode::SEE_OTHER); - assert_eq!(issue.location(), "/prices-api/?issue=failed"); + assert_eq!(issue.location(), "/api/prices-api/?issue=failed"); } // --------------------------------------------------------------------------- @@ -524,7 +524,7 @@ async fn a_complete_round_trip_signs_the_visitor_in() { // real deployment has — is covered by the `app_with_keys` tests below. assert_eq!( reply.location(), - "/prices-api/", + "/api/prices-api/", "the callback must land on the portal, and on a literal" ); @@ -655,7 +655,7 @@ async fn a_first_sign_in_issues_a_key_and_lands_on_the_welcome() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!(gateway.with(|s| s.create_calls), 1); assert_eq!(gateway.with(|s| s.named(&key_name()).len()), 1); @@ -673,7 +673,7 @@ async fn a_returning_sign_in_adopts_the_key_and_lands_plain() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/"); + assert_eq!(reply.location(), "/api/prices-api/"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!( gateway.with(|s| s.create_calls), @@ -704,7 +704,7 @@ async fn a_refused_sign_in_leaves_an_existing_session_alone() { let reply = sign_in_against(&app_against(&mock)).await; - assert_eq!(reply.location(), "/prices-api/?signin=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?signin=not_member"); // No session is issued … assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); // … and none is cleared either: the only `Set-Cookie` is the pending @@ -736,7 +736,7 @@ async fn an_unreadable_age_parameter_signs_the_visitor_in_without_a_key() { let reply = sign_in_against(&app_with_keys_and(&discord, &gateway, settings)).await; - assert_eq!(reply.location(), "/prices-api/"); + assert_eq!(reply.location(), "/api/prices-api/"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!(discord.member_calls(), 1, "membership is still proved"); assert_eq!( @@ -761,7 +761,7 @@ async fn an_unreadable_guild_parameter_still_refuses_sign_in() { let reply = sign_in_against(&app_with_keys_and(&discord, &gateway, settings)).await; - assert_eq!(reply.location(), "/prices-api/?signin=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?signin=unknown"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!(discord.member_calls(), 0); } @@ -780,7 +780,7 @@ async fn a_control_plane_failure_at_sign_in_lands_plain_with_a_session() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/"); + assert_eq!(reply.location(), "/api/prices-api/"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -807,7 +807,7 @@ async fn a_too_young_account_signs_in_but_gets_no_key() { assert!( reply .location() - .starts_with("/prices-api/?issue=too_young&wait_secs="), + .starts_with("/api/prices-api/?issue=too_young&wait_secs="), "{}", reply.location() ); @@ -839,7 +839,7 @@ async fn a_revoked_account_signs_in_and_lands_plain() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/"); + assert_eq!(reply.location(), "/api/prices-api/"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -859,7 +859,7 @@ async fn a_non_member_gets_no_key_either() { let reply = sign_in_against(&app_with_keys(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?signin=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?signin=not_member"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!(gateway.with(|s| s.list_calls), 0); assert_eq!(gateway.with(|s| s.create_calls), 0); @@ -896,7 +896,7 @@ async fn a_non_member_cannot_sign_in() { let reply = sign_in_against(&open).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?signin=not_member"); assert!( reply.cookie(cookies::SESSION_COOKIE).is_none(), "a refused sign-in must not leave a session behind" @@ -927,7 +927,7 @@ async fn a_member_still_in_screening_is_refused_as_pending_rules() { let reply = sign_in_against(&app_against(&mock)).await; - assert_eq!(reply.location(), "/prices-api/?signin=pending_rules"); + assert_eq!(reply.location(), "/api/prices-api/?signin=pending_rules"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert!(reply.clears(cookies::PENDING_COOKIE)); } @@ -955,7 +955,7 @@ async fn an_unanswerable_membership_question_refuses_without_accusing() { let reply = sign_in_against(&app_against(&mock)).await; - assert_eq!(reply.location(), "/prices-api/?signin=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?signin=unknown"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); } } @@ -996,7 +996,7 @@ async fn a_deployment_with_no_eligibility_settings_refuses_sign_in() { let reply = sign_in_against(&unwired).await; - assert_eq!(reply.location(), "/prices-api/?signin=not_open"); + assert_eq!(reply.location(), "/api/prices-api/?signin=not_open"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!( mock.member_calls(), @@ -1114,7 +1114,7 @@ async fn a_mismatched_state_is_rejected_and_issues_no_session() { // tab. What the refusal is made of has not changed — no session, no // exchange, the pending cookie untouched — only what the visitor is shown. assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); // And it leaves the browser's pending login alone — see // `an_unverifiable_callback_cannot_cancel_someone_elses_sign_in`. @@ -1135,7 +1135,7 @@ async fn a_callback_with_no_pending_cookie_is_rejected() { let reply = fetch(&open, &format!("{CALLBACK_PATH}?code=c&state={state}"), &[]).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!(mock.exchanges(), 0); } @@ -1159,7 +1159,7 @@ async fn replaying_a_callback_url_after_the_cookie_is_cleared_is_rejected() { // The replay: same URL, and the browser no longer holds the cookie. let second = fetch(&open, &uri, &[]).await; assert_eq!(second.status, StatusCode::SEE_OTHER); - assert_eq!(second.location(), "/prices-api/?signin=failed"); + assert_eq!(second.location(), "/api/prices-api/?signin=failed"); assert!(second.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!( mock.exchanges(), @@ -1187,7 +1187,7 @@ async fn a_forged_state_signed_with_another_key_is_rejected() { ) .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert_eq!(mock.exchanges(), 0); } @@ -1211,7 +1211,7 @@ async fn a_self_signed_pair_is_rejected() { ) .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert_eq!(mock.exchanges(), 0); } @@ -1239,7 +1239,7 @@ async fn a_cancelled_sign_in_returns_to_the_portal_saying_so() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=cancelled"); + assert_eq!(reply.location(), "/api/prices-api/?signin=cancelled"); assert!(reply.clears(cookies::PENDING_COOKIE)); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); } @@ -1345,7 +1345,7 @@ async fn a_verified_callback_always_drops_the_pending_cookie() { ) .await; assert_eq!(upstream.status, StatusCode::SEE_OTHER); - assert_eq!(upstream.location(), "/prices-api/?signin=failed"); + assert_eq!(upstream.location(), "/api/prices-api/?signin=failed"); assert!(upstream.clears(cookies::PENDING_COOKIE)); } @@ -1398,18 +1398,18 @@ async fn only_access_denied_is_reported_as_a_cancellation() { let open = signed_in_app(true); for (error, expected) in [ - ("access_denied", "/prices-api/?signin=cancelled"), - ("invalid_scope", "/prices-api/?signin=failed"), - ("server_error", "/prices-api/?signin=failed"), - ("temporarily_unavailable", "/prices-api/?signin=failed"), - ("invalid_request", "/prices-api/?signin=failed"), - ("unauthorized_client", "/prices-api/?signin=failed"), + ("access_denied", "/api/prices-api/?signin=cancelled"), + ("invalid_scope", "/api/prices-api/?signin=failed"), + ("server_error", "/api/prices-api/?signin=failed"), + ("temporarily_unavailable", "/api/prices-api/?signin=failed"), + ("invalid_request", "/api/prices-api/?signin=failed"), + ("unauthorized_client", "/api/prices-api/?signin=failed"), // Not in RFC 6749 §4.1.2.1 at all. Anything unrecognised is a failure, // not a cancellation — defaulting the other way is how a new Discord // error code would silently become "cancelled". ( "something_discord_invented_later", - "/prices-api/?signin=failed", + "/api/prices-api/?signin=failed", ), ] { let started = start_login(&open).await; @@ -1452,7 +1452,7 @@ async fn the_error_value_never_reaches_the_redirect_target() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(!reply.location().contains("evil")); assert!(reply.headers.get("x-injected").is_none()); } @@ -1527,7 +1527,11 @@ async fn a_grant_that_is_not_exactly_the_two_scopes_is_refused() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER, "scope={drifted}"); - assert_eq!(reply.location(), "/prices-api/?signin=failed", "{drifted}"); + assert_eq!( + reply.location(), + "/api/prices-api/?signin=failed", + "{drifted}" + ); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none(), "{drifted}"); } @@ -1542,7 +1546,7 @@ async fn a_grant_that_is_not_exactly_the_two_scopes_is_refused() { ) .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_ne!(reply.location(), "/prices-api/?signin=failed"); + assert_ne!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); } @@ -1570,7 +1574,7 @@ async fn a_failed_token_exchange_lands_with_no_upstream_detail() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); let body = String::from_utf8(reply.body.clone()).unwrap(); assert!(!body.contains("upstream said no"), "{body}"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); @@ -1733,7 +1737,7 @@ async fn an_open_portal_whose_load_fails_lands_on_a_retryable_failure() { let login = fetch(&router, LOGIN_PATH, &[]).await; assert_eq!(login.status, StatusCode::SEE_OTHER); - assert_eq!(login.location(), "/prices-api/?signin=failed"); + assert_eq!(login.location(), "/api/prices-api/?signin=failed"); // `/auth/me` cannot check a cookie without the signing key, and "nobody // is signed in" would be false for a visitor who is: a `503` the page @@ -1762,7 +1766,7 @@ async fn an_open_portal_loaded_without_credentials_lands_on_the_closed_card() { let login = fetch(&router, LOGIN_PATH, &[]).await; assert_eq!(login.status, StatusCode::SEE_OTHER); - assert_eq!(login.location(), "/prices-api/?signin=not_open"); + assert_eq!(login.location(), "/api/prices-api/?signin=not_open"); // No credentials means no sessions: a truthful "signed out". let me = fetch(&router, ME_PATH, &[]).await; @@ -1850,7 +1854,7 @@ async fn an_issue_callback_on_a_failed_load_lands_on_issue_failed() { for sent in [with_cookie, without] { let reply = fetch(&failing, &callback, sent).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert!( reply.headers.get(header::SET_COOKIE).is_none(), "an unverified callback must leave the pending cookie alone" @@ -1878,7 +1882,7 @@ async fn a_sign_in_callback_on_a_failed_load_lands_on_signin_failed() { ) .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.headers.get(header::SET_COOKIE).is_none()); assert_eq!(mock.exchanges(), 0); } @@ -1904,7 +1908,7 @@ async fn a_callback_whose_load_fits_the_allowance_still_exchanges_the_code() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_ne!(reply.location(), "/prices-api/?signin=failed"); + assert_ne!(reply.location(), "/api/prices-api/?signin=failed"); assert_eq!(mock.exchanges(), 1, "the token exchange must run"); } @@ -1931,7 +1935,7 @@ async fn a_callback_that_spent_its_allowance_loading_lands_on_signin_failed() { .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.clears(cookies::PENDING_COOKIE)); assert!(reply.cookie(cookies::SESSION_COOKIE).is_none()); assert_eq!(mock.exchanges(), 0, "the token exchange must not start"); @@ -1957,7 +1961,7 @@ async fn an_issue_callback_that_spent_its_allowance_loading_lands_on_issue_faile .await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert!(reply.clears(cookies::PENDING_COOKIE)); assert_eq!(mock.exchanges(), 0, "the token exchange must not start"); } @@ -2021,7 +2025,7 @@ async fn a_round_trip_lands_on_the_configured_web_origin() { assert_eq!(reply.status, StatusCode::SEE_OTHER); assert_eq!( reply.location(), - "https://sorobanscan.example/prices-api/?signin=cancelled" + "https://sorobanscan.example/api/prices-api/?signin=cancelled" ); // The cookie work is unchanged by the host: still dropped on this host. assert!(reply.set_cookies().iter().any(|c| c.contains("Max-Age=0"))); diff --git a/packages/prices-api/tests/portal_issue.rs b/packages/prices-api/tests/portal_issue.rs index 10ade41f..08b8be6c 100644 --- a/packages/prices-api/tests/portal_issue.rs +++ b/packages/prices-api/tests/portal_issue.rs @@ -126,7 +126,7 @@ async fn a_member_in_good_standing_gets_a_key_and_lands_on_issue_ok() { let reply = issue_round_trip(&app).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert!( reply.cookie(cookies::SESSION_COOKIE).is_some(), "the round-trip proves identity, so it signs the visitor in" @@ -171,11 +171,11 @@ async fn a_second_round_trip_returns_the_same_key() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!( @@ -205,13 +205,13 @@ async fn two_users_get_two_different_keys() { issue_round_trip(&issue_app(&mine, &gateway)) .await .location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!( issue_round_trip(&issue_app(&theirs, &gateway)) .await .location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let names: Vec = gateway.with(|s| s.keys.iter().map(|k| k.name.clone()).collect()); @@ -229,8 +229,8 @@ async fn two_simultaneous_round_trips_leave_exactly_one_key() { let app = issue_app(&discord, &gateway); let (a, b) = tokio::join!(issue_round_trip(&app), issue_round_trip(&app)); - assert_eq!(a.location(), "/prices-api/?issue=ok"); - assert_eq!(b.location(), "/prices-api/?issue=ok"); + assert_eq!(a.location(), "/api/prices-api/?issue=ok"); + assert_eq!(b.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.named(&key_name()).len()), @@ -265,7 +265,7 @@ async fn a_non_member_is_refused_and_no_key_is_created() { let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?issue=not_member"); assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); assert_eq!( @@ -295,7 +295,7 @@ async fn an_unknown_guild_code_is_also_refused_as_not_a_member() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=not_member"); + assert_eq!(reply.location(), "/api/prices-api/?issue=not_member"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -313,7 +313,7 @@ async fn a_404_with_an_unrecognised_code_is_unknown_not_an_accusation() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -341,7 +341,7 @@ async fn a_429_or_5xx_from_discord_refuses_without_claiming_non_membership() { let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!( reply.location(), - "/prices-api/?issue=unknown", + "/api/prices-api/?issue=unknown", "a {status} must be 'could not verify', never 'not a member'" ); assert_eq!(gateway.with(|s| s.create_calls), 0, "{status}"); @@ -366,7 +366,7 @@ async fn a_pending_member_is_refused_as_pending_rules() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=pending_rules"); + assert_eq!(reply.location(), "/api/prices-api/?issue=pending_rules"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -386,7 +386,7 @@ async fn an_absent_pending_field_does_not_silently_pass() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -403,7 +403,7 @@ async fn a_malformed_member_body_is_unknown() { let gateway = MockGateway::start().await; let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert_eq!(gateway.with(|s| s.create_calls), 0); } @@ -446,7 +446,11 @@ async fn a_grant_missing_the_member_scope_lands_on_denied_before_any_member_call let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER, "scope={drifted}"); - assert_eq!(reply.location(), "/prices-api/?issue=denied", "{drifted}"); + assert_eq!( + reply.location(), + "/api/prices-api/?issue=denied", + "{drifted}" + ); // Not the sign-in arm's error page: nothing renderable, no way back. assert!(reply.body.is_empty(), "{drifted}"); assert_eq!(discord.member_calls(), 0, "{drifted}"); @@ -467,7 +471,7 @@ async fn a_failed_token_exchange_on_an_issue_lands_on_unknown() { let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert!(!String::from_utf8_lossy(&reply.body).contains("upstream said no")); assert_eq!(discord.member_calls(), 0); assert_eq!(gateway.with(|s| s.create_calls), 0); @@ -495,7 +499,7 @@ async fn a_failed_identity_read_on_an_issue_lands_on_unknown() { let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?issue=unknown"); + assert_eq!(reply.location(), "/api/prices-api/?issue=unknown"); assert_eq!(discord.member_calls(), 1, "the membership call did happen"); // No identity means no key: the reconciler needs a `sub` to name one. assert_eq!(gateway.with(|s| s.create_calls), 0); @@ -557,7 +561,7 @@ async fn an_account_below_the_threshold_is_refused_with_the_time_remaining() { let (base, wait) = location .split_once("&wait_secs=") .unwrap_or_else(|| panic!("no wait_secs in {location}")); - assert_eq!(base, "/prices-api/?issue=too_young"); + assert_eq!(base, "/api/prices-api/?issue=too_young"); let wait: u64 = wait.parse().expect("wait_secs must be digits"); // ~180s remain; generous bounds absorb test-runner latency. assert!((150..=181).contains(&wait), "wait_secs={wait}"); @@ -588,7 +592,7 @@ async fn the_threshold_value_decides_the_verdict_at_action_time() { assert!( strict .location() - .starts_with("/prices-api/?issue=too_young"), + .starts_with("/api/prices-api/?issue=too_young"), "{}", strict.location() ); @@ -603,7 +607,7 @@ async fn the_threshold_value_decides_the_verdict_at_action_time() { ) .await; let lax = issue_round_trip(&issue_app_with(&lax_discord, &gateway, GUILD_ID, "0")).await; - assert_eq!(lax.location(), "/prices-api/?issue=ok"); + assert_eq!(lax.location(), "/api/prices-api/?issue=ok"); } // --------------------------------------------------------------------------- @@ -639,7 +643,7 @@ async fn a_session_for_someone_else_is_replaced_by_the_re_auth_identity() { ) .await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); let cookie = reply.cookie(cookies::SESSION_COOKIE).unwrap(); let session = Session::decode(SIGNING_KEY.as_bytes(), &cookie, state_token::now_secs()) .expect("the fresh session must verify"); @@ -681,7 +685,7 @@ async fn a_successful_issue_evicts_the_cached_no_key() { assert_eq!(before.json()["code"], "no_key"); let issued = issue_round_trip(&app).await; - assert_eq!(issued.location(), "/prices-api/?issue=ok"); + assert_eq!(issued.location(), "/api/prices-api/?issue=ok"); // Well inside the 60s TTL, so only the eviction can explain this. let after = call_path(app, "GET", USAGE_PATH, Some(&session)).await; @@ -717,7 +721,7 @@ async fn a_successful_issue_evicts_the_cached_no_plan() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| assert_eq!(s.attach_calls, 1, "the sign-in attached the key")); @@ -795,7 +799,7 @@ async fn an_unreadable_eligibility_parameter_is_unknown() { let reply = issue_round_trip(&app).await; assert_eq!( reply.location(), - "/prices-api/?issue=unknown", + "/api/prices-api/?issue=unknown", "guild={guild:?} age={age:?}" ); assert_eq!(gateway.with(|s| s.create_calls), 0); @@ -821,7 +825,7 @@ async fn a_control_plane_failure_after_eligibility_lands_on_issue_failed_not_unk gateway.with(|s| s.fail_list = true); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert_eq!(discord.member_calls(), 1, "eligibility ran and passed"); // Still signed in: identity and membership both proved. assert!(reply.cookie(cookies::SESSION_COOKIE).is_some()); @@ -836,7 +840,7 @@ async fn a_missing_usage_plan_is_issue_failed_without_minting_more_keys() { gateway.with(|s| s.attach_always_404 = true); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert_eq!(gateway.with(|s| s.create_calls), 1); } @@ -852,7 +856,7 @@ async fn a_stale_listing_bounds_the_attempts_and_lands_on_failed() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert_eq!( gateway.with(|s| s.list_calls), 2, @@ -874,7 +878,7 @@ async fn an_adopted_key_is_put_on_the_free_plan() { let orphan = gateway.with(|s| s.seed(&key_name(), 1_000)); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.plan_keys.clone()), vec![(PLAN_ID.to_string(), orphan.clone())], @@ -903,7 +907,7 @@ async fn duplicates_converge_on_the_earliest_and_the_losers_are_deleted() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); let mut deleted = gateway.with(|s| s.deleted.clone()); deleted.sort(); @@ -934,7 +938,7 @@ async fn the_reconciler_pages_get_api_keys_to_exhaustion_before_ranking() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.list_calls), 3, @@ -961,7 +965,7 @@ async fn a_key_that_vanishes_before_the_attach_is_not_a_dead_end() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); let survivors: Vec = gateway.with(|s| s.named(&key_name()).iter().map(|k| k.id.clone()).collect()); @@ -991,7 +995,7 @@ async fn a_key_the_listing_has_not_caught_up_with_is_not_orphaned() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.keys.len()), 1, @@ -1017,7 +1021,7 @@ async fn a_duplicate_that_will_not_delete_does_not_withhold_the_key() { }); let reply = issue_round_trip(&issue_app(&discord, &gateway)).await; - assert_eq!(reply.location(), "/prices-api/?issue=ok"); + assert_eq!(reply.location(), "/api/prices-api/?issue=ok"); assert_eq!( gateway.with(|s| s.keys.len()), 2, diff --git a/packages/prices-api/tests/portal_keys_logs.rs b/packages/prices-api/tests/portal_keys_logs.rs index 77423245..b61a7c26 100644 --- a/packages/prices-api/tests/portal_keys_logs.rs +++ b/packages/prices-api/tests/portal_keys_logs.rs @@ -115,7 +115,7 @@ async fn no_key_value_ever_reaches_the_logs() { Some(eligibility(GUILD_ID, "5")), ); let issued = issue_round_trip(&app).await; - assert_eq!(issued.location(), "/prices-api/?issue=ok"); + assert_eq!(issued.location(), "/api/prices-api/?issue=ok"); let body = reveal(&mock, USER_ID).await.json(); let text = String::from_utf8_lossy(&logs.0.lock().unwrap()).to_string(); diff --git a/packages/prices-api/tests/portal_lazy_load.rs b/packages/prices-api/tests/portal_lazy_load.rs index 784f5d7d..6bff6469 100644 --- a/packages/prices-api/tests/portal_lazy_load.rs +++ b/packages/prices-api/tests/portal_lazy_load.rs @@ -388,7 +388,7 @@ async fn a_callback_on_a_failed_load_that_claims_no_action_lands_on_signin_faile let reply = get(&router, "/api/auth/callback?code=c&state=s").await; assert_eq!(reply.status, StatusCode::SEE_OTHER); - assert_eq!(reply.location(), "/prices-api/?signin=failed"); + assert_eq!(reply.location(), "/api/prices-api/?signin=failed"); assert!(reply.headers.get(header::SET_COOKIE).is_none()); } @@ -405,8 +405,11 @@ async fn a_callback_on_a_failed_load_that_claims_no_action_lands_on_signin_faile #[tokio::test] async fn login_on_a_failed_load_lands_on_a_retryable_failure() { for (uri, landing) in [ - ("/api/auth/login", "/prices-api/?signin=failed"), - ("/api/auth/login?action=issue", "/prices-api/?issue=failed"), + ("/api/auth/login", "/api/prices-api/?signin=failed"), + ( + "/api/auth/login?action=issue", + "/api/prices-api/?issue=failed", + ), ] { let (sources, _) = scripted(vec![Step::Fail], Duration::ZERO); let router = router(true, sources); diff --git a/packages/prices-api/tests/portal_rework.rs b/packages/prices-api/tests/portal_rework.rs index 04cf42d1..8694989e 100644 --- a/packages/prices-api/tests/portal_rework.rs +++ b/packages/prices-api/tests/portal_rework.rs @@ -305,7 +305,7 @@ async fn a_partial_revocation_is_reported_as_partial() { // 1st. assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!( gateway.with(|s| s.create_calls), @@ -372,7 +372,7 @@ async fn the_revocation_instant_comes_from_the_control_plane_not_our_clock() { // rolled, so a key is due immediately. assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); } @@ -393,7 +393,7 @@ async fn an_undeletable_revoked_record_does_not_block_the_re_issue() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { let live: Vec<_> = s.keys.iter().filter(|k| k.enabled).collect(); @@ -551,7 +551,7 @@ async fn an_issue_after_a_revoke_in_the_same_period_is_capped_with_the_date() { assert_eq!(reply.status, StatusCode::SEE_OTHER); assert_eq!( reply.location(), - format!("/prices-api/?issue=capped&next_eligible_at={expected_date}") + format!("/api/prices-api/?issue=capped&next_eligible_at={expected_date}") ); assert_eq!( discord.member_calls(), @@ -584,7 +584,7 @@ async fn revoked_on_the_3rd_refuses_until_the_1st_and_issues_once_it_has_passed( let app = app_with_discord(&discord, &gateway); assert_eq!( issue_round_trip(&app).await.location(), - format!("/prices-api/?issue=capped&next_eligible_at={expected_date}") + format!("/api/prices-api/?issue=capped&next_eligible_at={expected_date}") ); assert_eq!(gateway.with(|s| s.create_calls), 0); let revealed = reveal_via(&app).await; @@ -597,7 +597,7 @@ async fn revoked_on_the_3rd_refuses_until_the_1st_and_issues_once_it_has_passed( let app = app_with_discord(&discord, &gateway); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!( @@ -713,7 +713,7 @@ async fn an_undated_duplicate_does_not_lock_the_owner_out_forever() { assert_eq!(revealed.json()["code"], "no_key"); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!(gateway.with(|s| s.create_calls), 1); } @@ -732,7 +732,7 @@ async fn the_latest_revocation_governs_the_cap() { issue_round_trip(&app) .await .location() - .starts_with("/prices-api/?issue=capped") + .starts_with("/api/prices-api/?issue=capped") ); assert_eq!(gateway.with(|s| s.create_calls), 0); assert_eq!(gateway.with(|s| s.deleted.len()), 0); @@ -755,7 +755,7 @@ async fn membership_is_still_checked_before_the_cap() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=not_member" + "/api/prices-api/?issue=not_member" ); assert_eq!(gateway.with(|s| s.list_calls), 0); } @@ -775,7 +775,7 @@ async fn a_live_key_beside_a_revoked_one_is_the_current_key() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!(s.deleted, vec![dead.clone()]); @@ -796,7 +796,7 @@ async fn a_live_key_is_adopted_as_before_the_cap_existed() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!(gateway.with(|s| s.create_calls), 0); assert_eq!(reveal_via(&app).await.json()["key_id"], key); @@ -814,7 +814,7 @@ async fn the_full_cycle_issue_revoke_wait_reissue() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let first = gateway.with(|s| s.keys[0].clone()); @@ -823,7 +823,7 @@ async fn the_full_cycle_issue_revoke_wait_reissue() { issue_round_trip(&app) .await .location() - .starts_with("/prices-api/?issue=capped") + .starts_with("/api/prices-api/?issue=capped") ); // The 1st arrives. @@ -832,7 +832,7 @@ async fn the_full_cycle_issue_revoke_wait_reissue() { }); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { @@ -931,7 +931,7 @@ async fn the_reveal_and_the_issue_agree_on_the_cap_with_mixed_period_revocations assert_eq!( issue_round_trip(&app).await.location(), - format!("/prices-api/?issue=capped&next_eligible_at={expected_date}") + format!("/api/prices-api/?issue=capped&next_eligible_at={expected_date}") ); // And the revoke's idempotent answer names the same instant. @@ -1011,7 +1011,7 @@ async fn a_stale_listing_after_the_roll_does_not_rank_the_deleted_record() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!(s.keys.len(), 1); @@ -1037,7 +1037,7 @@ async fn a_lost_create_response_is_not_retried_into_duplicates() { let app = app_with_discord(&discord, &gateway); let reply = issue_round_trip(&app).await; - assert_eq!(reply.location(), "/prices-api/?issue=failed"); + assert_eq!(reply.location(), "/api/prices-api/?issue=failed"); assert_eq!( gateway.with(|s| s.create_calls), 1, @@ -1046,7 +1046,7 @@ async fn a_lost_create_response_is_not_retried_into_duplicates() { // The next press adopts the key that landed, as before. assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); assert_eq!(gateway.with(|s| s.create_calls), 1); } @@ -1166,7 +1166,7 @@ async fn rework_rolled_on(plan: StoredPlan) -> (MockGateway, Router, String, Str let app = app_with_discord(&discord, &gateway); assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let new = gateway.with(|s| { assert_eq!(s.keys.len(), 1, "the revocation record is gone"); @@ -1286,7 +1286,7 @@ async fn a_crash_between_create_and_attach_retries_onto_the_previous_keys_plan() assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=failed" + "/api/prices-api/?issue=failed" ); let new = gateway.with(|s| { assert_eq!(s.keys.len(), 2, "the new key exists beside the revoked one"); @@ -1305,7 +1305,7 @@ async fn a_crash_between_create_and_attach_retries_onto_the_previous_keys_plan() assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert!( @@ -1347,7 +1347,7 @@ async fn a_live_key_already_on_a_plan_gets_no_attach() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok", + "/api/prices-api/?issue=ok", "{plan_id}" ); gateway.with(|s| { @@ -1389,7 +1389,7 @@ async fn a_rework_keeps_the_paid_plan_when_a_newer_revoked_record_is_on_free() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok", + "/api/prices-api/?issue=ok", "{newer_on:?}" ); gateway.with(|s| { @@ -1436,7 +1436,7 @@ async fn between_two_non_free_revoked_plans_the_newest_revocation_wins() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { let new = s @@ -1482,7 +1482,7 @@ async fn a_rework_whose_new_key_is_not_listed_yet_still_lands_on_the_paid_plan() assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let new = gateway.with(|s| { let new = s @@ -1509,7 +1509,7 @@ async fn a_rework_whose_new_key_is_not_listed_yet_still_lands_on_the_paid_plan() assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!( @@ -1552,7 +1552,7 @@ async fn an_attach_refused_for_another_plan_on_the_stage_keeps_that_plan() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!(s.attach_calls, 1, "the lagging lookup led to one attach"); @@ -1590,7 +1590,7 @@ async fn a_conflict_on_the_same_plan_is_settled_and_the_sweep_still_runs() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); gateway.with(|s| { assert_eq!(s.attach_calls, 1); @@ -1623,7 +1623,7 @@ async fn a_refused_attach_whose_plan_never_shows_is_not_handed_out() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=failed" + "/api/prices-api/?issue=failed" ); gateway.with(|s| { assert!(s.attach_calls >= 1); @@ -1659,7 +1659,7 @@ async fn after_a_second_rework_usage_reports_the_key_revoked_last() { assert_eq!( issue_round_trip(&app).await.location(), - "/prices-api/?issue=ok" + "/api/prices-api/?issue=ok" ); let new = gateway.with(|s| { assert!( diff --git a/skills/stellar-prices-api/SKILL.md b/skills/stellar-prices-api/SKILL.md index 8d9ba916..c98432a9 100644 --- a/skills/stellar-prices-api/SKILL.md +++ b/skills/stellar-prices-api/SKILL.md @@ -59,7 +59,7 @@ KEY=${STELLAR_PRICES_API_KEY:-$(sed -n 's/^\(export \)*STELLAR_PRICES_API_KEY=// **If the key is missing, stop and ask the user for it.** Do not call the API without a key, and do not guess or invent prices. Tell the user how to get one: -1. Open https://sorobanscan.rumblefish.dev/prices-api/?utm_source=stellar-skill +1. Open https://sorobanscan.rumblefish.dev/api/prices-api/?utm_source=stellar-skill 2. Click **Sign in with Discord**. The account must be a member of the [Stellar Developers Discord](https://discord.gg/stellardev), must have passed the server's membership screening, and must be older than 5 minutes. @@ -94,7 +94,7 @@ Handle the key like a secret: | Base URL | `https://prices-api.sorobanscan.rumblefish.dev/v1` | | Auth header | `x-api-key: `. `Authorization: Bearer` is rejected with 403. | | Spec (no key needed) | `https://prices-api.sorobanscan.rumblefish.dev/api-docs-json` (OpenAPI 3.1) | -| Human docs | https://sorobanscan.rumblefish.dev/prices-api/docs | +| Human docs | https://sorobanscan.rumblefish.dev/api/prices-api/docs | The spec describes every parameter and field in detail. Fetch it whenever something is not covered here. diff --git a/web/portal/index.html b/web/portal/index.html index 8690510f..84737f17 100644 --- a/web/portal/index.html +++ b/web/portal/index.html @@ -45,15 +45,15 @@ Stellar Prices API — API keys /,//p" \ docs/runbooks/0139-asset-id-migration.md | grep '^SELECT ' | while IFS= read -r q; do r=$(printf '%s' "$q" | curl --fail-with-body -sS --cert "$READ_CERT" --key "$READ_KEY" \ - --cacert "$PRICES_CA" "https://ch.sorobanscan.rumblefish.dev/${M:+?param_m=$M}" --data-binary @-) + --cacert "$PRICES_CA" "https://ch.sorobanscan.com/${M:+?param_m=$M}" --data-binary @-) printf '%s\t%s\n' "$r" "$q" done ``` diff --git a/docs/runbooks/0142-rollup-mv-reapply.md b/docs/runbooks/0142-rollup-mv-reapply.md index 75d1aa84..40a3b4fa 100644 --- a/docs/runbooks/0142-rollup-mv-reapply.md +++ b/docs/runbooks/0142-rollup-mv-reapply.md @@ -50,7 +50,7 @@ this is an operator procedure rather than something the apply path does. **local dev ClickHouse** and exits 0 — a clean result that says nothing whatever about ch-prod-01. Every command below is written for the Hetzner host. -Prod's HTTP endpoint (`ch.sorobanscan.rumblefish.dev`) is mTLS-only behind Caddy +Prod's HTTP endpoint (`ch.sorobanscan.com`) is mTLS-only behind Caddy and `prices_clickhouse::client()` builds a plaintext client, so there is no path from a local machine to prod for this binary. Run it **on the host, against the loopback port** — the same pattern as diff --git a/docs/runbooks/0151-zero-invariant-probe-rollout.md b/docs/runbooks/0151-zero-invariant-probe-rollout.md index 8e2f8be3..caf2e92a 100644 --- a/docs/runbooks/0151-zero-invariant-probe-rollout.md +++ b/docs/runbooks/0151-zero-invariant-probe-rollout.md @@ -153,7 +153,7 @@ A client pointed at `localhost:8123` reads the **local dev ClickHouse** and returns a clean result that says nothing whatever about production — check the host in the command before you read the answer. -Prod's HTTP endpoint (`ch.sorobanscan.rumblefish.dev`) is mTLS-only behind Caddy +Prod's HTTP endpoint (`ch.sorobanscan.com`) is mTLS-only behind Caddy (task 0276). Run the reads from a workstation with your **personal read certificate**, which maps to the read-only ClickHouse user `dev_read`: @@ -162,7 +162,7 @@ curl --fail-with-body -sS \ --cert "$READ_CERT" \ --key "$READ_KEY" \ --cacert "$PRICES_CA" \ - https://ch.sorobanscan.rumblefish.dev/ \ + https://ch.sorobanscan.com/ \ --data-binary @query.sql ``` diff --git a/docs/runbooks/0242-sac-identity-heal.md b/docs/runbooks/0242-sac-identity-heal.md index 83d0a31d..0436caac 100644 --- a/docs/runbooks/0242-sac-identity-heal.md +++ b/docs/runbooks/0242-sac-identity-heal.md @@ -395,7 +395,7 @@ bundle): ```bash curl -sS --fail-with-body --cert ~/prices-mtls/prices_writer.crt --key ~/prices-mtls/prices_writer.key \ - --cacert ~/prices-mtls/ca.crt https://ch.sorobanscan.rumblefish.dev/ \ + --cacert ~/prices-mtls/ca.crt https://ch.sorobanscan.com/ \ --data-binary "SELECT count() FROM (SELECT DISTINCT contract_id FROM default.soroban_contracts WHERE is_sac)" ``` diff --git a/docs/runbooks/0286-candle-definitions-rollout.md b/docs/runbooks/0286-candle-definitions-rollout.md index 74094435..829ad597 100644 --- a/docs/runbooks/0286-candle-definitions-rollout.md +++ b/docs/runbooks/0286-candle-definitions-rollout.md @@ -40,7 +40,7 @@ tmux new -s rollout0286 cd ~/stellar-prices-api && git checkout develop && git pull chadmin() { curl -sS --cert ~/prices-mtls/prices-admin-production.crt \ --key ~/prices-mtls/prices-admin-production.key --cacert ~/prices-mtls/ca.crt \ - "https://ch.sorobanscan.rumblefish.dev/" --data-binary "$1"; } + "https://ch.sorobanscan.com/" --data-binary "$1"; } chadmin "SELECT currentUser(), version() FORMAT TSVWithNames" ``` @@ -437,7 +437,7 @@ statement rather than editing the SQL, which is rendered from ```bash curl -sS --cert ~/prices-mtls/prices-admin-production.crt \ --key ~/prices-mtls/prices-admin-production.key --cacert ~/prices-mtls/ca.crt \ - "https://ch.sorobanscan.rumblefish.dev/?max_partitions_per_insert_block=1000" \ + "https://ch.sorobanscan.com/?max_partitions_per_insert_block=1000" \ --data-binary "$(sed -n '/^INSERT INTO prices.price_ohlcv_1M/,$p' \ packages/prices-clickhouse/schema/preroll.sql)" ``` diff --git a/docs/runbooks/0286-reingest-history.md b/docs/runbooks/0286-reingest-history.md index a4fd1d25..636029a0 100644 --- a/docs/runbooks/0286-reingest-history.md +++ b/docs/runbooks/0286-reingest-history.md @@ -317,7 +317,7 @@ Then, **one writer per source** (see the warning below): ```bash tmux new -s reingest -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt @@ -689,10 +689,10 @@ cargo build --release -p enrichment-worker --features aws-mtls --bin coarse-repa chadmin() { curl -sS --fail-with-body --cacert ~/prices-mtls/ca.crt \ --cert ~/prices-mtls/prices-admin-production.crt \ --key ~/prices-mtls/prices-admin-production.key \ - "https://ch.sorobanscan.rumblefish.dev/" --data-binary "$1"; } + "https://ch.sorobanscan.com/" --data-binary "$1"; } chadmin "SELECT currentUser() FORMAT TSV" # must print prices_admin -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt diff --git a/docs/runbooks/backfill-handoff-covering-operator.md b/docs/runbooks/backfill-handoff-covering-operator.md index 22d707c5..88f6586f 100644 --- a/docs/runbooks/backfill-handoff-covering-operator.md +++ b/docs/runbooks/backfill-handoff-covering-operator.md @@ -440,7 +440,7 @@ session: ```bash # certs + CH endpoint (the write path) -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt diff --git a/docs/runbooks/continue-soroban-backfill.md b/docs/runbooks/continue-soroban-backfill.md index 0576c308..ab9a5c28 100644 --- a/docs/runbooks/continue-soroban-backfill.md +++ b/docs/runbooks/continue-soroban-backfill.md @@ -185,7 +185,7 @@ the run — see §11 on why finishing matters): ```bash tmux new -s backfill # detach later with Ctrl-b then d; reattach: tmux attach -t backfill -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt @@ -431,7 +431,7 @@ let it run to `=== sdex-backfill complete ===`. | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------- | | Ledger bucket | `s3://aws-public-blockchain/v1.1/stellar/ledgers/pubnet/` — region **us-east-2**, anonymous (`--no-sign-request`) | | Writer secret | `prices/production/clickhouse-mtls-prices-ingestion-production` (region `eu-central-1`, acct `750702271865`) | -| CH domain | `ch.sorobanscan.rumblefish.dev` | +| CH domain | `ch.sorobanscan.com` | | Prod CH shell | `ssh -i ~/.ssh/sorban-prod_ed25519 deploy@168.119.73.161` → `docker exec -i app-clickhouse-1 clickhouse-client --query='…'` | | Activation ledger | `50457424` (split between `combined` and `sdex-only`) | | Combined floor (`--end` max) | `63352611` (live starts at `63352612`) | diff --git a/docs/runbooks/fix-backfill-history-loss-and-rerun.md b/docs/runbooks/fix-backfill-history-loss-and-rerun.md index 563deec2..07236e67 100644 --- a/docs/runbooks/fix-backfill-history-loss-and-rerun.md +++ b/docs/runbooks/fix-backfill-history-loss-and-rerun.md @@ -192,7 +192,7 @@ cat > ~/run-full-backfill.sh <<'SCRIPT' #!/usr/bin/env bash set -uo pipefail -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt @@ -312,7 +312,7 @@ Set `START = min(current_ledger over the combined streams) + 1` (≈ **62,592,00 ### 6b.2 Run the bounded gap-fill (tmux, same certs as 6.1) ```bash -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt diff --git a/docs/runbooks/repair-coarse-usd-values.md b/docs/runbooks/repair-coarse-usd-values.md index 936d7f95..06d34f1c 100644 --- a/docs/runbooks/repair-coarse-usd-values.md +++ b/docs/runbooks/repair-coarse-usd-values.md @@ -83,7 +83,7 @@ Binary lands at `./target/release/coarse-repair`. Point at the writer certs (same env as sdex-backfill): ```bash -export CH_DOMAIN=ch.sorobanscan.rumblefish.dev +export CH_DOMAIN=ch.sorobanscan.com export MTLS_CERT_PATH=$HOME/prices-mtls/prices_writer.crt export MTLS_KEY_PATH=$HOME/prices-mtls/prices_writer.key export MTLS_CA_PATH=$HOME/prices-mtls/ca.crt diff --git a/docs/runbooks/running-ingestion-components.md b/docs/runbooks/running-ingestion-components.md index d6180154..2f976fa9 100644 --- a/docs/runbooks/running-ingestion-components.md +++ b/docs/runbooks/running-ingestion-components.md @@ -192,7 +192,7 @@ no separate push step — so `/backfill/status` updates in real time. Needs the `aws-mtls` build and the operator's client bundle: ```bash -CH_DOMAIN=ch.sorobanscan.rumblefish.dev \ +CH_DOMAIN=ch.sorobanscan.com \ MTLS_CERT_PATH=… MTLS_KEY_PATH=… MTLS_CA_PATH=… \ target/release/sdex-backfill --transport hetzner \ --mode combined --start 50457424 --end --verbose diff --git a/docs/runbooks/seed-pool-registry.md b/docs/runbooks/seed-pool-registry.md index 6ae9d42b..3d324ab6 100644 --- a/docs/runbooks/seed-pool-registry.md +++ b/docs/runbooks/seed-pool-registry.md @@ -73,7 +73,7 @@ set -a; . ./.env.local; set +a cargo run --release -p pool-registry-seed -- \ --network mainnet \ - --ch-domain ch.sorobanscan.rumblefish.dev \ + --ch-domain ch.sorobanscan.com \ --mtls-cert-path /path/to/prices_writer.crt \ --mtls-key-path /path/to/prices_writer.key \ --mtls-ca-path /path/to/ca.crt diff --git a/packages/sdex-backfill/src/cli.rs b/packages/sdex-backfill/src/cli.rs index b8f582a6..39b76ba8 100644 --- a/packages/sdex-backfill/src/cli.rs +++ b/packages/sdex-backfill/src/cli.rs @@ -67,7 +67,7 @@ pub struct Cli { pub clickhouse_url: String, /// Caddy host fronting the Hetzner CH, for `--transport hetzner` (e.g. - /// ch.sorobanscan.rumblefish.dev). The client connects to `https://{domain}`. + /// ch.sorobanscan.com). The client connects to `https://{domain}`. #[arg(long, env = "CH_DOMAIN")] pub ch_domain: Option, diff --git a/tools/scripts/reingest_0286.py b/tools/scripts/reingest_0286.py index e318f5b4..9026712e 100755 --- a/tools/scripts/reingest_0286.py +++ b/tools/scripts/reingest_0286.py @@ -1253,7 +1253,7 @@ def build_parser(): p.add_argument("--state-dir", default="~/reingest-0286") p.add_argument("--repo", default=subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True).stdout.strip() or ".") - p.add_argument("--ch-url", default="https://ch.sorobanscan.rumblefish.dev") + p.add_argument("--ch-url", default="https://ch.sorobanscan.com") p.add_argument("--database", default="prices") # Defaults are the bundles ON the campaign machine (fishuser-hero), per the # 2026-09-21 decision: prices_admin, NOT dev_shared. The admin cert is the From 5a2dbfdf682463ed71613d77b929203deb7edee3 Mon Sep 17 00:00:00 2001 From: Marek Kowalski Date: Tue, 10 Nov 2026 10:11:20 +0100 Subject: [PATCH 139/144] docs(lore-0341,lore-0342): backlog SAC alias and RWA NAV requests from Inferera Hackathon feedback from Dominik (Inferera). External requests tracked in GH #416 (SAC alias) and #417 (NAV). Notify him when each ships. --- ..._sac-address-as-alias-for-classic-asset.md | 64 +++++++++++++++++++ ...42_FEATURE_nav-price-for-tokenised-rwas.md | 63 ++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md create mode 100644 lore/1-tasks/backlog/0342_FEATURE_nav-price-for-tokenised-rwas.md diff --git a/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md b/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md new file mode 100644 index 00000000..36b03e65 --- /dev/null +++ b/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md @@ -0,0 +1,64 @@ +--- +id: "0341" +title: "Accept a classic asset's SAC C… address as an alias — today it answers 404" +type: FEATURE +status: backlog +related_adr: [] +related_tasks: ["0242", "0139"] +tags: [layer-backend, api, identity, external-request, priority-medium, effort-small] +links: + - "https://github.com/rumblefishdev/stellar-prices-api/issues/416" + - "../../../packages/prices-api/src/identity.rs" + - "../../../packages/prices-api/src/assets/handlers.rs" + - "../../../packages/prices-ingest-core/src/writer.rs" +history: + - date: "2026-11-10" + status: backlog + who: mkowalski + note: > + Created from hackathon feedback by Dominik (Inferera). External request, + tracked in GH #416. Tell Dominik when it ships. +--- + +# Accept a classic asset's SAC C… address as an alias + +> **External request:** suggested by **Dominik (Inferera)** after using the API +> at the hackathon. Tracked in [GH #416](https://github.com/rumblefishdev/stellar-prices-api/issues/416). +> **When this is completed, let Dominik know** and close #416. + +## Summary + +A Soroban caller usually knows a classic asset only by its Stellar Asset +Contract (SAC) address. Asking for e.g. USDC by its `C…` address returns +`404`; only `CODE:ISSUER` works. Resolve a SAC address to the classic asset it +wraps, so both names return the same data. + +## Context + +- Documented behaviour today: `assets/handlers.rs` says classic assets are named + by `CODE:ISSUER` and their SAC addresses "as a rule answer `404`". +- The parser already accepts `C…` (`AssetIdentifier::Contract`, `identity.rs`). +- Ingest already collapses SAC legs into the classic asset and stores the SAC + that wraps each classic asset (`prices-ingest-core/src/writer.rs`, §12.4). +- USDC already resolves by its SAC through a special case (`handlers.rs`, + the id arm of the canonical-USDC check). This task generalises that. +- [[0242]] (SAC minted as a second identity) touches the same mapping. Check + its state first so the alias doesn't resolve to a split identity. + +## Implementation Plan + +1. In the identity resolver, look up a `Contract(c)` as a SAC of a held classic + asset first, and only then as a pure Soroban token. +2. Apply it to every route that takes `{asset_identifier}` (assets, oracles, + history), not only `/assets/{id}`. +3. Update the OpenAPI parameter descriptions that currently promise `404`. +4. Decide whether the response echoes the requested `C…` or the canonical + `CODE:ISSUER`, and document the choice. + +## Acceptance Criteria + +- [ ] `GET /assets/{SAC C…}` returns the same data as `GET /assets/{CODE:ISSUER}` for a held classic asset (incl. USDC, XLM's SAC → `native`) +- [ ] A pure Soroban token and an unknown `C…` behave as before +- [ ] OpenAPI docs no longer say SAC addresses answer `404` +- [ ] Integration test covering at least one non-USDC SAC +- [ ] Dominik (Inferera) informed; GH #416 closed diff --git a/lore/1-tasks/backlog/0342_FEATURE_nav-price-for-tokenised-rwas.md b/lore/1-tasks/backlog/0342_FEATURE_nav-price-for-tokenised-rwas.md new file mode 100644 index 00000000..5eb7fd99 --- /dev/null +++ b/lore/1-tasks/backlog/0342_FEATURE_nav-price-for-tokenised-rwas.md @@ -0,0 +1,63 @@ +--- +id: "0342" +title: "NAV / fundamental price for tokenised RWAs next to the market price — USDY trades below NAV, BENJI and WisdomTree come back unpriced" +type: FEATURE +status: backlog +related_adr: [] +related_tasks: [] +tags: [layer-backend, api, oracles, rwa, external-request, priority-medium, effort-large] +links: + - "https://github.com/rumblefishdev/stellar-prices-api/issues/417" + - "../../../packages/prices-ingest-core/src/writer.rs" + - "../../../packages/prices-api/src/oracles/handlers.rs" +history: + - date: "2026-11-10" + status: backlog + who: mkowalski + note: > + Created from hackathon feedback by Dominik (Inferera). External request, + tracked in GH #417. Tell Dominik when it ships. +--- + +# NAV / fundamental price for tokenised RWAs + +> **External request:** suggested by **Dominik (Inferera)** after using the API +> at the hackathon. Tracked in [GH #417](https://github.com/rumblefishdev/stellar-prices-api/issues/417). +> **When this is completed, let Dominik know** and close #417. + +## Summary + +For a tokenised fund, the trade price isn't the valuation. USDY trades 0.7–2% +below its NAV. Funds that are minted and redeemed through the issuer and rarely +or never trade (BENJI, WisdomTree) get no price at all. Add a NAV / "fundamental" +price, sourced from on-chain feeds such as RedStone, as a separate field next to +the market price. The API could then value fund shares, not only traded tokens. + +## Context + +- Our prices come from trades, so an asset with no trades gets no price. +- RedStone is already ingested, but its samples carry no asset and are stored + under the sentinel id `0` (`prices-ingest-core/src/writer.rs`, `oracle_row`). + Nothing links a feed to a token yet. +- Adding the field changes the public response shape, so it needs an ADR. + +## Implementation Plan + +1. **Research (gate):** list which NAV feeds RedStone (and other providers) + actually publish *on Stellar* for USDY, BENJI and WisdomTree tokens: contract, + feed id, update cadence, and how the value is defined (NAV per share vs. a + price). If no feeds exist on Stellar, decide whether to stop or use issuer data. +2. Map each feed to its Stellar asset identity. +3. ADR: field name (`nav_usd` / `fundamental_price_usd`), source, timestamp and + staleness semantics, and wording that it is issuer-reported, not a market price. +4. Ingest + store per asset; expose it on the asset endpoint(s). Assets with a + NAV but no trades must return a response instead of `404`. + +## Acceptance Criteria + +- [ ] Research note recording which NAV feeds exist on Stellar and for which assets +- [ ] ADR accepted for the new field +- [ ] USDY returns both market price and NAV +- [ ] BENJI / WisdomTree return a NAV even with no trades +- [ ] Docs explain market price vs. NAV +- [ ] Dominik (Inferera) informed; GH #417 closed From 8be13b5cb88dfd05d150d7f99ac19a24e153cacc Mon Sep 17 00:00:00 2001 From: Marek Kowalski Date: Tue, 10 Nov 2026 10:12:40 +0100 Subject: [PATCH 140/144] chore(lore): add mkowalski to team roster --- lore/0-session/team.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lore/0-session/team.yaml b/lore/0-session/team.yaml index 83b1b2c3..0953e2b6 100644 --- a/lore/0-session/team.yaml +++ b/lore/0-session/team.yaml @@ -12,3 +12,8 @@ stkrolikiewicz: name: 'Stanisław Królikiewicz' role: 'Developer' focus: 'Stellar prices API' + +mkowalski: + name: 'Marek Kowalski' + role: 'Developer' + focus: 'Stellar prices API' From 052dfffb8f65806df9277c7e8dcaf7882fc25c1b Mon Sep 17 00:00:00 2001 From: Marek Kowalski Date: Tue, 10 Nov 2026 10:15:25 +0100 Subject: [PATCH 141/144] chore(lore-0341): activate task --- .../0341_FEATURE_sac-address-as-alias-for-classic-asset.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lore/1-tasks/{backlog => active}/0341_FEATURE_sac-address-as-alias-for-classic-asset.md (100%) diff --git a/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md b/lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md similarity index 100% rename from lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md rename to lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md From 7cd1d8406822ae16c5a4fc65b5fbd86e292b0ef0 Mon Sep 17 00:00:00 2001 From: Marek Kowalski Date: Tue, 10 Nov 2026 10:16:05 +0100 Subject: [PATCH 142/144] chore(lore-0341): set status, assignee and history --- .../0341_FEATURE_sac-address-as-alias-for-classic-asset.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md b/lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md index 36b03e65..7817b9ee 100644 --- a/lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md +++ b/lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md @@ -2,7 +2,8 @@ id: "0341" title: "Accept a classic asset's SAC C… address as an alias — today it answers 404" type: FEATURE -status: backlog +status: active +assignee: mkowalski related_adr: [] related_tasks: ["0242", "0139"] tags: [layer-backend, api, identity, external-request, priority-medium, effort-small] @@ -12,6 +13,10 @@ links: - "../../../packages/prices-api/src/assets/handlers.rs" - "../../../packages/prices-ingest-core/src/writer.rs" history: + - date: "2026-11-10" + status: active + who: mkowalski + note: "Activated for implementation." - date: "2026-11-10" status: backlog who: mkowalski From b24425b5dceaab3dc8953d30ca49a727f8f18174 Mon Sep 17 00:00:00 2001 From: Marek Kowalski Date: Tue, 10 Nov 2026 10:28:19 +0100 Subject: [PATCH 143/144] chore(lore-0341): return task to backlog for handover No code written. The alias already exists on develop via 0242 (PR #390); what production runs is unclear. Findings recorded in the task. --- ..._sac-address-as-alias-for-classic-asset.md | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) rename lore/1-tasks/{active => backlog}/0341_FEATURE_sac-address-as-alias-for-classic-asset.md (65%) diff --git a/lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md b/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md similarity index 65% rename from lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md rename to lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md index 7817b9ee..eb874b4f 100644 --- a/lore/1-tasks/active/0341_FEATURE_sac-address-as-alias-for-classic-asset.md +++ b/lore/1-tasks/backlog/0341_FEATURE_sac-address-as-alias-for-classic-asset.md @@ -2,8 +2,7 @@ id: "0341" title: "Accept a classic asset's SAC C… address as an alias — today it answers 404" type: FEATURE -status: active -assignee: mkowalski +status: backlog related_adr: [] related_tasks: ["0242", "0139"] tags: [layer-backend, api, identity, external-request, priority-medium, effort-small] @@ -13,6 +12,14 @@ links: - "../../../packages/prices-api/src/assets/handlers.rs" - "../../../packages/prices-ingest-core/src/writer.rs" history: + - date: "2026-11-10" + status: backlog + who: mkowalski + note: > + Returned to backlog, unassigned; handed over to another team member. + No code written. Finding: the alias already exists on develop (0242 D5, + PR #390), but develop is 163 commits ahead of master and it is unclear + what production runs. See "Handover findings". - date: "2026-11-10" status: active who: mkowalski @@ -31,6 +38,27 @@ history: > at the hackathon. Tracked in [GH #416](https://github.com/rumblefishdev/stellar-prices-api/issues/416). > **When this is completed, let Dominik know** and close #416. +## Handover findings (2026-11-10) + +**The alias is already implemented on `develop`, so most likely no new code is +needed.** Check this before planning work: + +- [[0242]] decision D5, commit `3de11181`, merged to `develop` in PR #390 on + 2026-11-02: `resolve_sac_alias(es)` in + `packages/prices-api/src/assets/queries_ch.rs` maps a SAC `C…` address to its + classic asset (XLM's SAC → `native`). Single-asset, history, oracle and batch + handlers call it. The OpenAPI text says SAC addresses are "accepted as an + alias". `tests/endpoints_it.rs` covers XLM and XCR (incl. a pre-heal duplicate). +- **None of it is on `master`.** `develop` is 163 commits / 20 PRs ahead of + `master` (host move 0340, stellar-xdr 30 bump 0335, pricing fixes 0329/0331/0332, …). +- **It is unclear what production actually runs** (confirmed with the team), so + it's also unclear why the hackathon user got a 404. + +Suggested next steps: find out what production runs, call +`GET /assets/{XLM SAC}` against it, and then either ship via a develop → master +release or close this task as done by 0242. The Implementation Plan below was +written before this was known and is likely obsolete. + ## Summary A Soroban caller usually knows a classic asset only by its Stellar Asset From 667246652b142dce7630fcf936ded4d94a55cb79 Mon Sep 17 00:00:00 2001 From: karczuRF Date: Thu, 12 Nov 2026 18:13:29 +0100 Subject: [PATCH 144/144] docs(lore-0343): add task for the deploy/production release flow Production is deployed by hand from develop and master trails it. Describe the develop -> deploy/production -> master flow; no code yet. --- ...y-production-master-follows-each-deploy.md | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 lore/1-tasks/active/0343_FEATURE_release-through-deploy-production-master-follows-each-deploy.md diff --git a/lore/1-tasks/active/0343_FEATURE_release-through-deploy-production-master-follows-each-deploy.md b/lore/1-tasks/active/0343_FEATURE_release-through-deploy-production-master-follows-each-deploy.md new file mode 100644 index 00000000..a6c58aca --- /dev/null +++ b/lore/1-tasks/active/0343_FEATURE_release-through-deploy-production-master-follows-each-deploy.md @@ -0,0 +1,135 @@ +--- +id: "0343" +title: "Production is deployed by hand from develop and master trails it — release through deploy/production, and move master only after a successful deploy" +type: FEATURE +status: active +assignee: okarcz +related_adr: [] +related_tasks: ["0141", "0242"] +tags: ["deploy", "release", "ci", "branching", "priority-medium", "effort-medium"] +links: + - "../../../infra/Makefile" + - "../../../infra/src/lib/stacks/cicd-stack.ts" + - "../../../.github/workflows/ci.yml" + - https://github.com/rumblefishdev/stellar-x402/blob/develop/.github/workflows/deploy-production.yml +history: + - date: 2026-11-12 + status: active + who: okarcz + note: > + Task created and activated. Scope so far is the branch flow below; no + code yet. The mechanism (who runs the deploy, how the commit is stamped) + is still open and listed under Open decisions. +--- + +# Production is deployed by hand from develop and master trails it — release through deploy/production, and move master only after a successful deploy + +## Summary + +Anyone reading the repository should be able to trust one thing: the code on +`master` is the code running in production. Today that does not hold. This task +introduces a three-branch release flow, `develop` → `deploy/production` → +`master`, in which production is deployed only from `deploy/production` and +`master` moves only to a commit whose deploy succeeded. + +## Status: Active + +**Current state:** flow described, nothing implemented. Waiting for the +operator's decisions on the open points before any code. + +## Context + +Measured on 2026-11-12: + +- Production is deployed from an operator's machine with + `make deploy-production-*`, from `develop`, one stack at a time. +- `master` is updated later by a release PR. It was 164 commits behind + `develop` (release PR #418, open). +- `master` has no branch protection. The repository has no release tags. +- A deployed Lambda carries no commit id. Which commit the 2026-11-06 Compute + deploy was built from could only be answered from the deployer's reflog. +- One stack was held back on purpose (0242 D8: Observability, three alarms), so + "what `develop` describes" and "what is deployed" differed by design, with + the difference recorded only in a task file. +- `infra/src/lib/stacks/cicd-stack.ts` defines a GitHub OIDC deploy role scoped + to a `production` environment. No workflow uses it. Whether that stack is + deployed was not checked. + +`stellar-x402` already releases through a protected `deploy/production` branch: +a PR from `develop`, a deploy started by the merge, a build stamped with its +commit. This task brings the same order here and adds `master` as the record of +what is live. + +## The flow + +| Branch | Meaning | Who moves it | +| --- | --- | --- | +| `develop` | work in progress | feature PRs | +| `deploy/production` | approved for release | release PR from `develop`, approved, merge commit | +| `master` | confirmed live | the deploy, after it succeeds; fast-forward only | + +1. Feature PRs merge to `develop`, as now. Task documents keep going straight + to `develop`. +2. A release is a PR from `develop` to `deploy/production`. CI runs on it, and + the production `cdk diff` is read on it before approval. +3. The merge starts (or permits) the deploy of **all** stacks from that exact + commit. A stack with no change is a no-op. +4. On success, `master` is fast-forwarded to the deployed commit and the commit + is tagged (for example `prod-2026-11-12`). +5. On failure, `master` does not move. The fix or a revert goes to + `deploy/production` by PR and is deployed like any release. + +Rules that keep it true: + +- `master` takes no PRs and no direct commits; only the deploy moves it. +- Hotfixes go to `deploy/production` by PR and are merged back into `develop`. +- A rollback is a revert PR on `deploy/production`. `master` is never + force-moved. +- No stack is "held back". A resource that must wait is switched off by a + config flag, so every stack can always be deployed from the release commit. +- Changes that are not branch state (ClickHouse schema applies, Ansible runs, + the portal sync, data repairs) are applied from the release commit and noted + on the release tag. + +## Open decisions + +1. **Who runs the deploy.** GitHub Actions on `deploy/production` behind a + `production` environment approval, or the operator's machine with a guard in + `infra/Makefile` that refuses any other commit. Possibly the guard first and + the workflow second. +2. **How the deployed commit is stamped.** Per stack, since a five-stack deploy + can fail part-way. Candidate: an SSM parameter written by each stack. A + Lambda environment variable would restart every Lambda on every release. +3. **Drift check.** A scheduled job that compares the stamps with `master` and + reports a mismatch. +4. **What happens to the current branches.** How `deploy/production` is created + (from `master` after #418, or from `develop`), and how the one commit that + exists only on `master` (`7d52fa9b`) is carried over. +5. **Branch protection.** Rules for `deploy/production` and `master`, and which + identity is allowed to fast-forward `master`. +6. **Non-AWS targets.** How far the Hetzner side (ClickHouse, Ansible) is + brought under the same rule in this task. + +## Implementation Plan + +To be written once the open decisions are answered. + +## Acceptance Criteria + +- [ ] `deploy/production` exists, is protected, and takes only approved PRs +- [ ] A production deploy from any other commit is refused +- [ ] Every stack records the commit it was deployed from, readable without + access to the deployer's machine +- [ ] After a successful deploy `master` points at the deployed commit and + carries a release tag, with no manual step +- [ ] After a failed deploy `master` is unchanged +- [ ] A scheduled check reports when the deployed commit and `master` differ +- [ ] The flow, the hotfix path and the rollback path are documented where + deploys are documented (`infra/README.md`) + +## Notes + +- Out of scope: the catch-up release itself (PR #418) and the 0242 cleanup. + Both are handled separately and do not wait for this task. +- [[0141]] is the history behind `build-lambdas`: a deploy shipped stale Lambda + assets from a laptop. Building in CI removes that class of error.