From 3068880e53d891147f12058003d8d880cbdf53e7 Mon Sep 17 00:00:00 2001 From: Kevin Codex Date: Tue, 10 Nov 2026 23:01:11 +0800 Subject: [PATCH] feat(usd): price every trade and fee at the time it happened The all-time volume (and the fee and per-launch USD figures) priced every trade ever made at TODAY's quote price: queries.getLaunchTotals summed each quote's volume times its live price. So September trades moved with today's ETH and GITLAWB. Both sit near their lows since launch (ETH $2,063..$2,784, GITLAWB 6.2e-6..7.5e-5), so the figure was understated. This change records each trade's USD value at its own hour. The site still reads the old figures until a second change switches it, after the backfill below. - bb_quote_usd_hourly: one USD price per quote per UTC hour. The indexer writes the live price into the hour in progress (usdLedger.captureLiveUsd: at most once a minute per chain, always on a new hour, same sources as the lists via queries.quoteUsdNow; never throws). - bb_launch_swaps.usd / bb_launch_fee_events.usd: each trade's and fee's value at its own hour, written in the event's own transaction together with new running sums bb_launches.volume_usd / fees_collected_usd / fees_burned_usd. No price for that hour means NULL (never a guess), left for the backfill. Fees are quote side only, as the USD fee totals always were. - scripts/usd-backfill.ts writes ONE reviewed SQL file and touches no database: - hourly prices from launch day: ETH-USD and HYPE-USD from Coinbase (HYPE gaps from Hyperliquid); GITLAWB from its GITLAWB/WETH pool via GeckoTerminal (price in WETH, forward-filled between trades) times ETH-USD, the live method; TWIG = GITLAWB; stables $1 - --mode dry-run prints old method vs trade-time per chain and quote, totals and fees, then ROLLS BACK - --mode apply prices rows with no USD value, then recomputes launch totals under the rebuild-launch-totals.sql locks - It never replaces an hour captured live, and stops at the last completed hour (a partial candle would block the live capture for the rest of the hour). - LAUNCH.md: the runbook. Points scoring is unchanged this season. Stocks, community pairs and unlisted quotes have no price history: their older trades stay without a USD value. Verified on a throwaway Postgres 16: - the real migrate.mjs applies on a fresh database and again unchanged - a fixture of ETH, Arc USDC, GITLAWB and unlisted-quote launches with trades and fees: the dry run reported the hand-computed values and left the database untouched; apply wrote exactly those values, kept the live-captured hour, left the unlisted trade and the token-side burn NULL, and a second apply changed nothing - the real captureLiveUsd wrote live rows through the real driver; a live 2-ETH trade priced at 2 x the captured spot and the launch total moved by exactly that The backfilled GITLAWB and ETH for the latest hour match the live site's within 1%. tsc and lint clean; npm test 1404 (1393 pass, 11 skipped, 0 fail). Co-Authored-By: Claude Opus 5.5 --- app/LAUNCH.md | 25 ++ app/db/concurrent-indexes.sql | 3 + app/db/schema.sql | 24 ++ app/scripts/usd-backfill.ts | 125 ++++++++++ app/src/lib/launchpad/indexer.ts | 61 +++-- app/src/lib/launchpad/queries.ts | 14 ++ app/src/lib/launchpad/usd-history.test.ts | 157 ++++++++++++ app/src/lib/launchpad/usd-history.ts | 279 ++++++++++++++++++++++ app/src/lib/launchpad/usdLedger.ts | 55 +++++ 9 files changed, 725 insertions(+), 18 deletions(-) create mode 100644 app/scripts/usd-backfill.ts create mode 100644 app/src/lib/launchpad/usd-history.test.ts create mode 100644 app/src/lib/launchpad/usd-history.ts create mode 100644 app/src/lib/launchpad/usdLedger.ts diff --git a/app/LAUNCH.md b/app/LAUNCH.md index 2369e9c6..05fcec51 100644 --- a/app/LAUNCH.md +++ b/app/LAUNCH.md @@ -80,3 +80,28 @@ cluster with `fly ssh console -a -C "sh -c 'PGPASSWORD=\$OPERATOR_PASSWORD -U postgres -d basebid -c ...'"` (port 5433, not the socket). Destroy the rehearsal app afterwards. For a real recovery: restore to a new app, `fly postgres detach`/`attach` it to `basebid` (DATABASE_URL), redeploy, run `scripts/smoke.sh`. Point-in-time: `--restore-target-time ` instead of a name. + +## Trade-time USD backfill (2026-11-10) + +Every trade and fee carries its USD value **at the time it happened** (`bb_launch_swaps.usd`, `bb_launch_fee_events.usd`, +per-launch sums in `bb_launches.volume_usd` / `fees_collected_usd` / `fees_burned_usd`). The indexer prices new ones from +the hourly ledger `bb_quote_usd_hourly`, writing the live price into the hour in progress (`lib/launchpad/usd-history.ts`, +`usdLedger.ts`). Trades from before that went live, and any hour the indexer could not price (a price source down, the +indexer catching up), are filled by the backfill. It is safe to run again at any time: it only prices rows with no USD +value, never replaces an hour the indexer captured live, and stops at the last completed hour. + +```bash +cd app +# 1. generate (public price APIs only, no database access): dry run first +npx tsx --tsconfig tsconfig.test.json scripts/usd-backfill.ts --mode dry-run --out /tmp/usd-dry.sql +# 2. run it on the database machine: it prints the report (old method vs trade-time, per chain and quote, totals, fees) +# and ROLLS BACK, so nothing changes +fly ssh sftp shell -a basebid-db <<< "put /tmp/usd-dry.sql /tmp/usd-dry.sql" +fly ssh console -a basebid-db -C "sh -c 'PGPASSWORD=\$OPERATOR_PASSWORD psql -h localhost -p 5433 -U postgres -d basebid -f /tmp/usd-dry.sql'" +# 3. once the report looks right: generate and run --mode apply the same way (it writes, then recomputes every launch's +# USD totals under the same locks as scripts/rebuild-launch-totals.sql; the indexer waits a few seconds) +``` + +History sources: ETH-USD and HYPE-USD hourly closes from Coinbase Exchange (HYPE gaps from Hyperliquid), GITLAWB from its +GITLAWB/WETH pool on Base via GeckoTerminal (price in WETH between trades) × ETH-USD, TWIG = GITLAWB, stables $1. Stocks, +community pairs and unlisted quotes have no history there: their older trades stay without a USD value (`usd IS NULL`). diff --git a/app/db/concurrent-indexes.sql b/app/db/concurrent-indexes.sql index e02ee3a2..65ba506c 100644 --- a/app/db/concurrent-indexes.sql +++ b/app/db/concurrent-indexes.sql @@ -17,3 +17,6 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS bb_token_transfers_to_idx ON bb_token_tr CREATE INDEX CONCURRENTLY IF NOT EXISTS bb_token_transfers_from_idx ON bb_token_transfers (chain_id, token, from_addr, block_number); -- boosts (2026-10-15): a boost names its launch by position id; the lane joins on it CREATE INDEX CONCURRENTLY IF NOT EXISTS bb_launches_chain_token_id_idx ON bb_launches (chain_id, token_id); +-- trade-time USD (2026-11-10): the trades still without a USD value (every one until the first backfill, then only +-- those whose hour had no price yet), so a repeat backfill and the "partial" check never scan every swap +CREATE INDEX CONCURRENTLY IF NOT EXISTS bb_launch_swaps_usd_missing_idx ON bb_launch_swaps (chain_id, block_time) WHERE usd IS NULL; diff --git a/app/db/schema.sql b/app/db/schema.sql index 615dccec..4dd746d1 100644 --- a/app/db/schema.sql +++ b/app/db/schema.sql @@ -624,3 +624,27 @@ CREATE TABLE IF NOT EXISTS bb_admin_sessions ( revoked_at timestamptz ); CREATE INDEX IF NOT EXISTS bb_admin_sessions_wallet_idx ON bb_admin_sessions (wallet, created_at DESC); + +-- ── trade-time USD (2026-11-10, lib/launchpad/usd-history.ts) ─────────────── +-- Every trade's and fee's USD value at the time it happened, instead of its quote volume at today's price. One USD price +-- per quote per UTC hour: the live price the indexer saw during that hour (source 'live', updated until the hour ends, +-- never replaced afterwards), or a historical close filled in by scripts/usd-backfill.ts ('coinbase', 'geckoterminal', +-- 'fixed'). `decimals` is the quote's, so a raw amount converts without a second lookup. +CREATE TABLE IF NOT EXISTS bb_quote_usd_hourly ( + chain_id integer NOT NULL, + quote text NOT NULL, -- lowercase; 0x000… = the chain's native asset + hour timestamptz NOT NULL, -- start of the UTC hour + usd double precision NOT NULL CHECK (usd > 0), -- per whole quote unit + decimals integer NOT NULL CHECK (decimals BETWEEN 0 AND 36), + source text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (chain_id, quote, hour) +); +-- the value at the trade's own hour, fixed once written; NULL = no price for that hour yet (the backfill fills it) +ALTER TABLE bb_launch_swaps ADD COLUMN IF NOT EXISTS usd double precision; +-- quote side only, as the USD fee totals always were: 'collected' (quote_amount) and a 'burned' row in the launch's quote +ALTER TABLE bb_launch_fee_events ADD COLUMN IF NOT EXISTS usd double precision; +-- each launch's sums of those values, added to in the event's own transaction and recomputed by the backfill +ALTER TABLE bb_launches ADD COLUMN IF NOT EXISTS volume_usd double precision NOT NULL DEFAULT 0; +ALTER TABLE bb_launches ADD COLUMN IF NOT EXISTS fees_collected_usd double precision NOT NULL DEFAULT 0; +ALTER TABLE bb_launches ADD COLUMN IF NOT EXISTS fees_burned_usd double precision NOT NULL DEFAULT 0; diff --git a/app/scripts/usd-backfill.ts b/app/scripts/usd-backfill.ts new file mode 100644 index 00000000..66758d9e --- /dev/null +++ b/app/scripts/usd-backfill.ts @@ -0,0 +1,125 @@ +/** + * Trade-time USD backfill (lib/launchpad/usd-history.ts states the rule). Fetches hourly prices from launch day to now and + * writes ONE SQL file that fills bb_quote_usd_hourly, prices every trade and fee that has no USD value yet at its own + * hour, and recomputes each launch's USD totals. Nothing here touches a database: the file is reviewed, then run on the + * database machine exactly like scripts/rebuild-launch-totals.sql. + * + * npx tsx --tsconfig tsconfig.test.json scripts/usd-backfill.ts --mode dry-run [--from 2026-09-01] [--out file.sql] + * npx tsx --tsconfig tsconfig.test.json scripts/usd-backfill.ts --mode apply [--from 2026-09-01] [--out file.sql] + * + * Dry run: the SQL fills the ledger inside a transaction, prints the report (old method vs trade-time, per chain and + * quote, plus totals) and ROLLS BACK. Apply: same report, then writes the values and recomputes the totals. Both are safe + * to run again: only rows with no USD value are priced, and an hour the indexer captured live is never replaced. + * + * Sources (all public, no key): ETH-USD and HYPE-USD hourly closes from Coinbase Exchange (the live site prices ETH from + * Coinbase too), HYPE hours Coinbase lacks from Hyperliquid; GITLAWB from its GITLAWB/WETH Uniswap v4 pool on Base via + * GeckoTerminal (price in WETH, forward-filled between trades: a pool's price only moves when it trades) times ETH-USD, + * the live method (gitlawb.ts). TWIG takes GITLAWB's price (1:1 wrapper). Stables are $1. Stocks, community pairs and + * unlisted quotes have no history here: their past trades stay without a USD value and count as partial. + */ +import { writeFileSync } from "node:fs"; +import { GITLAWB_POOL_ID } from "../src/lib/launchpad/gitlawb.ts"; +import { + GITLAWB_HISTORY, HOUR_MS, backfillHours, buildBackfillSql, fillHours, multiplySeries, parseCoinbaseCandles, parseGeckoOhlcv, + parseHyperliquidCandles, quoteSeriesRows, type BackfillMode, type HourPoint, type SeriesKey, +} from "../src/lib/launchpad/usd-history.ts"; + +const args = new Map(); +for (let i = 2; i < process.argv.length; i += 2) args.set(process.argv[i].replace(/^--/, ""), process.argv[i + 1] ?? ""); +const mode = args.get("mode") as BackfillMode; +if (mode !== "dry-run" && mode !== "apply") throw new Error("--mode dry-run | apply"); +// up to the last COMPLETED hour: the hour in progress belongs to the indexer's live capture (usd-history.backfillHours) +const { from, to } = backfillHours(args.get("from") || "2026-09-01T00:00:00Z", Date.now()); +const out = args.get("out") || `usd-backfill.${mode}.sql`; +const UA = { "user-agent": "openlaunch-usd-backfill" }; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +async function getJson(url: string, init?: RequestInit, tries = 4): Promise { + for (let i = 0; ; i++) { + const res = await fetch(url, { ...init, headers: { ...UA, ...(init?.headers ?? {}) }, signal: AbortSignal.timeout(20_000) }); + if (res.ok) return res.json(); + if (i + 1 >= tries || (res.status !== 429 && res.status < 500)) throw new Error(`${url.split("?")[0]}: HTTP ${res.status}`); + await sleep(2_000 * (i + 1)); + } +} + +/** Coinbase Exchange: at most 300 candles per request. */ +async function coinbaseHourly(product: string): Promise { + const pts: HourPoint[] = []; + for (let start = from; start <= to; start += 300 * HOUR_MS) { + const end = Math.min(to, start + 299 * HOUR_MS); + const url = `https://api.exchange.coinbase.com/products/${product}/candles?granularity=3600&start=${new Date(start).toISOString()}&end=${new Date(end).toISOString()}`; + pts.push(...parseCoinbaseCandles(await getJson(url))); + await sleep(250); + } + return pts; +} + +/** Hyperliquid candleSnapshot: up to 5000 candles per request. */ +async function hyperliquidHourly(coin: string): Promise { + const pts: HourPoint[] = []; + for (let start = from; start <= to; start += 4_000 * HOUR_MS) { + const end = Math.min(to + HOUR_MS - 1, start + 4_000 * HOUR_MS - 1); + const body = { type: "candleSnapshot", req: { coin, interval: "1h", startTime: start, endTime: end } }; + pts.push(...parseHyperliquidCandles(await getJson("https://api.hyperliquid.xyz/info", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }))); + await sleep(250); + } + return pts; +} + +/** GeckoTerminal pool OHLCV, newest first, paged backwards with before_timestamp (1000 per page). Price in the pool's other token. */ +async function geckoHourlyInPair(network: string, pool: string, token: string): Promise { + const pts: HourPoint[] = []; + let before = Math.floor((to + HOUR_MS) / 1000); + for (let page = 0; page < 20; page++) { + const url = `https://api.geckoterminal.com/api/v2/networks/${network}/pools/${pool}/ohlcv/hour?aggregate=1&limit=1000¤cy=token&token=${token}&before_timestamp=${before}`; + const got = parseGeckoOhlcv(await getJson(url, { headers: { accept: "application/json" } })); + if (got.length === 0) break; + pts.push(...got); + const oldest = got[0].hour; + if (oldest <= from) break; + before = Math.floor(oldest / 1000); + await sleep(2_500); // GeckoTerminal's public API allows about 30 calls a minute + } + return pts; +} + +function summary(name: string, m: ReadonlyMap): string { + const v = [...m.values()]; + const hours = (to - from) / HOUR_MS + 1; + return `${name.padEnd(12)} ${String(m.size).padStart(5)}/${hours} hours min ${Math.min(...v).toPrecision(4)} max ${Math.max(...v).toPrecision(4)}`; +} + +async function main(): Promise { + const quotes = quoteSeriesRows(); + for (const q of quotes) if (q.series === "USD" && q.fixedUsd !== 1) throw new Error(`fixed quote ${q.chain}:${q.quote} is not $1 (${q.fixedUsd}); give it its own series`); + + console.error(`hours ${new Date(from).toISOString()} .. ${new Date(to).toISOString()}`); + const eth = fillHours(await coinbaseHourly("ETH-USD"), from, to, 6); + const hypeCb = await coinbaseHourly("HYPE-USD"); + const hypeHl = await hyperliquidHourly("HYPE"); + // Coinbase first (the live HYPE price's primary source); Hyperliquid only for the hours Coinbase has no candle for + const cbHours = new Set(hypeCb.map((p) => p.hour)); + const hype = fillHours([...hypeHl.filter((p) => !cbHours.has(p.hour)), ...hypeCb], from, to, 6); + const gitlawbInWeth = fillHours(await geckoHourlyInPair(GITLAWB_HISTORY.network, GITLAWB_POOL_ID, GITLAWB_HISTORY.token), from, to, 24 * 365); + const gitlawb = multiplySeries(gitlawbInWeth, eth); + const usd = new Map(); + for (let h = from; h <= to; h += HOUR_MS) usd.set(h, 1); + + // sanity bounds: a unit mix-up (a price in wei, a series in the wrong currency) fails here, before anything is written + const bounds: Record = { USD: [1, 1], "ETH-USD": [100, 100_000], "HYPE-USD": [1, 10_000], "GITLAWB-USD": [1e-12, 1] }; + const series = new Map>([["USD", usd], ["ETH-USD", eth], ["HYPE-USD", hype], ["GITLAWB-USD", gitlawb]]); + for (const [k, m] of series) { + console.error(summary(k, m)); + for (const v of m.values()) if (v < bounds[k][0] || v > bounds[k][1]) throw new Error(`${k}: ${v} is outside ${bounds[k].join("..")}`); + } + + const sql = buildBackfillSql({ mode, generatedAt: new Date().toISOString(), from, to, quotes, series }); + writeFileSync(out, sql); + console.error(`wrote ${out} (${(sql.length / 1024).toFixed(0)} KiB, ${quotes.length} quotes). Run it on the database machine as LAUNCH.md describes.`); +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); +}); diff --git a/app/src/lib/launchpad/indexer.ts b/app/src/lib/launchpad/indexer.ts index 6d6999fa..dba713ec 100644 --- a/app/src/lib/launchpad/indexer.ts +++ b/app/src/lib/launchpad/indexer.ts @@ -15,6 +15,8 @@ import { TWIG_ADDRESSES } from "./twig"; import { boostFromTransfer } from "./boost"; import { pairBurnFromTransfer } from "./pairApply"; import { recordPairBurn } from "./pairApplyServer"; +import { captureLiveUsd } from "./usdLedger"; +import { hourIso } from "./usd-history"; /** * Launchpad chain → Postgres indexer. @@ -174,7 +176,7 @@ async function applyLaunched(db: Db, chain: ChainKey, log: Log & { args: Launche type Swap = { id: Hex; sender: Address; amount0: bigint; amount1: bigint; sqrtPriceX96: bigint; liquidity: bigint; tick: number; fee: number }; /** Record one PoolManager swap of a launched token (credited to its trader via traderOf) and roll the launch's totals forward, atomically. */ -async function applySwap(db: Db, chain: ChainKey, log: Log & { args: Swap }, poolToToken: Map): Promise { +async function applySwap(db: Db, chain: ChainKey, log: Log & { args: Swap }, poolToToken: Map, tokenQuote: Map): Promise { const a = log.args; const poolId = a.id.toLowerCase(); const token = poolToToken.get(poolId); @@ -185,20 +187,26 @@ async function applySwap(db: Db, chain: ChainKey, log: Log & { args: Swap }, poo const [time, who] = await Promise.all([timeOf(chain, log.blockNumber!), traderOf(chain, log.transactionHash!, log.logIndex!, token)]); const bn = log.blockNumber!; const li = log.logIndex!; + // the trade's USD value at its own hour (bb_quote_usd_hourly; usd-history.ts), fixed once written; no price for that hour = NULL + const quote = tokenQuote.get(token) ?? ""; // One transaction: the swap row and the launch totals commit together or not at all (a failure between // the two used to leave a recorded trade whose totals were never applied — and the dedupe then skips it). return db.begin(async (tx) => { const t = tx as unknown as Db; const inserted = await t` - INSERT INTO bb_launch_swaps (chain_id, tx_hash, log_index, token, pool_id, trader, tx_from, trader_via, amount0, amount1, sqrt_price_x96, tick, is_buy, block_number, block_time) + INSERT INTO bb_launch_swaps (chain_id, tx_hash, log_index, token, pool_id, trader, tx_from, trader_via, amount0, amount1, sqrt_price_x96, tick, is_buy, block_number, block_time, usd) VALUES (${cid}, ${log.transactionHash!.toLowerCase()}, ${log.logIndex!}, ${token}, ${poolId}, ${who.trader}, ${who.tx_from}, ${who.via}, ${a.amount0.toString()}, ${a.amount1.toString()}, - ${a.sqrtPriceX96.toString()}, ${a.tick}, ${isBuy}, ${log.blockNumber!}, ${time}) + ${a.sqrtPriceX96.toString()}, ${a.tick}, ${isBuy}, ${log.blockNumber!}, ${time}, + (SELECT ${absQuote.toString()}::numeric / power(10::numeric, p.decimals) * p.usd FROM bb_quote_usd_hourly p + WHERE p.chain_id = ${cid} AND p.quote = ${quote} AND p.hour = ${hourIso(time)}::timestamptz)) ON CONFLICT DO NOTHING - RETURNING tx_hash`; + RETURNING tx_hash, usd`; if (inserted.length === 0) return false; + const usd = (inserted[0] as { usd: number | null }).usd ?? 0; await t` UPDATE bb_launches SET volume_quote = volume_quote + ${absQuote.toString()}::numeric, + volume_usd = volume_usd + ${usd}::float8, buys = buys + ${isBuy ? 1 : 0}, sells = sells + ${isBuy ? 0 : 1}, last_trade_at = GREATEST(COALESCE(last_trade_at, ${time}::timestamptz), ${time}::timestamptz), @@ -264,13 +272,19 @@ async function applyFeeIn(db: Db, chain: ChainKey, cid: number, time: string, lo const bn = log.blockNumber!; if (log.eventName === "Collected") { const t = log.args.token.toLowerCase(); + const quote = tokenIdToToken.get(log.args.tokenId.toString())?.quote ?? ""; + // the quote side's USD value at the collect's own hour (usd-history.ts), as for trades const r = await db` - INSERT INTO bb_launch_fee_events (chain_id, tx_hash, log_index, kind, token_id, token, quote_amount, token_amount, block_number, block_time) - VALUES (${cid}, ${tx}, ${li}, 'collected', ${log.args.tokenId}, ${t}, ${log.args.quoteAmount.toString()}, ${log.args.tokenAmount.toString()}, ${bn}, ${time}) - ON CONFLICT DO NOTHING RETURNING tx_hash`; + INSERT INTO bb_launch_fee_events (chain_id, tx_hash, log_index, kind, token_id, token, quote_amount, token_amount, block_number, block_time, usd) + VALUES (${cid}, ${tx}, ${li}, 'collected', ${log.args.tokenId}, ${t}, ${log.args.quoteAmount.toString()}, ${log.args.tokenAmount.toString()}, ${bn}, ${time}, + (SELECT ${log.args.quoteAmount.toString()}::numeric / power(10::numeric, p.decimals) * p.usd FROM bb_quote_usd_hourly p + WHERE p.chain_id = ${cid} AND p.quote = ${quote} AND p.hour = ${hourIso(time)}::timestamptz)) + ON CONFLICT DO NOTHING RETURNING tx_hash, usd`; if (r.length === 0) return false; + const usd = (r[0] as { usd: number | null }).usd ?? 0; await db`UPDATE bb_launches SET fees_quote_collected = fees_quote_collected + ${log.args.quoteAmount.toString()}::numeric, - fees_token_collected = fees_token_collected + ${log.args.tokenAmount.toString()}::numeric WHERE chain_id = ${cid} AND token = ${t}`; + fees_token_collected = fees_token_collected + ${log.args.tokenAmount.toString()}::numeric, + fees_collected_usd = fees_collected_usd + ${usd}::float8 WHERE chain_id = ${cid} AND token = ${t}`; return true; } if (log.eventName === "Burned") { @@ -280,12 +294,18 @@ async function applyFeeIn(db: Db, chain: ChainKey, cid: number, time: string, lo // the rebuild script joins on token), so leave it for the poller, which indexes the launch first. if (!info) return false; const cur = log.args.currency.toLowerCase(); + // a burn in the launch's quote gets its USD value at its own hour; a burn of the launched token stays NULL (quote side only) + const priceIn = cur === info.quote ? cur : ""; const r = await db` - INSERT INTO bb_launch_fee_events (chain_id, tx_hash, log_index, kind, token_id, token, currency, amount, block_number, block_time) - VALUES (${cid}, ${tx}, ${li}, 'burned', ${log.args.tokenId}, ${info.token}, ${cur}, ${log.args.amount.toString()}, ${bn}, ${time}) - ON CONFLICT DO NOTHING RETURNING tx_hash`; + INSERT INTO bb_launch_fee_events (chain_id, tx_hash, log_index, kind, token_id, token, currency, amount, block_number, block_time, usd) + VALUES (${cid}, ${tx}, ${li}, 'burned', ${log.args.tokenId}, ${info.token}, ${cur}, ${log.args.amount.toString()}, ${bn}, ${time}, + (SELECT ${log.args.amount.toString()}::numeric / power(10::numeric, p.decimals) * p.usd FROM bb_quote_usd_hourly p + WHERE p.chain_id = ${cid} AND p.quote = ${priceIn} AND p.hour = ${hourIso(time)}::timestamptz)) + ON CONFLICT DO NOTHING RETURNING tx_hash, usd`; if (r.length === 0) return false; - if (cur === info.quote) await db`UPDATE bb_launches SET fees_quote_burned = fees_quote_burned + ${log.args.amount.toString()}::numeric WHERE chain_id = ${cid} AND token = ${info.token}`; + const usd = (r[0] as { usd: number | null }).usd ?? 0; + if (cur === info.quote) await db`UPDATE bb_launches SET fees_quote_burned = fees_quote_burned + ${log.args.amount.toString()}::numeric, + fees_burned_usd = fees_burned_usd + ${usd}::float8 WHERE chain_id = ${cid} AND token = ${info.token}`; else await db`UPDATE bb_launches SET fees_token_burned = fees_token_burned + ${log.args.amount.toString()}::numeric WHERE chain_id = ${cid} AND token = ${info.token}`; return true; } @@ -305,15 +325,17 @@ async function applyFeeIn(db: Db, chain: ChainKey, cid: number, time: string, lo return r.length > 0; } -async function poolMaps(db: Db, chain: ChainKey): Promise<{ poolToToken: Map; tokenIdToToken: Map }> { +async function poolMaps(db: Db, chain: ChainKey): Promise<{ poolToToken: Map; tokenIdToToken: Map; tokenQuote: Map }> { const rows = await db<{ token: string; pool_id: string; token_id: bigint; quote: string }[]>`SELECT token, pool_id, token_id, quote FROM bb_launches WHERE chain_id = ${chainIdOf(chain)}`; const poolToToken = new Map(); const tokenIdToToken = new Map(); + const tokenQuote = new Map(); for (const r of rows) { poolToToken.set(r.pool_id, r.token); tokenIdToToken.set(r.token_id.toString(), { token: r.token, quote: r.quote }); + tokenQuote.set(r.token, r.quote); } - return { poolToToken, tokenIdToToken }; + return { poolToToken, tokenIdToToken, tokenQuote }; } /** @@ -337,7 +359,9 @@ async function applyRange(db: Db, chain: ChainKey, from: bigint, to: bigint): Pr const launchedLogs = await fetchLogsSplit((f, t) => client.getLogs({ address: factory, event: LAUNCHED_EVENT, fromBlock: f, toBlock: t }), from, to); for (const l of launchedLogs) if (await applyLaunched(db, chain, l as Log & { args: Launched })) launches++; - const { poolToToken, tokenIdToToken } = await poolMaps(db, chain); + const { poolToToken, tokenIdToToken, tokenQuote } = await poolMaps(db, chain); + // this hour's live USD for every quote in use, before the range's trades are priced at their hours (never throws) + await captureLiveUsd(db, chain, tokenQuote.values()); if (poolToToken.size > 0) { const ids = [...poolToToken.keys()] as Hex[]; // every pool id is one topic selector: past the node's cap the id list goes out in slices (the 1001st Robinhood launch stopped the indexer) @@ -345,7 +369,7 @@ async function applyRange(db: Db, chain: ChainKey, from: bigint, to: bigint): Pr const swapLogs = byChainOrder( (await fetchBySelectors(SELECTOR_CAPS, chain, ids, (slice) => fetchLogsSplit((f, t) => client.getLogs({ address: pm, event: POOL_SWAP_EVENT, args: { id: slice }, fromBlock: f, toBlock: t }), from, to))) as Log[], ); - for (const l of swapLogs) if (await applySwap(db, chain, l as Log & { args: Swap }, poolToToken)) swaps++; + for (const l of swapLogs) if (await applySwap(db, chain, l as Log & { args: Swap }, poolToToken, tokenQuote)) swaps++; } const feeLogs = await fetchLogsSplit((f, t) => client.getLogs({ address: locker, events: LOCKER_EVENTS, fromBlock: f, toBlock: t }), from, to); for (const l of feeLogs) if (await applyFee(db, chain, l as unknown as FeeLog, tokenIdToToken)) fees++; @@ -876,10 +900,11 @@ export async function applyLaunchTx(chain: ChainKey, hash: Hex): Promise l.address.toLowerCase() === pm) }); for (const l of swapLogs) { - if (await applySwap(db, chain, l as unknown as Log & { args: Swap }, poolToToken)) { + if (await applySwap(db, chain, l as unknown as Log & { args: Swap }, poolToToken, tokenQuote)) { swaps++; const t = poolToToken.get(l.args.id.toLowerCase()); if (t && !tokens.includes(t)) tokens.push(t); diff --git a/app/src/lib/launchpad/queries.ts b/app/src/lib/launchpad/queries.ts index a1167ff4..d45d61e8 100644 --- a/app/src/lib/launchpad/queries.ts +++ b/app/src/lib/launchpad/queries.ts @@ -828,6 +828,20 @@ export async function getTrending(ethUsd: number | null = null): Promise { decimals: number; decimalsKnown: boolean; usd: number | null }> { + await withStocks(); + const snap = priceState(); + return (chain, quote) => { + const q = quoteInfo(chain, quote, snap); + return { decimals: q.decimals, decimalsKnown: q.decimalsKnown !== false, usd: quoteUsd(q, ethUsd) }; + }; +} + /** * The quote pricing the lists use, as a function, for code that prices many launches at once (season points): * decimals and USD per whole quote unit (null = unpriced) for a launch's quote on its chain. `priced` is false for a diff --git a/app/src/lib/launchpad/usd-history.test.ts b/app/src/lib/launchpad/usd-history.test.ts new file mode 100644 index 00000000..9430c813 --- /dev/null +++ b/app/src/lib/launchpad/usd-history.test.ts @@ -0,0 +1,157 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { + HOUR_MS, backfillHours, buildBackfillSql, captureDue, fillHours, hourIso, hourSql, hourStartMs, multiplySeries, parseCoinbaseCandles, + parseGeckoOhlcv, parseHyperliquidCandles, quoteSeriesRows, type SeriesKey, +} from "./usd-history.ts"; + +const H = (iso: string) => Date.parse(iso); + +test("hour buckets are UTC hour starts, from ms, ISO or Date", () => { + assert.equal(hourIso("2026-09-06T10:59:59.999Z"), "2026-09-06T10:00:00.000Z"); + assert.equal(hourIso(H("2026-09-06T11:00:00Z")), "2026-09-06T11:00:00.000Z"); + assert.equal(hourIso(new Date("2026-09-06T11:30:00+05:30")), "2026-09-06T06:00:00.000Z", "offsets resolve to the UTC hour"); + assert.throws(() => hourStartMs("not a date")); + // the SQL bucket is zone independent (date_trunc on timestamptz truncates in the session zone) and refuses odd input + assert.equal(hourSql("s.block_time"), "to_timestamp(floor(extract(epoch FROM s.block_time) / 3600) * 3600)"); + assert.throws(() => hourSql("s.block_time); DROP TABLE x; --")); +}); + +test("a backfill stops at the last completed hour: the hour in progress is the live capture's", () => { + const r = backfillHours("2026-09-01T00:00:00Z", H("2026-11-10T14:37:00Z")); + assert.equal(new Date(r.from).toISOString(), "2026-09-01T00:00:00.000Z"); + assert.equal(new Date(r.to).toISOString(), "2026-11-10T13:00:00.000Z"); + assert.equal(new Date(backfillHours("2026-09-01T00:00:00Z", H("2026-11-10T14:00:00Z")).to).toISOString(), "2026-11-10T13:00:00.000Z", "even at the very start of an hour"); +}); + +test("the live capture runs at most once a minute, and always on a new hour", () => { + const m = 60_000; + assert.equal(captureDue(undefined, H("2026-11-10T10:00:10Z"), m), true, "first run"); + assert.equal(captureDue(H("2026-11-10T10:00:10Z"), H("2026-11-10T10:00:40Z"), m), false, "inside the minute"); + assert.equal(captureDue(H("2026-11-10T10:00:10Z"), H("2026-11-10T10:01:10Z"), m), true, "a minute later"); + assert.equal(captureDue(H("2026-11-10T10:59:50Z"), H("2026-11-10T11:00:02Z"), m), true, "a new hour, even inside the minute"); +}); + +test("candle parsers take each source's live shape, keep the close, sort by hour and skip junk", () => { + // Coinbase: [time s, low, high, open, close, volume], newest first + assert.deepEqual(parseCoinbaseCandles([[1789264800, 2400, 2500, 2450, 2490.4, 10], [1789261200, 2300, 2460, 2400, 2450, 9], ["x"], [1789258000, 1, 1, 1, 0, 1]]), [ + { hour: 1789261200_000, usd: 2450 }, + { hour: 1789264800_000, usd: 2490.4 }, + ]); + assert.deepEqual(parseCoinbaseCandles({ message: "rate limited" }), []); + // Hyperliquid: { t: open ms, c: close string } + assert.deepEqual(parseHyperliquidCandles([{ t: 1789264800000, c: "79.223" }, { t: 1789261200000, c: "0" }, null, { c: "1" }]), [{ hour: 1789264800000, usd: 79.223 }]); + // GeckoTerminal: data.attributes.ohlcv_list [[time s, o, h, l, c, v]], newest first + assert.deepEqual(parseGeckoOhlcv({ data: { attributes: { ohlcv_list: [[1789264800, 1, 1, 1, 3e-9, 1], [1789261200, 1, 1, 1, 2.5e-9, 1]] } } }), [ + { hour: 1789261200_000, usd: 2.5e-9 }, + { hour: 1789264800_000, usd: 3e-9 }, + ]); + assert.deepEqual(parseGeckoOhlcv({ errors: [{ status: "404" }] }), []); +}); + +test("fillHours: an hour's own close, else the previous close within the gap limit, else nothing", () => { + const t0 = H("2026-09-06T00:00:00Z"); + const pts = [{ hour: t0 - HOUR_MS, usd: 9 }, { hour: t0, usd: 10 }, { hour: t0 + 3 * HOUR_MS, usd: 13 }, { hour: t0 + 3 * HOUR_MS, usd: 14 }]; + const m = fillHours(pts, t0, t0 + 7 * HOUR_MS, 2); + assert.equal(m.get(t0), 10); + assert.equal(m.get(t0 + HOUR_MS), 10, "carried 1h"); + assert.equal(m.get(t0 + 2 * HOUR_MS), 10, "carried 2h"); + assert.equal(m.get(t0 + 3 * HOUR_MS), 14, "a later point for the same hour wins"); + assert.equal(m.get(t0 + 5 * HOUR_MS), 14); + assert.equal(m.has(t0 + 6 * HOUR_MS), false, "past the gap limit: no price, never a stale guess"); + // a point before the range carries into it + assert.equal(fillHours([{ hour: t0 - HOUR_MS, usd: 9 }], t0, t0, 2).get(t0), 9); + const product = multiplySeries(new Map([[t0, 2e-9], [t0 + HOUR_MS, 3e-9]]), new Map([[t0, 2000]])); + assert.deepEqual([...product.keys()], [t0], "an hour missing on either side has no product"); + assert.ok(Math.abs(product.get(t0)! - 4e-6) < 1e-18); +}); + +test("quoteSeriesRows: every quote the backfill prices comes from the app's own quote config, once", () => { + const rows = quoteSeriesRows(); + const by = (chainId: number, quote: string) => rows.filter((r) => r.chainId === chainId && r.quote === quote.toLowerCase()); + const one = (chainId: number, quote: string) => { + const r = by(chainId, quote); + assert.equal(r.length, 1, `${chainId}:${quote}`); + return r[0]; + }; + const NATIVE = "0x0000000000000000000000000000000000000000"; + assert.deepEqual([one(8453, NATIVE).series, one(8453, NATIVE).decimals], ["ETH-USD", 18]); + assert.equal(one(8453, "0x4200000000000000000000000000000000000006").series, "ETH-USD", "WETH at ETH's price"); + assert.equal(one(4663, NATIVE).series, "ETH-USD"); + assert.deepEqual([one(5042, NATIVE).series, one(5042, NATIVE).decimals], ["USD", 18], "Arc's native asset is USDC, never ETH"); + assert.deepEqual([one(5042, "0x3600000000000000000000000000000000000000").series, one(5042, "0x3600000000000000000000000000000000000000").decimals], ["USD", 6]); + assert.equal(one(99901, NATIVE).series, "HYPE-USD"); + assert.equal(one(99901, "0xbA876237B0F021A6c6D852d6c26E107997cC5E81").series, "HYPE-USD", "WHYPE at HYPE's price"); + assert.deepEqual([one(99901, "0xb47c4C8F73F009CC0C0d3f92B388d13E293F0B65").series, one(99901, "0xb47c4C8F73F009CC0C0d3f92B388d13E293F0B65").decimals], ["USD", 6]); + assert.equal(one(4663, "0x5fc5360D0400a0Fd4f2af552ADD042D716F1d168").decimals, 6, "USDG"); + assert.equal(one(8453, "0x5F980Dcfc4c0fa3911554cf5ab288ed0eb13DBa3").series, "GITLAWB-USD"); + assert.equal(one(4663, "0xd1b0d44E4f6ed940fcC7A9F59Bf30Daf62cCFe3D").series, "GITLAWB-USD", "the Robinhood OFT is the same GITLAWB"); + assert.equal(one(8453, "0x6aC18bcf4eDE02591d4917452700ea5eaAea13c1").series, "GITLAWB-USD", "one TWIG unwraps to one GITLAWB"); + assert.equal(new Set(rows.map((r) => `${r.chainId}:${r.quote}`)).size, rows.length, "no quote twice"); + for (const r of rows) assert.equal(r.quote, r.quote.toLowerCase()); + for (const r of rows.filter((x) => x.series === "USD")) assert.equal(r.fixedUsd, 1); +}); + +function sql(mode: "dry-run" | "apply") { + const t0 = H("2026-09-06T00:00:00Z"); + const series = new Map>([["ETH-USD", new Map([[t0, 2490.4]])], ["USD", new Map([[t0, 1]])]]); + return buildBackfillSql({ mode, generatedAt: "2026-11-10T14:00:00Z", from: t0, to: t0, quotes: quoteSeriesRows(), series }); +} + +test("a dry run writes nothing: it reports and rolls back", () => { + const s = sql("dry-run"); + assert.match(s, /MODE: DRY-RUN/); + assert.match(s, /^\\set ON_ERROR_STOP on$/m, "psql stops at the first error"); + assert.match(s, /ROLLBACK; -- dry run: nothing was written\n$/); + assert.doesNotMatch(s, /UPDATE bb_|LOCK TABLE|^COMMIT;/m, "no write and no commit (the temp tables' ON COMMIT DROP is not one)"); + assert.match(s, /ALL-TIME VOLUME/); + assert.match(s, /usd_at_latest_price/); + assert.match(s, /usd_at_trade_time/); + // the ledger fill never replaces an hour the indexer captured live + assert.match(s, /INSERT INTO bb_quote_usd_hourly \(chain_id, quote, hour, usd, decimals, source\)[\s\S]*?ON CONFLICT \(chain_id, quote, hour\) DO NOTHING;/); + assert.match(s, /\('ETH-USD', '2026-09-06T00:00:00\.000Z', 2490\.40000000000, 'coinbase'\)/); +}); + +test("apply prices only rows with no USD yet, at their own hour, then recomputes totals under the rebuild script's locks", () => { + const s = sql("apply"); + const updates = s.match(/UPDATE bb_launch_(swaps|fee_events) [\s\S]*?;/g) ?? []; + assert.equal(updates.length, 3, "trades, collected fees, burned fees"); + for (const u of updates) { + assert.match(u, /\.usd IS NULL/, "never re-prices a row that has a value"); + assert.match(u, /p\.hour = to_timestamp\(floor\(extract\(epoch FROM [sf]\.block_time\) \/ 3600\) \* 3600\)/, "its own hour"); + } + assert.match(updates[2], /f\.kind = 'burned' AND f\.currency = l\.quote/, "a burn of the launched token is never priced as the quote"); + const lockAt = s.indexOf("LOCK TABLE bb_launch_swaps IN SHARE MODE"); + const recomputeAt = s.indexOf("UPDATE bb_launches l SET"); + assert.ok(lockAt > 0 && recomputeAt > lockAt, "the totals are recomputed only under the locks"); + assert.match(s, /LOCK TABLE bb_launches IN EXCLUSIVE MODE/); + assert.equal(s.match(/^COMMIT;$/gm)?.length, 2); + assert.match(s, /AFTER APPLY/); +}); + +test("prices must be positive and finite, and text is quoted safely", () => { + const t0 = H("2026-09-06T00:00:00Z"); + for (const bad of [0, -1, Number.NaN, Infinity]) { + assert.throws(() => buildBackfillSql({ mode: "dry-run", generatedAt: "x", from: t0, to: t0, quotes: [], series: new Map([["ETH-USD", new Map([[t0, bad]])]]) }), /bad price/); + } + const s = buildBackfillSql({ mode: "dry-run", generatedAt: "x", from: t0, to: t0, quotes: [{ chain: "base", chainId: 8453, quote: "o'brien", series: "USD", decimals: 6, fixedUsd: 1 }], series: new Map() }); + assert.match(s, /'o''brien'/); +}); + +test("the indexer prices each trade and fee at its own hour, in the same transaction as the totals", () => { + const src = readFileSync(path.join(process.cwd(), "src/lib/launchpad/indexer.ts"), "utf8"); + const swap = src.slice(src.indexOf("async function applySwap("), src.indexOf("type TransferArgs")); + assert.match(swap, /return db\.begin\(async \(tx\) => \{[\s\S]*INSERT INTO bb_launch_swaps \([^)]*, usd\)[\s\S]*FROM bb_quote_usd_hourly p[\s\S]*p\.hour = \$\{hourIso\(time\)\}::timestamptz[\s\S]*RETURNING tx_hash, usd[\s\S]*volume_usd = volume_usd \+ \$\{usd\}::float8/); + const fee = src.slice(src.indexOf("async function applyFeeIn("), src.indexOf("async function poolMaps(")); + assert.match(fee, /'collected'[\s\S]*FROM bb_quote_usd_hourly p[\s\S]*fees_collected_usd = fees_collected_usd \+ \$\{usd\}::float8/); + assert.match(fee, /const priceIn = cur === info\.quote \? cur : "";[\s\S]*p\.quote = \$\{priceIn\}[\s\S]*fees_burned_usd = fees_burned_usd \+ \$\{usd\}::float8/); + // this hour's live prices are captured before any trade in the range is priced, on both ingestion paths + for (const fn of ["async function applyRange(", "export async function applyLaunchTx("]) { + const body = src.slice(src.indexOf(fn), src.indexOf(fn) + 4000); + const capture = body.indexOf("await captureLiveUsd(db, chain, tokenQuote.values())"); + const swaps = body.indexOf("applySwap(db, chain,"); + assert.ok(capture > 0 && swaps > capture, `${fn}: capture before swaps`); + } +}); diff --git a/app/src/lib/launchpad/usd-history.ts b/app/src/lib/launchpad/usd-history.ts new file mode 100644 index 00000000..9598c99c --- /dev/null +++ b/app/src/lib/launchpad/usd-history.ts @@ -0,0 +1,279 @@ +/** + * Trade-time USD, pure (node --test loads it directly): hour buckets, historical candle parsers, gap filling, which price + * series prices which quote, and the backfill SQL that scripts/usd-backfill.ts writes. + * + * The rule: a trade's (or fee's) USD value is its quote amount times the quote's USD price in the trade's own UTC hour, + * fixed once written (bb_launch_swaps.usd, bb_launch_fee_events.usd). bb_quote_usd_hourly holds one price per quote per + * hour: the live price the indexer saw during that hour (source 'live', updated until the hour ends, so a trade indexed + * seconds after it happened is priced at that spot), or a historical close filled in by the backfill. + */ +import type { ChainKey } from "../chainKeys.ts"; +import { CHAIN_IDS, CHAIN_KEYS } from "../chainKeys.ts"; +import { fixedUsdQuotes, quoteInfo, quotesWithKey } from "./config.ts"; +import { GITLAWB_ADDRESS } from "./gitlawb.ts"; + +export const HOUR_MS = 3_600_000; + +/** Start of the UTC hour containing `t` (ms, ISO string or Date), as ms. */ +export function hourStartMs(t: number | string | Date): number { + const ms = typeof t === "number" ? t : new Date(t).getTime(); + if (!Number.isFinite(ms)) throw new Error(`bad time: ${String(t)}`); + return Math.floor(ms / HOUR_MS) * HOUR_MS; +} +/** Start of the UTC hour containing `t`, as an ISO string (the bb_quote_usd_hourly.hour key). */ +export function hourIso(t: number | string | Date): string { + return new Date(hourStartMs(t)).toISOString(); +} +/** + * The same bucket in SQL, for a timestamptz column. Time-zone independent on purpose: date_trunc('hour', timestamptz) + * truncates in the session's zone, which would key a half-hour-offset session to the wrong row. + */ +export function hourSql(col: string): string { + if (!/^[a-z_][a-z0-9_.]*$/i.test(col)) throw new Error(`bad column: ${col}`); + return `to_timestamp(floor(extract(epoch FROM ${col}) / 3600) * 3600)`; +} + +/** + * The hours a backfill covers: from `from` to the last COMPLETED hour before `now`. The hour in progress belongs to the + * indexer's live capture: a partial candle written for it would block the capture (it never replaces a non-live row), + * and every trade for the rest of that hour would be priced at that stale close. + */ +export function backfillHours(from: number | string, now: number): { from: number; to: number } { + return { from: hourStartMs(from), to: hourStartMs(now) - HOUR_MS }; +} + +/** The live capture's rhythm (usdLedger.ts): at most once per `everyMs`, and always on the first run of a new UTC hour. */ +export function captureDue(last: number | undefined, now: number, everyMs: number): boolean { + return last === undefined || now - last >= everyMs || hourStartMs(now) !== hourStartMs(last); +} + +export type HourPoint = { hour: number; usd: number }; + +function positive(v: unknown): number | null { + const n = typeof v === "string" ? Number(v) : typeof v === "number" ? v : NaN; + return Number.isFinite(n) && n > 0 ? n : null; +} + +/** Coinbase Exchange candles: [[time s, low, high, open, close, volume], …] (newest first). The hour's close. */ +export function parseCoinbaseCandles(body: unknown): HourPoint[] { + if (!Array.isArray(body)) return []; + const out: HourPoint[] = []; + for (const c of body) { + if (!Array.isArray(c) || c.length < 5) continue; + const t = Number(c[0]); + const close = positive(c[4]); + if (Number.isFinite(t) && close !== null) out.push({ hour: hourStartMs(t * 1000), usd: close }); + } + return out.sort((a, b) => a.hour - b.hour); +} + +/** Hyperliquid candleSnapshot: [{ t: open ms, c: close as a string, … }, …]. The hour's close. */ +export function parseHyperliquidCandles(body: unknown): HourPoint[] { + if (!Array.isArray(body)) return []; + const out: HourPoint[] = []; + for (const c of body) { + if (typeof c !== "object" || c === null) continue; + const t = Number((c as { t?: unknown }).t); + const close = positive((c as { c?: unknown }).c); + if (Number.isFinite(t) && close !== null) out.push({ hour: hourStartMs(t), usd: close }); + } + return out.sort((a, b) => a.hour - b.hour); +} + +/** GeckoTerminal pool OHLCV: { data: { attributes: { ohlcv_list: [[time s, o, h, l, c, v], …] } } }. The hour's close. */ +export function parseGeckoOhlcv(body: unknown): HourPoint[] { + const list = (body as { data?: { attributes?: { ohlcv_list?: unknown } } } | null)?.data?.attributes?.ohlcv_list; + if (!Array.isArray(list)) return []; + const out: HourPoint[] = []; + for (const c of list) { + if (!Array.isArray(c) || c.length < 5) continue; + const t = Number(c[0]); + const close = positive(c[4]); + if (Number.isFinite(t) && close !== null) out.push({ hour: hourStartMs(t * 1000), usd: close }); + } + return out.sort((a, b) => a.hour - b.hour); +} + +/** + * One value per hour from `from` to `to` (inclusive, hour starts): each hour takes its own point, else the last earlier + * point at most `maxGapHours` back (a market with no trade in an hour still has the previous close as its price), else + * no value. Later points for the same hour win. + */ +export function fillHours(points: readonly HourPoint[], from: number, to: number, maxGapHours: number): Map { + const byHour = new Map(); + for (const p of [...points].sort((a, b) => a.hour - b.hour)) byHour.set(hourStartMs(p.hour), p.usd); + const out = new Map(); + let last: { hour: number; usd: number } | null = null; + // a point before the range still carries into it + for (const [h, v] of [...byHour].sort((a, b) => a[0] - b[0])) if (h < hourStartMs(from)) last = { hour: h, usd: v }; + for (let h = hourStartMs(from); h <= hourStartMs(to); h += HOUR_MS) { + const own = byHour.get(h); + if (own !== undefined) last = { hour: h, usd: own }; + if (last && (h - last.hour) / HOUR_MS <= maxGapHours) out.set(h, last.usd); + } + return out; +} + +/** Hour-by-hour product of two series (GITLAWB per WETH × ETH per USD); an hour missing from either has no value. */ +export function multiplySeries(a: ReadonlyMap, b: ReadonlyMap): Map { + const out = new Map(); + for (const [h, v] of a) { + const w = b.get(h); + if (w !== undefined) out.set(h, v * w); + } + return out; +} + +/** The historical series the backfill knows. A quote whose key maps to none (stocks, community pairs, unlisted) has no history here. */ +export type SeriesKey = "USD" | "ETH-USD" | "HYPE-USD" | "GITLAWB-USD"; +export const SERIES_SOURCE: Record = { + USD: "fixed", + "ETH-USD": "coinbase", + "HYPE-USD": "coinbase", + "GITLAWB-USD": "geckoterminal", +}; +/** The GITLAWB/WETH Uniswap v4 pool on Base and the GITLAWB token, for GeckoTerminal's OHLCV (price in WETH, × ETH-USD). */ +export const GITLAWB_HISTORY = { network: "base", token: GITLAWB_ADDRESS }; + +export type QuoteSeriesRow = { chain: ChainKey; chainId: number; quote: string; series: SeriesKey; decimals: number; fixedUsd: number | null }; + +/** + * Every quote the backfill can price, from the app's own quote config (so it can never price a quote the site does not): + * the fixed-price quotes (stables, Arc's native USDC) at their fixed price; ETH and WETH from ETH-USD; HYPE and WHYPE from + * HYPE-USD; GITLAWB (Base and the Robinhood OFT) and TWIG (one TWIG unwraps to one GITLAWB) from GITLAWB-USD. + */ +export function quoteSeriesRows(): QuoteSeriesRow[] { + const rows: QuoteSeriesRow[] = []; + const seen = new Set(); + const add = (chain: ChainKey, address: string, series: SeriesKey, fixedUsd: number | null) => { + const quote = address.toLowerCase(); + const k = `${chain}:${quote}`; + if (seen.has(k)) return; + seen.add(k); + rows.push({ chain, chainId: CHAIN_IDS[chain], quote, series, decimals: quoteInfo(chain, quote).decimals, fixedUsd }); + }; + for (const f of fixedUsdQuotes()) add(f.chain, f.address, "USD", f.usd); + for (const q of quotesWithKey("eth")) add(q.chain, q.address, "ETH-USD", null); + for (const q of quotesWithKey("hype")) add(q.chain, q.address, "HYPE-USD", null); + for (const key of ["gitlawb", "twig"] as const) for (const q of quotesWithKey(key)) add(q.chain, q.address, "GITLAWB-USD", null); + // stable output order: chain registry order, then series, then address + const order = new Map(CHAIN_KEYS.map((k, i) => [k, i])); + return rows.sort((a, b) => (order.get(a.chain)! - order.get(b.chain)!) || a.series.localeCompare(b.series) || a.quote.localeCompare(b.quote)); +} + +export type BackfillMode = "dry-run" | "apply"; + +/** The SQL file scripts/usd-backfill.ts writes. Pure so the tests can read exactly what will run against production. */ +export function buildBackfillSql(opts: { + mode: BackfillMode; + generatedAt: string; + from: number; + to: number; + quotes: readonly QuoteSeriesRow[]; + series: ReadonlyMap>; +}): string { + const lit = (s: string) => `'${s.replace(/'/g, "''")}'`; + const num = (n: number) => { + if (!Number.isFinite(n) || n <= 0) throw new Error(`bad price: ${n}`); + return n.toPrecision(15); + }; + const out: string[] = []; + out.push(`-- Trade-time USD backfill, generated by scripts/usd-backfill.ts at ${opts.generatedAt}. MODE: ${opts.mode.toUpperCase()}.`); + out.push(`-- Hours ${new Date(hourStartMs(opts.from)).toISOString()} .. ${new Date(hourStartMs(opts.to)).toISOString()} (UTC). Idempotent: prices only rows that have no USD value yet,`); + out.push(`-- and never overwrites an hour the indexer captured live. ${opts.mode === "dry-run" ? "Dry run: everything is ROLLED BACK at the end; only the report is real." : "Apply: prices are written and every launch's USD totals are recomputed."}`); + out.push(`\\set ON_ERROR_STOP on`); + out.push(`BEGIN;`); + out.push(`CREATE TEMP TABLE usd_quote_map (chain_id integer, quote text, series text, decimals integer) ON COMMIT DROP;`); + if (opts.quotes.length > 0) { + out.push(`INSERT INTO usd_quote_map (chain_id, quote, series, decimals) VALUES`); + out.push(opts.quotes.map((q) => ` (${q.chainId}, ${lit(q.quote)}, ${lit(q.series)}, ${q.decimals})`).join(",\n") + ";"); + } + out.push(`CREATE TEMP TABLE usd_series (series text, hour timestamptz, usd double precision, source text) ON COMMIT DROP;`); + for (const [key, hours] of opts.series) { + const rows = [...hours].sort((a, b) => a[0] - b[0]); + for (let i = 0; i < rows.length; i += 1000) { + const part = rows.slice(i, i + 1000); + out.push(`INSERT INTO usd_series (series, hour, usd, source) VALUES`); + out.push(part.map(([h, v]) => ` (${lit(key)}, ${lit(new Date(h).toISOString())}, ${num(v)}, ${lit(SERIES_SOURCE[key])})`).join(",\n") + ";"); + } + } + out.push(`-- the hourly ledger: fill the hours the indexer did not capture live (a live capture is the spot at index time and wins)`); + out.push(`INSERT INTO bb_quote_usd_hourly (chain_id, quote, hour, usd, decimals, source)`); + out.push(` SELECT m.chain_id, m.quote, s.hour, s.usd, m.decimals, s.source FROM usd_series s JOIN usd_quote_map m ON m.series = s.series`); + out.push(` ON CONFLICT (chain_id, quote, hour) DO NOTHING;`); + const hour = (col: string) => hourSql(col); + out.push(`-- REPORT: every trade at today's price (the old method) vs at its own hour's price (the new one), per chain and quote`); + out.push(`WITH latest AS ( + SELECT DISTINCT ON (chain_id, quote) chain_id, quote, usd, decimals FROM bb_quote_usd_hourly ORDER BY chain_id, quote, hour DESC +), sw AS ( + SELECT s.chain_id, l.quote, abs(s.amount0) AS amt, COALESCE(s.usd, abs(s.amount0) / power(10::numeric, p.decimals) * p.usd) AS usd_trade_time + FROM bb_launch_swaps s + JOIN bb_launches l ON l.chain_id = s.chain_id AND l.token = s.token + LEFT JOIN bb_quote_usd_hourly p ON p.chain_id = s.chain_id AND p.quote = l.quote AND p.hour = ${hour("s.block_time")} +) +SELECT sw.chain_id, sw.quote, count(*) AS trades, + count(*) FILTER (WHERE sw.usd_trade_time IS NULL) AS trades_without_price, + round(sum(sw.amt / power(10::numeric, lt.decimals))::numeric, 4) AS volume_in_quote, + round(sum(sw.amt / power(10::numeric, lt.decimals) * lt.usd)::numeric) AS usd_at_latest_price, + round(sum(sw.usd_trade_time)::numeric) AS usd_at_trade_time + FROM sw LEFT JOIN latest lt ON lt.chain_id = sw.chain_id AND lt.quote = sw.quote + GROUP BY 1, 2 + ORDER BY usd_at_trade_time DESC NULLS LAST;`); + out.push(`WITH latest AS ( + SELECT DISTINCT ON (chain_id, quote) chain_id, quote, usd, decimals FROM bb_quote_usd_hourly ORDER BY chain_id, quote, hour DESC +), sw AS ( + SELECT abs(s.amount0) / power(10::numeric, lt.decimals) * lt.usd AS usd_latest, + COALESCE(s.usd, abs(s.amount0) / power(10::numeric, p.decimals) * p.usd) AS usd_trade_time + FROM bb_launch_swaps s + JOIN bb_launches l ON l.chain_id = s.chain_id AND l.token = s.token + LEFT JOIN latest lt ON lt.chain_id = s.chain_id AND lt.quote = l.quote + LEFT JOIN bb_quote_usd_hourly p ON p.chain_id = s.chain_id AND p.quote = l.quote AND p.hour = ${hour("s.block_time")} +) +SELECT 'ALL-TIME VOLUME' AS total, round(sum(usd_latest)::numeric) AS usd_at_latest_price, round(sum(usd_trade_time)::numeric) AS usd_at_trade_time, + count(*) FILTER (WHERE usd_trade_time IS NULL) AS trades_without_price, count(*) AS trades + FROM sw;`); + out.push(`WITH fe AS ( + SELECT f.kind, COALESCE(f.usd, CASE WHEN f.kind = 'collected' THEN f.quote_amount ELSE f.amount END / power(10::numeric, p.decimals) * p.usd) AS usd_trade_time + FROM bb_launch_fee_events f + JOIN bb_launches l ON l.chain_id = f.chain_id AND l.token = f.token + LEFT JOIN bb_quote_usd_hourly p ON p.chain_id = f.chain_id AND p.quote = l.quote AND p.hour = ${hour("f.block_time")} + WHERE f.kind = 'collected' OR (f.kind = 'burned' AND f.currency = l.quote) +) +SELECT 'FEES (quote side)' AS total, kind, round(sum(usd_trade_time)::numeric) AS usd_at_trade_time, + count(*) FILTER (WHERE usd_trade_time IS NULL) AS events_without_price, count(*) AS events + FROM fe GROUP BY kind ORDER BY kind;`); + if (opts.mode === "dry-run") { + out.push(`ROLLBACK; -- dry run: nothing was written`); + return out.join("\n") + "\n"; + } + out.push(`-- price every trade and fee that has no USD value yet, at its own hour`); + out.push(`UPDATE bb_launch_swaps s SET usd = abs(s.amount0) / power(10::numeric, p.decimals) * p.usd + FROM bb_launches l, bb_quote_usd_hourly p + WHERE s.usd IS NULL AND l.chain_id = s.chain_id AND l.token = s.token + AND p.chain_id = s.chain_id AND p.quote = l.quote AND p.hour = ${hour("s.block_time")};`); + out.push(`UPDATE bb_launch_fee_events f SET usd = f.quote_amount / power(10::numeric, p.decimals) * p.usd + FROM bb_launches l, bb_quote_usd_hourly p + WHERE f.usd IS NULL AND f.kind = 'collected' AND l.chain_id = f.chain_id AND l.token = f.token + AND p.chain_id = f.chain_id AND p.quote = l.quote AND p.hour = ${hour("f.block_time")};`); + out.push(`UPDATE bb_launch_fee_events f SET usd = f.amount / power(10::numeric, p.decimals) * p.usd + FROM bb_launches l, bb_quote_usd_hourly p + WHERE f.usd IS NULL AND f.kind = 'burned' AND f.currency = l.quote AND l.chain_id = f.chain_id AND l.token = f.token + AND p.chain_id = f.chain_id AND p.quote = l.quote AND p.hour = ${hour("f.block_time")};`); + out.push(`COMMIT;`); + out.push(`-- every launch's USD totals, recomputed from the priced rows. Same locking as scripts/rebuild-launch-totals.sql: the indexer's`); + out.push(`-- ingestion transactions (INSERT event + UPDATE totals) wait the few seconds this takes, then apply on top. Nothing is lost.`); + out.push(`BEGIN;`); + out.push(`LOCK TABLE bb_launch_swaps IN SHARE MODE;`); + out.push(`LOCK TABLE bb_launch_fee_events IN SHARE MODE;`); + out.push(`LOCK TABLE bb_launches IN EXCLUSIVE MODE;`); + out.push(`UPDATE bb_launches l SET + volume_usd = COALESCE((SELECT sum(s.usd) FROM bb_launch_swaps s WHERE s.chain_id = l.chain_id AND s.token = l.token), 0), + fees_collected_usd = COALESCE((SELECT sum(f.usd) FROM bb_launch_fee_events f WHERE f.chain_id = l.chain_id AND f.token = l.token AND f.kind = 'collected'), 0), + fees_burned_usd = COALESCE((SELECT sum(f.usd) FROM bb_launch_fee_events f WHERE f.chain_id = l.chain_id AND f.token = l.token AND f.kind = 'burned' AND f.currency = l.quote), 0);`); + out.push(`SELECT 'AFTER APPLY' AS total, round(sum(volume_usd)::numeric) AS volume_usd, round(sum(fees_collected_usd)::numeric) AS fees_collected_usd, + round(sum(fees_burned_usd)::numeric) AS fees_burned_usd, + (SELECT count(*) FROM bb_launch_swaps WHERE usd IS NULL) AS trades_still_without_price + FROM bb_launches;`); + out.push(`COMMIT;`); + return out.join("\n") + "\n"; +} diff --git a/app/src/lib/launchpad/usdLedger.ts b/app/src/lib/launchpad/usdLedger.ts new file mode 100644 index 00000000..67241d59 --- /dev/null +++ b/app/src/lib/launchpad/usdLedger.ts @@ -0,0 +1,55 @@ +import "server-only"; +import { chainIdOf, type ChainKey } from "@/lib/chainPublic"; +import { errMessage, type Db } from "@/lib/db"; +import { ethUsd } from "./ethPrice"; +import { quoteUsdNow } from "./queries"; +import { captureDue, hourIso } from "./usd-history"; + +/** + * Live capture for trade-time USD (usd-history.ts states the rule). Before the indexer applies a range, it writes the + * live USD price of every quote in use on the chain into the current UTC hour's row of bb_quote_usd_hourly (source + * 'live'), so a trade indexed seconds after it happened is priced at that spot. The row keeps updating until the hour + * ends; after that it is frozen, and it is never replaced by a historical close (the backfill fills only hours that have + * no row). Prices come from the same sources the lists use (queries.quoteUsdNow). A quote with no price right now is not + * written: its trades keep usd NULL until scripts/usd-backfill.ts fills that hour. Display only, like every USD figure. + */ +export const LIVE_CAPTURE_EVERY_MS = 60_000; +const lastCapture = new Map(); + +/** Test-only: forget when each chain was last captured. */ +export function resetLiveCapture(): void { + lastCapture.clear(); +} + +/** + * Capture at most once a minute per chain, and always on the first run of a new hour (so a trade in its first seconds + * finds its hour's row). Never throws: a failed capture only leaves that minute's trades for the backfill. + */ +export async function captureLiveUsd(db: Db, chain: ChainKey, quotes: Iterable, now = Date.now()): Promise { + const last = lastCapture.get(chain); + if (!captureDue(last, now, LIVE_CAPTURE_EVERY_MS)) return 0; + lastCapture.set(chain, now); + try { + const wanted = [...new Set([...quotes].map((q) => q.toLowerCase()))]; + if (wanted.length === 0) return 0; + const price = await quoteUsdNow(await ethUsd()); + const rows = wanted.flatMap((quote) => { + const p = price(chain, quote); + return p.usd !== null && Number.isFinite(p.usd) && p.usd > 0 && p.decimalsKnown ? [{ quote, usd: p.usd, decimals: p.decimals }] : []; + }); + if (rows.length === 0) return 0; + await db` + INSERT INTO bb_quote_usd_hourly (chain_id, quote, hour, usd, decimals, source) + SELECT ${chainIdOf(chain)}, u.q, ${hourIso(now)}::timestamptz, u.usd, u.dec, 'live' + FROM unnest(${rows.map((r) => r.quote)}::text[], ${rows.map((r) => r.usd)}::float8[], ${rows.map((r) => r.decimals)}::int[]) AS u(q, usd, dec) + ON CONFLICT (chain_id, quote, hour) DO UPDATE SET usd = EXCLUDED.usd, decimals = EXCLUDED.decimals, updated_at = now() + WHERE bb_quote_usd_hourly.source = 'live'`; + return rows.length; + } catch (err) { + // retry on the next run instead of waiting out the minute + if (last === undefined) lastCapture.delete(chain); + else lastCapture.set(chain, last); + console.warn(`[usd] live price capture failed on ${chain}: ${errMessage(err)}`); + return 0; + } +}