diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bdaaa8e..6ce8ebc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,3 +31,8 @@ jobs: - name: Tests run: npm test + + - name: Ad reporting and crawler limits (PostgreSQL + Redis) + run: | + node scripts/test-ad-token-earnings.mjs + node --import tsx scripts/test-crawl-limits.mjs diff --git a/app/(app)/dashboard/ads/earnings/page.tsx b/app/(app)/dashboard/ads/earnings/page.tsx index ac8620a9..7b912d7d 100644 --- a/app/(app)/dashboard/ads/earnings/page.tsx +++ b/app/(app)/dashboard/ads/earnings/page.tsx @@ -21,6 +21,8 @@ const EMPTY: EarningsModel = { // Signed out: nothing was attempted, so nothing failed. statsUnavailable: false, totals: { + advBilledClicks: 0, advFreeClicks: 0, pubBilledClicks: 0, pubFreeClicks: 0, + advPaidImpressions: 0, advFreeImpressions: 0, pubPaidImpressions: 0, pubFreeImpressions: 0, spentCents: 0, earnedCents: 0, netCents: 0, diff --git a/app/a/[id]/route.ts b/app/a/[id]/route.ts index 6fe742b3..797a0bd5 100644 --- a/app/a/[id]/route.ts +++ b/app/a/[id]/route.ts @@ -17,6 +17,8 @@ import { adClickIp } from "@/lib/ads/client-ip"; import { parseDevice } from "@/lib/tracker/device"; import { isShortCode } from "@/lib/ads/shortcode"; import { env } from "@/lib/env"; +import { gate } from "@/lib/crawl-gateway"; +import { crawlerFamily } from "@/lib/crawl-policy"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -67,6 +69,8 @@ async function findImpression( } export async function GET(request: NextRequest, ctx: { params: Promise<{ id: string }> }) { + const denied = await gate(request); + if (denied) return denied; const fallback = env.siteUrl || "https://crawlproof.com"; try { const { id } = await ctx.params; @@ -82,12 +86,13 @@ export async function GET(request: NextRequest, ctx: { params: Promise<{ id: str if (!imp) return NextResponse.redirect(fallback, { status: 302 }); const ip = adClickIp(request.headers); - const geo = await lookupGeo(ip).catch(() => null); + const crawler = crawlerFamily(request.headers.get("user-agent")); + const geo = crawler ? null : await lookupGeo(ip).catch(() => null); // Deliberately the STRICT classification here, unlike /api/ads/motd: a // terminal ad is served to curl, but it's clicked from a browser when the // reader follows the link. Anyone can curl this URL in a loop, so scripted // hits stay unbilled (recorded with valid=false) rather than paying out. - const device = parseDevice(request.headers.get("user-agent")).deviceType; + const device = crawler ? "bot" : parseDevice(request.headers.get("user-agent")).deviceType; const dest = await resolveClick({ impressionId: imp.id, @@ -103,6 +108,7 @@ export async function GET(request: NextRequest, ctx: { params: Promise<{ id: str }); if (!dest) return NextResponse.redirect(fallback, { status: 302 }); + if (crawler) return NextResponse.redirect(dest, { status: 302, headers: { "cache-control": "no-store", "x-robots-tag": "noindex, nofollow" } }); // Terminal traffic is invisible in an advertiser's analytics without a tag // — there's no referrer from a shell. resolveClick already appended diff --git a/app/api/admin/crawl-activity/route.ts b/app/api/admin/crawl-activity/route.ts new file mode 100644 index 00000000..305a2bb0 --- /dev/null +++ b/app/api/admin/crawl-activity/route.ts @@ -0,0 +1,30 @@ +import { NextRequest, NextResponse } from "next/server"; +import { authenticateBearer } from "@/lib/sp/apiAuth"; +import { createClient } from "@/lib/supabase/server"; +import { serviceClient } from "@/lib/supabase/service"; +import { readCrawlerActivity } from "@/lib/crawl-limits"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +export async function GET(request: NextRequest) { + let userId: string | undefined; + if (request.headers.has("authorization")) { + const auth = await authenticateBearer(request); + if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); + userId = auth.userId; + } else { + const sb = await createClient(); + userId = (await sb.auth.getUser()).data.user?.id; + } + if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + const { data: profile, error } = await serviceClient().from("profiles").select("is_admin").eq("id", userId).maybeSingle(); + if (error || !profile?.is_admin) return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + const requested = Number(request.nextUrl.searchParams.get("days") ?? 7); + const days = Number.isInteger(requested) ? Math.min(31, Math.max(1, requested)) : 7; + try { + return NextResponse.json({ scope: "network", rangeDays: days, daily: await readCrawlerActivity(days) }, { headers: { "cache-control": "private, no-store" } }); + } catch { + return NextResponse.json({ error: "Crawler activity is temporarily unavailable" }, { status: 503 }); + } +} diff --git a/app/api/ads/click/route.ts b/app/api/ads/click/route.ts index 2b28496b..402dffef 100644 --- a/app/api/ads/click/route.ts +++ b/app/api/ads/click/route.ts @@ -8,11 +8,15 @@ import { lookupGeo } from "@/lib/tracker/geo"; import { adClickIp } from "@/lib/ads/client-ip"; import { parseDevice } from "@/lib/tracker/device"; import { env } from "@/lib/env"; +import { gate } from "@/lib/crawl-gateway"; +import { crawlerFamily } from "@/lib/crawl-policy"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; export async function GET(request: NextRequest) { + const denied = await gate(request); + if (denied) return denied; const fallback = env.siteUrl || "https://crawlproof.com"; try { const url = new URL(request.url); @@ -23,8 +27,9 @@ export async function GET(request: NextRequest) { const visitorId = url.searchParams.get("v"); const ip = adClickIp(request.headers); - const geo = await lookupGeo(ip).catch(() => null); - const device = parseDevice(request.headers.get("user-agent")).deviceType; + const crawler = crawlerFamily(request.headers.get("user-agent")); + const geo = crawler ? null : await lookupGeo(ip).catch(() => null); + const device = crawler ? "bot" : parseDevice(request.headers.get("user-agent")).deviceType; const dest = await resolveClick({ impressionId, diff --git a/app/api/ads/v1/earnings/route.ts b/app/api/ads/v1/earnings/route.ts index 7d7666c9..0867ff90 100644 --- a/app/api/ads/v1/earnings/route.ts +++ b/app/api/ads/v1/earnings/route.ts @@ -1,134 +1,32 @@ -// /api/ads/v1/earnings — the account's ad money and delivery, for a token caller. -// -// GET ?days=7|30|90|365 -// -// The same model /dashboard/ads/earnings renders, for something holding an API -// token. It exists because the alternative for a client that wants fleet totals -// is one /campaigns/[id] request per campaign, and the account is past 170 of -// them — see `crawlproof dashboard`, which polls this on a timer. -// -// Same auth as the rest of /api/ads/v1/*: `Authorization: Bearer crp_…`. - import { NextResponse, type NextRequest } from "next/server"; - import { serviceClient } from "@/lib/supabase/service"; import { authenticateBearer } from "@/lib/sp/apiAuth"; import { loadEarnings } from "@/lib/ads/earnings-data"; +import { loadTokenDelivery } from "@/lib/ads/token-earnings"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; -const ALLOWED_DAYS = [7, 30, 90, 365]; - -/** An unknown window falls back to 30 rather than reaching the query planner. */ export function parseDays(raw: string | null): number { const n = Number(raw); - return ALLOWED_DAYS.includes(n) ? n : 30; + return [7, 30, 90, 365].includes(n) ? n : 30; } export async function GET(req: NextRequest) { const auth = await authenticateBearer(req); if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status }); - const days = parseDays(req.nextUrl.searchParams.get("days")); - const sb = serviceClient(); - // The service client has no RLS. loadEarnings filters every table by - // owner_id itself, which is what makes passing it here safe. - const model = await loadEarnings(sb, auth.userId, days); - - // The windowed delivery figures come from RPCs that are `security definer` - // and filter on `auth.uid()`. A service client has no auth.uid(), so they - // return nothing and every impression count arrives as a confident zero. - // The stats views are `security_invoker` and granted to service_role, so - // they can be read directly. They are lifetime rather than windowed, which - // is why this only replaces figures that came back empty, and why the answer - // says which it gave you. - const delivery = await lifetimeDelivery(sb, model); - return NextResponse.json({ ...model, ...delivery }); -} - -type StatsRow = { - impressions: number | null; - free_impressions: number | null; - clicks: number | null; - free_clicks: number | null; -}; - -/** - * What the stats views actually mean, which is not what the column names - * suggest and is worth writing down once: - * - * impressions paid tier, non-duplicate - * free_impressions free tier, non-duplicate -> delivery is the SUM of both - * clicks **valid** clicks, either tier -> delivery is this alone - * free_clicks free tier and **not valid** -> fraud/duplicate, NOT delivery - * - * So impressions add up and clicks do not. Adding `free_clicks` into clicks - * would fold invalid clicks into the CTR, which is the one number a click - * fraud problem would show up in. - */ -const sum = (rows: StatsRow[], key: keyof StatsRow) => - rows.reduce((total, row) => total + (Number(row[key]) || 0), 0); - -/** - * Read a stats view for a list of ids, in chunks. - * - * PostgREST puts `in.(…)` in the query string, and this account is past 180 - * campaigns, so one call is a 7KB URL that comes back empty rather than - * erroring. That empty answer is exactly what a network with no delivery looks - * like, which is how it went unnoticed. - */ -async function readStats( - sb: ReturnType, - view: "ad_campaign_stats" | "ad_slot_stats", - key: "campaign_id" | "slot_id", - ids: string[], -): Promise<{ rows: StatsRow[]; failed: boolean }> { - const rows: StatsRow[] = []; - let failed = false; - const CHUNK = 50; - for (let i = 0; i < ids.length; i += CHUNK) { - const { data, error } = await sb - .from(view) - .select("impressions, free_impressions, clicks, free_clicks") - .in(key, ids.slice(i, i + CHUNK)); - if (error) failed = true; - else rows.push(...((data ?? []) as StatsRow[])); + try { + const sb = serviceClient(); + const model = await loadEarnings(sb, auth.userId, days, loadTokenDelivery(sb, auth.userId, days)); + if (model.statsUnavailable) throw new Error("incomplete_reporting"); + return NextResponse.json({ ...model, deliveryWindow: "range", clickSemantics: "accepted_billed_plus_free" }, { + headers: { "cache-control": "private, no-store" }, + }); + } catch { + console.error("[ads] Earnings response unavailable"); + return NextResponse.json({ error: "Ad reporting is temporarily unavailable. Please retry.", statsUnavailable: true }, { + status: 503, headers: { "retry-after": "5", "cache-control": "no-store" }, + }); } - return { rows, failed }; -} - -async function lifetimeDelivery( - sb: ReturnType, - model: Awaited>, -) { - const t = model.totals; - const empty = !t.advImpressions && !t.advClicks && !t.pubImpressions && !t.pubClicks; - if (!empty) return { deliveryWindow: "range" as const }; - - const campaignIds = model.campaigns.map((c) => c.id); - const slotIds = model.slots.map((s) => s.id); - if (!campaignIds.length && !slotIds.length) return { deliveryWindow: "range" as const }; - - const [c, s] = await Promise.all([ - readStats(sb, "ad_campaign_stats", "campaign_id", campaignIds), - readStats(sb, "ad_slot_stats", "slot_id", slotIds), - ]); - - return { - deliveryWindow: "lifetime" as const, - statsUnavailable: model.statsUnavailable || c.failed || s.failed, - totals: { - ...t, - advImpressions: sum(c.rows, "impressions") + sum(c.rows, "free_impressions"), - advClicks: sum(c.rows, "clicks"), - advFreeImpressions: sum(c.rows, "free_impressions"), - advPaidImpressions: sum(c.rows, "impressions"), - pubImpressions: sum(s.rows, "impressions") + sum(s.rows, "free_impressions"), - pubClicks: sum(s.rows, "clicks"), - pubFreeImpressions: sum(s.rows, "free_impressions"), - pubPaidImpressions: sum(s.rows, "impressions"), - invalidClicks: sum(c.rows, "free_clicks"), - }, - }; } diff --git a/app/robots.txt/route.ts b/app/robots.txt/route.ts index 319d762f..15c4bd63 100644 --- a/app/robots.txt/route.ts +++ b/app/robots.txt/route.ts @@ -5,5 +5,5 @@ import { gateway } from "@/lib/crawl-gateway"; // crawlers are refused everywhere but /crawl (where they can buy a pass), // retrieval crawlers are named as welcome, everyone else gets the rules below. export const GET = robotsRoute(gateway, { - disallow: ["/api/"], + disallow: ["/api/", "/a/"], }); diff --git a/cli/dashboard.ts b/cli/dashboard.ts index 7d76d5ec..b964db64 100644 --- a/cli/dashboard.ts +++ b/cli/dashboard.ts @@ -9,7 +9,7 @@ import type { Container, RenderArgs, Theme } from "@profullstack/hqtui"; -import { collectDashboard, type CoinPayAuth, type DashboardSnapshot, type SiteStats } from "../lib/dashboard/collect"; +import { collectDashboard, type CoinPayAuth, type DashboardSnapshot, type SiteStats, type FeedName, type FeedProgress } from "../lib/dashboard/collect"; import { AD_TARGET_CTR, AD_TARGET_IMPRESSIONS, adTargets } from "../lib/dashboard/roi"; import { buildSiteDetail, type SiteDetail } from "../lib/dashboard/site"; import type { Component } from "../lib/dashboard/score"; @@ -87,6 +87,10 @@ export type State = { who: string; snapshot: DashboardSnapshot | null; loading: boolean; + feeds: Record; + refreshStartedAt: Date | null; + refreshQueued: boolean; + refreshMessage: string | null; lastRefresh: Date | null; error: string | null; paused: boolean; @@ -357,7 +361,7 @@ function trafficScreen(ui: Container, state: State, theme: Theme): void { { title: `Sites · ${state.range} · ${state.who}`, subtitle: `${s.roi.attention.sitesReporting} of ${rows.length} reporting · by ${state.sort}`, - footer: "↑/↓ select · Enter opens · s sorts", + footer: "Click / Enter opens · ↑/↓ select · s sorts", }, (p) => { const view = pane(state, "sites", rows.length); @@ -517,8 +521,9 @@ function domainScreen(ui: Container, state: State, theme: Theme): void { const t = detail.traffic; const m = detail.money; const score = detail.score; + const adsKnown = Boolean(state.snapshot?.ads && !state.snapshot.ads.statsUnavailable); - ui.grid({ columns: ["1fr", "1fr", "1fr"], rows: [12, "1fr"], gap: 1 }, (grid) => { + ui.grid({ columns: ["1fr", "1fr", "1fr"], rows: [14, "1fr"], gap: 1 }, (grid) => { grid.panel( { title: detail.site, @@ -559,16 +564,16 @@ function domainScreen(ui: Container, state: State, theme: Theme): void { // Short subtitles on purpose: hqtui draws the title and the subtitle in the // same border row and the subtitle wins, so a long one costs the panel its // own name at a narrow width. - grid.panel({ title: "Money", subtitle: `${detail.window.range} · ${detail.window.financeDays}d bank` }, (p) => { + grid.panel({ title: "Money", subtitle: `${detail.window.range} · ${detail.window.financeDays}d bank`, footer: adsKnown ? `Ads · ${state.snapshot?.ads?.rangeDays ?? detail.window.financeDays}d · internal` : "Ads unavailable · r retries" }, (p) => { p.keyValues( [ { - label: "Cost · by views", + label: "Est. cost/views", value: m.costByViewsUsd === null ? "—" : money(m.costByViewsUsd, { cents: true }), color: theme.danger, }, { - label: "Cost · by visits", + label: "Est. cost/visits", value: m.costByVisitsUsd === null ? "—" : money(m.costByVisitsUsd, { cents: true }), color: theme.muted, }, @@ -586,12 +591,17 @@ function domainScreen(ui: Container, state: State, theme: Theme): void { label: "Per 1k humans", value: m.rpmUsd === null ? "—" : money(m.rpmUsd, { cents: true }), }, + { + label: `Volume · ${m.observedDays ?? detail.window.financeDays}d`, + value: m.grossVolumeUsd === null ? "—" : money(m.grossVolumeUsd), + }, + { label: "Payments", value: m.transactions === null ? "—" : count(m.transactions) }, { label: "", value: "" }, // Internal by construction: one account owns the slot and the // campaign, so this is the same dollar in two pockets. - { label: "Ad earned (int.)", value: money(m.adEarnedUsd, { cents: true }), color: theme.muted }, - { label: "Ad spent (int.)", value: money(m.adSpentUsd, { cents: true }), color: theme.muted }, - { label: "Impressions", value: count(m.adImpressions), color: theme.muted }, + { label: "Ad earned (int.)", value: adsKnown ? money(m.adEarnedUsd, { cents: true }) : "—", color: theme.muted }, + { label: "Ad spent (int.)", value: adsKnown ? money(m.adSpentUsd, { cents: true }) : "—", color: theme.muted }, + { label: "Impressions", value: adsKnown ? count(m.adImpressions) : "—", color: theme.muted }, ], { labelWidth: 17 }, ); @@ -604,10 +614,14 @@ function domainScreen(ui: Container, state: State, theme: Theme): void { subtitleColor: score.provisional ? theme.warning : theme.muted, }, (p) => { - p.text(score.score === null ? " —" : ` ${score.score.toFixed(0)}`, { + p.text(score.score === null ? " —" : ` ${score.score.toFixed(0)} / 100`, { bold: true, fg: scoreColor(theme, score.score), }); + p.text(score.score === null ? "No signal yet" : score.provisional ? "Early signal · small sample" : score.score >= 60 ? "Promising" : score.score >= 35 ? "Worth watching" : "Limited signal", { + fg: scoreColor(theme, score.score), + }); + p.text("Higher = more potential · heuristic", { fg: theme.muted }); p.meters( [ { label: "viral", value: score.viral, max: 1, text: pct(score.viral, 0) }, @@ -622,6 +636,7 @@ function domainScreen(ui: Container, state: State, theme: Theme): void { grid.panel({ title: "Why it scores that", subtitle: "weights", colSpan: 2 }, (p) => { p.table({ + size: 5, columns: [ { key: "part", title: "Viral", width: 23 }, { key: "value", title: "", align: "right", width: 6 }, @@ -631,6 +646,7 @@ function domainScreen(ui: Container, state: State, theme: Theme): void { rowColor: (row: ComponentRow) => row.color, }); p.table({ + size: 5, columns: [ { key: "part", title: "Risk", width: 23 }, { key: "value", title: "", align: "right", width: 6 }, @@ -680,6 +696,11 @@ function adsScreen(ui: Container, state: State, theme: Theme): void { if (!ads) { ui.panel({ title: "Ads" }, (p) => { + if (feedBusy(state.feeds.ads)) { + p.spinner({ label: state.feeds.ads.detail, color: theme.primary }); + p.text("Waiting for ad earnings and delivery…", { fg: theme.muted }); + return; + } p.text(s.errors.ads ?? "Ad earnings unavailable.", { fg: theme.danger }); p.text("Press r to retry.", { fg: theme.muted }); }); @@ -701,23 +722,23 @@ function adsScreen(ui: Container, state: State, theme: Theme): void { { label: "Impressions", value: count(t.impressions), color: theme.primary }, { label: " free", value: count(t.freeImpressions), color: theme.success }, { label: " paid", value: count(t.paidImpressions), color: theme.muted }, - { label: "Clicks", value: count(t.clicks) }, + { label: "Accepted clicks", value: count(t.clicks) }, + { label: " billed", value: count(t.billedClicks) }, + { label: " free", value: count(t.freeClicks) }, { label: "CTR", value: pct(t.ctr, 3) }, { - label: "Invalid clicks", + label: "Rejected clicks", value: count(t.invalidClicks), color: t.invalidClicks > t.clicks ? theme.danger : theme.warning, }, ], { labelWidth: 16 }, ); - if (t.invalidClicks > t.clicks && t.clicks > 0) { - p.text(`${Math.round(t.invalidClicks / t.clicks)}x more invalid than valid.`, { fg: theme.danger }); - } + p.text("Rejected: historical bots + refused clicks.", { fg: theme.muted }); }); grid.panel( - { title: "Toward the target", subtitle: `${count(t.targetImpressions)}/mo · ${pct(t.targetCtr, 0)} CTR` }, + { title: "Toward the target", subtitle: `${count(t.windowTargetImpressions)}/${window} · ${pct(t.targetCtr, 0)} CTR` }, (p) => { p.meters( [ @@ -733,13 +754,13 @@ function adsScreen(ui: Container, state: State, theme: Theme): void { ); p.keyValues( [ - { label: "Short by", value: count(Math.max(0, t.targetImpressions - t.impressions)) }, + { label: "Short by", value: count(Math.max(0, t.windowTargetImpressions - t.impressions)) }, { label: "Cost per click", - value: t.cpcCents === null ? "nothing charged yet" : `${t.cpcCents.toFixed(1)}c`, + value: t.cpcCents === null ? "no billed clicks in range" : `${t.cpcCents.toFixed(1)}c`, }, { - label: "At target", + label: "If all paid", value: t.projectedMonthlyUsd === null ? "-" : `${money(t.projectedMonthlyUsd)}/mo`, color: theme.success, }, @@ -796,6 +817,10 @@ function moneyScreen(ui: Container, state: State, theme: Theme): void { if (!f) { ui.panel({ title: "Money" }, (p) => { + if (feedBusy(state.feeds.finance)) { + p.spinner({ label: state.feeds.finance.detail, color: theme.primary }); + return; + } p.text(s.errors.finance ?? "CoinPay unavailable.", { fg: theme.danger }); p.text("`coinpay auth login` writes the session this reads.", { fg: theme.muted }); }); @@ -981,7 +1006,41 @@ const SCREENS = [roiScreen, trafficScreen, adsScreen, moneyScreen, spendScreen]; * between them is state, not a tab. Exported so the render tests draw exactly * what the app draws. */ +function feedBusy(feed: FeedProgress): boolean { + return feed.status === "loading" || feed.status === "retrying"; +} + +/** Each request has a visible lifecycle on every screen, including the first load. */ +export function renderFetchStatus(ui: Container, state: State, theme: Theme): void { + ui.row({ size: 1, gap: 2 }, (row) => { + for (const name of ["traffic", "ads", "finance"] as const) { + const feed = state.feeds[name]; + const label = name === "finance" ? "CoinPay" : name === "ads" ? "Ads" : "Traffic"; + row.column({}, (cell) => { + if (feedBusy(feed)) cell.spinner({ label: feed.detail, color: theme.primary }); + else cell.text(`${feed.status === "error" ? "!" : feed.status === "success" ? "✓" : "·"} ${feed.status === "idle" ? `${label} ${state.snapshot ? "ready" : "waiting"}` : feed.detail}`, { + fg: feed.status === "error" ? theme.warning : feed.status === "success" ? theme.success : theme.muted, + }); + }); + } + }); + if (state.loading) { + const elapsed = state.refreshStartedAt ? Math.floor((Date.now() - state.refreshStartedAt.getTime()) / 1000) : 0; + ui.text(`Refreshing… ${elapsed}s${state.refreshQueued ? " · next refresh queued" : ""}${state.snapshot ? " · showing previous snapshot until complete" : ""}`, { size: 1, fg: theme.primary }); + } else if (state.refreshMessage) { + ui.text(state.refreshMessage, { size: 1, fg: state.error || Object.keys(state.snapshot?.errors ?? {}).length ? theme.warning : theme.success }); + } + if (state.snapshot?.adsStale) { + ui.text(`Ads: saved data from ${state.snapshot.adsUpdatedAt ?? state.snapshot.generatedAt} · r retries`, { size: 1, fg: theme.warning }); + } +} + export function renderBody(ui: Container, state: State, theme: Theme): void { + renderFetchStatus(ui, state, theme); + ui.column({ size: "fill" }, (body) => renderContent(body, state, theme)); +} + +function renderContent(ui: Container, state: State, theme: Theme): void { if (!state.snapshot) { ui.panel({ title: "Spend & ROI" }, (p) => { if (state.error) { @@ -1009,6 +1068,14 @@ export function initialState(overrides: Partial = {}): State { who: "humans", snapshot: null, loading: false, + feeds: { + traffic: { status: "idle", detail: "Traffic waiting" }, + ads: { status: "idle", detail: "Ads waiting" }, + finance: { status: "idle", detail: "CoinPay waiting" }, + }, + refreshStartedAt: null, + refreshQueued: false, + refreshMessage: null, lastRefresh: null, error: null, paused: false, @@ -1192,6 +1259,73 @@ export type DashboardOptions = { sort?: string; }; +/** Manual retries queue once during an active read, including range/filter changes. */ +export function createRefreshController( + state: State, + opts: DashboardOptions, + invalidate: () => void, + collect: typeof collectDashboard = collectDashboard, +): () => Promise { + let active: Promise | null = null; + return function refresh(): Promise { + if (active) { + state.refreshQueued = true; + invalidate(); + return active; + } + active = (async () => { + do { + state.refreshQueued = false; + state.loading = true; + state.error = null; + state.refreshStartedAt = new Date(); + state.refreshMessage = null; + state.feeds = { + traffic: { status: "loading", detail: "Fetching traffic" }, + ads: { status: "loading", detail: "Fetching ads" }, + finance: { status: "loading", detail: "Fetching CoinPay" }, + }; + invalidate(); + try { + state.snapshot = await collect({ + baseUrl: opts.baseUrl, + token: opts.token, + range: state.range, + who: state.who, + financeDays: FINANCE_DAYS[state.range] ?? 30, + concurrency: opts.concurrency ?? 8, + coinpay: opts.coinpay, + only: opts.only ?? null, + previous: state.snapshot, + onProgress: (feed, progress) => { + state.feeds[feed] = progress; + invalidate(); + }, + }); + state.lastRefresh = new Date(); + const failures = Object.keys(state.snapshot.errors); + state.refreshMessage = failures.length + ? `Refresh finished · ${failures.join(", ")} unavailable · r retries` + : `Refresh complete at ${state.lastRefresh.toLocaleTimeString("en-US", { hour12: false })} · all sources loaded`; + } catch (err) { + state.error = err instanceof Error ? err.message : String(err); + state.refreshMessage = `Refresh failed: ${state.error} · r retries`; + } finally { + // Stop every spinner even when collection itself failed unexpectedly. + for (const name of ["traffic", "ads", "finance"] as const) { + if (feedBusy(state.feeds[name])) { + state.feeds[name] = { status: state.error ? "error" : "success", detail: state.error ? `${name} failed` : `${name} refreshed` }; + } + } + state.loading = false; + invalidate(); + } + } while (state.refreshQueued); + })().finally(() => { active = null; }); + return active; + }; +} + export async function runDashboard(opts: DashboardOptions): Promise { const hqtui = await loadHqtui(); const app = await hqtui.createApp({ @@ -1208,38 +1342,11 @@ export async function runDashboard(opts: DashboardOptions): Promise { ...(opts.sort && SORTS.includes(opts.sort as never) ? { sort: opts.sort as Sort } : {}), }); - let refreshing = false; - - async function refresh(): Promise { - if (refreshing) return; - refreshing = true; - state.loading = true; - app.invalidate(); - try { - state.snapshot = await collectDashboard({ - baseUrl: opts.baseUrl, - token: opts.token, - range: state.range, - who: state.who, - financeDays: FINANCE_DAYS[state.range] ?? 30, - concurrency: opts.concurrency ?? 8, - coinpay: opts.coinpay, - only: opts.only ?? null, - }); - state.error = null; - state.lastRefresh = new Date(); - } catch (err) { - state.error = err instanceof Error ? err.message : String(err); - } finally { - state.loading = false; - refreshing = false; - app.invalidate(); - } - } + const refresh = createRefreshController(state, opts, () => app.invalidate()); const interval = Math.max(10, opts.interval ?? 60); const poll = setInterval(() => { - if (!state.paused) void refresh(); + if (!state.paused && !state.loading) void refresh(); }, interval * 1000); poll.unref?.(); @@ -1258,6 +1365,7 @@ export async function runDashboard(opts: DashboardOptions): Promise { active: state.tab, onSelect: (index: number) => { state.tab = index; + state.domain = null; }, }); const right = [ @@ -1283,14 +1391,14 @@ export async function runDashboard(opts: DashboardOptions): Promise { ...(onList ? [{ key: "↵", label: "Open site" }] : []), ...(state.domain ? [{ key: "esc", label: "Back", active: true }] : []), ...(onList ? [{ key: "s", label: `Sort ${state.sort}` }] : []), - { key: "r", label: "Refresh" }, + { key: "r", label: state.loading ? state.refreshQueued ? "Queued" : "Refreshing" : "Refresh", active: state.loading }, { key: "w", label: `Window ${state.range}` }, { key: "b", label: state.who }, { key: "p", label: state.paused ? "Resume" : "Pause", active: state.paused }, { key: "?", label: "Help" }, { key: "q", label: "Quit" }, ], - right: errorCount + right: state.loading ? [{ label: state.refreshQueued ? "refresh queued" : "refreshing…", color: theme.primary }] : state.error ? [{ label: "refresh failed", color: theme.danger }] : errorCount ? [{ label: `${errorCount} source${errorCount > 1 ? "s" : ""} unavailable`, color: theme.warning }] : [{ label: "all sources live", color: theme.success }], }); diff --git a/docs/crawler-access.md b/docs/crawler-access.md new file mode 100644 index 00000000..c40cde66 --- /dev/null +++ b/docs/crawler-access.md @@ -0,0 +1,28 @@ +# Crawler access and ad reporting + +Commercial crawlers (including SemrushBot and SiteAuditBot) join the existing training-crawler policy in `@profullstack/x402-gateway`. The configured offer remains **$1 for a 24-hour pass**, settled by the existing CoinPay integration. The `/crawl` page explains how to buy and present the module's signed `x-crawl-pass`. This grants public crawl access; application authentication still protects account resources. + +All recognized crawlers, including paid crawlers, have a shared Redis limit of **12 requests per minute per source IP** and **600 requests per hour per crawler family** across replicas and public/ad surfaces. Excess requests receive `429` with `Retry-After`. Rejected requests do not extend the window. Public ad redirect requests also have a 60/minute IP ceiling for browser-classified clients, in addition to the existing five-second accepted-click cooldown. Railway's `X-Real-IP` provides the rate-limit identity; caller-supplied Cloudflare/forwarded headers cannot override it. + +Crawler identities in policy are user-agent classifications, not verified source attribution. Being a recognized crawler provides no rate-limit exemption. Concealing a crawler as a browser does not bypass the shared IP ceiling on ad redirects, but this policy alone is not comprehensive bot detection. + +`robots.txt` excludes `/a/` and `/api/` in every relevant group. Commercial and training crawlers are directed to `/crawl`. Before any impression lookup, unpaid commercial crawlers receive the gateway's payment offer; other unpaid crawlers receive `403` for ad redirects. Valid paid passes may resolve ad links under the rate limits, but those requests create **no ad click row, cash charge, paper charge, publisher earnings, or campaign referral attribution**. Search/retrieval crawlers retain free public-page access under the same crawler limits. + +Redis stores daily aggregate request outcomes by bounded crawler family and `page`/`ad` surface. It retains these for 366 days without storing IPs or full user agents in the metrics. `/api/admin/crawl-activity?days=7` exposes up to 31 days to administrators through either their session or API token. The counter labels are `requests`, `throttled`, `payment_required`, `pass_issued`, `blocked`, and `passed`; issuing/reissuing a pass is not a revenue count. Actual revenue comes from settlement and the existing sale hook. + +If Redis is unavailable, recognized crawlers receive `503` and a retry delay. Human redirects remain available; the existing click admission independently withholds charges when it cannot validate them. + +The earnings API uses the service-only `ad_token_earnings(owner, days)` RPC. It reads closed-day rollups plus current-day records and returns one JSON document, avoiding both missing `auth.uid()` and PostgREST row truncation. Header totals and domain rows share the requested window. Accepted clicks are billed plus free; rejected clicks remain separate. Lifetime wallet/balance totals remain explicitly lifetime. A failed reporting or balance query returns `503` without fabricated totals, allowing the TUI to retain visibly stale data. + +Apply `supabase/migrations/20260913170000_ad_token_earnings.sql` before deploying the application. It adds a function and grants execution only to `service_role`; existing browser RPCs and historical click rows are preserved. + +Validation commands: + +```sh +npm test +npm run typecheck +node scripts/test-ad-token-earnings.mjs +node --import tsx scripts/test-crawl-limits.mjs +``` + +The SQL and Redis scripts use disposable local Docker containers. They verify owner isolation, role permissions, date boundaries, billed/free/rejected semantics, atomic rate admission, fixed retry windows and limits across source IPs. diff --git a/lib/ads/earnings-data.ts b/lib/ads/earnings-data.ts index bad2b745..0cb9b5bf 100644 --- a/lib/ads/earnings-data.ts +++ b/lib/ads/earnings-data.ts @@ -1,4 +1,5 @@ import type { SupabaseClient } from "@supabase/supabase-js"; +import type { TokenDelivery } from "./token-earnings"; import { deliveredClicks, deliveredImpressions, @@ -93,6 +94,14 @@ export type EarningsModel = { advClicks: number; pubImpressions: number; pubClicks: number; + advBilledClicks: number; + advFreeClicks: number; + pubBilledClicks: number; + pubFreeClicks: number; + advPaidImpressions: number; + advFreeImpressions: number; + pubPaidImpressions: number; + pubFreeImpressions: number; /** * Clicks recorded in the range and deliberately not counted as delivery: * bot, duplicate, forged, or against a campaign that was not servable. @@ -131,19 +140,20 @@ export async function loadEarnings( supabase: SupabaseClient, userId: string, days = 30, + tokenDelivery?: Promise, ): Promise { // The window the tables and the impression/click totals cover. Money is not // scoped to it — see the note on EarningsModel.totals. const since = sinceForDays(days); const [ - { data: campaignsData }, + { data: campaignsData, error: campaignsError }, campaignTotals, - { data: projectsData }, - { data: slotsData }, + { data: projectsData, error: projectsError }, + { data: slotsData, error: slotsError }, slotTotals, - { data: ledgerData }, - { data: payoutsData }, + { data: ledgerData, error: ledgerError }, + { data: payoutsData, error: payoutsError }, ] = await Promise.all([ supabase .from("ad_campaigns") @@ -153,11 +163,11 @@ export async function loadEarnings( // only tier 'paid', so on a network running entirely on free backfill they // report zero for every campaign and every site. The RPCs take a window and // return both tiers. - getCampaignTotalsSince(supabase, since), + tokenDelivery ? tokenDelivery.then((d) => d.campaignTotals) : getCampaignTotalsSince(supabase, since), // Monetization is owner-only (payouts go to the slot owner), like /ads/slots. supabase.from("projects").select("id, name").eq("owner_id", userId), supabase.from("ad_slots").select("id, project_id, status").eq("owner_id", userId), - getSlotTotalsSince(supabase, since), + tokenDelivery ? tokenDelivery.then((d) => d.slotTotals) : getSlotTotalsSince(supabase, since), supabase.from("ad_ledger").select("slot_id, amount_cents").eq("kind", "publisher_accrual").eq("owner_id", userId), supabase .from("ad_payouts") @@ -223,20 +233,20 @@ export async function loadEarnings( .reduce((a, c) => a + (c.spend_today_cents ?? 0), 0); // Daily series (spend from campaigns, earnings from slots), merged by day. - const [campaignSeries, slotSeries] = await Promise.all([ + const [campaignSeries, slotSeries] = tokenDelivery ? [null, null] : await Promise.all([ getCampaignDailySeries(supabase, campaignRows.map((c) => c.id), days), getSlotDailySeries(supabase, slotRows.map((s) => s.id), days), ]); - const daily = mergeMoneySeries(campaignSeries.data, slotSeries.data, days); + const daily = tokenDelivery ? (await tokenDelivery).daily : mergeMoneySeries(campaignSeries!.data, slotSeries!.data, days); const earnedTodayCents = daily.length ? daily[daily.length - 1].earnedCents : 0; return { rangeDays: days, statsUnavailable: + Boolean(campaignsError || projectsError || slotsError || ledgerError || payoutsError) || campaignTotals.failed || slotTotals.failed || - campaignSeries.failed || - slotSeries.failed, + Boolean(campaignSeries?.failed || slotSeries?.failed), totals: { spentCents, earnedCents, @@ -249,6 +259,14 @@ export async function loadEarnings( advClicks: campaignRows.reduce((a, c) => a + c.clicks, 0), pubImpressions: slotRows.reduce((a, s) => a + s.impressions, 0), pubClicks: slotRows.reduce((a, s) => a + s.clicks, 0), + advBilledClicks: campaigns.reduce((a, c) => a + (campaignTotals.data.get(c.id)?.clicks ?? 0), 0), + advFreeClicks: campaigns.reduce((a, c) => a + (campaignTotals.data.get(c.id)?.freeClicks ?? 0), 0), + pubBilledClicks: slots.reduce((a, s) => a + (slotTotals.data.get(s.id)?.clicks ?? 0), 0), + pubFreeClicks: slots.reduce((a, s) => a + (slotTotals.data.get(s.id)?.freeClicks ?? 0), 0), + advPaidImpressions: campaigns.reduce((a, c) => a + (campaignTotals.data.get(c.id)?.impressions ?? 0), 0), + advFreeImpressions: campaigns.reduce((a, c) => a + (campaignTotals.data.get(c.id)?.freeImpressions ?? 0), 0), + pubPaidImpressions: slots.reduce((a, s) => a + (slotTotals.data.get(s.id)?.impressions ?? 0), 0), + pubFreeImpressions: slots.reduce((a, s) => a + (slotTotals.data.get(s.id)?.freeImpressions ?? 0), 0), invalidClicks: slots.reduce( (a, sl) => a + (slotTotals.data.get(sl.id)?.invalidClicks ?? 0), 0, diff --git a/lib/ads/series.ts b/lib/ads/series.ts index 7b2bf8c6..52a71d11 100644 --- a/lib/ads/series.ts +++ b/lib/ads/series.ts @@ -310,7 +310,7 @@ function dayKey(iso: string): string { } /** Build a zero-filled list of the last `days` UTC calendar days, oldest first. */ -function dayAxis(days: number): string[] { +export function dayAxis(days: number): string[] { const out: string[] = []; const now = new Date(); for (let i = days - 1; i >= 0; i--) { diff --git a/lib/ads/serve.ts b/lib/ads/serve.ts index 60ff9846..94111b03 100644 --- a/lib/ads/serve.ts +++ b/lib/ads/serve.ts @@ -400,6 +400,7 @@ export async function serveAd( if (!pick) return houseFill(format, theme); const campaign = oneCampaign(pick.ad_campaigns); if (!campaign) return null; + // The winner's own answer wins over which pool it came from: a promo // campaign can win the paid auction and must still book as free. tier = tierByCampaign.get(campaign.id) ?? tier; @@ -621,6 +622,8 @@ export async function resolveClick(input: { if (!campaign) return null; // slot_id must be present (ad_clicks.slot_id NOT NULL) to record a click. + // Paid crawlers may resolve links, but never create cash/paper ad activity. + if (input.ctx?.device === "bot") return campaign.destination_url; if (input.slotId) { const visitorId = await resolveClickVisitor(sb, input.impressionId, input.ctx?.visitorId); // The row is stored under today's salt; the dedupe lookup has to consider diff --git a/lib/ads/token-earnings.ts b/lib/ads/token-earnings.ts new file mode 100644 index 00000000..438eb4eb --- /dev/null +++ b/lib/ads/token-earnings.ts @@ -0,0 +1,41 @@ +import type { SupabaseClient } from "@supabase/supabase-js"; +import { dayAxis, type RangeTotals, type SlotTotals } from "./series"; + +export type TokenDelivery = { + campaignTotals: { data: Map; failed: false }; + slotTotals: { data: Map; failed: false }; + daily: Array<{ date: string; spentCents: number; earnedCents: number }>; +}; + +type Row = Record; +const n = (value: unknown) => Number(value) || 0; + +/** An error is unavailable data, never a successful zero-filled response. */ +export async function loadTokenDelivery(sb: SupabaseClient, ownerId: string, days: number): Promise { + const { data, error } = await sb.rpc("ad_token_earnings", { p_owner: ownerId, p_days: days }); + if (error || !data || !Array.isArray(data.campaigns) || !Array.isArray(data.slots) || !Array.isArray(data.daily)) { + console.error("[ads] Token reporting failed", { code: error?.code ?? "invalid_response" }); + throw new Error("Ad reporting is temporarily unavailable. Please retry."); + } + const campaignTotals = new Map(); + for (const row of data.campaigns as Row[]) { + campaignTotals.set(String(row.campaign_id), { + impressions: n(row.impressions), freeImpressions: n(row.free_impressions), + clicks: n(row.clicks), freeClicks: n(row.free_clicks), spentCents: n(row.spent_cents), + }); + } + const slotTotals = new Map(); + for (const row of data.slots as Row[]) { + slotTotals.set(String(row.slot_id), { + impressions: n(row.impressions), freeImpressions: n(row.free_impressions), + clicks: n(row.clicks), freeClicks: n(row.free_clicks), + invalidClicks: n(row.invalid_clicks), earnedCents: n(row.earned_cents), + }); + } + const money = new Map((data.daily as Row[]).map((r) => [String(r.date), r])); + return { + campaignTotals: { data: campaignTotals, failed: false }, + slotTotals: { data: slotTotals, failed: false }, + daily: dayAxis(days).map((date) => ({ date, spentCents: n(money.get(date)?.spentCents), earnedCents: n(money.get(date)?.earnedCents) })), + }; +} diff --git a/lib/crawl-gateway.ts b/lib/crawl-gateway.ts index 1211128d..96620d70 100644 --- a/lib/crawl-gateway.ts +++ b/lib/crawl-gateway.ts @@ -1,13 +1,11 @@ -import { createGateway, type Sale } from "@profullstack/x402-gateway"; +import { createGateway, readPass, renderPage, type Sale } from "@profullstack/x402-gateway"; import { x402Proxy } from "@profullstack/x402-gateway/next"; +import { crawlerFamily, isAdClickPath, isPaidCrawler, PAID_CRAWLERS } from "./crawl-policy"; +import { limitCrawlRequest, recordCrawlerOutcome, CRAWLER_IP_PER_MINUTE, CRAWLER_FAMILY_PER_HOUR } from "./crawl-limits"; /** - * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, - * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over - * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval - * crawlers behind AI search pass through untouched. - * - * Runs inside the middleware, so nothing here may import Node-only modules. + * Sells day passes to commercial and training crawlers using x402-gateway. + * Next 16's Node proxy enforces shared limits before payment or app queries. * The env is read through a non-literal key on purpose: Next inlines * `process.env.NAME` at build time, and these are runtime secrets. Without * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training @@ -54,7 +52,47 @@ export const gateway = createGateway({ payTo: env("CRAWL_PAY_TO"), contact: "mailto:support@crawlproof.com", onSale: recordSale, + training: PAID_CRAWLERS, + isPaidAgent: isPaidCrawler, + page: (ctx) => renderPage(ctx) + .replace("Training crawlers pay for access here.", "Commercial and training crawlers pay for access here.") + .replace("A crawler that copies pages into a training corpus sends nobody back, so it pays for the time it spends.", "Commercial data collection and training access require a paid pass.") + .replace("

How it works

", `

All crawlers, including paid crawlers, are limited to ${CRAWLER_IP_PER_MINUTE} requests per minute per IP and ${CRAWLER_FAMILY_PER_HOUR} requests per hour per crawler family. Honour Retry-After. A pass grants access to public resources; it does not grant account access. Automated ad-link requests never count as ad clicks or earn publisher payouts.

How it works

`), }); -/** Resolves to a Response for a refused crawler, or undefined to carry on. */ -export const gate = x402Proxy(gateway); +const paidGate = x402Proxy(gateway); + +async function hasPass(request: Request): Promise { + const token = request.headers.get(gateway.options.header)?.trim() + || /^Bearer\s+(cp_[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$/i.exec(request.headers.get("authorization") ?? "")?.[1]; + const secret = gateway.options.secret || gateway.options.coinpay.apiKey; + return Boolean(secret && token && await readPass(token, { secret })); +} + +/** A commercial crawler pays; its signed pass does not waive rate limits. */ +export async function gate(request: Request): Promise { + const path = new URL(request.url).pathname; + const ad = isAdClickPath(path); + const family = crawlerFamily(request.headers.get("user-agent")); + // Service APIs are independently authenticated. Don't throttle an entire + // publisher's feed-serving integration as if it were a public page crawl. + const throttle = ad || path === "/crawl" || (family && !path.startsWith("/api/")); + if (throttle) { + const refused = await limitCrawlRequest(request, family, ad ? "ad" : "page"); + if (refused) return refused; + } + const answer = await paidGate(request); + if (answer) { + if (family && throttle) await recordCrawlerOutcome(family, ad ? "ad" : "page", answer.status === 200 ? "pass_issued" : "payment_required"); + return answer; + } + // Search crawlers may read content free, but ad redirects require a pass. + // This happens before impression/campaign lookups and before click writes. + if (ad && family && !(await hasPass(request))) { + await recordCrawlerOutcome(family, "ad", "blocked"); + return Response.json({ error: "ad_redirect_crawl_disallowed", crawl_access: "/crawl" }, { + status: 403, headers: { "cache-control": "no-store", "x-robots-tag": "noindex, nofollow" }, + }); + } + if (family && throttle) await recordCrawlerOutcome(family, ad ? "ad" : "page", "passed"); +} diff --git a/lib/crawl-limits.ts b/lib/crawl-limits.ts new file mode 100644 index 00000000..03870cf9 --- /dev/null +++ b/lib/crawl-limits.ts @@ -0,0 +1,102 @@ +import Redis from "ioredis"; +import { isIP } from "node:net"; +import { hashIp } from "@/lib/ipHash"; + +export const CRAWLER_IP_PER_MINUTE = 12; +export const CRAWLER_FAMILY_PER_HOUR = 600; +export const AD_IP_PER_MINUTE = 60; +const METRICS_TTL = 366 * 86400; + +// All counters share Redis across replicas. Rejected requests never prolong +// the window. Metrics use fixed family/surface/outcome fields, never raw URLs. +export const REQUEST_LIMIT_LUA = ` +local metrics = KEYS[#KEYS] +if ARGV[1] ~= '' then + redis.call('HINCRBY', metrics, ARGV[1] .. ':requests', 1) + redis.call('EXPIRE', metrics, ARGV[2]) +end +local retry = 0 +for i = 1, #KEYS - 1 do + local limit = tonumber(ARGV[2 * i + 1]) + if tonumber(redis.call('GET', KEYS[i]) or '0') >= limit then + retry = math.max(retry, redis.call('PTTL', KEYS[i])) + end +end +if retry > 0 then + if ARGV[1] ~= '' then redis.call('HINCRBY', metrics, ARGV[1] .. ':throttled', 1) end + return retry +end +for i = 1, #KEYS - 1 do + local count = redis.call('INCR', KEYS[i]) + if count == 1 then redis.call('PEXPIRE', KEYS[i], ARGV[2 * i + 2]) end +end +return 0 +`; +let redis: Redis | undefined; +function connection(): Redis | null { + const url = process.env.REDIS_URL; + if (!url) return null; + if (!redis || redis.status === "end") { + redis = new Redis(url, { lazyConnect: true, maxRetriesPerRequest: 0, connectTimeout: 1000, commandTimeout: 1500, retryStrategy: () => null }); + redis.on("error", () => {}); + } + return redis; +} + +export const activityKey = (date = new Date()) => `crawl:activity:${date.toISOString().slice(0, 10)}`; + +/** Railway supplies X-Real-IP. Do not let a forged CF header override it. */ +export function sourceIp(headers: Headers): string | null { + const ip = headers.get("x-real-ip")?.trim(); + return ip && isIP(ip) ? ip : null; +} + +export async function limitCrawlRequest(request: Request, family: string | null, surface: "ad" | "page"): Promise { + const ip = sourceIp(request.headers); + const keys = [`crawl:request:ip:${hashIp(ip ?? "unknown")}`]; + const field = family ? `${family}:${surface}` : ""; + const args: Array = [field, METRICS_TTL, family ? CRAWLER_IP_PER_MINUTE : AD_IP_PER_MINUTE, 60_000]; + if (family) { + keys.push(`crawl:request:family:${family}`); + args.push(CRAWLER_FAMILY_PER_HOUR, 3_600_000); + } + keys.push(activityKey()); + try { + const client = connection(); + if (!client) throw new Error("unavailable"); + const retryMs = Number(await client.eval(REQUEST_LIMIT_LUA, keys.length, ...keys, ...args)); + if (!Number.isFinite(retryMs)) throw new Error("invalid_response"); + if (retryMs === 0) return; + return refusal(429, "crawler_rate_limited", Math.max(1, Math.ceil(retryMs / 1000))); + } catch { + // Preserve human redirects during Redis outages; the existing click + // admission still withholds billing unless its own check succeeds. + if (!family) return; + return refusal(503, "crawler_limits_unavailable", 60); + } +} + +function refusal(status: number, error: string, retry: number): Response { + return Response.json({ error, retry_after_seconds: retry, crawl_access: "/crawl" }, { + status, headers: { "retry-after": String(retry), "cache-control": "no-store", "x-robots-tag": "noindex, nofollow" }, + }); +} + +export async function recordCrawlerOutcome(family: string, surface: "ad" | "page", outcome: "payment_required" | "pass_issued" | "blocked" | "passed"): Promise { + try { + const client = connection(); + if (client) await client.multi().hincrby(activityKey(), `${family}:${surface}:${outcome}`, 1).expire(activityKey(), METRICS_TTL).exec(); + } catch { /* Metrics do not determine access or successful payment. */ } +} + +/** Operations-only aggregates. No IP addresses, cookies, URLs, or visitor IDs. */ +export async function readCrawlerActivity(days: number) { + const client = connection(); + if (!client) throw new Error("Crawler counters unavailable"); + const dates = Array.from({ length: Math.min(31, Math.max(1, days)) }, (_, i) => new Date(Date.now() - i * 86400000).toISOString().slice(0, 10)); + const pipeline = client.pipeline(); + for (const date of dates) pipeline.hgetall(`crawl:activity:${date}`); + const rows = await pipeline.exec(); + if (!rows || rows.some(([error]) => error)) throw new Error("Crawler counters unavailable"); + return dates.map((date, i) => ({ date, counts: Object.fromEntries(Object.entries(rows[i][1] as Record).map(([key, count]) => [key, Number(count)])) })); +} diff --git a/lib/crawl-policy.ts b/lib/crawl-policy.ts new file mode 100644 index 00000000..9252322e --- /dev/null +++ b/lib/crawl-policy.ts @@ -0,0 +1,17 @@ +import { TRAINING_AGENTS, isTrainingAgent } from "@profullstack/x402-gateway"; +import { parseDevice } from "@/lib/tracker/device"; + +export const COMMERCIAL_CRAWLERS = ["SemrushBot", "SiteAuditBot", "AhrefsBot", "AhrefsSiteAudit", "MJ12bot", "DotBot", "BLEXBot", "DataForSeoBot"]; +export const PAID_CRAWLERS = [...TRAINING_AGENTS, ...COMMERCIAL_CRAWLERS]; +const FAMILIES = [...PAID_CRAWLERS, "Amazonbot", "Googlebot", "Bingbot", "Applebot", "OAI-SearchBot", "ChatGPT-User", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User"]; + +/** Labels bound counter cardinality. A user agent identifies policy, not ownership. */ +export function crawlerFamily(ua: string | null): string | null { + const text = (ua ?? "").slice(0, 1024).toLowerCase(); + const named = FAMILIES.find((name) => text.includes(name.toLowerCase())); + if (named) return named.toLowerCase(); + return parseDevice(text).deviceType === "bot" ? "other" : null; +} + +export const isPaidCrawler = (ua: string) => isTrainingAgent(ua, PAID_CRAWLERS); +export const isAdClickPath = (path: string) => path.startsWith("/a/") || path === "/api/ads/click"; diff --git a/lib/dashboard/collect.ts b/lib/dashboard/collect.ts index 9e0eb7e0..961f552d 100644 --- a/lib/dashboard/collect.ts +++ b/lib/dashboard/collect.ts @@ -6,8 +6,8 @@ // request is how the tracker RPCs have timed out before, and a slow client is // a much better failure than a route that 504s for everybody. // -// CoinPay is one call into its own SDK, which is the whole point: the finance -// dashboard already exists and this reads it rather than reimplementing it. +// CoinPay's SDK supplies the fleet snapshot and business-specific analytics +// for the domain view. The latter use a bounded fan-out too. // // Nothing here throws for a partial answer. A source that fails lands in // `errors` and its panel says so, because a dashboard that hides a dead feed @@ -16,6 +16,7 @@ import { buildRoi, type AdsInput, + type BusinessRevenue, type FinanceInput, type RoiModel, type SiteTraffic, @@ -24,6 +25,12 @@ import type { ScoreModel } from "./score"; import { buildSiteDetail, type SiteMix, type SitePoint } from "./site"; export type ListItem = { label: string; value: number }; +export type FeedName = "traffic" | "ads" | "finance"; +export type FeedProgress = { + status: "idle" | "loading" | "retrying" | "success" | "error"; + detail: string; +}; +export type ProgressListener = (feed: FeedName, progress: FeedProgress) => void; export type SiteStats = SiteTraffic & { id?: string; @@ -49,6 +56,9 @@ export type DashboardSnapshot = { sites: SiteStats[]; fleet: { sources: ListItem[]; referrers: ListItem[]; pages: ListItem[] }; ads: AdsInput | null; + /** Last successful ads read; retained across a transient failure in the same window. */ + adsUpdatedAt?: string; + adsStale?: boolean; finance: FinanceInput | null; roi: RoiModel; /** Source name → why it is missing. Empty when everything answered. */ @@ -56,6 +66,13 @@ export type DashboardSnapshot = { }; const TIMEOUT_MS = 20_000; +export const ADS_TIMEOUT_MS = 60_000; + +class FeedError extends Error { + constructor(message: string, readonly retryable: boolean) { + super(message); + } +} async function fetchJson(url: string, token: string, timeoutMs = TIMEOUT_MS): Promise { const controller = new AbortController(); @@ -70,18 +87,46 @@ async function fetchJson(url: string, token: string, timeoutMs = TIMEOUT_MS): try { body = text ? JSON.parse(text) : {}; } catch { - throw new Error(`${res.status} ${res.statusText}: not JSON`); + throw new FeedError(`${res.status} ${res.statusText}: not JSON`, res.status >= 500); } if (!res.ok) { const message = (body as { error?: string })?.error ?? `${res.status} ${res.statusText}`; - throw new Error(message); + throw new FeedError(message, res.status === 408 || res.status === 429 || res.status >= 500); } return body as T; + } catch (err) { + if (controller.signal.aborted) throw new FeedError(`Request timed out after ${timeoutMs / 1000}s`, true); + throw err; } finally { clearTimeout(timer); } } +/** Ads aggregate hundreds of campaigns. Retry one transient or partial read. */ +export async function collectAds(baseUrl: string, token: string, days: number, progress?: (value: FeedProgress) => void): Promise { + const url = `${baseUrl}/api/ads/v1/earnings?days=${encodeURIComponent(String(days))}`; + let partial: AdsInput | null = null; + for (let attempt = 0; attempt < 2; attempt++) { + progress?.({ status: attempt ? "retrying" : "loading", detail: attempt ? "Ads retry 2/2" : "Fetching ads" }); + try { + const ads = await fetchJson(url, token, ADS_TIMEOUT_MS); + if (!ads.statsUnavailable) return ads; + partial = ads; + progress?.({ status: "retrying", detail: "Ads incomplete; retrying" }); + } catch (err) { + const retryable = err instanceof FeedError ? err.retryable : err instanceof TypeError; + if (!retryable) throw err; + if (attempt === 1) { + if (partial) return partial; + throw err; + } + progress?.({ status: "retrying", detail: "Ads request failed; retrying" }); + } + if (attempt === 0) await new Promise((resolve) => setTimeout(resolve, 1000)); + } + return partial as AdsInput; +} + /** Run `fn` over `items`, at most `limit` in flight. */ export async function mapLimit( items: T[], @@ -174,6 +219,45 @@ async function statsForSite( export type CoinPayAuth = { token: string; baseUrl: string }; +/** Keep each business separate, including failures, so no fleet total leaks into a domain. */ +export async function collectBusinessRevenue( + businesses: NonNullable, + days: number, + analyticsFor: (businessId: string) => Promise>, + progress?: (completed: number, total: number) => void, +): Promise> { + const ids = [...new Set(businesses.flatMap((b) => b.id ? [b.id] : []))]; + let completed = 0; + progress?.(completed, ids.length); + const entries = await mapLimit(ids, 4, async (id): Promise<[string, BusinessRevenue]> => { + try { + const analytics = await analyticsFor(id); + const series = analytics?.series as { points?: Array> } | undefined; + if (!Array.isArray(series?.points)) throw new Error("CoinPay returned no windowed series"); + const totals = { commissionUsd: 0, grossVolumeUsd: 0, transactions: 0 }; + for (const point of series.points) { + for (const [key, field] of [ + ["commissionUsd", "total_commission_usd"], + ["grossVolumeUsd", "total_volume_usd"], + ["transactions", "total_count"], + ] as const) { + const value = point?.[field]; + if (value == null || value === "" || !Number.isFinite(Number(value))) { + throw new Error("CoinPay returned incomplete windowed analytics"); + } + totals[key] += Number(value); + } + } + return [id, { windowDays: days, ...totals }]; + } catch (err) { + return [id, { windowDays: days, error: err instanceof Error ? err.message : String(err) }]; + } finally { + progress?.(++completed, ids.length); + } + }); + return Object.fromEntries(entries); +} + /** * The CoinPay finance snapshot, via CoinPay's own SDK. * @@ -183,7 +267,9 @@ export type CoinPayAuth = { token: string; baseUrl: string }; export async function collectFinance( auth: CoinPayAuth, days: number, + progress?: (value: FeedProgress) => void, ): Promise { + progress?.({ status: "loading", detail: "CoinPay bank & payments" }); const [{ default: CoinPayClient }, finances] = await Promise.all([ import("@profullstack/coinpay"), import("@profullstack/coinpay/finances"), @@ -191,7 +277,16 @@ export async function collectFinance( const client = new CoinPayClient({ apiKey: auth.token, baseUrl: auth.baseUrl }); // 500 rather than the default page: the vendor breakdown is only honest if // the ledger it groups covers the whole window. - return (await finances.collectFinanceSnapshot(client, { days, limit: 500 })) as FinanceInput; + const snapshot = await finances.collectFinanceSnapshot(client, { days, limit: 500 }); + // CoinPay's route accepts day/week/month/year, while this SDK's periodForDays + // emits 7d/30d, which the route treats as all-time. Use the server's presets. + const period = days <= 1 ? "day" : days <= 7 ? "week" : days <= 30 ? "month" : "year"; + const windowDays = days <= 1 ? 1 : days <= 7 ? 7 : days <= 30 ? 30 : 365; + const businessRevenue = await collectBusinessRevenue(snapshot.businesses, windowDays, (businessId) => + finances.getFinanceAnalytics(client, { period, businessId }), + (completed, total) => progress?.({ status: "loading", detail: `CoinPay businesses ${completed}/${total}` }), + ); + return { ...snapshot, businessRevenue } as FinanceInput; } export type CollectOptions = { @@ -204,32 +299,45 @@ export type CollectOptions = { coinpay: CoinPayAuth | null; /** Limit the fan-out to these site names or ids. */ only?: string[] | null; + /** Preserve a successful ads read if this refresh fails in the same finance window. */ + previous?: DashboardSnapshot | null; + onProgress?: ProgressListener; }; export async function collectDashboard(opts: CollectOptions): Promise { const errors: Record = {}; + const report = (feed: FeedName, status: FeedProgress["status"], detail: string) => opts.onProgress?.(feed, { status, detail }); + report("traffic", "loading", "Listing domains"); const sitesPromise = listSites(opts.baseUrl, opts.token).catch((err: unknown) => { errors.sites = err instanceof Error ? err.message : String(err); return [] as SiteRow[]; }); - const adsPromise = fetchJson( - `${opts.baseUrl}/api/ads/v1/earnings?days=${encodeURIComponent(String(opts.financeDays))}`, - opts.token, - ).catch((err: unknown) => { + const adsPromise = collectAds(opts.baseUrl, opts.token, opts.financeDays, (p) => opts.onProgress?.("ads", p)).then((ads) => { + report("ads", ads.statsUnavailable ? "error" : "success", ads.statsUnavailable ? "Ads partially loaded" : "Ads refreshed"); + return ads; + }).catch((err: unknown) => { errors.ads = err instanceof Error ? err.message : String(err); + report("ads", "error", `Ads failed: ${errors.ads}`); return null; }); const financePromise = opts.coinpay - ? collectFinance(opts.coinpay, opts.financeDays).catch((err: unknown) => { + ? collectFinance(opts.coinpay, opts.financeDays, (p) => opts.onProgress?.("finance", p)).then((finance) => { + const failures = Object.keys(finance.errors ?? {}).length + Object.values(finance.businessRevenue ?? {}).filter((b) => b.error).length; + if (failures) errors.finance = `${failures} CoinPay sources unavailable`; + report("finance", failures ? "error" : "success", failures ? errors.finance! : "CoinPay refreshed"); + return finance; + }).catch((err: unknown) => { errors.finance = err instanceof Error ? err.message : String(err); + report("finance", "error", `CoinPay failed: ${errors.finance}`); return null; }) : Promise.resolve(null); if (!opts.coinpay) { errors.finance = "No CoinPay session. Run `coinpay auth login`, or set COINPAY_SESSION_TOKEN."; + report("finance", "error", "CoinPay: no session"); } let siteRows = await sitesPromise; @@ -238,15 +346,31 @@ export async function collectDashboard(opts: CollectOptions): Promise wanted.has(s.name.toLowerCase()) || wanted.has(s.id)); } - const sites = await mapLimit(siteRows, opts.concurrency ?? 8, (site) => - statsForSite(opts.baseUrl, opts.token, site, opts.range, opts.who), - ); + let completed = 0; + report("traffic", "loading", `Traffic ${completed}/${siteRows.length}`); + const sites = await mapLimit(siteRows, opts.concurrency ?? 8, async (site) => { + const stats = await statsForSite(opts.baseUrl, opts.token, site, opts.range, opts.who); + report("traffic", "loading", `Traffic ${++completed}/${siteRows.length}`); + return stats; + }); sites.sort((a, b) => b.visitors - a.visitors || a.site.localeCompare(b.site)); const failed = sites.filter((s) => s.error).length; if (failed) errors.stats = `${failed} of ${sites.length} sites did not answer`; + report("traffic", failed || errors.sites ? "error" : "success", errors.sites ? "Domain list failed" : failed ? `Traffic: ${failed} sites failed` : `Traffic ${sites.length}/${sites.length} refreshed`); - const [ads, finance] = await Promise.all([adsPromise, financePromise]); + const [fetchedAds, finance] = await Promise.all([adsPromise, financePromise]); + let ads = fetchedAds; + if (ads?.statsUnavailable) errors.ads = "Some ad queries failed; domain ad money and delivery are unavailable."; + let adsUpdatedAt = ads ? new Date().toISOString() : undefined; + let adsStale = false; + const previous = opts.previous; + if ((!ads || ads.statsUnavailable) && previous?.ads && !previous.ads.statsUnavailable && previous.window.financeDays === opts.financeDays) { + ads = previous.ads; + adsUpdatedAt = previous.adsUpdatedAt ?? previous.generatedAt; + adsStale = true; + report("ads", "error", "Ads failed; showing saved data"); + } const roi = buildRoi({ traffic: { range: opts.range, who: opts.who, sites }, @@ -254,9 +378,8 @@ export async function collectDashboard(opts: CollectOptions): Promise s.pages)), }, ads, + adsUpdatedAt, + adsStale, finance, roi, errors, diff --git a/lib/dashboard/roi.ts b/lib/dashboard/roi.ts index df47dcb1..b67b0d4f 100644 --- a/lib/dashboard/roi.ts +++ b/lib/dashboard/roi.ts @@ -93,18 +93,31 @@ export type AdsInput = { advClicks?: number; pubImpressions?: number; pubClicks?: number; + pubBilledClicks?: number; + pubFreeClicks?: number; + advBilledClicks?: number; invalidClicks?: number; }; }; +/** Windowed CoinPay analytics fetched with an explicit business_id. */ +export type BusinessRevenue = { + windowDays: number; + commissionUsd?: number; + grossVolumeUsd?: number; + transactions?: number; + error?: string; +}; + /** The subset of the CoinPay finance snapshot this module reads. */ export type FinanceInput = { windowDays?: number; /** - * The merchant's businesses. Read by lib/dashboard/site.ts, which can only - * attribute commission to a domain when there is exactly one of them. + * The merchant's businesses, matched to a property by domain/name. */ businesses?: Array<{ id?: string; name?: string }>; + businessRevenue?: Record; + errors?: Record; /** * The headline earnings figures, which are **lifetime and not windowed**. * @@ -323,6 +336,8 @@ export const AD_TARGET_CTR = 0.05; export type AdTargets = { impressions: number; clicks: number; + billedClicks: number; + freeClicks: number; ctr: number | null; invalidClicks: number; freeImpressions: number; @@ -331,6 +346,7 @@ export type AdTargets = { impressionProgress: number; ctrProgress: number; targetImpressions: number; + windowTargetImpressions: number; targetCtr: number; /** Cents earned per valid click today, if any money has moved at all. */ cpcCents: number | null; @@ -362,22 +378,27 @@ export function adTargets( const t = ads?.totals ?? {}; const impressions = n(t.pubImpressions); const clicks = n(t.pubClicks); - const spent = n(t.spentCents); + const spent = (ads?.campaigns ?? []).reduce((total, c) => total + n(c.spentCents), 0); const ctr = impressions > 0 ? clicks / impressions : null; - const derivedCpc = clicks > 0 && spent > 0 ? spent / clicks : null; + const billed = n(t.advBilledClicks); + const derivedCpc = billed > 0 && spent > 0 ? spent / billed : null; const cpc = cpcCents ?? derivedCpc; + const windowTargetImpressions = targetImpressions * Math.max(1, ads?.rangeDays ?? 30) / 30; return { impressions, clicks, + billedClicks: n(t.pubBilledClicks), + freeClicks: n(t.pubFreeClicks), ctr, invalidClicks: n(t.invalidClicks), freeImpressions: n((t as { pubFreeImpressions?: number }).pubFreeImpressions), paidImpressions: n((t as { pubPaidImpressions?: number }).pubPaidImpressions), - impressionProgress: targetImpressions > 0 ? impressions / targetImpressions : 0, + impressionProgress: windowTargetImpressions > 0 ? impressions / windowTargetImpressions : 0, ctrProgress: ctr !== null && targetCtr > 0 ? ctr / targetCtr : 0, targetImpressions, + windowTargetImpressions, targetCtr, cpcCents: cpc, projectedMonthlyUsd: cpc === null ? null : (targetImpressions * targetCtr * cpc) / 100, diff --git a/lib/dashboard/score.ts b/lib/dashboard/score.ts index 82076de4..53ba052b 100644 --- a/lib/dashboard/score.ts +++ b/lib/dashboard/score.ts @@ -123,7 +123,7 @@ export type ScoreInput = { /** Arrival channels for the window, as the dashboard already carries them. */ sources?: ScoreItem[]; /** Money attributable to this property over the window, in dollars. */ - revenueUsd?: number; + revenueUsd?: number | null; /** * Human visits over the window, when a truer total than the series sum is * known. The series is filtered to whichever side was asked for, so under @@ -185,7 +185,8 @@ export function growthRate(series: number[]): number | null { if (points.length < 4) return null; const mid = Math.floor(points.length / 2); const prior = sum(points.slice(0, mid)); - const recent = sum(points.slice(mid)); + // Equal-sized halves: an extra bucket must not turn flat traffic into growth. + const recent = sum(points.slice(-mid)); if (prior <= 0 && recent <= 0) return null; // max(prior, 1) rather than a guard: from nothing to something is growth, and // dividing by zero to say so is not. @@ -271,8 +272,10 @@ export function scoreSite(input: ScoreInput): ScoreModel { const concentrationRaw = topSourceShare(input.sources); const cv = coefficientOfVariation(humansSeries); + const revenueKnown = input.revenueUsd != null && Number.isFinite(input.revenueUsd); const revenueUsd = num(input.revenueUsd); - const rpm = safeDiv(revenueUsd * 1000, humans); + const rpm = revenueKnown ? safeDiv(revenueUsd * 1000, humans) : null; + if (!revenueKnown) notes.push("Revenue is unavailable; money and monetisation risk are unscored."); const money = rpm === null ? null : clamp01(rpm / TARGET_RPM_USD); const viralComponents: Component[] = [ @@ -310,7 +313,7 @@ export function scoreSite(input: ScoreInput): ScoreModel { weight: VIRAL_WEIGHTS.money, detail: rpm === null - ? "no human visits to divide by" + ? revenueKnown ? "no human visits to divide by" : "revenue unavailable" : `$${rpm.toFixed(2)} per 1k human visits (target $${TARGET_RPM_USD.toFixed(2)})`, }, ]; diff --git a/lib/dashboard/site.ts b/lib/dashboard/site.ts index cea4acf3..fde6c438 100644 --- a/lib/dashboard/site.ts +++ b/lib/dashboard/site.ts @@ -18,10 +18,8 @@ // destination host. Both exact. Both internal: this network has // one account on both sides, so neither is revenue. // revenue CoinPay commission, which is the only money from outside the -// fleet — and is attributable to a domain only when the merchant -// account has exactly one business and it is this one. Otherwise -// it is null and the screen says why rather than dividing the -// fleet's revenue by a number of sites. +// fleet — matched by business name/domain and fetched with that +// business's id. Missing or unmatched analytics stay unknown. import type { AdsInput, FinanceInput, RoiModel } from "./roi"; import { scoreSite, type ScoreItem, type ScoreModel } from "./score"; @@ -58,6 +56,10 @@ export type SiteMoney = { revenueUsd: number | null; /** How `revenueUsd` was arrived at, for the screen to print. */ revenueBasis: string; + /** Actual payment volume and count over the CoinPay observation window. */ + grossVolumeUsd: number | null; + transactions: number | null; + observedDays: number | null; /** Revenue per 1,000 human visits, when both halves exist. */ rpmUsd: number | null; /** Revenue less the cost-by-views share. Null when either half is null. */ @@ -127,36 +129,75 @@ export function sameProperty(site: SiteLike, url: string | null | undefined): bo return site.site.toLowerCase() === target; } +/** Self-referrals and loopback checks are existing traffic, not new discovery. */ +export function sourcesForScore(site: SiteLike): ScoreItem[] { + const grouped = new Map(); + for (const source of site.sources ?? []) { + const referral = /^referral\s*(?:·|:)\s*(.+)$/i.exec(source.label)?.[1]; + const host = hostFrom(referral); + const local = host === "localhost" || host === "0.0.0.0" || host === "[::1]" || /^127\./.test(host ?? ""); + const label = referral && (sameProperty(site, referral) || local) ? "Internal referral" : source.label; + grouped.set(label, (grouped.get(label) ?? 0) + num(source.value)); + } + return [...grouped].map(([label, value]) => ({ label, value })); +} + /** * CoinPay commission attributable to one property. * - * Only when the merchant account has exactly one business and it is this one. - * With several, the snapshot carries a fleet total and no per-business split — - * `getFinanceAnalytics` takes a `businessId` but the dashboard makes one call, - * not one per business — so anything else would be the fleet's revenue divided - * by a guess. + * Prefer analytics requested with a business id. Older snapshots can supply a + * windowed series only if their sole business matches this property. */ export function coinpayRevenueForSite( finance: FinanceInput | null, roi: RoiModel, site: SiteLike, -): { usd: number | null; basis: string } { - const businesses = (finance as { businesses?: Array<{ id?: string; name?: string }> } | null)?.businesses ?? []; - if (!finance) return { usd: null, basis: "no CoinPay session" }; - if (businesses.length !== 1) { +): { usd: number | null; basis: string; grossVolumeUsd: number | null; transactions: number | null; observedDays: number | null } { + const unknown = (basis: string) => ({ usd: null, basis, grossVolumeUsd: null, transactions: null, observedDays: null }); + const businesses = finance?.businesses ?? []; + if (!finance) return unknown("no CoinPay session"); + const matches = businesses.filter((b) => + sameProperty(site, b.name) || site.site.toLowerCase() === b.name?.toLowerCase(), + ); + if (finance.businessRevenue) { + if (!matches.length) return unknown("no CoinPay business matches this domain"); + const rows = matches.map((b) => b.id ? finance.businessRevenue?.[b.id] : undefined); + for (const row of rows) { + if (!row || row.error || row.commissionUsd == null || row.grossVolumeUsd == null || row.transactions == null || !(row.windowDays > 0)) { + return unknown(row?.error ?? "business analytics unavailable"); + } + } + const observedDays = rows[0]!.windowDays; + if (rows.some((row) => row!.windowDays !== observedDays)) return unknown("business windows do not match"); + const commission = rows.reduce((total, row) => total + num(row!.commissionUsd), 0); return { - usd: null, - basis: businesses.length - ? `${businesses.length} CoinPay businesses, no per-business split in the snapshot` - : "CoinPay reported no businesses", + usd: commission * roi.window.days / observedDays, + basis: `CoinPay: ${matches.map((b) => b.name).join(", ")} · ${observedDays}d commission prorated to ${roi.window.range}`, + grossVolumeUsd: rows.reduce((total, row) => total + num(row!.grossVolumeUsd), 0), + transactions: rows.reduce((total, row) => total + num(row!.transactions), 0), + observedDays, }; } + if (businesses.length !== 1) { + return unknown(businesses.length + ? `${businesses.length} CoinPay businesses, no per-business split in the snapshot` + : "CoinPay reported no businesses"); + } const only = businesses[0] as { id?: string; name?: string }; const name = String(only?.name ?? ""); if (!(sameProperty(site, name) || site.site.toLowerCase() === name.toLowerCase())) { - return { usd: null, basis: `all commission belongs to ${name || "another business"}` }; + return unknown(`all commission belongs to ${name || "another business"}`); + } + if (finance.errors?.analytics || !finance.series?.length) { + return unknown(finance.errors?.analytics ?? "no windowed CoinPay series"); } - return { usd: num(roi.revenue.windowUsd), basis: `all CoinPay commission (${name})` }; + return { + usd: num(roi.revenue.windowUsd), + basis: `all CoinPay commission (${name}) · ${roi.revenue.observedDays}d prorated to ${roi.window.range}`, + grossVolumeUsd: finance.series.reduce((total, point) => total + num(point.volumeUsd), 0), + transactions: finance.series.reduce((total, point) => total + num(point.count), 0), + observedDays: roi.revenue.observedDays, + }; } /** Ad money and delivery for one property, joined exactly rather than shared out. */ @@ -217,13 +258,16 @@ export function buildSiteDetail(input: BuildSiteDetailInput): SiteDetail { const visitShare = share(site.visitors, roi.attention.visitors); const viewShare = share(site.pageviews, roi.attention.pageviews); const costWindow = num(roi.cost.windowUsd); - const costByViews = roi.attention.pageviews > 0 ? costWindow * viewShare : null; - const costByVisits = roi.attention.visitors > 0 ? costWindow * visitShare : null; + const costKnown = Boolean(input.finance?.position && !input.finance.errors?.summary && !site.error); + const costByViews = costKnown && roi.attention.pageviews > 0 ? costWindow * viewShare : null; + const costByVisits = costKnown && roi.attention.visitors > 0 ? costWindow * visitShare : null; const ad = adMoneyForSite(input.ads, site); const revenue = coinpayRevenueForSite(input.finance, roi, site); if (revenue.usd === null) { gaps.push(`Revenue is not attributable to one domain here: ${revenue.basis}.`); + } else { + gaps.push(revenue.basis); } // The earn rail is a network-wide pool — crawler pass revenue funds it with @@ -231,6 +275,10 @@ export function buildSiteDetail(input: BuildSiteDetailInput): SiteDetail { // figure to show. Named rather than omitted, because a missing money line on // a money screen reads as a zero. gaps.push("Earn-rail rewards are pooled network-wide; there is no per-domain share to report."); + const scoreSources = input.window.who === "bots" ? [] : sourcesForScore(site); + if (scoreSources.some((s) => s.label === "Internal referral")) { + gaps.push("Self-referrals and localhost traffic do not count as discovery."); + } // Human visits are the denominator for a per-reader figure, and the score // half that pays attention to money uses the same one. Ad earnings are @@ -249,8 +297,8 @@ export function buildSiteDetail(input: BuildSiteDetailInput): SiteDetail { : scoreSite({ humans: series.map((p) => num(p.humans)), bots: series.map((p) => num(p.bots)), - sources: site.sources ?? [], - revenueUsd: revenue.usd ?? 0, + sources: scoreSources, + revenueUsd: revenue.usd, mixKnown, ...(mixKnown ? { humansTotal: humans, botsTotal: bots } : {}), }); @@ -296,6 +344,9 @@ export function buildSiteDetail(input: BuildSiteDetailInput): SiteDetail { adClicks: ad.clicks, revenueUsd: revenue.usd, revenueBasis: revenue.basis, + grossVolumeUsd: revenue.grossVolumeUsd, + transactions: revenue.transactions, + observedDays: revenue.observedDays, rpmUsd, netUsd: revenue.usd === null || costByViews === null ? null : revenue.usd - costByViews, }, diff --git a/package-lock.json b/package-lock.json index 56962f8d..8fd556de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "@modelcontextprotocol/sdk": "^1.26.0", "@profullstack/autoblog": "github:profullstack/autoblog#75e54af", "@profullstack/coinpay": "^0.9.0", - "@profullstack/hqtui": "^0.5.0", + "@profullstack/hqtui": "^0.6.2", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", "@profullstack/x402-client": "^0.2.0", @@ -2185,9 +2185,9 @@ "license": "MIT" }, "node_modules/@profullstack/hqtui": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@profullstack/hqtui/-/hqtui-0.5.0.tgz", - "integrity": "sha512-jDwILBmdQx8pQA0ao/tzzty8u5nFNS4XhvU9xXhZvdIaMeiF3HBJIVtP6cS6kigS4lXmHR7sztbb9QJpaWwhig==", + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@profullstack/hqtui/-/hqtui-0.6.2.tgz", + "integrity": "sha512-r8w5PtAclzhxvjXrnVOIDD6pqTN812flNPN6ujH9cnJYIIfsiHZAu4b6dFuUI4Quv9dWCFS1RU/xxK7ORHr9Xg==", "license": "MIT", "bin": { "hqtui": "bin/hqtui.mjs" diff --git a/package.json b/package.json index d6f268a9..d5f8ce51 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@modelcontextprotocol/sdk": "^1.26.0", "@profullstack/autoblog": "github:profullstack/autoblog#75e54af", "@profullstack/coinpay": "^0.9.0", - "@profullstack/hqtui": "^0.5.0", + "@profullstack/hqtui": "^0.6.2", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", "@profullstack/x402-client": "^0.2.0", diff --git a/packages/cli/README.md b/packages/cli/README.md index 6e4c16b6..bc9e1d6c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -22,14 +22,32 @@ and CoinPay for what the bank actually did. `1`–`5` or Tab switches screens, `w` cycles the window, `b` cycles humans / all / bots, `r` refreshes, `?` explains the arithmetic, `q` quits. +Traffic, ads and CoinPay each show an animated spinner while fetching, with +domain/business progress and a visible completion or failure result. Pressing +`r` during a fetch queues one more refresh; changing the window during a fetch +also queues the newly selected window. The previous snapshot stays visible +until collection finishes. + +Ads get a 60-second request deadline and one automatic retry for timeouts, +transient errors or partial responses. If that still fails, the dashboard keeps +the last successful ads data for the same finance window and displays its saved +timestamp. A different window never inherits those cached figures. + ## One property at a time On **Traffic**, `↑`/`↓` pick a site and `Enter` — or a click on the row — opens it. That screen is only that domain: its pageviews, visits, humans against bots, AI referrals and where they arrived from; the burn prorated onto it by both denominators; its ad earnings and spend, joined by project and by where -the campaign points; and the commission, when there is exactly one merchant -business to attribute it to. `Esc`, `←` or `2` comes back to the list. +the campaign points; and CoinPay payment volume, payment count and commission +for businesses whose names match the domain (ignoring `www`). Analytics are +fetched separately for each business, so another property's earnings stay out +of this view. Unmatched businesses and failed requests show `—` with a reason. +`Esc`, `←` or `2` comes back to the list. + +Shared bank costs are estimates allocated by traffic share. CoinPay volume and +payment count cover the labeled bank window; commission is prorated onto the +traffic window. Ad earnings, spend and impressions cover the labeled ad window. ## The risk-to-viral score @@ -46,8 +64,8 @@ risk = volatility .40 + concentration .30 + bot dependence .20 + unmonetised .1 | Component | What it is | | --- | --- | -| momentum | Human visits in the recent half of the window against the earlier half. Flat scores 0.5, doubling scores 1. | -| discovery | Share of arrivals through search, social, an AI assistant, an ad or another site's link, rather than direct. | +| momentum | Human visits in equal-sized recent and earlier halves of the window, skipping the middle bucket when needed. Flat scores 0.5, doubling scores 1. | +| discovery | Share of arrivals through search, social, an AI assistant, an ad or another site's link. Direct traffic, self-referrals and localhost referrals do not count as discovery; bots-only windows leave it unscored. | | humanity | Humans over humans plus bots, from an unfiltered read — never from a filtered one, whose bot column is zero by construction. | | money | Revenue per 1,000 human visits against a $2 target. | | volatility | Coefficient of variation of the human series. Scale-free, so a small site is not penalised for being small. | @@ -59,6 +77,11 @@ A component with no data behind it is **dropped and its weight redistributed**, never counted as a zero, and the domain screen prints the raw figure under each one. A trailing `~` marks fewer than 25 human visits in the window — too small a sample to lean on. A site whose stats call failed is not scored at all. +Missing revenue leaves money and monetisation risk unscored; observed zero +revenue counts as unmonetised. Higher scores mean more potential under this +heuristic, not a probability of going viral. The domain view labels scores of +60+ as Promising, 35–59 as Worth watching, and lower scores as Limited signal; +small samples are labeled Early signal. `s` cycles the order: score, visitors, pageviews. `--sort=score` starts there. `--json` carries `.sites[].score` with every component, for a script. diff --git a/packages/cli/package.json b/packages/cli/package.json index a6e7dcc0..b340c416 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -21,7 +21,7 @@ }, "dependencies": { "@profullstack/coinpay": "^0.9.0", - "@profullstack/hqtui": "^0.5.0" + "@profullstack/hqtui": "^0.6.2" }, "devDependencies": { "esbuild": "^0.25.0" diff --git a/proxy.ts b/proxy.ts index f9e594ec..dcd9a437 100644 --- a/proxy.ts +++ b/proxy.ts @@ -1,4 +1,5 @@ import { gate } from "@/lib/crawl-gateway"; +import { isAdClickPath } from "@/lib/crawl-policy"; import { NextResponse, type NextRequest } from "next/server"; import { createServerClient, type CookieOptions } from "@supabase/ssr"; import { trackReferralCode } from "@profullstack/stack/referrals"; @@ -7,6 +8,9 @@ import { isNavigation } from "@/lib/affiliate/cookie"; type Cookie = { name: string; value: string; options?: CookieOptions }; export async function proxy(request: NextRequest) { + // Ad routes enforce their gate themselves, before DB work. Avoid counting a + // request twice or making an unrelated Supabase Auth call for each click. + if (isAdClickPath(request.nextUrl.pathname)) return NextResponse.next(); // Crawl gateway first: AI training crawlers get 402 Payment Required (or the // sales page at /crawl) unless they present a paid pass. People, Googlebot // and retrieval crawlers fall through to everything below. diff --git a/scripts/test-ad-token-earnings.mjs b/scripts/test-ad-token-earnings.mjs new file mode 100644 index 00000000..e4919946 --- /dev/null +++ b/scripts/test-ad-token-earnings.mjs @@ -0,0 +1,20 @@ +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +const name = `crawlproof-earnings-test-${process.pid}`; +try { + execFileSync('docker', ['run', '-d', '--rm', '--name', name, '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', 'postgres:17-alpine'], { stdio: 'pipe' }); + let ready = false; + for (let i = 0; i < 60; i++) { + // Initialization briefly starts a socket-only server, then restarts it. + // TCP readiness identifies the final server and avoids racing that restart. + try { execFileSync('docker', ['exec', name, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres'], { stdio: 'pipe' }); ready = true; break; } + catch { await new Promise((r) => setTimeout(r, 250)); } + } + if (!ready) throw new Error('Test database did not start'); + const migration = readFileSync(new URL('../supabase/migrations/20260913170000_ad_token_earnings.sql', import.meta.url), 'utf8'); + const fixture = readFileSync(new URL('../tests/sql/ad-token-earnings.sql', import.meta.url), 'utf8'); + execFileSync('docker', ['exec', '-i', name, 'psql', '-h', '127.0.0.1', '-U', 'postgres', '-v', 'ON_ERROR_STOP=1'], { input: fixture.replace('-- APPLY MIGRATION HERE', migration), stdio: ['pipe', 'inherit', 'inherit'] }); + console.log('Owner isolation, role permissions, reporting windows and click accounting passed.'); +} finally { + try { execFileSync('docker', ['stop', name], { stdio: 'pipe' }); } catch {} +} diff --git a/scripts/test-crawl-limits.mjs b/scripts/test-crawl-limits.mjs new file mode 100644 index 00000000..11d3396d --- /dev/null +++ b/scripts/test-crawl-limits.mjs @@ -0,0 +1,30 @@ +// Run: node --import tsx scripts/test-crawl-limits.mjs +import { execFileSync } from 'node:child_process'; +import assert from 'node:assert/strict'; +import Redis from 'ioredis'; +import { REQUEST_LIMIT_LUA } from '../lib/crawl-limits.ts'; +const name = `crawlproof-rate-test-${process.pid}`; +let redis; +try { + execFileSync('docker', ['run', '-d', '--rm', '--name', name, '-p', '127.0.0.1::6379', 'redis:8-alpine'], { stdio: 'pipe' }); + const mapping = execFileSync('docker', ['port', name, '6379/tcp'], { encoding: 'utf8' }).trim(); + redis = new Redis(`redis://${mapping}`, { maxRetriesPerRequest: 1, connectTimeout: 2000 }); + const args = ['semrushbot:ad', 3600, 12, 60_000, 600, 3_600_000]; + const keys = ['test:ip', 'test:family', 'test:metrics']; + const claim = () => redis.eval(REQUEST_LIMIT_LUA, keys.length, ...keys, ...args); + const concurrent = await Promise.all(Array.from({ length: 30 }, claim)); + assert.equal(concurrent.filter((x) => x === 0).length, 12, 'atomic IP admission'); + assert.equal(concurrent.filter((x) => x > 0).length, 18, 'excess requests rejected'); + assert.equal(await redis.hget('test:metrics', 'semrushbot:ad:requests'), '30'); + assert.equal(await redis.hget('test:metrics', 'semrushbot:ad:throttled'), '18'); + const before = await redis.pttl('test:ip'); + await claim(); + assert.ok(await redis.pttl('test:ip') <= before, 'rejects must not extend the deadline'); + await redis.set('test:family', '600', 'PX', 3_600_000); + const acrossIps = await redis.eval(REQUEST_LIMIT_LUA, 3, 'test:other-ip', 'test:family', 'test:metrics', ...args); + assert.ok(acrossIps > 60_000, 'rotating IPs must still hit family budget'); + console.log('Redis atomic concurrency, bounded counters, fixed deadlines and family throttling passed.'); +} finally { + if (redis) redis.disconnect(); + try { execFileSync('docker', ['stop', name], { stdio: 'pipe' }); } catch {} +} diff --git a/supabase/migrations/20260913170000_ad_token_earnings.sql b/supabase/migrations/20260913170000_ad_token_earnings.sql new file mode 100644 index 00000000..4d1db71f --- /dev/null +++ b/supabase/migrations/20260913170000_ad_token_earnings.sql @@ -0,0 +1,97 @@ +-- API tokens authenticate outside Supabase Auth. Use an explicit owner rather +-- than auth.uid(), and read closed-day rollups plus today's events once. +-- One JSON document avoids PostgREST's row cap on campaign/day series. +create or replace function public.ad_token_earnings(p_owner uuid, p_days integer) +returns jsonb +language plpgsql stable security definer +set search_path = public +as $fn$ +declare + today date := (now() at time zone 'UTC')::date; + today_start timestamptz := today::timestamp at time zone 'UTC'; + from_day date; +begin + if p_owner is null or p_days is null or p_days not in (7, 30, 90, 365) then + raise exception 'An owner and supported reporting window are required' using errcode = '22023'; + end if; + from_day := today - (p_days - 1); + return ( + with campaigns as materialized ( + select id from public.ad_campaigns where owner_id = p_owner + ), slots as materialized ( + select s.id from public.ad_slots s + join public.projects p on p.id = s.project_id and p.owner_id = p_owner + where s.owner_id = p_owner + ), campaign_events as ( + select r.campaign_id, r.day, r.paid_impressions as paid, r.free_impressions as free, + r.valid_clicks as billed, r.free_clicks as unbilled, r.spent_cents as spent + from public.ad_stats_campaign_daily r join campaigns c on c.id = r.campaign_id + where r.day >= from_day and r.day < today + union all + select i.campaign_id, today, (i.tier <> 'free')::int, (i.tier = 'free')::int, 0, 0, 0 + from public.ad_impressions i join campaigns c on c.id = i.campaign_id + where not i.duplicate and i.ts >= today_start and i.ts <= now() + union all + select cl.campaign_id, today, 0, 0, cl.valid::int, + (not cl.valid and cl.tier = 'free')::int, + case when cl.valid then coalesce(cl.charged_cents, 0) else 0 end + from public.ad_clicks cl join campaigns c on c.id = cl.campaign_id + where cl.ts >= today_start and cl.ts <= now() + ), campaign_days as materialized ( + select campaign_id, day, sum(paid)::bigint as paid, sum(free)::bigint as free, + sum(billed)::bigint as billed, sum(unbilled)::bigint as unbilled, + sum(spent)::bigint as spent + from campaign_events group by campaign_id, day + ), slot_events as ( + select r.slot_id, r.day, r.paid_impressions as paid, r.free_impressions as free, + r.valid_clicks as billed, r.free_clicks as unbilled, + r.invalid_clicks as rejected, r.earned_cents as earned + from public.ad_stats_slot_daily r join slots s on s.id = r.slot_id + where r.day >= from_day and r.day < today + union all + select i.slot_id, today, (i.tier <> 'free')::int, (i.tier = 'free')::int, 0, 0, 0, 0 + from public.ad_impressions i join slots s on s.id = i.slot_id + where not i.duplicate and i.ts >= today_start and i.ts <= now() + union all + select cl.slot_id, today, 0, 0, cl.valid::int, + (not cl.valid and cl.tier = 'free')::int, + (not cl.valid and cl.tier <> 'free')::int, + case when cl.valid then coalesce(cl.publisher_earn_cents, 0) else 0 end + from public.ad_clicks cl join slots s on s.id = cl.slot_id + where cl.ts >= today_start and cl.ts <= now() + ), slot_days as materialized ( + select slot_id, day, sum(paid)::bigint as paid, sum(free)::bigint as free, + sum(billed)::bigint as billed, sum(unbilled)::bigint as unbilled, + sum(rejected)::bigint as rejected, sum(earned)::bigint as earned + from slot_events group by slot_id, day + ), campaign_totals as ( + select campaign_id, sum(paid)::bigint as impressions, sum(free)::bigint as free_impressions, + sum(billed)::bigint as clicks, sum(unbilled)::bigint as free_clicks, + sum(spent)::bigint as spent_cents + from campaign_days group by campaign_id + ), slot_totals as ( + select slot_id, sum(paid)::bigint as impressions, sum(free)::bigint as free_impressions, + sum(billed)::bigint as clicks, sum(unbilled)::bigint as free_clicks, + sum(rejected)::bigint as invalid_clicks, sum(earned)::bigint as earned_cents + from slot_days group by slot_id + ), money as ( + select day, spent, 0::bigint as earned from campaign_days + union all select day, 0::bigint, earned from slot_days + ), daily as ( + select day as date, sum(spent)::bigint as "spentCents", sum(earned)::bigint as "earnedCents" + from money group by day + ) + select jsonb_build_object( + 'asOf', now(), 'since', from_day, 'rangeDays', p_days, + 'campaigns', coalesce((select jsonb_agg(to_jsonb(c) order by campaign_id) from campaign_totals c), '[]'::jsonb), + 'slots', coalesce((select jsonb_agg(to_jsonb(s) order by slot_id) from slot_totals s), '[]'::jsonb), + 'daily', coalesce((select jsonb_agg(to_jsonb(d) order by date) from daily d), '[]'::jsonb) + ) + ); +end; +$fn$; + +-- This function accepts an owner from the trusted API, never from a browser. +revoke all on function public.ad_token_earnings(uuid, integer) from public, anon, authenticated; +grant execute on function public.ad_token_earnings(uuid, integer) to service_role; +notify pgrst, 'reload schema'; diff --git a/tests/ads-earnings-route.test.ts b/tests/ads-earnings-route.test.ts new file mode 100644 index 00000000..a349a218 --- /dev/null +++ b/tests/ads-earnings-route.test.ts @@ -0,0 +1,33 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; +const mocks = vi.hoisted(() => ({ auth: vi.fn(), model: vi.fn(), delivery: vi.fn() })); +vi.mock("@/lib/sp/apiAuth", () => ({ authenticateBearer: mocks.auth })); +vi.mock("@/lib/supabase/service", () => ({ serviceClient: () => ({}) })); +vi.mock("@/lib/ads/earnings-data", () => ({ loadEarnings: mocks.model })); +vi.mock("@/lib/ads/token-earnings", () => ({ loadTokenDelivery: mocks.delivery })); +import { GET } from "@/app/api/ads/v1/earnings/route"; +beforeEach(() => { + mocks.auth.mockReset().mockResolvedValue({ ok: true, userId: "authenticated-owner" }); + mocks.delivery.mockReset().mockResolvedValue({}); + mocks.model.mockReset().mockResolvedValue({ statsUnavailable: false, totals: { pubClicks: 123 } }); +}); +describe("earnings response integrity", () => { + it("uses the token owner, ignoring caller-supplied ownership", async () => { + const r = await GET(new NextRequest("https://crawlproof.com/api/ads/v1/earnings?days=7&owner=attacker")); + expect(r.status).toBe(200); + expect(mocks.delivery).toHaveBeenCalledWith({}, "authenticated-owner", 7); + expect(await r.json()).toMatchObject({ deliveryWindow: "range", totals: { pubClicks: 123 } }); + }); + it("returns retryable 503 with no fabricated totals for partial data", async () => { + mocks.model.mockResolvedValue({ statsUnavailable: true, totals: { pubClicks: 0 } }); + const r = await GET(new NextRequest("https://crawlproof.com/api/ads/v1/earnings?days=7")); + expect(r.status).toBe(503); + expect(r.headers.get("retry-after")).toBe("5"); + expect((await r.json()).totals).toBeUndefined(); + }); + it("does not run reporting for an unauthenticated request", async () => { + mocks.auth.mockResolvedValue({ ok: false, status: 401, error: "Unauthorized" }); + expect((await GET(new NextRequest("https://crawlproof.com/api/ads/v1/earnings"))).status).toBe(401); + expect(mocks.delivery).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/ads-token-earnings.test.ts b/tests/ads-token-earnings.test.ts new file mode 100644 index 00000000..b07a80c6 --- /dev/null +++ b/tests/ads-token-earnings.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it, vi } from "vitest"; +import { loadTokenDelivery } from "@/lib/ads/token-earnings"; +import { loadEarnings } from "@/lib/ads/earnings-data"; + +function stub(failMetadata = false) { + const rows: Record = { + ad_campaigns: [{ id: "campaign", name: "Campaign", status: "active", destination_url: "https://example.com", total_spent_cents: 1550 }], + projects: [{ id: "project", name: "example.com" }], + ad_slots: [{ id: "slot", project_id: "project", status: "active" }], ad_ledger: [], ad_payouts: [], + }; + const rpc = vi.fn().mockResolvedValue({ data: { + campaigns: [{ campaign_id: "campaign", impressions: "10", free_impressions: "100", clicks: "2", free_clicks: "12", spent_cents: "40" }], + slots: [{ slot_id: "slot", impressions: "10", free_impressions: "100", clicks: "2", free_clicks: "12", invalid_clicks: "250", earned_cents: "20" }], daily: [], + }, error: null }); + const filters: Array<[string, string, unknown]> = []; + const from = (table: string) => { + const result: any = { select: () => result, eq: (field: string, value: unknown) => { filters.push([table, field, value]); return result; }, order: () => result, + then: (resolve: (value: unknown) => void) => resolve({ data: rows[table], error: failMetadata ? { code: "57014" } : null }), + }; + return result; + }; + return { client: { rpc, from } as any, rpc, filters }; +} + +describe("API-token ad earnings", () => { + it("keeps billed, free and rejected counts separate and populates domain rows", async () => { + const { client, rpc, filters } = stub(); + const model = await loadEarnings(client, "owner", 7, loadTokenDelivery(client, "owner", 7)); + expect(rpc).toHaveBeenCalledExactlyOnceWith("ad_token_earnings", { p_owner: "owner", p_days: 7 }); + expect(model.statsUnavailable).toBe(false); + expect(model.totals).toMatchObject({ pubImpressions: 110, pubClicks: 14, pubBilledClicks: 2, pubFreeClicks: 12, invalidClicks: 250, spentCents: 1550 }); + expect(model.campaigns[0]).toMatchObject({ clicks: 14, impressions: 110, spentCents: 40 }); + expect(model.slots[0]).toMatchObject({ clicks: 14, impressions: 110, earnedCents: 20 }); + expect(model.daily).toHaveLength(7); + for (const [table, field, value] of filters.filter(([, field]) => field === "owner_id")) expect(value, table).toBe("owner"); + }); + it("rejects failed RPCs instead of returning zero delivery", async () => { + const { client, rpc } = stub(); rpc.mockResolvedValue({ data: null, error: { code: "57014" } }); + await expect(loadTokenDelivery(client, "owner", 7)).rejects.toThrow("temporarily unavailable"); + }); + it("does not mistake a malformed response for an empty account", async () => { + const { client, rpc } = stub(); rpc.mockResolvedValue({ data: {}, error: null }); + await expect(loadTokenDelivery(client, "owner", 7)).rejects.toThrow("temporarily unavailable"); + }); + it("flags failed balances or metadata as unavailable", async () => { + const { client } = stub(true); + expect((await loadEarnings(client, "owner", 7, loadTokenDelivery(client, "owner", 7))).statsUnavailable).toBe(true); + }); +}); diff --git a/tests/contract/ads-crawler-metering.test.ts b/tests/contract/ads-crawler-metering.test.ts new file mode 100644 index 00000000..b3031c23 --- /dev/null +++ b/tests/contract/ads-crawler-metering.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it, vi } from "vitest"; +const db = vi.hoisted(() => ({ from: vi.fn(), rpc: vi.fn() })); +vi.mock("@/lib/supabase/service", () => ({ serviceClient: () => db })); +import { resolveClick } from "@/lib/ads/serve"; + +describe("paid crawler link resolution", () => { + it("resolves the destination without a click write, attribution, fraud query or charge", async () => { + const chain: any = { select: () => chain, eq: () => chain, maybeSingle: async () => ({ data: { id: "campaign", destination_url: "https://advertiser.example/", ref_slug: "ad-ref", status: "active", bid_credits: 1 } }) }; + db.from.mockImplementation((table) => { if (table !== "ad_campaigns") throw new Error(`Unexpected metering table: ${table}`); return chain; }); + const dest = await resolveClick({ campaignId: "campaign", impressionId: "impression", slotId: "slot", ctx: { device: "bot", ip: "85.208.96.196" } }); + expect(dest).toBe("https://advertiser.example/"); + expect(db.from).toHaveBeenCalledExactlyOnceWith("ad_campaigns"); + expect(db.rpc).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/crawl-activity-route.test.ts b/tests/crawl-activity-route.test.ts new file mode 100644 index 00000000..7ba54b74 --- /dev/null +++ b/tests/crawl-activity-route.test.ts @@ -0,0 +1,28 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; +const mocks = vi.hoisted(() => ({ auth: vi.fn(), profile: vi.fn(), counters: vi.fn() })); +vi.mock("@/lib/sp/apiAuth", () => ({ authenticateBearer: mocks.auth })); +vi.mock("@/lib/supabase/server", () => ({ createClient: async () => ({ auth: { getUser: async () => ({ data: { user: null } }) } }) })); +vi.mock("@/lib/supabase/service", () => ({ serviceClient: () => ({ from: () => ({ select: () => ({ eq: () => ({ maybeSingle: mocks.profile }) }) }) }) })); +vi.mock("@/lib/crawl-limits", () => ({ readCrawlerActivity: mocks.counters })); +import { GET } from "@/app/api/admin/crawl-activity/route"; +beforeEach(() => { + mocks.auth.mockReset().mockResolvedValue({ ok: true, userId: "owner" }); + mocks.profile.mockReset().mockResolvedValue({ data: { is_admin: false }, error: null }); + mocks.counters.mockReset().mockResolvedValue([]); +}); +const req = () => new NextRequest("https://crawlproof.com/api/admin/crawl-activity?days=7", { headers: { authorization: "Bearer test" } }); +describe("network crawler diagnostics access", () => { + it("denies non-admin tokens without reading network counters", async () => { + expect((await GET(req())).status).toBe(403); expect(mocks.counters).not.toHaveBeenCalled(); + }); + it("denies anonymous requests", async () => { + expect((await GET(new NextRequest("https://crawlproof.com/api/admin/crawl-activity"))).status).toBe(401); + expect(mocks.counters).not.toHaveBeenCalled(); + }); + it("returns explicitly network-scoped data to admins", async () => { + mocks.profile.mockResolvedValue({ data: { is_admin: true }, error: null }); + const response = await GET(req()); expect(response.status).toBe(200); + expect(await response.json()).toEqual({ scope: "network", rangeDays: 7, daily: [] }); + }); +}); diff --git a/tests/crawl-gateway.test.ts b/tests/crawl-gateway.test.ts new file mode 100644 index 00000000..f5562225 --- /dev/null +++ b/tests/crawl-gateway.test.ts @@ -0,0 +1,70 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mintPass } from "@profullstack/x402-gateway"; + +const mocks = vi.hoisted(() => ({ limit: vi.fn(), record: vi.fn() })); +vi.mock("@/lib/crawl-limits", () => ({ + limitCrawlRequest: mocks.limit, recordCrawlerOutcome: mocks.record, + CRAWLER_IP_PER_MINUTE: 12, CRAWLER_FAMILY_PER_HOUR: 600, +})); +const SECRET = "test-crawl-secret-not-a-real-key"; +const SEMRUSH = "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)"; +function request(path = "/a/AbCdEf123456", ua = SEMRUSH, pass?: string) { + return new Request(`https://crawlproof.com${path}`, { headers: { + "user-agent": ua, "x-real-ip": "85.208.96.196", ...(pass ? { "x-crawl-pass": pass } : {}), + } }); +} +beforeEach(() => { + vi.resetModules(); mocks.limit.mockReset().mockResolvedValue(undefined); mocks.record.mockReset().mockResolvedValue(undefined); + vi.stubEnv("COINPAY_X402_KEY", SECRET); vi.stubEnv("CRAWL_PAY_TO", `0x${"1".repeat(40)}`); +}); +afterEach(() => vi.unstubAllEnvs()); + +describe("crawler access through x402-gateway", () => { + it("offers a priced module day pass to commercial crawlers", async () => { + const { gate } = await import("@/lib/crawl-gateway"); + const response = await gate(request()); + expect(response?.status).toBe(402); + const body = await response!.json(); + expect(body.accepts.length).toBeGreaterThan(0); + expect(body.pass.minutes).toBe(1440); + expect(body.pass.price).toBe("1.00 USD"); + expect(mocks.limit).toHaveBeenCalledWith(expect.any(Request), "semrushbot", "ad"); + }); + it("admits a valid signed pass and rejects a forged one", async () => { + const { gate } = await import("@/lib/crawl-gateway"); + const pass = await mintPass({ secret: SECRET, ref: "test", expiresAt: Math.floor(Date.now() / 1000) + 3600 }); + expect(await gate(request("/a/AbCdEf123456", SEMRUSH, pass.token))).toBeUndefined(); + expect((await gate(request("/a/AbCdEf123456", SEMRUSH, "cp_forged.signature")))?.status).toBe(402); + }); + it("throttles even paid crawlers before processing their pass or payment", async () => { + const { gate } = await import("@/lib/crawl-gateway"); + const pass = await mintPass({ secret: SECRET, ref: "test", expiresAt: Math.floor(Date.now() / 1000) + 3600 }); + mocks.limit.mockResolvedValue(new Response(null, { status: 429, headers: { "retry-after": "60" } })); + expect((await gate(request("/a/AbCdEf123456", SEMRUSH, pass.token)))?.status).toBe(429); + expect(mocks.record).not.toHaveBeenCalled(); + }); + it("keeps human ad redirects free but limits their request rate", async () => { + const { gate } = await import("@/lib/crawl-gateway"); + expect(await gate(request("/a/AbCdEf123456", "Mozilla/5.0 Chrome/120.0 Safari/537.36"))).toBeUndefined(); + expect(mocks.limit).toHaveBeenCalledWith(expect.any(Request), null, "ad"); + }); + it("allows search indexing of content while refusing unpaid ad redirects", async () => { + const { gate } = await import("@/lib/crawl-gateway"); + expect(await gate(request("/blog", "Googlebot"))).toBeUndefined(); + expect((await gate(request("/a/AbCdEf123456", "Googlebot")))?.status).toBe(403); + }); + it("keeps all generated robots groups away from ad redirects", async () => { + const { gateway } = await import("@/lib/crawl-gateway"); + const txt = gateway.robotsTxt({ disallow: ["/api/", "/a/"] }); + for (const group of txt.split(/\n\n+/).filter((g) => g.startsWith("User-agent:"))) { + expect(group).toMatch(/Disallow: \/a\/|Disallow: \/\n/); + } + expect(txt).toContain("User-agent: SemrushBot\nDisallow: /\nAllow: /crawl"); + }); + it("explains the same rate limits to paying operators", async () => { + const { gateway } = await import("@/lib/crawl-gateway"); + expect(gateway.page()).toContain("Commercial and training crawlers"); + expect(gateway.page()).toContain("600 requests per hour"); + expect(gateway.page()).toContain("never count as ad clicks"); + }); +}); diff --git a/tests/dashboard-finance.test.ts b/tests/dashboard-finance.test.ts new file mode 100644 index 00000000..381df450 --- /dev/null +++ b/tests/dashboard-finance.test.ts @@ -0,0 +1,53 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { collectBusinessRevenue, collectFinance } from "@/lib/dashboard/collect"; + +const sdk = vi.hoisted(() => ({ snapshot: vi.fn(), analytics: vi.fn() })); +vi.mock("@profullstack/coinpay", () => ({ default: class CoinPayClient {} })); +vi.mock("@profullstack/coinpay/finances", () => ({ + collectFinanceSnapshot: sdk.snapshot, + getFinanceAnalytics: sdk.analytics, +})); +afterEach(() => vi.resetAllMocks()); + +const analytics = (commission: number) => ({ + combined: { total_fees_usd: 999999 }, // Headline lifetime totals are not a rate. + series: { points: [ + { total_commission_usd: commission, total_volume_usd: "100", total_count: 2 }, + { total_commission_usd: commission, total_volume_usd: "200", total_count: 3 }, + ] }, +}); + +describe("collectBusinessRevenue", () => { + it("keeps windowed totals under their business ids and isolates a failed request", async () => { + const fetchAnalytics = vi.fn(async (id: string) => { + if (id === "down") throw new Error("timeout"); + return analytics(id === "a" ? 2 : 10); + }); + const result = await collectBusinessRevenue([{ id: "a" }, { id: "b" }, { id: "down" }, { id: "a" }, {}], 7, fetchAnalytics); + expect(result.a).toEqual({ windowDays: 7, commissionUsd: 4, grossVolumeUsd: 300, transactions: 5 }); + expect(result.b?.commissionUsd).toBe(20); + expect(result.down).toEqual({ windowDays: 7, error: "timeout" }); + expect(fetchAnalytics).toHaveBeenCalledTimes(3); + }); + + it("distinguishes a measured empty window from an absent or incomplete response", async () => { + const empty = await collectBusinessRevenue([{ id: "a" }], 7, async () => ({ series: { points: [] } })); + expect(empty.a?.commissionUsd).toBe(0); + for (const response of [{}, { series: { points: [{ total_volume_usd: 100 }] } }]) { + const result = await collectBusinessRevenue([{ id: "a" }], 7, async () => response); + expect(result.a?.error).toBeTruthy(); + expect(result.a?.commissionUsd).toBeUndefined(); + } + }); +}); + +describe("collectFinance", () => { + it.each([[7, "week"], [30, "month"]])("requests business ids with the server's %s-day preset", async (days, period) => { + sdk.snapshot.mockResolvedValue({ businesses: [{ id: "a", name: "a.dev" }, { id: "b", name: "b.dev" }] }); + sdk.analytics.mockResolvedValue(analytics(2)); + const result = await collectFinance({ token: "test", baseUrl: "https://coinpay.test/api" }, days as number); + expect(sdk.analytics).toHaveBeenCalledWith(expect.anything(), { businessId: "a", period }); + expect(sdk.analytics).toHaveBeenCalledWith(expect.anything(), { businessId: "b", period }); + expect(result.businessRevenue?.a?.commissionUsd).toBe(4); + }); +}); diff --git a/tests/dashboard-loading.test.ts b/tests/dashboard-loading.test.ts new file mode 100644 index 00000000..84786034 --- /dev/null +++ b/tests/dashboard-loading.test.ts @@ -0,0 +1,152 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { renderToScreen } from "@profullstack/hqtui/testing"; +import { ADS_TIMEOUT_MS, collectAds, collectDashboard, type DashboardSnapshot, type FeedProgress } from "@/lib/dashboard/collect"; +import { createRefreshController, handleKey, initialState, renderBody, renderFetchStatus } from "@/cli/dashboard"; +import { buildRoi } from "@/lib/dashboard/roi"; + +const ads = { rangeDays: 7, statsUnavailable: false, totals: { pubImpressions: 12345, pubClicks: 83 } }; +const options = { baseUrl: "https://crawlproof.test", token: "test", range: "1w", who: "humans", financeDays: 7, coinpay: null }; +const snapshot = (): DashboardSnapshot => ({ + generatedAt: "2026-09-13T10:00:00.000Z", + window: { range: "1w", who: "humans", financeDays: 7 }, + sites: [], fleet: { sources: [], pages: [], referrers: [] }, ads, finance: null, errors: {}, + roi: buildRoi({ traffic: { range: "1w", who: "humans", sites: [] }, ads, finance: null }), +}); + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +describe("ads retries", () => { + it("allows a healthy 25-second ads response to finish", async () => { + vi.useFakeTimers(); + const fetcher = vi.fn((_url, init) => new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve(Response.json(ads)), 25000); + init.signal.addEventListener("abort", () => { clearTimeout(timer); reject(new DOMException("aborted", "AbortError")); }); + })); + vi.stubGlobal("fetch", fetcher); + const result = collectAds(options.baseUrl, "test", 7); + await vi.advanceTimersByTimeAsync(25000); + expect(await result).toEqual(ads); + expect(fetcher).toHaveBeenCalledTimes(1); + }); + + it("retries a timeout once and reports the attempt visibly", async () => { + vi.useFakeTimers(); + const fetcher = vi.fn() + .mockImplementationOnce((_url, init) => new Promise((_resolve, reject) => { + init.signal.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError"))); + })) + .mockResolvedValueOnce(Response.json(ads)); + vi.stubGlobal("fetch", fetcher); + const progress: FeedProgress[] = []; + const result = collectAds(options.baseUrl, "test", 7, (value) => progress.push(value)); + await vi.advanceTimersByTimeAsync(ADS_TIMEOUT_MS + 1000); + expect(await result).toEqual(ads); + expect(fetcher).toHaveBeenCalledTimes(2); + expect(progress).toContainEqual({ status: "retrying", detail: "Ads retry 2/2" }); + }); + + it("retries partial data and transient HTTP errors, but not an invalid token", async () => { + vi.useFakeTimers(); + for (const first of [Response.json({ ...ads, statsUnavailable: true }), Response.json({ error: "busy" }, { status: 503 })]) { + const fetcher = vi.fn().mockResolvedValueOnce(first).mockResolvedValueOnce(Response.json(ads)); + vi.stubGlobal("fetch", fetcher); + const result = collectAds(options.baseUrl, "test", 7); + await vi.runAllTimersAsync(); + expect(await result).toEqual(ads); + expect(fetcher).toHaveBeenCalledTimes(2); + } + const fetcher = vi.fn().mockResolvedValue(Response.json({ error: "Invalid token" }, { status: 401 })); + vi.stubGlobal("fetch", fetcher); + await expect(collectAds(options.baseUrl, "test", 7)).rejects.toThrow("Invalid token"); + expect(fetcher).toHaveBeenCalledTimes(1); + }); +}); + +describe("failed ads refresh", () => { + it("retains the last successful response and timestamp only for the same window", async () => { + vi.useFakeTimers(); + vi.stubGlobal("fetch", vi.fn(async (url: string) => url.includes("/sites") + ? Response.json({ sites: [] }) + : Response.json({ error: "upstream unavailable" }, { status: 503 }))); + const previous = snapshot(); + for (const financeDays of [7, 30]) { + const pending = collectDashboard({ ...options, financeDays, previous }); + await vi.runAllTimersAsync(); + const result = await pending; + expect(result.errors.ads).toBe("upstream unavailable"); + if (financeDays === 7) { + expect(result.ads).toEqual(ads); + expect(result.adsStale).toBe(true); + expect(result.adsUpdatedAt).toBe(previous.generatedAt); + const state = initialState({ tab: 2, snapshot: result }); + const screen = renderToScreen(({ ui, theme }) => renderBody(ui, state, theme), { width: 140, height: 40 }); + expect(screen.text()).toContain("Ads: saved data from 2026-09-13T10:00:00.000Z"); + expect(screen.text()).toContain("12,345"); + } else { + expect(result.ads).toBeNull(); + expect(result.adsStale).toBe(false); + } + } + }); +}); + +describe("visible refresh lifecycle", () => { + it("r immediately starts animated feed spinners and then shows success", async () => { + const state = initialState({ tab: 2, snapshot: snapshot() }); + let finish!: (value: DashboardSnapshot) => void; + const collector = vi.fn(() => new Promise((resolve) => { finish = resolve; })); + const refresh = createRefreshController(state, options, vi.fn(), collector); + handleKey(state, { name: "r" }, { refresh }); + expect(state.loading).toBe(true); + const draw = (elapsed: number) => renderToScreen(({ ui, theme }) => renderFetchStatus(ui, state, theme), { width: 120, height: 4, elapsed }).text(); + expect(draw(0)).toContain("Fetching traffic"); + expect(draw(0)).toContain("Fetching ads"); + expect(draw(0)).toContain("Fetching CoinPay"); + expect(draw(0)).toContain("Refreshing…"); + expect(draw(0)).not.toEqual(draw(80)); + for (const feed of ["traffic", "ads", "finance"] as const) { + collector.mock.calls[0]![0].onProgress?.(feed, { status: "success", detail: `${feed} refreshed` }); + } + finish(snapshot()); + await vi.waitFor(() => expect(state.loading).toBe(false)); + expect(state.refreshMessage).toContain("Refresh complete"); + expect(draw(0)).toEqual(draw(80)); + }); + + it("queues repeated r presses once and applies a changed window", async () => { + const state = initialState({ range: "1w" }); + let finish!: (value: DashboardSnapshot) => void; + const collector = vi.fn() + .mockImplementationOnce(() => new Promise((resolve) => { finish = resolve; })) + .mockResolvedValue(snapshot()); + const refresh = createRefreshController(state, options, vi.fn(), collector); + const running = refresh(); + state.range = "1m"; + void refresh(); + void refresh(); + expect(state.refreshQueued).toBe(true); + expect(collector).toHaveBeenCalledTimes(1); + const text = renderToScreen(({ ui, theme }) => renderFetchStatus(ui, state, theme), { width: 120, height: 4 }).text(); + expect(text).toContain("next refresh queued"); + finish(snapshot()); + await running; + expect(collector).toHaveBeenCalledTimes(2); + expect(collector.mock.calls[1]![0].financeDays).toBe(30); + expect(state.loading).toBe(false); + expect(state.refreshQueued).toBe(false); + }); + + it("stops spinners and shows a failure instead of pretending the refresh worked", async () => { + const previous = snapshot(); + const state = initialState({ snapshot: previous }); + const collector = vi.fn().mockRejectedValue(new Error("network down")); + await createRefreshController(state, options, vi.fn(), collector)(); + expect(state.snapshot).toBe(previous); + expect(state.loading).toBe(false); + expect(state.refreshMessage).toBe("Refresh failed: network down · r retries"); + expect(Object.values(state.feeds).every((feed) => feed.status === "error")).toBe(true); + }); +}); diff --git a/tests/dashboard-roi.test.ts b/tests/dashboard-roi.test.ts index 31ac7ea5..5fe298e3 100644 --- a/tests/dashboard-roi.test.ts +++ b/tests/dashboard-roi.test.ts @@ -146,11 +146,13 @@ describe("vendors", () => { describe("adTargets", () => { const delivered = { + campaigns: [{ spentCents: 1500 }], totals: { pubImpressions: 220_000, pubPaidImpressions: 17_000, pubFreeImpressions: 203_000, pubClicks: 80, + advBilledClicks: 80, invalidClicks: 9_700, spentCents: 1_500, }, @@ -183,6 +185,25 @@ describe("adTargets", () => { expect(t.projectedMonthlyUsd).toBeNull(); }); + it("uses a matching window and advertiser population for paid CPC", () => { + const t = adTargets({ rangeDays: 7, campaigns: [{ spentCents: 90 }], totals: { + spentCents: 1550, advBilledClicks: 3, pubImpressions: 700000, + pubClicks: 500, pubBilledClicks: 0, pubFreeClicks: 500, + } }); + expect(t.cpcCents).toBe(30); + expect(t.windowTargetImpressions).toBe(700000); + expect(t.impressionProgress).toBe(1); + expect(t.freeClicks).toBe(500); + }); + + it("does not divide lifetime spend by this week's free clicks", () => { + const t = adTargets({ rangeDays: 7, campaigns: [{ spentCents: 0 }], totals: { + spentCents: 1550, advBilledClicks: 0, pubClicks: 13484, pubFreeClicks: 13484, + } }); + expect(t.cpcCents).toBeNull(); + expect(t.projectedMonthlyUsd).toBeNull(); + }); + it("takes overridden targets", () => { const t = adTargets(delivered, { targetImpressions: 1_000_000, targetCtr: 0.06 }); expect(t.impressionProgress).toBeCloseTo(0.22); diff --git a/tests/dashboard-score.test.ts b/tests/dashboard-score.test.ts index 9c5e6bbf..9f4e2036 100644 --- a/tests/dashboard-score.test.ts +++ b/tests/dashboard-score.test.ts @@ -60,6 +60,11 @@ describe("growthRate", () => { expect(g).toBeGreaterThan(0); }); + it("compares equal durations when the series has an odd number of buckets", () => { + expect(growthRate([5, 5, 5, 5, 5])).toBe(0); + expect(growthRate([2, 2, 99, 4, 4])).toBe(1); + }); + it("is null with too few buckets, or with no traffic at all", () => { expect(growthRate([1, 2, 3])).toBeNull(); expect(growthRate([])).toBeNull(); @@ -208,6 +213,18 @@ describe("scoreSite", () => { expect(component(over.riskComponents, "unmonetised")?.value).toBe(0); }); + it("leaves missing revenue unscored, while measured zero is unmonetised", () => { + for (const revenueUsd of [undefined, null, Number.NaN]) { + const unknown = scoreSite({ ...growing(), revenueUsd }); + expect(component(unknown.viralComponents, "money")?.value).toBeNull(); + expect(component(unknown.riskComponents, "unmonetised")?.value).toBeNull(); + expect(unknown.coverage).toBeCloseTo(0.9); + } + const zero = scoreSite({ ...growing(), revenueUsd: 0 }); + expect(component(zero.viralComponents, "money")?.value).toBe(0); + expect(component(zero.riskComponents, "unmonetised")?.value).toBe(1); + }); + it("flags a sample too small to lean on without hiding the number", () => { const tiny = scoreSite({ humans: [1, 1, 2, 1], mixKnown: true, bots: [0, 0, 0, 0] }); expect(tiny.provisional).toBe(true); diff --git a/tests/dashboard-screens.test.ts b/tests/dashboard-screens.test.ts index 8f2f7f8d..8d8ab0e7 100644 --- a/tests/dashboard-screens.test.ts +++ b/tests/dashboard-screens.test.ts @@ -113,6 +113,23 @@ describe("Traffic list", () => { expect(screen.regions.some((r) => typeof r.onClick === "function")).toBe(true); }); + it("opens the clicked domain after scrolling and ignores the header", () => { + const rows = Array.from({ length: 40 }, (_, i) => siteRow({ + site: `domain${i}.dev`, id: `p-${i}`, url: `https://domain${i}.dev`, visitors: 1000 - i, + })); + const state = stateWith({ sort: "visitors", panes: { sites: { selected: 20, offset: 20, total: 40 } } }, rows); + const screen = renderToScreen(({ ui, theme }) => renderBody(ui, state, theme), { width: 160, height: 12 }); + const point = screen.find("domain21.dev"); + expect(point).not.toBeNull(); + const region = screen.regions.find((r) => point && point.x >= r.rect.x && point.x < r.rect.x + r.rect.width && point.y >= r.rect.y && point.y < r.rect.y + r.rect.height)!; + region.onClick?.(0, 0, "left"); + expect(state.domain).toBeNull(); + region.onClick?.(point!.x - region.rect.x, point!.y - region.rect.y, "left"); + expect(state.domain).toBe("domain21.dev"); + expect(state.panes.sites?.selected).toBe(21); + expect(draw(state).text()).not.toContain("domain20.dev"); + }); + it("marks a site that did not answer rather than drawing it as a quiet one", () => { const rows = [siteRow(), siteRow({ site: "down.dev", id: "p-2", url: "https://down.dev", error: "504 Gateway Timeout", visitors: 0, pageviews: 0, series: [], mix: undefined })]; const text = draw(stateWith({}, rows)).text(); @@ -201,11 +218,37 @@ describe("the domain screen", () => { it("shows the money for that domain, cost by both denominators", () => { const out = text(); - expect(out).toContain("Cost · by views"); - expect(out).toContain("Cost · by visits"); + expect(out).toContain("Est. cost/views"); + expect(out).toContain("Est. cost/visits"); expect(out).toContain("Ad earned"); }); + it("shows only the opened domain's CoinPay volume, payments and commission", () => { + const state = stateWith({ domain: "nichedb.dev" }); + const fin = state.snapshot!.finance!; + fin.businesses = [{ id: "b-1", name: "nichedb.dev" }, { id: "b-2", name: "other.dev" }]; + fin.businessRevenue = { + "b-1": { windowDays: 30, commissionUsd: 12, grossVolumeUsd: 4321, transactions: 7 }, + "b-2": { windowDays: 30, commissionUsd: 999, grossVolumeUsd: 987654, transactions: 900 }, + }; + const out = draw(state).text(); + expect(out).toMatch(/Revenue\s+\$12.00/); + expect(out).toMatch(/Volume · 30d\s+\$4,321/); + expect(out).toMatch(/Payments\s+7/); + expect(out).not.toContain("987,654"); + expect(out).toContain("CoinPay: nichedb.dev"); + }); + + it("marks partial ad stats unavailable instead of showing zero-filled earnings", () => { + const state = stateWith({ domain: "nichedb.dev" }); + state.snapshot!.ads!.statsUnavailable = true; + const out = draw(state).text(); + expect(out).toMatch(/Ad earned \(int\.\)\s+—/); + expect(out).toMatch(/Ad spent \(int\.\)\s+—/); + expect(out).toMatch(/Impressions\s+—/); + expect(out).toContain("Ads unavailable"); + }); + it("shows the score and every component behind it", () => { const out = text(); expect(out).toContain("Risk-to-viral"); diff --git a/tests/dashboard-site.test.ts b/tests/dashboard-site.test.ts index 6a74c89c..05c11179 100644 --- a/tests/dashboard-site.test.ts +++ b/tests/dashboard-site.test.ts @@ -16,6 +16,7 @@ import { coinpayRevenueForSite, hostFrom, sameProperty, + sourcesForScore, type SiteLike, } from "@/lib/dashboard/site"; @@ -119,7 +120,56 @@ describe("adMoneyForSite", () => { }); }); +describe("sourcesForScore", () => { + it("keeps self and local referrals in the denominator without treating them as discovery", () => { + const input = site({ sources: [ + { label: "Referral · www.nichedb.dev", value: 50 }, + { label: "Referral · 127.0.0.1:3000", value: 10 }, + { label: "referral:localhost", value: 10 }, + { label: "Social · reddit", value: 30 }, + ] }); + expect(sourcesForScore(input)).toEqual([ + { label: "Internal referral", value: 70 }, + { label: "Social · reddit", value: 30 }, + ]); + const detail = buildSiteDetail({ site: input, roi: roiFor([input]), ads: ads(), finance: null, window: { range: "1m", who: "humans", financeDays: 30 } }); + expect(detail.score.viralComponents.find((c) => c.key === "discovery")?.value).toBeCloseTo(0.3); + expect(detail.traffic.sources).toEqual(input.sources); + }); +}); + describe("coinpayRevenueForSite", () => { + it("uses only the matching business and prorates its commission onto the traffic window", () => { + const fin = finance([{ id: "b-1", name: "www.nichedb.dev" }, { id: "b-2", name: "other.dev" }]); + fin.businessRevenue = { + "b-1": { windowDays: 7, commissionUsd: 14, grossVolumeUsd: 1000, transactions: 3 }, + "b-2": { windowDays: 7, commissionUsd: 9000, grossVolumeUsd: 50000, transactions: 99 }, + }; + const roi = roiFor([site()], fin); + roi.window.days = 1; + roi.window.range = "1d"; + const result = coinpayRevenueForSite(fin, roi, site()); + expect(result.usd).toBe(2); + expect(result.grossVolumeUsd).toBe(1000); + expect(result.transactions).toBe(3); + expect(result.observedDays).toBe(7); + expect(result.basis).toContain("prorated to 1d"); + }); + + it("never falls back to fleet revenue when a business request fails or the domain is unmatched", () => { + const fin = finance([{ id: "b-1", name: "nichedb.dev" }]); + fin.businessRevenue = { "b-1": { windowDays: 7, error: "timeout" } }; + expect(coinpayRevenueForSite(fin, roiFor([site()], fin), site()).usd).toBeNull(); + expect(coinpayRevenueForSite(fin, roiFor([site()], fin), site()).basis).toBe("timeout"); + expect(coinpayRevenueForSite(fin, roiFor([site()], fin), site({ site: "other.dev", url: "https://other.dev" })).usd).toBeNull(); + }); + + it("does not rescale lifetime earnings if an older snapshot has no day series", () => { + const fin = finance([{ name: "nichedb.dev" }]); + fin.series = []; + expect(coinpayRevenueForSite(fin, roiFor([site()], fin), site()).usd).toBeNull(); + }); + it("attributes the whole commission when there is exactly one matching business", () => { const fin = finance([{ id: "b-1", name: "nichedb.dev" }]); const result = coinpayRevenueForSite(fin, roiFor([site()], fin), site()); @@ -189,6 +239,15 @@ describe("buildSiteDetail", () => { expect(detail.money.rpmUsd).toBeNull(); expect(detail.money.netUsd).toBeNull(); expect(detail.gaps.join(" ")).toMatch(/not attributable/); + expect(detail.score.viralComponents.find((c) => c.key === "money")?.value).toBeNull(); + }); + + it("leaves cost unknown when CoinPay has no bank position", () => { + expect(build({}, null).money.costByViewsUsd).toBeNull(); + expect(build({}, null).money.costByVisitsUsd).toBeNull(); + const fin = finance([]); + fin.errors = { summary: "bank offline" }; + expect(build({}, fin).money.costByViewsUsd).toBeNull(); }); it("computes revenue per 1k humans once there is revenue to divide", () => { @@ -224,5 +283,6 @@ describe("buildSiteDetail", () => { window: { range: "1m", who: "bots", financeDays: 30 }, }); expect(detail.gaps.join(" ")).toMatch(/bots-only/); + expect(detail.score.viralComponents.find((c) => c.key === "discovery")?.value).toBeNull(); }); }); diff --git a/tests/sql/ad-token-earnings.sql b/tests/sql/ad-token-earnings.sql new file mode 100644 index 00000000..30513ed9 --- /dev/null +++ b/tests/sql/ad-token-earnings.sql @@ -0,0 +1,78 @@ +-- Run with node scripts/test-ad-token-earnings.mjs (disposable PostgreSQL). +begin; +create role anon; +create role authenticated; +create role service_role; +create table ad_campaigns (id uuid primary key, owner_id uuid); +create table projects (id uuid primary key, owner_id uuid); +create table ad_slots (id uuid primary key, owner_id uuid, project_id uuid); +create table ad_impressions (campaign_id uuid, slot_id uuid, ts timestamptz, tier text, duplicate boolean); +create table ad_clicks (campaign_id uuid, slot_id uuid, ts timestamptz, tier text, valid boolean, charged_cents int, publisher_earn_cents int); +create table ad_stats_campaign_daily (campaign_id uuid, day date, paid_impressions bigint, free_impressions bigint, valid_clicks bigint, free_clicks bigint, spent_cents bigint); +create table ad_stats_slot_daily (slot_id uuid, day date, paid_impressions bigint, free_impressions bigint, valid_clicks bigint, free_clicks bigint, invalid_clicks bigint, earned_cents bigint); +insert into ad_campaigns values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000010'), ('00000000-0000-0000-0000-000000000002','00000000-0000-0000-0000-000000000020'); +insert into projects values ('00000000-0000-0000-0000-000000000003','00000000-0000-0000-0000-000000000010'), ('00000000-0000-0000-0000-000000000004','00000000-0000-0000-0000-000000000020'); +insert into ad_slots values ('00000000-0000-0000-0000-000000000005','00000000-0000-0000-0000-000000000010','00000000-0000-0000-0000-000000000003'), ('00000000-0000-0000-0000-000000000006','00000000-0000-0000-0000-000000000020','00000000-0000-0000-0000-000000000004'); +insert into ad_stats_campaign_daily values + ('00000000-0000-0000-0000-000000000001', (now() at time zone 'UTC')::date-1,10,20,2,3,100), + ('00000000-0000-0000-0000-000000000001', (now() at time zone 'UTC')::date-7,999,999,999,999,999), + ('00000000-0000-0000-0000-000000000002', (now() at time zone 'UTC')::date-1,999,999,999,999,999); +insert into ad_stats_slot_daily values + ('00000000-0000-0000-0000-000000000005', (now() at time zone 'UTC')::date-1,10,20,2,3,7,50), + ('00000000-0000-0000-0000-000000000005', (now() at time zone 'UTC')::date-7,999,999,999,999,999,999), + ('00000000-0000-0000-0000-000000000006', (now() at time zone 'UTC')::date-1,999,999,999,999,999,999); +insert into ad_impressions values + ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000005',now(),'paid',false), + ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000005',now(),'free',false), + ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000005',now(),'free',true), + ('00000000-0000-0000-0000-000000000002','00000000-0000-0000-0000-000000000006',now(),'free',false); +insert into ad_clicks values + ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000005',now(),'paid',true,25,10), + ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000005',now(),'free',false,0,0), + ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000005',now(),'paid',false,0,0), + ('00000000-0000-0000-0000-000000000002','00000000-0000-0000-0000-000000000006',now(),'paid',true,999,999); +-- APPLY MIGRATION HERE +set role service_role; +do $$ +declare j jsonb; c jsonb; s jsonb; +begin + j := public.ad_token_earnings('00000000-0000-0000-0000-000000000010',7); + assert jsonb_array_length(j->'campaigns')=1, 'tenant scope leaked campaigns'; + assert jsonb_array_length(j->'slots')=1, 'tenant scope leaked slots'; + c := j->'campaigns'->0; s := j->'slots'->0; + assert (c->>'impressions')::int=11 and (c->>'free_impressions')::int=21, 'wrong window/duplicate impressions'; + assert (c->>'clicks')::int=3 and (c->>'free_clicks')::int=4, 'billed/free classification'; + assert (c->>'spent_cents')::int=125, 'wrong windowed spend'; + assert (s->>'invalid_clicks')::int=8, 'rejected/free classification'; + assert (s->>'earned_cents')::int=60, 'wrong windowed earnings'; + assert jsonb_array_length(j->'daily')=2, 'daily boundary mismatch'; + assert (select sum((d->>'spentCents')::int) from jsonb_array_elements(j->'daily') d)=125, 'daily spend mismatch'; + j := public.ad_token_earnings('00000000-0000-0000-0000-000000000030',7); + assert j->'campaigns'='[]'::jsonb and j->'slots'='[]'::jsonb, 'empty owner leaked data'; + begin + perform public.ad_token_earnings(null,7); + raise exception 'null owner accepted'; + exception when invalid_parameter_value then null; end; + begin + perform public.ad_token_earnings('00000000-0000-0000-0000-000000000010',999999); + raise exception 'unbounded window accepted'; + exception when invalid_parameter_value then null; end; +end $$; +reset role; +set role authenticated; +do $$ begin + begin + perform public.ad_token_earnings('00000000-0000-0000-0000-000000000010',7); + raise exception 'authenticated user can choose another owner'; + exception when insufficient_privilege then null; end; +end $$; +reset role; +set role anon; +do $$ begin + begin + perform public.ad_token_earnings('00000000-0000-0000-0000-000000000010',7); + raise exception 'anonymous user can read earnings'; + exception when insufficient_privilege then null; end; +end $$; +reset role; +rollback;