Feature: feat/telemetry-prod-grading to dev

monitoring/README.md

@@ -42,6 +42,12 @@ number of instances is irrelevant:
 - `dsgt_members_active`, `dsgt_members_lapsed`
 - `dsgt_bootcamp_enrolled{term}`
 - `dsgt_payments_by_plan{plan}` — the yearly/semester split
+- `dsgt_hackathon_participants{edition,phase,status}`, `dsgt_hackathon_projects`,
+  `dsgt_hackathon_checkins_today`, `dsgt_hackathon_judging_projects`,
+  `dsgt_hackathon_votes`, `dsgt_hackathon_judging_active` — only editions that
+  are `open`, `closed`, or `in_progress`. A completed edition disappears on
+  the next refresh. Badge scans are hackathon QR rows since midnight Eastern,
+  not club door check-ins.
 - `dsgt_payments_unlinked` — paid, claimed by nobody. This is **not** zero and
   is not supposed to be: as of August 2026 it sits around 428 of 443 paid rows,
   nearly all from the August–September 2025 and January 2026 drives. Those are
@@ -85,6 +91,8 @@ The alerts that matter:
 - `MembershipGrantFailing` — a payment was recorded and the membership was not.
 - `UnlinkedPaymentsClimbing` — the link paths are not claiming payments.
 - `ReconcileRecoveringOften` — the backstop is carrying the fast path.
+- `HackathonJudgingIdle` — judging is on, tables are assigned, and no scores
+  have landed for 30 minutes.
 
 To route them into ClickUp, run Alertmanager with a webhook receiver and pipe
 its payload to `node scripts/clickup-task.mjs --stdin`. That step is not built

monitoring/clickhouse/init/02-hackathon.sql

@@ -0,0 +1,57 @@
+-- Hackathon history. Same reason as the payment tables: aggregate it here,
+-- not on the 0.5 GB application database. `export:clickhouse` creates these
+-- too, so a volume that already ran 01-schema.sql still picks them up.
+
+CREATE DATABASE IF NOT EXISTS dsgt;
+
+CREATE TABLE IF NOT EXISTS dsgt.hackathon_registrations
+(
+    id            String,
+    hackathon     String,
+    status        LowCardinality(String),
+    registered_at DateTime64(3),
+    updated_at    DateTime64(3)
+)
+ENGINE = ReplacingMergeTree(updated_at)
+ORDER BY (hackathon, id);
+
+CREATE TABLE IF NOT EXISTS dsgt.hackathon_projects
+(
+    id           String,
+    hackathon    String,
+    status       LowCardinality(String),
+    submitted_at Nullable(DateTime64(3)),
+    updated_at   DateTime64(3)
+)
+ENGINE = ReplacingMergeTree(updated_at)
+ORDER BY (hackathon, id);
+
+CREATE TABLE IF NOT EXISTS dsgt.hackathon_scans
+(
+    id            String,
+    hackathon     String,
+    event_name    String,
+    checked_in_at DateTime64(3)
+)
+ENGINE = ReplacingMergeTree(checked_in_at)
+ORDER BY (hackathon, id);
+
+CREATE TABLE IF NOT EXISTS dsgt.hackathon_votes
+(
+    id         String,
+    hackathon  String,
+    score      Int32,
+    voted_at   DateTime64(3),
+    updated_at DateTime64(3)
+)
+ENGINE = ReplacingMergeTree(updated_at)
+ORDER BY (hackathon, id);
+
+CREATE VIEW IF NOT EXISTS dsgt.hackathon_registrations_by_status AS
+SELECT
+    hackathon,
+    status,
+    count() AS registrations
+FROM dsgt.hackathon_registrations FINAL
+GROUP BY hackathon, status
+ORDER BY hackathon, status;

monitoring/grafana/dashboards/dsgt-portal.json

@@ -144,17 +144,112 @@
     },
     {
       "type": "row",
-      "title": "Payments (per-instance counters — a sample in production)",
+      "title": "Hackathon (read from the database — correct in production)",
       "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 },
       "collapsed": false,
       "panels": []
     },
+    {
+      "type": "stat",
+      "title": "Checked in",
+      "description": "Accepted participants whose badge was scanned, for each live edition.",
+      "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
+      "gridPos": { "h": 5, "w": 4, "x": 0, "y": 7 },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (edition) (dsgt_hackathon_participants{env=~\"$env\",status=\"checked_in\"})",
+          "legendFormat": "{{edition}}"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Approved, not checked in",
+      "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
+      "gridPos": { "h": 5, "w": 4, "x": 4, "y": 7 },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (edition) (dsgt_hackathon_participants{env=~\"$env\",status=\"approved\"})",
+          "legendFormat": "{{edition}}"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Projects submitted",
+      "description": "Not drafts, and not withdrawn by an organiser.",
+      "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
+      "gridPos": { "h": 5, "w": 4, "x": 8, "y": 7 },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (edition) (dsgt_hackathon_projects{env=~\"$env\"})",
+          "legendFormat": "{{edition}}"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Badge scans today",
+      "description": "Hackathon QR scans since midnight Eastern. Club door check-ins are not in this number.",
+      "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
+      "gridPos": { "h": 5, "w": 4, "x": 12, "y": 7 },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (edition) (dsgt_hackathon_checkins_today{env=~\"$env\"})",
+          "legendFormat": "{{edition}}"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Judging tables",
+      "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
+      "gridPos": { "h": 5, "w": 4, "x": 16, "y": 7 },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (edition) (dsgt_hackathon_judging_projects{env=~\"$env\"})",
+          "legendFormat": "{{edition}}"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Judge scores",
+      "description": "Zero while judging is switched on is what HackathonJudgingIdle fires on.",
+      "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
+      "gridPos": { "h": 5, "w": 4, "x": 20, "y": 7 },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (edition) (dsgt_hackathon_votes{env=~\"$env\"})",
+          "legendFormat": "{{edition}}"
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "Payments (per-instance counters — a sample in production)",
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 },
+      "collapsed": false,
+      "panels": []
+    },
     {
       "type": "timeseries",
       "title": "Grant failures",
       "description": "Payment recorded, membership not. Non-zero means money taken with nothing given.",
       "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
-      "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 },
+      "gridPos": { "h": 7, "w": 8, "x": 0, "y": 13 },
       "fieldConfig": {
         "defaults": {
           "custom": { "drawStyle": "bars", "fillOpacity": 60, "lineWidth": 1 },
@@ -174,7 +269,7 @@
       "type": "timeseries",
       "title": "Grants by source",
       "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
-      "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 },
+      "gridPos": { "h": 7, "w": 8, "x": 8, "y": 13 },
       "fieldConfig": {
         "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 50, "stacking": { "mode": "normal" } } },
         "overrides": []
@@ -191,7 +286,7 @@
       "type": "timeseries",
       "title": "Intents minted, by what was bought",
       "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
-      "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 },
+      "gridPos": { "h": 7, "w": 8, "x": 16, "y": 13 },
       "fieldConfig": {
         "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 50 } },
         "overrides": []
@@ -207,15 +302,15 @@
     {
       "type": "row",
       "title": "App health",
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 },
       "collapsed": false,
       "panels": []
     },
     {
       "type": "timeseries",
       "title": "Portal API calls / sec",
       "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
-      "gridPos": { "h": 7, "w": 8, "x": 0, "y": 15 },
+      "gridPos": { "h": 7, "w": 8, "x": 0, "y": 21 },
       "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
       "targets": [
         {
@@ -229,7 +324,7 @@
       "type": "timeseries",
       "title": "Latency p50 / p95 / p99",
       "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
-      "gridPos": { "h": 7, "w": 8, "x": 8, "y": 15 },
+      "gridPos": { "h": 7, "w": 8, "x": 8, "y": 21 },
       "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
       "targets": [
         {
@@ -253,7 +348,7 @@
       "type": "timeseries",
       "title": "Slowest procedures (p99)",
       "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
-      "gridPos": { "h": 7, "w": 8, "x": 16, "y": 15 },
+      "gridPos": { "h": 7, "w": 8, "x": 16, "y": 21 },
       "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
       "targets": [
         {
@@ -267,7 +362,7 @@
       "type": "timeseries",
       "title": "Process memory",
       "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
-      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 22 },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 28 },
       "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
       "targets": [
         {
@@ -286,7 +381,7 @@
       "type": "timeseries",
       "title": "Event loop lag (p99)",
       "datasource": { "type": "prometheus", "uid": "dsgt-prometheus" },
-      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 22 },
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 28 },
       "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
       "targets": [
         {

monitoring/prometheus.yml

@@ -29,8 +29,9 @@ scrape_configs:
   #
   # Read the numbers it returns with the instance model in mind. App Hosting
   # autoscales, so each scrape lands on one arbitrary instance:
-  #   - dsgt_members_*, dsgt_payments_* are read from the database on scrape and
-  #     are therefore correct regardless of which instance answered.
+  #   - dsgt_members_*, dsgt_payments_*, and dsgt_hackathon_* are read from the
+  #     database on scrape and are therefore correct regardless of which
+  #     instance answered.
   #   - counters and histograms are that one instance's, since its own start,
   #     and vanish when it scales down. Treat them as a sample, not a total.
   #

monitoring/rules/hackathon.yml

@@ -0,0 +1,26 @@
+# Alerts for a live hackathon edition. The series are database gauges, so
+# they are the same number on every instance. See monitoring/README.md.
+
+groups:
+  - name: hackathon
+    rules:
+      # Judging switched on, tables assigned, and still no scores. Thirty
+      # minutes is long enough that flipping the switch before judges reach
+      # the room does not page, and short enough that a broken submit does
+      # not sit through a whole judging block.
+      - alert: HackathonJudgingIdle
+        expr: >-
+          (
+            dsgt_hackathon_judging_active > 0
+            and on (edition) (dsgt_hackathon_judging_projects > 0)
+          )
+          unless on (edition) (dsgt_hackathon_votes > 0)
+        for: 30m
+        labels:
+          severity: page
+        annotations:
+          summary: "Judging is on for {{ $labels.edition }} and no scores have landed"
+          description: >-
+            The edition has judging switched on and projects on the floor, and
+            the vote count is still zero. Judges may be blocked from
+            submitting, or judging was opened before anyone was at a table.

packages/api/src/.internal-tests/hackathon-admin-edge.test.ts

@@ -1417,17 +1417,17 @@ describe("Hackathon admin management edge cases", () => {
 
   // =====================================================================
   describe("Day-of dashboard", () => {
-    // BUG: analyticsOverview counts eventCheckIns (the club check-in table),
-    // not hackathonEventAttendees, which is what scanParticipantPass writes.
-    it("counts today's hackathon badge scans", async () => {
+    it("counts today's hackathon badge scans apart from club door check-ins", async () => {
       const caller = adminCaller({}, "admin");
       mockCount.mockImplementation((t: string) =>
-        t === "hackathonEventAttendees" ? 7 : 0,
+        t === "hackathonEventAttendees" ? 7 : t === "eventCheckIns" ? 2 : 0,
       );
 
       const res = await caller.admin.analyticsOverview();
 
-      expect(res.checkinsToday).toBe(7);
+      expect(res.badgeScansToday).toBe(7);
+      expect(res.doorCheckinsToday).toBe(2);
+      expect(res.checkinsToday).toBe(9);
     });
   });
 });

packages/api/src/routers/admin.ts

@@ -18,6 +18,7 @@ import { compareTerms, currentTerm } from "@query/db/services/membership";
 import { isExpiredAdmin, isStaffRole } from "../types/portal-context";
 import type { DrizzleDB } from "@query/db";
 import { startOfEasternDay } from "../services/eastern-time";
+import { readLiveHackathons } from "../services/metrics";
 
 export const adminRouter = createTRPCRouter({
   isAdmin: protectedProcedure.query(async ({ ctx }) => {
@@ -78,6 +79,8 @@ export const adminRouter = createTRPCRouter({
       totalEvents: number;
       totalHackathons: number;
       checkinsToday: number;
+      badgeScansToday: number;
+      doorCheckinsToday: number;
     }>(
       cacheKey,
       async () => {
@@ -114,13 +117,19 @@ export const adminRouter = createTRPCRouter({
             .where(gte(eventCheckIns.checkedInAt, startOfToday)),
         ]);
 
+        const badgeScansToday = badgeScansResult[0]?.count ?? 0;
+        const doorCheckinsToday = doorCheckinsResult[0]?.count ?? 0;
+
         const result = {
           totalParticipants: participantsResult[0]?.count ?? 0,
           totalEvents: eventsResult[0]?.count ?? 0,
           totalHackathons: hackathonsResult[0]?.count ?? 0,
-          checkinsToday:
-            (badgeScansResult[0]?.count ?? 0) +
-            (doorCheckinsResult[0]?.count ?? 0),
+          // Kept as the sum so a caller that only knew the blended number
+          // still moves. The page reads the two halves, because a hackathon
+          // weekend and a club meeting are not the same turnout.
+          badgeScansToday,
+          doorCheckinsToday,
+          checkinsToday: badgeScansToday + doorCheckinsToday,
         };
 
         return result;
@@ -129,6 +138,16 @@ export const adminRouter = createTRPCRouter({
     );
   }),
 
+  // The same counts the Prometheus gauges use, for the analytics page. A
+  // scrape is optional; this is what staff open during the weekend.
+  hackathonLive: isAdmin.query(async ({ ctx }) => {
+    return await ctx.cache.getOrSet(
+      "admin:hackathon-live",
+      () => readLiveHackathons(ctx.db as DrizzleDB),
+      15,
+    );
+  }),
+
   // Membership growth for the Insights page. One pass over the member table
   // instead of a bucket query per month: the club is a few thousand rows, and
   // the cumulative series has to see everything before the window anyway.