diff --git a/.gitignore b/.gitignore index 70d93d7..8885d76 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ coverage/ # 变异测试（Stryker） .stryker-tmp/ reports/ +# 本地性能评测数据与生成报告 +/benchmarks/results/ # 日志 *.log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 191652c..8b3750c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,54 @@ pnpm run build:packages 完整 PR 检查的定义以 [test.yml](.github/workflows/test.yml) 为准。 文档修改重点检查命令、示例和相对链接；无需为纯文字改动重复运行全部运行时测试。 +## HTTP 接入性能评测 + +`pnpm run bench` 测参数校验。比较接入 erest 前后的 HTTP 开销，先按上面的环境准备步骤 +安装依赖并构建核心包与适配器，再执行以下命令（macOS、Linux 均可）： + +```bash +node benchmarks/run.mjs +node benchmarks/report.mjs benchmarks/results/latest.json +``` + +脚本对 Express、Koa、@leizm/web、H3 分别比较原生路由、原生路由加同等 Zod 校验、erest， +覆盖无 schema 的 GET、带路径/query/body 校验的 POST，以及增加 5ms 异步等待的 POST。 +默认单并发与 50 并发、预热 1.5 秒、采样 3 秒、重复三轮，完整执行约需十二分钟。 +报告包含吞吐、延迟分位数、客户端/服务端 CPU、内存快照和轮间波动。 + +macOS 本地复测可先聚焦 50 并发的 POST，增加预热和重复轮数（四框架共 84 次采样，约需十二分钟）： + +```bash +BENCH_SCENARIOS=post BENCH_CONCURRENCY=50 \ + BENCH_ROUNDS=7 BENCH_MS=5000 BENCH_WARMUP_MS=3000 \ + BENCH_OUT=benchmarks/results/mac-post.json node benchmarks/run.mjs +node benchmarks/report.mjs benchmarks/results/mac-post.json +open benchmarks/results/mac-post.html +``` + +同一轮对照使用相同 Node 版本和 lockfile，尽量减少其他 CPU 密集任务。比较 Mac 和云端 +结果时，重点看各自环境中“原生 + Zod → erest”的相对变化、每请求 CPU 和轮间波动， +不要直接比较两台机器的绝对 QPS；客户端 CPU 接近饱和时，观察到的吞吐也可能受发压能力限制。 + +Linux 下可选用两个不同的、当前进程允许使用的 CPU 固定客户端和服务端；macOS 不设置 +`BENCH_SERVER_CPU`，由操作系统调度： + +```bash +BENCH_SERVER_CPU=0 taskset -c 1 node benchmarks/run.mjs +``` + +`BENCH_FRAMEWORKS=express`、`BENCH_SCENARIOS=post`、`BENCH_CONCURRENCY=50` 可筛选场景； +`BENCH_ROUNDS`、`BENCH_MS`、`BENCH_WARMUP_MS` 调整轮数和时长，`BENCH_OUT` 指定 JSON 输出。 +快速试跑只检查响应一致性和非法输入拒绝，不能用于性能结论： + +```bash +BENCH_ROUNDS=1 BENCH_MS=100 BENCH_WARMUP_MS=100 \ + BENCH_OUT=benchmarks/results/smoke.json node benchmarks/run.mjs +``` + +`benchmarks/results/` 下的 JSON、CSV 和 HTML 都是被忽略的本地临时生成物，不随工具提交。 +短测结果不是线上容量结论；只有在独立复测中出现一致趋势后，才适合讨论运行时优化。 + ## 合约专项 `test:contract` 在 Express、Koa、LeizmWeb、H3 下执行同一套 GET/POST/PUT/PATCH/DELETE 定义。 diff --git a/benchmarks/fixture.mjs b/benchmarks/fixture.mjs new file mode 100644 index 0000000..f68556b --- /dev/null +++ b/benchmarks/fixture.mjs @@ -0,0 +1,40 @@ +import { z } from "zod"; + +export const schemas = { + params: z.object({ id: z.string().min(1) }), + query: z.object({ limit: z.coerce.number().int().min(1).max(100).default(10) }), + body: z.object({ + name: z.string().min(1).max(100), + age: z.number().int().min(0).max(150), + email: z.string().email(), + tags: z.array(z.string()).max(10), + address: z.object({ city: z.string(), zip: z.string() }), + }), +}; +export const payload = { + name: "Tom", + age: 25, + email: "tom@example.com", + tags: ["node", "typescript", "api"], + address: { city: "Shenzhen", zip: "518000" }, +}; +export const path = "/users/u123?limit=20"; +export function output(req) { + return { id: req.params.id, limit: Number(req.query.limit), ...req.body }; +} +export const expectedPost = output({ params: { id: "u123" }, query: { limit: 20 }, body: payload }); +export const expectedGet = { ok: true, message: "hello erest" }; + +export function validate(input) { + const result = {}; + for (const key of ["params", "query", "body"]) { + const parsed = schemas[key].safeParse(input[key]); + if (!parsed.success) { + const error = new Error("invalid input"); + error.status = 400; + throw error; + } + result[key] = parsed.data; + } + return result; +} diff --git a/benchmarks/load.mjs b/benchmarks/load.mjs new file mode 100644 index 0000000..9f229d2 --- /dev/null +++ b/benchmarks/load.mjs @@ -0,0 +1,92 @@ +import http from "node:http"; +import { performance } from "node:perf_hooks"; +import { payload, path } from "./fixture.mjs"; + +// Closed-loop HTTP/1.1 keep-alive, one outstanding request per connection. +// Separate client/server processes; report client CPU to expose generator limits. +// Includes full response consumption, but correctness is checked before timing. +export async function load(port, scenario, concurrency, durationMs) { + const agent = new http.Agent({ keepAlive: true, maxSockets: concurrency, maxFreeSockets: concurrency }); + const body = scenario === "get" ? undefined : JSON.stringify(payload); + const options = { + host: "127.0.0.1", + port, + agent, + method: body ? "POST" : "GET", + path: body ? path : "/ping", + headers: body + ? { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + } + : {}, + }; + // 1us histogram resolution through 2s; last bucket counts longer responses. + const histogram = new Uint32Array(2_000_001); + let count = 0, + errors = 0, + totalLatency = 0, + maxLatency = 0; + const cpuStart = process.cpuUsage(); + const start = performance.now(); + const deadline = start + durationMs; + await Promise.all( + Array.from( + { length: concurrency }, + () => + new Promise((resolve) => { + function next() { + if (performance.now() >= deadline) return resolve(); + const requestStart = performance.now(); + let finished = false; + function complete(failed) { + if (finished) return; + finished = true; + const latency = performance.now() - requestStart; + count++; + if (failed) errors++; + totalLatency += latency; + maxLatency = Math.max(maxLatency, latency); + histogram[Math.min(2_000_000, Math.floor(latency * 1000))]++; + next(); + } + const req = http.request(options, (res) => { + res.resume(); + res.once("end", () => complete(res.statusCode !== 200)); + res.once("error", () => complete(true)); + }); + req.setTimeout(5000, () => req.destroy(new Error("request timeout"))); + req.once("error", () => complete(true)); + req.end(body); + } + next(); + }) + ) + ); + const elapsedMs = performance.now() - start; + const cpu = process.cpuUsage(cpuStart); + agent.destroy(); + function percentile(p) { + const target = Math.ceil(count * p); + let n = 0; + for (let i = 0; i < histogram.length; i++) { + n += histogram[i]; + if (n >= target) return i / 1000; + } + return null; + } + return { + requests: count, + errors, + elapsedMs, + rps: (count * 1000) / elapsedMs, + latencyMs: { + mean: totalLatency / count, + p50: percentile(0.5), + p95: percentile(0.95), + p99: percentile(0.99), + max: maxLatency, + }, + clientCpuPercent: (cpu.user + cpu.system) / (elapsedMs * 10), + }; +} diff --git a/benchmarks/report.mjs b/benchmarks/report.mjs new file mode 100644 index 0000000..fd80f00 --- /dev/null +++ b/benchmarks/report.mjs @@ -0,0 +1,274 @@ +// node benchmarks/report.mjs benchmarks/results/latest.json +// Produces self-contained HTML plus summary JSON and CSV beside the raw samples. +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve, basename } from "node:path"; +import assert from "node:assert/strict"; + +const input = resolve(process.argv[2] || "benchmarks/results/latest.json"); +const data = JSON.parse(readFileSync(input, "utf8")); +assert(data.finishedAt, "The benchmark is still running; do not publish a partial report."); +const median = (values) => { + const sorted = values.toSorted((a, b) => a - b); + const i = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[i] : (sorted[i - 1] + sorted[i]) / 2; +}; +const grouped = new Map(); +for (const sample of data.samples) { + const key = [sample.framework, sample.scenario, sample.concurrency, sample.mode].join("/"); + if (!grouped.has(key)) grouped.set(key, []); + grouped.get(key).push(sample); +} +const fields = { + rps: (s) => s.rps, + meanMs: (s) => s.latencyMs.mean, + p50Ms: (s) => s.latencyMs.p50, + p95Ms: (s) => s.latencyMs.p95, + p99Ms: (s) => s.latencyMs.p99, + serverCpuPercent: (s) => s.serverCpuPercent, + clientCpuPercent: (s) => s.clientCpuPercent, + cpuUsPerRequest: (s) => s.serverCpuUsPerRequest, + rssMiB: (s) => s.serverRssMiB, + retainedHeapMiB: (s) => s.serverRetainedHeapMiB, + idleRssMiB: (s) => s.idleRssMiB, +}; +const groups = [...grouped.values()].map((samples) => ({ + framework: samples[0].framework, + scenario: samples[0].scenario, + concurrency: samples[0].concurrency, + mode: samples[0].mode, + rounds: samples.length, + requests: samples.reduce((sum, s) => sum + s.requests, 0), + errors: samples.reduce((sum, s) => sum + s.errors, 0), + ...Object.fromEntries(Object.entries(fields).map(([name, getter]) => [name, median(samples.map(getter))])), + rpsMin: Math.min(...samples.map((s) => s.rps)), + rpsMax: Math.max(...samples.map((s) => s.rps)), + rpsByRound: samples.map((s) => ({ round: s.round, rps: s.rps })), +})); +const comparisons = []; +for (const erest of groups.filter((g) => g.mode === "erest")) { + for (const baseline of groups.filter( + (g) => + g.mode !== "erest" && + g.framework === erest.framework && + g.scenario === erest.scenario && + g.concurrency === erest.concurrency + )) { + const paired = erest.rpsByRound + .map((entry) => { + const base = baseline.rpsByRound.find((r) => r.round === entry.round); + return base ? (entry.rps / base.rps - 1) * 100 : null; + }) + .filter((v) => v !== null); + comparisons.push({ + framework: erest.framework, + scenario: erest.scenario, + concurrency: erest.concurrency, + baseline: baseline.mode, + throughputChangePercent: (erest.rps / baseline.rps - 1) * 100, + pairedChangePercent: { + median: median(paired), + min: Math.min(...paired), + max: Math.max(...paired), + values: paired, + }, + meanLatencyDeltaMs: erest.meanMs - baseline.meanMs, + p95DeltaMs: erest.p95Ms - baseline.p95Ms, + cpuDeltaUsPerRequest: erest.cpuUsPerRequest - baseline.cpuUsPerRequest, + retainedHeapDeltaMiB: erest.retainedHeapMiB - baseline.retainedHeapMiB, + rssDeltaMiB: erest.rssMiB - baseline.rssMiB, + }); + } +} +const summary = { input: basename(input), commit: data.commit, config: data.config, groups, comparisons }; +const prefix = input.replace(/\.json$/, ""); +writeFileSync(`${prefix}-summary.json`, `${JSON.stringify(summary, null, 2)}\n`); +const csvFields = Object.keys(groups[0]); +writeFileSync( + `${prefix}-summary.csv`, + [ + csvFields.join(","), + ...groups.map((g) => + csvFields + .map((key) => (typeof g[key] === "object" ? `"${JSON.stringify(g[key]).replaceAll('"', '""')}"` : g[key])) + .join(",") + ), + ].join("\n") + "\n" +); + +const escape = (value) => + String(value).replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """); +const n = (value, digits = 2) => + Number(value).toLocaleString("en-US", { minimumFractionDigits: digits, maximumFractionDigits: digits }); +const signed = (value, digits = 2) => `${value >= 0 ? "+" : ""}${n(value, digits)}`; +const label = { + get: "GET · 无 schema", + post: "POST · params + query + 嵌套 body", + io5: "POST · 同等校验 + 5ms 异步等待", +}; +const modes = { native: "原生 · 无校验", "native-zod": "原生 + Zod", erest: "erest" }; +const total = data.samples.reduce((sum, s) => sum + s.requests, 0); +const errors = data.samples.reduce((sum, s) => sum + s.errors, 0); +const table = (heads, rows) => + `
${heads.map((h) => ``).join("")}${rows.join("")}
${h}
`; +const row = (g, cells) => + `${cells.map((c) => `${c}`).join("")}`; +const comparisonTable = table( + [ + "框架", + "场景", + "并发", + "比较基线", + "吞吐变化", + "配对轮次变化范围", + "平均延迟变化 ms", + "P95 变化 ms", + "CPU 增量 μs/请求", + "GC 后堆增量 MiB", + ], + comparisons.map((c) => + row(c, [ + c.framework, + label[c.scenario], + c.concurrency, + modes[c.baseline], + `${signed(c.throughputChangePercent)}%`, + `${signed(c.pairedChangePercent.min)}% ～ ${signed(c.pairedChangePercent.max)}%`, + signed(c.meanLatencyDeltaMs, 3), + signed(c.p95DeltaMs, 3), + signed(c.cpuDeltaUsPerRequest), + signed(c.retainedHeapDeltaMiB), + ]) + ) +); +const detailTable = table( + [ + "框架", + "场景", + "并发", + "实现", + "req/s 中位数", + "req/s 范围", + "平均 ms", + "P50 ms", + "P95 ms", + "P99 ms", + "CPU μs/请求", + "服务端 CPU %", + "客户端 CPU %", + "RSS MiB", + "GC 后堆 MiB", + "错误", + ], + groups.map((g) => + row(g, [ + g.framework, + label[g.scenario], + g.concurrency, + modes[g.mode], + n(g.rps, 0), + `${n(g.rpsMin, 0)} ～ ${n(g.rpsMax, 0)}`, + n(g.meanMs, 3), + n(g.p50Ms, 3), + n(g.p95Ms, 3), + n(g.p99Ms, 3), + n(g.cpuUsPerRequest), + n(g.serverCpuPercent, 1), + n(g.clientCpuPercent, 1), + n(g.rssMiB), + n(g.retainedHeapMiB), + g.errors, + ]) + ) +); +const charts = data.config.frameworks + .map((framework) => { + const selected = groups.filter((g) => g.framework === framework && g.concurrency === 50 && g.scenario === "post"); + const max = Math.max(...selected.map((g) => g.rps)); + return `

${framework} · POST · 50 并发

${selected.map((g) => `
${modes[g.mode]}
${n(g.rps, 0)}
`).join("")}

单位 req/s；${data.config.rounds} 轮中位数，越高越好。

`; + }) + .join(""); + +const findings = comparisons + .filter((c) => c.scenario === "post" && c.concurrency === 50 && c.baseline === "native-zod") + .map( + (c) => + `
  • ${c.framework}：同等请求校验下，erest 吞吐变化 ${signed(c.throughputChangePercent)}%，` + + `每请求服务端 CPU 时间变化 ${signed(c.cpuDeltaUsPerRequest)} μs；平均延迟变化 ${signed(c.meanLatencyDeltaMs, 3)} ms。` + + (c.pairedChangePercent.min <= 0 && c.pairedChangePercent.max >= 0 + ? "配对轮次差异跨过零，尚不能确定稳定的吞吐变化方向。" + : `本次各配对轮次方向一致，但 ${data.config.rounds} 轮不足以给出统计显著性结论。`) + + "
    • " + ) + .join(""); +const ioFindings = comparisons + .filter((c) => c.scenario === "io5" && c.concurrency === 50) + .map((c) => `${c.framework} ${signed(c.throughputChangePercent)}%`) + .join("，"); + +const shellQuote = (value) => "'" + String(value).replaceAll("'", "'\\''") + "'"; +const settings = [ + ["BENCH_FRAMEWORKS", data.config.frameworks.join(",")], + ["BENCH_SCENARIOS", data.config.scenarios.join(",")], + ["BENCH_CONCURRENCY", data.config.concurrency.join(",")], + ["BENCH_ROUNDS", data.config.rounds], + ["BENCH_MS", data.config.durationMs], + ["BENCH_WARMUP_MS", data.config.warmupMs], + ["BENCH_OUT", `benchmarks/results/${basename(input)}`], +] + .map(([name, value]) => `${name}=${shellQuote(value)}`) + .join(" "); +const command = `pnpm run build +pnpm run build:packages + +# macOS / Linux：用本报告的场景、时长和轮数复测 +${settings} node benchmarks/run.mjs +node benchmarks/report.mjs ${shellQuote(`benchmarks/results/${basename(input)}`)} + +# Linux 可选 CPU 绑定；核心编号按本机允许列表调整 +BENCH_SERVER_CPU=${shellQuote(data.config.serverCpu ?? "0")} ${settings} taskset -c ${shellQuote(data.environment.clientAffinity ?? "1")} node benchmarks/run.mjs + +# 快速试跑（仅检查脚本，不用于性能结论） +BENCH_ROUNDS=1 BENCH_MS=100 BENCH_WARMUP_MS=100 BENCH_OUT=/tmp/erest-smoke.json node benchmarks/run.mjs`; +const html = `erest ${escape(data.environment.versions.erest)} 性能实测 + +

    erest 接入前后性能实测

    +

    完成时间 ${escape(data.finishedAt)} · erest ${escape(data.environment.versions.erest)} · commit ${escape(data.commit)}

    +
    ${data.samples.length}独立进程采样 · ${data.config.rounds} 轮重复
    ${n(total, 0)}计时区间 HTTP 请求
    ${errors}计时区间 HTTP / 网络错误
    +

    重点看“原生 + Zod → erest”：这衡量已有同等校验时的接入成本。“原生无校验 → erest”还包含新增参数校验的成本。具体差异见下表；接近轮间波动的差异不能当成稳定提升或回退。

    +

    典型 POST 的吞吐量

    ${charts}
    +

    本次观察

    +

    加入 5ms 异步等待后，同等校验基线下的吞吐变化为：${ioFindings || "本次未测"}。这只是固定等待模型；线上数据库、连接池和下游服务的瓶颈会改变结果。

    +

    接入影响与原始指标

    +

    吞吐变化 = erest 吞吐中位数 ÷ 基线吞吐中位数 − 1；负值表示下降。配对范围按同轮次计算，不是置信区间。延迟是客户端从发起请求到读完响应的时间；50 并发时包含排队与调度，不能把延迟差当作纯 erest 执行时间。表内延迟分位数为各轮分位数的中位数，不是合并所有请求后的分位数。

    + + + +${comparisonTable}

    各组完整指标

    ${detailTable} +

    方法与边界

    +

    环境与复现

    ${escape(data.environment.cpu)} · ${escape(data.environment.node)} · ${escape(data.environment.os)}
    +cgroup CPU 配额 ${escape(data.environment.cpuMax ?? "未提供")}；内存限额 ${escape(data.environment.memoryMax ?? "未提供")} 字节；服务端 CPU ${escape(data.config.serverCpu ?? "未固定")}；客户端 CPU ${escape(data.environment.clientAffinity ?? "由操作系统调度")}。

    +
    ${escape(JSON.stringify(data.environment.versions, null, 2))}
    ${escape(command)}
    +

    可通过 BENCH_FRAMEWORKS、BENCH_SCENARIOS、BENCH_CONCURRENCY 筛选；BENCH_ROUNDS、BENCH_MS、BENCH_WARMUP_MS 调整重复次数和时长；BENCH_OUT 指定输出路径。默认四框架完整运行约需 12 分钟（取决于机器）。CPU 绑定仅用于 Linux，需按机器允许的核心编号调整。

    +

    原始数据：${escape(basename(input))} · 汇总 CSV · 汇总 JSON。原始数据记录了采样脚本 SHA-256，便于核对复现版本。

    + +`; +writeFileSync(`${prefix}.html`, html); +console.log(`Report: ${prefix}.html`); +console.log( + JSON.stringify( + comparisons.filter((c) => c.concurrency === 50), + null, + 2 + ) +); diff --git a/benchmarks/run.mjs b/benchmarks/run.mjs new file mode 100644 index 0000000..96b19d2 --- /dev/null +++ b/benchmarks/run.mjs @@ -0,0 +1,209 @@ +// Build first: pnpm run build && pnpm run build:packages +// Full run: node benchmarks/run.mjs +// Smoke: BENCH_ROUNDS=1 BENCH_MS=100 BENCH_WARMUP_MS=100 node benchmarks/run.mjs +// Filter: BENCH_FRAMEWORKS=express BENCH_SCENARIOS=post BENCH_CONCURRENCY=50 ... +import { fork, execFileSync } from "node:child_process"; +import { once } from "node:events"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import os from "node:os"; +import assert from "node:assert/strict"; +import { performance } from "node:perf_hooks"; +import { createHash } from "node:crypto"; +import { expectedGet, expectedPost, payload, path } from "./fixture.mjs"; +import { load } from "./load.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const list = (name, fallback) => (process.env[name] || fallback).split(","); +const config = { + frameworks: list("BENCH_FRAMEWORKS", "express,koa,leizmweb,h3"), + scenarios: list("BENCH_SCENARIOS", "get,post,io5"), + concurrency: list("BENCH_CONCURRENCY", "1,50").map(Number), + rounds: Number(process.env.BENCH_ROUNDS || 3), + durationMs: Number(process.env.BENCH_MS || 3000), + warmupMs: Number(process.env.BENCH_WARMUP_MS || 1500), + serverCpu: process.env.BENCH_SERVER_CPU || null, +}; +assert(Number.isInteger(config.rounds) && config.rounds > 0, "BENCH_ROUNDS must be a positive integer"); +assert(Number.isFinite(config.durationMs) && config.durationMs > 0, "BENCH_MS must be positive and finite"); +assert(Number.isFinite(config.warmupMs) && config.warmupMs > 0, "BENCH_WARMUP_MS must be positive and finite"); +assert(config.concurrency.every((c) => Number.isInteger(c) && c > 0)); +assert( + config.frameworks.every((f) => ["express", "koa", "leizmweb", "h3"].includes(f)), + "Unknown BENCH_FRAMEWORKS value" +); +assert( + config.scenarios.every((s) => ["get", "post", "io5"].includes(s)), + "Unknown BENCH_SCENARIOS value" +); +assert(!config.serverCpu || process.platform === "linux", "BENCH_SERVER_CPU uses Linux taskset; omit it on macOS"); +const destination = resolve(root, process.env.BENCH_OUT || "benchmarks/results/latest.json"); +mkdirSync(dirname(destination), { recursive: true }); +function read(filePath) { + try { + return readFileSync(filePath, "utf8").trim(); + } catch { + return null; + } +} +function version(filePath) { + return JSON.parse(read(resolve(root, filePath))).version; +} +const result = { + startedAt: new Date().toISOString(), + commit: execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(), + harnessSha256: Object.fromEntries( + ["fixture.mjs", "load.mjs", "server.mjs", "run.mjs"].map((name) => [ + name, + createHash("sha256") + .update(readFileSync(resolve(root, "benchmarks", name))) + .digest("hex"), + ]) + ), + environment: { + node: process.version, + os: `${os.type()} ${os.release()} ${os.arch()}`, + platform: os.platform(), + cpu: os.cpus()[0].model, + logicalCpus: os.cpus().length, + availableParallelism: os.availableParallelism(), + systemMemoryBytes: os.totalmem(), + cpuMax: read("/sys/fs/cgroup/cpu.max"), + memoryMax: read("/sys/fs/cgroup/memory.max"), + clientAffinity: read("/proc/self/status")?.match(/^Cpus_allowed_list:\s*(.+)$/m)?.[1], + versions: Object.fromEntries( + [ + "", + "packages/erest-express/", + "packages/erest-koa/", + "packages/erest-leizmweb/", + "node_modules/express/", + "node_modules/koa/", + "node_modules/koa-router/", + "node_modules/koa-bodyparser/", + "node_modules/@leizm/web/", + "node_modules/zod/", + ...(config.frameworks.includes("h3") ? ["packages/erest-h3/", "node_modules/h3/"] : []), + ].map((p) => [p || "erest", version(`${p}package.json`)]) + ), + }, + config, + samples: [], +}; +function save() { + writeFileSync(destination, `${JSON.stringify(result, null, 2)}\n`); +} +function message(child, type, send) { + return new Promise((accept, reject) => { + const timer = setTimeout(() => done(new Error(`IPC timeout: ${type}`)), 15000); + function done(error, value) { + clearTimeout(timer); + child.off("message", received); + child.off("exit", exited); + child.off("error", done); + if (error) reject(error); + else accept(value); + } + function received(value) { + if (value.type === type) done(null, value); + } + function exited(code) { + done(new Error(`Server exited before ${type}: ${code}`)); + } + child.on("message", received); + child.once("exit", exited); + child.once("error", done); + if (send) child.send({ type: send }); + }); +} +async function sample(framework, scenario, concurrency, mode, round) { + const child = fork(resolve(root, "benchmarks/server.mjs"), [framework, mode, scenario], { + cwd: root, + ...(config.serverCpu + ? { execPath: "/usr/bin/taskset", execArgv: ["-c", config.serverCpu, process.execPath, "--expose-gc"] } + : { execArgv: ["--expose-gc"] }), + env: { ...process.env, NODE_ENV: "production", DEBUG: "" }, + stdio: ["ignore", "ignore", "inherit", "ipc"], + }); + try { + const ready = await message(child, "ready"); + const url = `http://127.0.0.1:${ready.port}`; + const options = + scenario === "get" + ? {} + : { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload) }; + const response = await fetch(url + (scenario === "get" ? "/ping" : path), options); + assert.equal(response.status, 200, "Unexpected preflight status"); + assert.deepEqual(await response.json(), scenario === "get" ? expectedGet : expectedPost); + // Verify that both validated paths actually reject bad input. + if (scenario !== "get" && mode !== "native") { + const invalid = await fetch(url + path, { ...options, body: JSON.stringify({ ...payload, age: "bad" }) }); + assert(invalid.status >= 400, "Validation did not reject invalid input"); + await invalid.arrayBuffer(); + } + await load(ready.port, scenario, concurrency, config.warmupMs); + await message(child, "started", "start"); + const client = await load(ready.port, scenario, concurrency, config.durationMs); + const server = await message(child, "stats", "stop"); + assert.equal(client.errors, 0, "Load generated HTTP/network errors"); + const measurement = { + framework, + scenario, + concurrency, + mode, + round, + ...client, + serverCpuPercent: (server.cpu.user + server.cpu.system) / (server.elapsedMs * 10), + serverCpuUsPerRequest: (server.cpu.user + server.cpu.system) / client.requests, + serverRssMiB: server.memory.rss / 2 ** 20, + serverRetainedHeapMiB: server.retainedMemory.heapUsed / 2 ** 20, + idleRssMiB: ready.idleMemory.rss / 2 ** 20, + server, + idleMemory: ready.idleMemory, + }; + result.samples.push(measurement); + save(); + console.log( + `${result.samples.length}: ${framework} ${scenario} c${concurrency} ${mode} r${round}: ` + + `${client.rps.toFixed(0)} req/s, p95 ${client.latencyMs.p95.toFixed(3)}ms, ` + + `server ${measurement.serverCpuPercent.toFixed(0)}% client ${client.clientCpuPercent.toFixed(0)}%` + ); + } finally { + if (child.exitCode === null && child.connected) { + const exited = once(child, "exit"); + child.send({ type: "close" }); + const timer = setTimeout(() => child.kill("SIGKILL"), 2000); + await exited; + clearTimeout(timer); + } + } +} + +const start = performance.now(); +save(); +// Rotate mode order each round, and framework order, to reduce temporal bias. +for (let round = 1; round <= config.rounds; round++) { + const offset = (round - 1) % config.frameworks.length; + for (const framework of [...config.frameworks.slice(offset), ...config.frameworks.slice(0, offset)]) { + for (const scenario of config.scenarios) { + const modes = + scenario === "get" + ? ["native", "erest"] + : scenario === "io5" + ? ["native-zod", "erest"] + : ["native", "native-zod", "erest"]; + for (const concurrency of config.concurrency) { + if (scenario === "io5" && concurrency === 1) continue; + const shift = (round - 1) % modes.length; + for (const mode of [...modes.slice(shift), ...modes.slice(0, shift)]) { + await sample(framework, scenario, concurrency, mode, round); + } + } + } + } +} +result.finishedAt = new Date().toISOString(); +result.elapsedSeconds = (performance.now() - start) / 1000; +save(); +console.log(`Saved ${result.samples.length} samples to ${destination}`); diff --git a/benchmarks/server.mjs b/benchmarks/server.mjs new file mode 100644 index 0000000..0bd9811 --- /dev/null +++ b/benchmarks/server.mjs @@ -0,0 +1,166 @@ +// A fresh production-JS process is used for every sample. No ts-node/Vitest overhead. +import { createServer } from "node:http"; +import { performance } from "node:perf_hooks"; +import { setTimeout as delay } from "node:timers/promises"; +import { expectedGet, output, schemas, validate } from "./fixture.mjs"; + +const [framework, mode, scenario] = process.argv.slice(2); +const isGet = scenario === "get"; +const method = isGet ? "get" : "post"; +const route = isGet ? "/ping" : "/users/:id"; +const compute = (req) => (isGet ? expectedGet : output(req)); +const work = (req) => (scenario === "io5" ? delay(5).then(() => compute(req)) : compute(req)); +const checked = (req) => (mode === "native-zod" ? validate(req) : req); +let app, router, server, adapter; +const sendKoa = (ctx, result) => { + // Match the adapter's serialization and Content-Type behavior. + ctx.type = "application/json"; + ctx.body = JSON.stringify(result); +}; + +if (framework === "express") { + const { default: express } = await import("express"); + app = express(); + app.use(express.json()); + router = app; + server = createServer(app); + if (mode === "erest") { + const mod = await import("../packages/erest-express/dist/index.js"); + adapter = new mod.ExpressAdapter(); + } else { + router[method]( + route, + scenario === "io5" + ? async (req, res) => res.json(await work(checked(req))) + : (req, res) => res.json(compute(checked(req))) + ); + } +} else if (framework === "koa") { + const [{ default: Koa }, { default: Router }, { default: bodyParser }] = await Promise.all([ + import("koa"), + import("koa-router"), + import("koa-bodyparser"), + ]); + app = new Koa(); + app.silent = true; + router = new Router(); + app.use(bodyParser()); + if (mode === "erest") { + const mod = await import("../packages/erest-koa/dist/index.js"); + adapter = new mod.KoaAdapter(); + } else { + const input = (ctx) => checked({ params: ctx.params, query: ctx.query, body: ctx.request.body }); + router[method]( + route, + scenario === "io5" + ? async (ctx) => sendKoa(ctx, await work(input(ctx))) + : (ctx) => sendKoa(ctx, compute(input(ctx))) + ); + } + app.use(router.routes()); + server = createServer(app.callback()); +} else if (framework === "leizmweb") { + const { Application, Router, component } = await import("@leizm/web"); + app = new Application(); + router = new Router(); + app.use("/", component.bodyParser.json()); + if (mode === "erest") { + const mod = await import("../packages/erest-leizmweb/dist/index.js"); + adapter = new mod.LeizmWebAdapter(); + } else { + router[method]( + route, + scenario === "io5" + ? async (ctx) => ctx.response.json(await work(checked(ctx.request))) + : (ctx) => ctx.response.json(compute(checked(ctx.request))) + ); + } + app.use("/", router); + server = app.server; +} else if (framework === "h3") { + const [{ H3, getQuery, getRouterParams, readBody }, { toNodeHandler }] = await Promise.all([ + import("h3"), + import("h3/node"), + ]); + app = new H3({ silent: true }); + router = app; + if (mode === "erest") { + const mod = await import("../packages/erest-h3/dist/index.js"); + adapter = new mod.H3Adapter(); + } else { + const send = (event, result) => { + // Match the adapter's serialization and Content-Type behavior. + event.res.headers.set("content-type", "application/json; charset=utf-8"); + return JSON.stringify(result); + }; + router.on( + method.toUpperCase(), + route, + isGet + ? (event) => send(event, expectedGet) + : async (event) => { + const input = checked({ + params: getRouterParams(event, { decode: true }), + query: getQuery(event), + body: (await readBody(event)) ?? {}, + }); + return send(event, scenario === "io5" ? await work(input) : compute(input)); + } + ); + } + server = createServer(toNodeHandler(app)); +} else { + throw new Error(`Unknown framework: ${framework}`); +} + +if (mode === "erest") { + const { ERest } = await import("../dist/lib/index.js"); + const service = new ERest({ info: { basePath: "" }, groups: { bench: "Benchmark" }, docs: { markdown: false } }); + const api = service.api[method](route).group("bench"); + if (isGet) api.register((ctx) => ctx.reply.json(expectedGet)); + else + api.registerTyped( + schemas, + scenario === "io5" + ? async (req, ctx) => ctx.reply.json(await work(req)) + : (req, ctx) => ctx.reply.json(compute(req)) + ); + service.bind({ adapter, router }); +} + +if (framework === "express") { + app.use((error, _req, res, _next) => + res.status(error.status || error.statusCode || 400).json({ error: error.message }) + ); +} + +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +global.gc?.(); +const idleMemory = process.memoryUsage(); +process.send({ type: "ready", port: server.address().port, idleMemory }); +let cpuStart, wallStart; +process.on("message", (message) => { + if (message.type === "start") { + // GC outside the measured interval; no forced GC during requests. + global.gc?.(); + cpuStart = process.cpuUsage(); + wallStart = performance.now(); + process.send({ type: "started" }); + } else if (message.type === "stop") { + const elapsedMs = performance.now() - wallStart; + const cpu = process.cpuUsage(cpuStart); + const memory = process.memoryUsage(); + global.gc?.(); + process.send({ + type: "stats", + elapsedMs, + cpu, + memory, + retainedMemory: process.memoryUsage(), + maxRssKiB: process.resourceUsage().maxRSS, + }); + } else if (message.type === "close") { + server.close(() => process.exit(0)); + server.closeAllConnections(); + } +});