From c2d000cc4878416c3c32ec4d84e0eb3a24ff7573 Mon Sep 17 00:00:00 2001 From: hz Date: Mon, 20 Jul 2026 23:54:11 -0500 Subject: [PATCH] Add automated UAT smoke tests --- .gitea/scripts/deploy-environment.sh | 11 ++ .gitea/scripts/run-uat-smoke.sh | 25 +++ .gitea/workflows/deploy-uat.yml | 10 ++ .gitea/workflows/smoke-uat.yml | 30 ++++ .gitignore | 5 + docs/UAT_SMOKE_TESTS.md | 38 +++++ tests/smoke/Dockerfile | 8 + tests/smoke/package-lock.json | 79 +++++++++ tests/smoke/package.json | 13 ++ tests/smoke/playwright.config.mjs | 24 +++ tests/smoke/uat-smoke.spec.mjs | 233 +++++++++++++++++++++++++++ 11 files changed, 476 insertions(+) create mode 100644 .gitea/scripts/run-uat-smoke.sh create mode 100644 .gitea/workflows/smoke-uat.yml create mode 100644 docs/UAT_SMOKE_TESTS.md create mode 100644 tests/smoke/Dockerfile create mode 100644 tests/smoke/package-lock.json create mode 100644 tests/smoke/package.json create mode 100644 tests/smoke/playwright.config.mjs create mode 100644 tests/smoke/uat-smoke.spec.mjs diff --git a/.gitea/scripts/deploy-environment.sh b/.gitea/scripts/deploy-environment.sh index 658bd56..32345e9 100644 --- a/.gitea/scripts/deploy-environment.sh +++ b/.gitea/scripts/deploy-environment.sh @@ -73,4 +73,15 @@ kubectl get --raw "/api/v1/namespaces/${KUBE_NAMESPACE}/services/step3-api:http/ kubectl rollout status deployment/step3-web -n "${KUBE_NAMESPACE}" --timeout=240s kubectl get --raw "/api/v1/namespaces/${KUBE_NAMESPACE}/services/step3-web:http/proxy/healthz" >/dev/null +if [ "${DEPLOYMENT_ENVIRONMENT}" = "uat" ]; then + # The images are already imported on every Kubernetes node. Reclaim runner + # space before pulling the Playwright browser image. + docker image rm "${API_IMAGE}" "${WEB_IMAGE}" "${TEST_IMAGE}" 2>/dev/null || true + docker builder prune -af >/dev/null || true + UAT_API_ORIGIN="${PUBLIC_API_ORIGIN}" \ + UAT_WEB_ORIGIN="${PUBLIC_WEB_ORIGIN:-https://ustep3.ranaze.com}" \ + EXPECTED_REVISION="${BUILD_TAG}" \ + bash .gitea/scripts/run-uat-smoke.sh +fi + echo "Deployed ${DEPLOYMENT_ENVIRONMENT} revision ${BUILD_TAG}." diff --git a/.gitea/scripts/run-uat-smoke.sh b/.gitea/scripts/run-uat-smoke.sh new file mode 100644 index 0000000..a95a047 --- /dev/null +++ b/.gitea/scripts/run-uat-smoke.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${UAT_API_ORIGIN:=https://us3q.ranaze.com}" +: "${UAT_WEB_ORIGIN:=https://ustep3.ranaze.com}" + +IMAGE_TAG="step3-uat-smoke:${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_NUMBER:-0}" +ARTIFACT_DIR="${SMOKE_ARTIFACT_DIR:-artifacts/uat-smoke}" +ARTIFACT_ABS="$(pwd)/${ARTIFACT_DIR}" + +cleanup() { + docker image rm "${IMAGE_TAG}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +rm -rf "${ARTIFACT_DIR}" +mkdir -p "${ARTIFACT_DIR}" + +docker build -f tests/smoke/Dockerfile -t "${IMAGE_TAG}" tests/smoke +docker run --rm --ipc=host \ + -e UAT_API_ORIGIN="${UAT_API_ORIGIN}" \ + -e UAT_WEB_ORIGIN="${UAT_WEB_ORIGIN}" \ + -e EXPECTED_REVISION="${EXPECTED_REVISION:-}" \ + -v "${ARTIFACT_ABS}:/work/test-results" \ + "${IMAGE_TAG}" diff --git a/.gitea/workflows/deploy-uat.yml b/.gitea/workflows/deploy-uat.yml index aea475c..4641bec 100644 --- a/.gitea/workflows/deploy-uat.yml +++ b/.gitea/workflows/deploy-uat.yml @@ -32,6 +32,7 @@ jobs: DEPLOYMENT_ENVIRONMENT: uat KUBE_NAMESPACE: step3-uat PUBLIC_API_ORIGIN: https://us3q.ranaze.com + PUBLIC_WEB_ORIGIN: https://ustep3.ranaze.com MONGODB_CONNECTION_STRING: ${{ secrets.UAT_MONGODB_CONNECTION_STRING }} AUTH_JWT_KEY: ${{ secrets.UAT_AUTH_JWT_KEY }} INTERNAL_PURGE_SECRET: ${{ secrets.UAT_INTERNAL_PURGE_SECRET }} @@ -40,3 +41,12 @@ jobs: GOOGLE_CLIENT_ID: ${{ secrets.UAT_GOOGLE_CLIENT_ID }} GOOGLE_CLIENT_SECRET: ${{ secrets.UAT_GOOGLE_CLIENT_SECRET }} run: bash .gitea/scripts/deploy-environment.sh + + - name: Upload UAT smoke-test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: uat-smoke-${{ github.run_number }}-${{ github.run_attempt }} + path: artifacts/uat-smoke + if-no-files-found: warn + retention-days: 14 diff --git a/.gitea/workflows/smoke-uat.yml b/.gitea/workflows/smoke-uat.yml new file mode 100644 index 0000000..b40273f --- /dev/null +++ b/.gitea/workflows/smoke-uat.yml @@ -0,0 +1,30 @@ +name: Smoke Test STEP3 UAT + +on: + workflow_dispatch: {} + +concurrency: + group: step3-uat-smoke + cancel-in-progress: false + +jobs: + smoke: + runs-on: self-hosted + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run UAT smoke tests + env: + UAT_API_ORIGIN: https://us3q.ranaze.com + UAT_WEB_ORIGIN: https://ustep3.ranaze.com + run: bash .gitea/scripts/run-uat-smoke.sh + + - name: Upload smoke-test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: uat-smoke-${{ github.run_number }}-${{ github.run_attempt }} + path: artifacts/uat-smoke + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 7fe0a3a..69ecdaa 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ # Local editor/user files *.user *.suo + +# UAT smoke-test output +/artifacts/uat-smoke/ +**/node_modules/ +/tests/smoke/local-results/ diff --git a/docs/UAT_SMOKE_TESTS.md b/docs/UAT_SMOKE_TESTS.md new file mode 100644 index 0000000..6787d99 --- /dev/null +++ b/docs/UAT_SMOKE_TESTS.md @@ -0,0 +1,38 @@ +# UAT smoke tests + +The UAT smoke suite runs after each successful UAT deployment and can also be +started manually with the `Smoke Test STEP3 UAT` Gitea workflow. + +It checks: + +- public API health, readiness, environment, and deployed revision; +- legal metadata and configured Steam and Google authorization URLs; +- registration, login, refresh, authenticated profile, dashboard, ratings, + review queue, and randomized question retrieval; +- random CCS attempt creation and abandonment; +- public web/legal assets, completion of the Godot loader, and a visible, + non-uniform canvas without browser or critical asset errors. + +Each run creates a unique temporary learner account. The test abandons its CCS +attempt and deletes the account even when an assertion fails. It does not +submit question answers, complete OAuth, or open Stripe Checkout, so it does +not alter question statistics, learner ratings, donations, or provider +accounts. + +Failure evidence is retained for 14 days under the workflow's +`uat-smoke--` artifact. It includes JUnit XML, an HTML report, the +rendered canvas image, and Playwright traces/screenshots for failed browser +tests. + +No secrets are required. The public UAT origins are supplied by the workflow. +The registration endpoint permits five registrations per source IP in 15 +minutes, so repeatedly dispatching the smoke workflow in a short interval may +receive HTTP 429 until that window resets. + +To run the same suite from a Linux host with Docker: + +```bash +UAT_API_ORIGIN=https://us3q.ranaze.com \ +UAT_WEB_ORIGIN=https://ustep3.ranaze.com \ +bash .gitea/scripts/run-uat-smoke.sh +``` diff --git a/tests/smoke/Dockerfile b/tests/smoke/Dockerfile new file mode 100644 index 0000000..4ba1d91 --- /dev/null +++ b/tests/smoke/Dockerfile @@ -0,0 +1,8 @@ +FROM mcr.microsoft.com/playwright:v1.61.1-noble + +WORKDIR /work +COPY package.json package-lock.json ./ +RUN npm ci --ignore-scripts --no-audit --no-fund +COPY playwright.config.mjs uat-smoke.spec.mjs ./ + +CMD ["npm", "test"] diff --git a/tests/smoke/package-lock.json b/tests/smoke/package-lock.json new file mode 100644 index 0000000..6809a10 --- /dev/null +++ b/tests/smoke/package-lock.json @@ -0,0 +1,79 @@ +{ + "name": "ranaze-uat-smoke", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ranaze-uat-smoke", + "version": "1.0.0", + "dependencies": { + "@playwright/test": "1.61.1", + "pngjs": "7.0.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "engines": { + "node": ">=14.19.0" + } + } + } +} diff --git a/tests/smoke/package.json b/tests/smoke/package.json new file mode 100644 index 0000000..f142720 --- /dev/null +++ b/tests/smoke/package.json @@ -0,0 +1,13 @@ +{ + "name": "ranaze-uat-smoke", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "test": "playwright test" + }, + "dependencies": { + "@playwright/test": "1.61.1", + "pngjs": "7.0.0" + } +} diff --git a/tests/smoke/playwright.config.mjs b/tests/smoke/playwright.config.mjs new file mode 100644 index 0000000..f093cb1 --- /dev/null +++ b/tests/smoke/playwright.config.mjs @@ -0,0 +1,24 @@ +import { defineConfig } from "@playwright/test"; + +const resultsDir = process.env.SMOKE_OUTPUT_DIR || "/work/test-results"; + +export default defineConfig({ + testDir: ".", + testMatch: "uat-smoke.spec.mjs", + fullyParallel: false, + workers: 1, + retries: 0, + timeout: 120_000, + expect: { timeout: 15_000 }, + outputDir: `${resultsDir}/raw`, + reporter: [ + ["line"], + ["junit", { outputFile: `${resultsDir}/junit.xml` }], + ["html", { outputFolder: `${resultsDir}/html`, open: "never" }] + ], + use: { + viewport: { width: 1440, height: 900 }, + screenshot: "only-on-failure", + trace: "retain-on-failure" + } +}); diff --git a/tests/smoke/uat-smoke.spec.mjs b/tests/smoke/uat-smoke.spec.mjs new file mode 100644 index 0000000..d3255b6 --- /dev/null +++ b/tests/smoke/uat-smoke.spec.mjs @@ -0,0 +1,233 @@ +import crypto from "node:crypto"; +import { expect, test } from "@playwright/test"; +import { PNG } from "pngjs"; + +const API_ORIGIN = requiredOrigin("UAT_API_ORIGIN"); +const WEB_ORIGIN = requiredOrigin("UAT_WEB_ORIGIN"); +const EXPECTED_REVISION = (process.env.EXPECTED_REVISION || "").trim(); + +function requiredOrigin(name) { + const value = (process.env[name] || "").trim().replace(/\/$/, ""); + if (!value.startsWith("https://")) throw new Error(`${name} must be an https origin`); + return value; +} + +async function call(path, options = {}) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), options.timeout ?? 20_000); + try { + const response = await fetch(`${API_ORIGIN}${path}`, { + method: options.method || "GET", + headers: { + Accept: "application/json", + ...(options.body === undefined ? {} : { "Content-Type": "application/json" }), + ...(options.token ? { Authorization: `Bearer ${options.token}` } : {}) + }, + body: options.body === undefined ? undefined : JSON.stringify(options.body), + redirect: "manual", + signal: controller.signal + }); + const text = await response.text(); + let data = null; + if (text) { + try { data = JSON.parse(text); } catch { data = text; } + } + const expected = options.expected || [200]; + if (!expected.includes(response.status)) { + throw new Error(`${options.method || "GET"} ${path} returned ${response.status}: ${text.slice(0, 500)}`); + } + return { response, data }; + } finally { + clearTimeout(timeout); + } +} + +function providerIds(payload) { + return (payload?.providers || []).map((provider) => provider.id); +} + +function validateExternalUrl(provider, authUrl) { + const url = new URL(authUrl); + if (provider === "google") { + expect(url.hostname).toBe("accounts.google.com"); + expect(url.searchParams.get("client_id")).toBeTruthy(); + expect(url.searchParams.get("redirect_uri")).toBe(`${API_ORIGIN}/api/Auth/external/google/callback`); + } else { + expect(url.hostname).toBe("steamcommunity.com"); + const returnTo = new URL(url.searchParams.get("openid.return_to")); + expect(returnTo.origin + returnTo.pathname).toBe(`${API_ORIGIN}/api/Auth/external/steam/callback`); + expect(url.searchParams.get("openid.realm")).toBe(API_ORIGIN); + } +} + +test("public API and disposable learner journey", async () => { + const suffix = `${Date.now().toString(36)}${crypto.randomBytes(3).toString("hex")}`; + const username = `smoke_${suffix}`.slice(0, 28); + const email = `${username}@smoke.invalid`; + const password = `Smk!${crypto.randomBytes(18).toString("base64url")}`; + let token = ""; + let refreshToken = ""; + let accountCreated = false; + let activeCcsAttempt = ""; + let mainError = null; + let cleanupError = null; + + try { + const health = await call("/healthz"); + expect(String(health.data).trim()).toBe("ok"); + const ready = await call("/readyz"); + expect(String(ready.data).trim()).toBe("ready"); + + const { data: runtime } = await call("/api/runtime"); + expect(String(runtime.environment).toLowerCase()).toBe("uat"); + if (EXPECTED_REVISION) expect(runtime.buildRevision).toBe(EXPECTED_REVISION); + + const { data: legal } = await call("/api/Auth/legal"); + expect(legal.termsUrl).toBeTruthy(); + expect(legal.privacyUrl).toBeTruthy(); + + const { data: providers } = await call("/api/Auth/external/providers"); + expect(providerIds(providers)).toEqual(expect.arrayContaining(["steam", "google"])); + for (const provider of ["steam", "google"]) { + const { data: start } = await call(`/api/Auth/external/${provider}/start`, { + method: "POST", body: { purpose: "login" } + }); + expect(start.transactionId).toBeTruthy(); + validateExternalUrl(provider, start.authUrl); + } + + await call("/api/Auth/register", { + method: "POST", body: { username, email, password, acceptLegal: true } + }); + accountCreated = true; + + const { data: session } = await call("/api/Auth/login", { + method: "POST", body: { username, password } + }); + token = session.accessToken; + refreshToken = session.refreshToken; + expect(token).toBeTruthy(); + expect(refreshToken).toBeTruthy(); + + const { data: me } = await call("/api/Auth/me", { token }); + expect(me.username).toBe(username); + expect(me.availableProviders).toEqual(expect.arrayContaining(["steam", "google"])); + + const { data: refreshed } = await call("/api/Auth/refresh", { + method: "POST", body: { username, refreshToken } + }); + token = refreshed.accessToken; + expect(token).toBeTruthy(); + + await call("/api/profile/me/preferences", { token }); + await call("/api/dashboard", { token }); + await call("/api/ratings/me", { token }); + + const { data: queue } = await call("/api/review-queue/next", { + method: "POST", token, body: { excludeExternalIds: [], supportsChoicePools: true } + }); + expect(Number.isInteger(queue.pendingCount)).toBeTruthy(); + + const { data: questions } = await call("/api/questions/search", { + method: "POST", + token, + body: { limit: 1, supportsChoicePools: true, requireMedia: false, excludeExternalIds: [] } + }); + expect(Array.isArray(questions)).toBeTruthy(); + expect(questions).toHaveLength(1); + expect(questions[0].externalId).toBeTruthy(); + expect(questions[0].stem).toBeTruthy(); + expect(questions[0].choices.length).toBeGreaterThanOrEqual(5); + expect(questions[0].choices.map((choice) => choice.id)).toContain(questions[0].correctChoiceId); + + const { data: cases } = await call("/api/ccs/cases", { token }); + expect(Array.isArray(cases)).toBeTruthy(); + expect(cases.length).toBeGreaterThan(0); + const { data: attempt } = await call("/api/ccs/attempts", { + method: "POST", token, body: { caseExternalId: "", mode: "practice" } + }); + activeCcsAttempt = attempt.id; + expect(activeCcsAttempt).toBeTruthy(); + expect(attempt.caseExternalId).toBeTruthy(); + expect(attempt.state).toBe("active"); + await call(`/api/ccs/attempts/${activeCcsAttempt}/abandon`, { + method: "POST", token, expected: [204] + }); + activeCcsAttempt = ""; + } catch (error) { + mainError = error; + } finally { + try { + if (accountCreated && !token) { + const login = await call("/api/Auth/login", { + method: "POST", body: { username, password } + }); + token = login.data.accessToken; + } + if (activeCcsAttempt && token) { + await call(`/api/ccs/attempts/${activeCcsAttempt}/abandon`, { + method: "POST", token, expected: [204, 404] + }); + } + if (accountCreated && token) { + await call("/api/Auth/me", { + method: "DELETE", token, body: { password, confirmation: "DELETE" }, expected: [204] + }); + } + } catch (error) { + cleanupError = error; + } + } + + if (mainError && cleanupError) throw new AggregateError([mainError, cleanupError], "Smoke journey and cleanup failed"); + if (mainError) throw mainError; + if (cleanupError) throw cleanupError; +}); + +test("web export boots a nonblank Godot canvas", async ({ page }, testInfo) => { + const consoleErrors = []; + const requestFailures = []; + page.on("pageerror", (error) => consoleErrors.push(error.message)); + page.on("console", (message) => { + if (message.type() === "error") consoleErrors.push(message.text()); + }); + page.on("requestfailed", (request) => { + const url = new URL(request.url()); + const failure = request.failure()?.errorText || "failed"; + // Godot cancels redundant range/download requests once its streamed asset + // loader has the complete payload. A removed loader plus a rendered canvas + // proves those expected ERR_ABORTED requests were not load failures. + if (url.origin === WEB_ORIGIN && failure !== "net::ERR_ABORTED" && + /\.(js|wasm|pck|ttf|png)$/.test(url.pathname)) { + requestFailures.push(`${url.pathname}: ${failure}`); + } + }); + + for (const path of ["/healthz", "/terms.html", "/privacy.html", "/legal/legal_documents.json", "/ranaze-loader-icon-v1.png"]) { + const response = await page.request.get(`${WEB_ORIGIN}${path}`); + expect(response.ok(), `${path} returned ${response.status()}`).toBeTruthy(); + } + + const response = await page.goto(WEB_ORIGIN, { waitUntil: "domcontentloaded", timeout: 30_000 }); + expect(response?.ok()).toBeTruthy(); + await page.locator("#status").waitFor({ state: "detached", timeout: 90_000 }); + const canvas = page.locator("#canvas"); + await expect(canvas).toBeVisible(); + const bounds = await canvas.boundingBox(); + expect(bounds.width).toBeGreaterThan(1000); + expect(bounds.height).toBeGreaterThan(600); + await page.waitForTimeout(1_000); + + const screenshot = await canvas.screenshot({ path: testInfo.outputPath("uat-canvas.png") }); + const png = PNG.sync.read(screenshot); + const colors = new Set(); + for (let y = 0; y < png.height; y += 12) { + for (let x = 0; x < png.width; x += 12) { + const offset = (png.width * y + x) * 4; + colors.add(`${png.data[offset]},${png.data[offset + 1]},${png.data[offset + 2]}`); + } + } + expect(colors.size, "Godot canvas appears blank or uniform").toBeGreaterThan(40); + expect(requestFailures, "critical web assets failed to load").toEqual([]); + expect(consoleErrors, "browser console errors were emitted").toEqual([]); +});