Add automated UAT smoke tests

.gitea/scripts/deploy-environment.sh

@@ -73,4 +73,15 @@ kubectl get --raw "/api/v1/namespaces/${KUBE_NAMESPACE}/services/step3-api:http/
 kubectl rollout status deployment/step3-web -n "${KUBE_NAMESPACE}" --timeout=240s
 kubectl get --raw "/api/v1/namespaces/${KUBE_NAMESPACE}/services/step3-web:http/proxy/healthz" >/dev/null
 
+if [ "${DEPLOYMENT_ENVIRONMENT}" = "uat" ]; then
+  # The images are already imported on every Kubernetes node. Reclaim runner
+  # space before pulling the Playwright browser image.
+  docker image rm "${API_IMAGE}" "${WEB_IMAGE}" "${TEST_IMAGE}" 2>/dev/null || true
+  docker builder prune -af >/dev/null || true
+  UAT_API_ORIGIN="${PUBLIC_API_ORIGIN}" \
+  UAT_WEB_ORIGIN="${PUBLIC_WEB_ORIGIN:-https://ustep3.ranaze.com}" \
+  EXPECTED_REVISION="${BUILD_TAG}" \
+    bash .gitea/scripts/run-uat-smoke.sh
+fi
+
 echo "Deployed ${DEPLOYMENT_ENVIRONMENT} revision ${BUILD_TAG}."

.gitea/scripts/run-uat-smoke.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${UAT_API_ORIGIN:=https://us3q.ranaze.com}"
+: "${UAT_WEB_ORIGIN:=https://ustep3.ranaze.com}"
+
+IMAGE_TAG="step3-uat-smoke:${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_NUMBER:-0}"
+ARTIFACT_DIR="${SMOKE_ARTIFACT_DIR:-artifacts/uat-smoke}"
+ARTIFACT_ABS="$(pwd)/${ARTIFACT_DIR}"
+
+cleanup() {
+  docker image rm "${IMAGE_TAG}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+rm -rf "${ARTIFACT_DIR}"
+mkdir -p "${ARTIFACT_DIR}"
+
+docker build -f tests/smoke/Dockerfile -t "${IMAGE_TAG}" tests/smoke
+docker run --rm --ipc=host \
+  -e UAT_API_ORIGIN="${UAT_API_ORIGIN}" \
+  -e UAT_WEB_ORIGIN="${UAT_WEB_ORIGIN}" \
+  -e EXPECTED_REVISION="${EXPECTED_REVISION:-}" \
+  -v "${ARTIFACT_ABS}:/work/test-results" \
+  "${IMAGE_TAG}"

.gitea/workflows/deploy-uat.yml

@@ -32,6 +32,7 @@ jobs:
           DEPLOYMENT_ENVIRONMENT: uat
           KUBE_NAMESPACE: step3-uat
           PUBLIC_API_ORIGIN: https://us3q.ranaze.com
+          PUBLIC_WEB_ORIGIN: https://ustep3.ranaze.com
           MONGODB_CONNECTION_STRING: ${{ secrets.UAT_MONGODB_CONNECTION_STRING }}
           AUTH_JWT_KEY: ${{ secrets.UAT_AUTH_JWT_KEY }}
           INTERNAL_PURGE_SECRET: ${{ secrets.UAT_INTERNAL_PURGE_SECRET }}
@@ -40,3 +41,12 @@ jobs:
           GOOGLE_CLIENT_ID: ${{ secrets.UAT_GOOGLE_CLIENT_ID }}
           GOOGLE_CLIENT_SECRET: ${{ secrets.UAT_GOOGLE_CLIENT_SECRET }}
         run: bash .gitea/scripts/deploy-environment.sh
+
+      - name: Upload UAT smoke-test artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: uat-smoke-${{ github.run_number }}-${{ github.run_attempt }}
+          path: artifacts/uat-smoke
+          if-no-files-found: warn
+          retention-days: 14

.gitea/workflows/smoke-uat.yml

@@ -0,0 +1,30 @@
+name: Smoke Test STEP3 UAT
+
+on:
+  workflow_dispatch: {}
+
+concurrency:
+  group: step3-uat-smoke
+  cancel-in-progress: false
+
+jobs:
+  smoke:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run UAT smoke tests
+        env:
+          UAT_API_ORIGIN: https://us3q.ranaze.com
+          UAT_WEB_ORIGIN: https://ustep3.ranaze.com
+        run: bash .gitea/scripts/run-uat-smoke.sh
+
+      - name: Upload smoke-test artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: uat-smoke-${{ github.run_number }}-${{ github.run_attempt }}
+          path: artifacts/uat-smoke
+          if-no-files-found: warn
+          retention-days: 14

.gitignore

@@ -10,3 +10,8 @@
 # Local editor/user files
 *.user
 *.suo
+
+# UAT smoke-test output
+/artifacts/uat-smoke/
+**/node_modules/
+/tests/smoke/local-results/

docs/UAT_SMOKE_TESTS.md

@@ -0,0 +1,38 @@
+# UAT smoke tests
+
+The UAT smoke suite runs after each successful UAT deployment and can also be
+started manually with the `Smoke Test STEP3 UAT` Gitea workflow.
+
+It checks:
+
+- public API health, readiness, environment, and deployed revision;
+- legal metadata and configured Steam and Google authorization URLs;
+- registration, login, refresh, authenticated profile, dashboard, ratings,
+  review queue, and randomized question retrieval;
+- random CCS attempt creation and abandonment;
+- public web/legal assets, completion of the Godot loader, and a visible,
+  non-uniform canvas without browser or critical asset errors.
+
+Each run creates a unique temporary learner account. The test abandons its CCS
+attempt and deletes the account even when an assertion fails. It does not
+submit question answers, complete OAuth, or open Stripe Checkout, so it does
+not alter question statistics, learner ratings, donations, or provider
+accounts.
+
+Failure evidence is retained for 14 days under the workflow's
+`uat-smoke-<run number>-<attempt>` artifact. It includes JUnit XML, an HTML report, the
+rendered canvas image, and Playwright traces/screenshots for failed browser
+tests.
+
+No secrets are required. The public UAT origins are supplied by the workflow.
+The registration endpoint permits five registrations per source IP in 15
+minutes, so repeatedly dispatching the smoke workflow in a short interval may
+receive HTTP 429 until that window resets.
+
+To run the same suite from a Linux host with Docker:
+
+```bash
+UAT_API_ORIGIN=https://us3q.ranaze.com \
+UAT_WEB_ORIGIN=https://ustep3.ranaze.com \
+bash .gitea/scripts/run-uat-smoke.sh
+```

tests/smoke/Dockerfile

@@ -0,0 +1,8 @@
+FROM mcr.microsoft.com/playwright:v1.61.1-noble
+
+WORKDIR /work
+COPY package.json package-lock.json ./
+RUN npm ci --ignore-scripts --no-audit --no-fund
+COPY playwright.config.mjs uat-smoke.spec.mjs ./
+
+CMD ["npm", "test"]

tests/smoke/package-lock.json

@@ -0,0 +1,79 @@
+{
+  "name": "ranaze-uat-smoke",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "ranaze-uat-smoke",
+      "version": "1.0.0",
+      "dependencies": {
+        "@playwright/test": "1.61.1",
+        "pngjs": "7.0.0"
+      }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz",
+      "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==",
+      "dependencies": {
+        "playwright": "1.61.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz",
+      "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==",
+      "dependencies": {
+        "playwright-core": "1.61.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz",
+      "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/pngjs": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz",
+      "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==",
+      "engines": {
+        "node": ">=14.19.0"
+      }
+    }
+  }
+}

tests/smoke/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "ranaze-uat-smoke",
+  "version": "1.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "playwright test"
+  },
+  "dependencies": {
+    "@playwright/test": "1.61.1",
+    "pngjs": "7.0.0"
+  }
+}

tests/smoke/playwright.config.mjs

@@ -0,0 +1,24 @@
+import { defineConfig } from "@playwright/test";
+
+const resultsDir = process.env.SMOKE_OUTPUT_DIR || "/work/test-results";
+
+export default defineConfig({
+  testDir: ".",
+  testMatch: "uat-smoke.spec.mjs",
+  fullyParallel: false,
+  workers: 1,
+  retries: 0,
+  timeout: 120_000,
+  expect: { timeout: 15_000 },
+  outputDir: `${resultsDir}/raw`,
+  reporter: [
+    ["line"],
+    ["junit", { outputFile: `${resultsDir}/junit.xml` }],
+    ["html", { outputFolder: `${resultsDir}/html`, open: "never" }]
+  ],
+  use: {
+    viewport: { width: 1440, height: 900 },
+    screenshot: "only-on-failure",
+    trace: "retain-on-failure"
+  }
+});

tests/smoke/uat-smoke.spec.mjs

@@ -0,0 +1,233 @@
+import crypto from "node:crypto";
+import { expect, test } from "@playwright/test";
+import { PNG } from "pngjs";
+
+const API_ORIGIN = requiredOrigin("UAT_API_ORIGIN");
+const WEB_ORIGIN = requiredOrigin("UAT_WEB_ORIGIN");
+const EXPECTED_REVISION = (process.env.EXPECTED_REVISION || "").trim();
+
+function requiredOrigin(name) {
+  const value = (process.env[name] || "").trim().replace(/\/$/, "");
+  if (!value.startsWith("https://")) throw new Error(`${name} must be an https origin`);
+  return value;
+}
+
+async function call(path, options = {}) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), options.timeout ?? 20_000);
+  try {
+    const response = await fetch(`${API_ORIGIN}${path}`, {
+      method: options.method || "GET",
+      headers: {
+        Accept: "application/json",
+        ...(options.body === undefined ? {} : { "Content-Type": "application/json" }),
+        ...(options.token ? { Authorization: `Bearer ${options.token}` } : {})
+      },
+      body: options.body === undefined ? undefined : JSON.stringify(options.body),
+      redirect: "manual",
+      signal: controller.signal
+    });
+    const text = await response.text();
+    let data = null;
+    if (text) {
+      try { data = JSON.parse(text); } catch { data = text; }
+    }
+    const expected = options.expected || [200];
+    if (!expected.includes(response.status)) {
+      throw new Error(`${options.method || "GET"} ${path} returned ${response.status}: ${text.slice(0, 500)}`);
+    }
+    return { response, data };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function providerIds(payload) {
+  return (payload?.providers || []).map((provider) => provider.id);
+}
+
+function validateExternalUrl(provider, authUrl) {
+  const url = new URL(authUrl);
+  if (provider === "google") {
+    expect(url.hostname).toBe("accounts.google.com");
+    expect(url.searchParams.get("client_id")).toBeTruthy();
+    expect(url.searchParams.get("redirect_uri")).toBe(`${API_ORIGIN}/api/Auth/external/google/callback`);
+  } else {
+    expect(url.hostname).toBe("steamcommunity.com");
+    const returnTo = new URL(url.searchParams.get("openid.return_to"));
+    expect(returnTo.origin + returnTo.pathname).toBe(`${API_ORIGIN}/api/Auth/external/steam/callback`);
+    expect(url.searchParams.get("openid.realm")).toBe(API_ORIGIN);
+  }
+}
+
+test("public API and disposable learner journey", async () => {
+  const suffix = `${Date.now().toString(36)}${crypto.randomBytes(3).toString("hex")}`;
+  const username = `smoke_${suffix}`.slice(0, 28);
+  const email = `${username}@smoke.invalid`;
+  const password = `Smk!${crypto.randomBytes(18).toString("base64url")}`;
+  let token = "";
+  let refreshToken = "";
+  let accountCreated = false;
+  let activeCcsAttempt = "";
+  let mainError = null;
+  let cleanupError = null;
+
+  try {
+    const health = await call("/healthz");
+    expect(String(health.data).trim()).toBe("ok");
+    const ready = await call("/readyz");
+    expect(String(ready.data).trim()).toBe("ready");
+
+    const { data: runtime } = await call("/api/runtime");
+    expect(String(runtime.environment).toLowerCase()).toBe("uat");
+    if (EXPECTED_REVISION) expect(runtime.buildRevision).toBe(EXPECTED_REVISION);
+
+    const { data: legal } = await call("/api/Auth/legal");
+    expect(legal.termsUrl).toBeTruthy();
+    expect(legal.privacyUrl).toBeTruthy();
+
+    const { data: providers } = await call("/api/Auth/external/providers");
+    expect(providerIds(providers)).toEqual(expect.arrayContaining(["steam", "google"]));
+    for (const provider of ["steam", "google"]) {
+      const { data: start } = await call(`/api/Auth/external/${provider}/start`, {
+        method: "POST", body: { purpose: "login" }
+      });
+      expect(start.transactionId).toBeTruthy();
+      validateExternalUrl(provider, start.authUrl);
+    }
+
+    await call("/api/Auth/register", {
+      method: "POST", body: { username, email, password, acceptLegal: true }
+    });
+    accountCreated = true;
+
+    const { data: session } = await call("/api/Auth/login", {
+      method: "POST", body: { username, password }
+    });
+    token = session.accessToken;
+    refreshToken = session.refreshToken;
+    expect(token).toBeTruthy();
+    expect(refreshToken).toBeTruthy();
+
+    const { data: me } = await call("/api/Auth/me", { token });
+    expect(me.username).toBe(username);
+    expect(me.availableProviders).toEqual(expect.arrayContaining(["steam", "google"]));
+
+    const { data: refreshed } = await call("/api/Auth/refresh", {
+      method: "POST", body: { username, refreshToken }
+    });
+    token = refreshed.accessToken;
+    expect(token).toBeTruthy();
+
+    await call("/api/profile/me/preferences", { token });
+    await call("/api/dashboard", { token });
+    await call("/api/ratings/me", { token });
+
+    const { data: queue } = await call("/api/review-queue/next", {
+      method: "POST", token, body: { excludeExternalIds: [], supportsChoicePools: true }
+    });
+    expect(Number.isInteger(queue.pendingCount)).toBeTruthy();
+
+    const { data: questions } = await call("/api/questions/search", {
+      method: "POST",
+      token,
+      body: { limit: 1, supportsChoicePools: true, requireMedia: false, excludeExternalIds: [] }
+    });
+    expect(Array.isArray(questions)).toBeTruthy();
+    expect(questions).toHaveLength(1);
+    expect(questions[0].externalId).toBeTruthy();
+    expect(questions[0].stem).toBeTruthy();
+    expect(questions[0].choices.length).toBeGreaterThanOrEqual(5);
+    expect(questions[0].choices.map((choice) => choice.id)).toContain(questions[0].correctChoiceId);
+
+    const { data: cases } = await call("/api/ccs/cases", { token });
+    expect(Array.isArray(cases)).toBeTruthy();
+    expect(cases.length).toBeGreaterThan(0);
+    const { data: attempt } = await call("/api/ccs/attempts", {
+      method: "POST", token, body: { caseExternalId: "", mode: "practice" }
+    });
+    activeCcsAttempt = attempt.id;
+    expect(activeCcsAttempt).toBeTruthy();
+    expect(attempt.caseExternalId).toBeTruthy();
+    expect(attempt.state).toBe("active");
+    await call(`/api/ccs/attempts/${activeCcsAttempt}/abandon`, {
+      method: "POST", token, expected: [204]
+    });
+    activeCcsAttempt = "";
+  } catch (error) {
+    mainError = error;
+  } finally {
+    try {
+      if (accountCreated && !token) {
+        const login = await call("/api/Auth/login", {
+          method: "POST", body: { username, password }
+        });
+        token = login.data.accessToken;
+      }
+      if (activeCcsAttempt && token) {
+        await call(`/api/ccs/attempts/${activeCcsAttempt}/abandon`, {
+          method: "POST", token, expected: [204, 404]
+        });
+      }
+      if (accountCreated && token) {
+        await call("/api/Auth/me", {
+          method: "DELETE", token, body: { password, confirmation: "DELETE" }, expected: [204]
+        });
+      }
+    } catch (error) {
+      cleanupError = error;
+    }
+  }
+
+  if (mainError && cleanupError) throw new AggregateError([mainError, cleanupError], "Smoke journey and cleanup failed");
+  if (mainError) throw mainError;
+  if (cleanupError) throw cleanupError;
+});
+
+test("web export boots a nonblank Godot canvas", async ({ page }, testInfo) => {
+  const consoleErrors = [];
+  const requestFailures = [];
+  page.on("pageerror", (error) => consoleErrors.push(error.message));
+  page.on("console", (message) => {
+    if (message.type() === "error") consoleErrors.push(message.text());
+  });
+  page.on("requestfailed", (request) => {
+    const url = new URL(request.url());
+    const failure = request.failure()?.errorText || "failed";
+    // Godot cancels redundant range/download requests once its streamed asset
+    // loader has the complete payload. A removed loader plus a rendered canvas
+    // proves those expected ERR_ABORTED requests were not load failures.
+    if (url.origin === WEB_ORIGIN && failure !== "net::ERR_ABORTED" &&
+        /\.(js|wasm|pck|ttf|png)$/.test(url.pathname)) {
+      requestFailures.push(`${url.pathname}: ${failure}`);
+    }
+  });
+
+  for (const path of ["/healthz", "/terms.html", "/privacy.html", "/legal/legal_documents.json", "/ranaze-loader-icon-v1.png"]) {
+    const response = await page.request.get(`${WEB_ORIGIN}${path}`);
+    expect(response.ok(), `${path} returned ${response.status()}`).toBeTruthy();
+  }
+
+  const response = await page.goto(WEB_ORIGIN, { waitUntil: "domcontentloaded", timeout: 30_000 });
+  expect(response?.ok()).toBeTruthy();
+  await page.locator("#status").waitFor({ state: "detached", timeout: 90_000 });
+  const canvas = page.locator("#canvas");
+  await expect(canvas).toBeVisible();
+  const bounds = await canvas.boundingBox();
+  expect(bounds.width).toBeGreaterThan(1000);
+  expect(bounds.height).toBeGreaterThan(600);
+  await page.waitForTimeout(1_000);
+
+  const screenshot = await canvas.screenshot({ path: testInfo.outputPath("uat-canvas.png") });
+  const png = PNG.sync.read(screenshot);
+  const colors = new Set();
+  for (let y = 0; y < png.height; y += 12) {
+    for (let x = 0; x < png.width; x += 12) {
+      const offset = (png.width * y + x) * 4;
+      colors.add(`${png.data[offset]},${png.data[offset + 1]},${png.data[offset + 2]}`);
+    }
+  }
+  expect(colors.size, "Godot canvas appears blank or uniform").toBeGreaterThan(40);
+  expect(requestFailures, "critical web assets failed to load").toEqual([]);
+  expect(consoleErrors, "browser console errors were emitted").toEqual([]);
+});