Add automated UAT smoke tests

2026-07-20 23:54:11 -05:00
parent e5b567eec8
commit c2d000cc48
11 changed files with 476 additions and 0 deletions
@@ -0,0 +1,8 @@
+FROM mcr.microsoft.com/playwright:v1.61.1-noble
+
+WORKDIR /work
+COPY package.json package-lock.json ./
+RUN npm ci --ignore-scripts --no-audit --no-fund
+COPY playwright.config.mjs uat-smoke.spec.mjs ./
+
+CMD ["npm", "test"]
@@ -0,0 +1,79 @@
+{
+  "name": "ranaze-uat-smoke",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "ranaze-uat-smoke",
+      "version": "1.0.0",
+      "dependencies": {
+        "@playwright/test": "1.61.1",
+        "pngjs": "7.0.0"
+      }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz",
+      "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==",
+      "dependencies": {
+        "playwright": "1.61.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz",
+      "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==",
+      "dependencies": {
+        "playwright-core": "1.61.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.61.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz",
+      "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/pngjs": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz",
+      "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==",
+      "engines": {
+        "node": ">=14.19.0"
+      }
+    }
+  }
+}
@@ -0,0 +1,13 @@
+{
+  "name": "ranaze-uat-smoke",
+  "version": "1.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "playwright test"
+  },
+  "dependencies": {
+    "@playwright/test": "1.61.1",
+    "pngjs": "7.0.0"
+  }
+}
@@ -0,0 +1,24 @@
+import { defineConfig } from "@playwright/test";
+
+const resultsDir = process.env.SMOKE_OUTPUT_DIR || "/work/test-results";
+
+export default defineConfig({
+  testDir: ".",
+  testMatch: "uat-smoke.spec.mjs",
+  fullyParallel: false,
+  workers: 1,
+  retries: 0,
+  timeout: 120_000,
+  expect: { timeout: 15_000 },
+  outputDir: `${resultsDir}/raw`,
+  reporter: [
+    ["line"],
+    ["junit", { outputFile: `${resultsDir}/junit.xml` }],
+    ["html", { outputFolder: `${resultsDir}/html`, open: "never" }]
+  ],
+  use: {
+    viewport: { width: 1440, height: 900 },
+    screenshot: "only-on-failure",
+    trace: "retain-on-failure"
+  }
+});
@@ -0,0 +1,233 @@
+import crypto from "node:crypto";
+import { expect, test } from "@playwright/test";
+import { PNG } from "pngjs";
+
+const API_ORIGIN = requiredOrigin("UAT_API_ORIGIN");
+const WEB_ORIGIN = requiredOrigin("UAT_WEB_ORIGIN");
+const EXPECTED_REVISION = (process.env.EXPECTED_REVISION || "").trim();
+
+function requiredOrigin(name) {
+  const value = (process.env[name] || "").trim().replace(/\/$/, "");
+  if (!value.startsWith("https://")) throw new Error(`${name} must be an https origin`);
+  return value;
+}
+
+async function call(path, options = {}) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), options.timeout ?? 20_000);
+  try {
+    const response = await fetch(`${API_ORIGIN}${path}`, {
+      method: options.method || "GET",
+      headers: {
+        Accept: "application/json",
+        ...(options.body === undefined ? {} : { "Content-Type": "application/json" }),
+        ...(options.token ? { Authorization: `Bearer ${options.token}` } : {})
+      },
+      body: options.body === undefined ? undefined : JSON.stringify(options.body),
+      redirect: "manual",
+      signal: controller.signal
+    });
+    const text = await response.text();
+    let data = null;
+    if (text) {
+      try { data = JSON.parse(text); } catch { data = text; }
+    }
+    const expected = options.expected || [200];
+    if (!expected.includes(response.status)) {
+      throw new Error(`${options.method || "GET"} ${path} returned ${response.status}: ${text.slice(0, 500)}`);
+    }
+    return { response, data };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function providerIds(payload) {
+  return (payload?.providers || []).map((provider) => provider.id);
+}
+
+function validateExternalUrl(provider, authUrl) {
+  const url = new URL(authUrl);
+  if (provider === "google") {
+    expect(url.hostname).toBe("accounts.google.com");
+    expect(url.searchParams.get("client_id")).toBeTruthy();
+    expect(url.searchParams.get("redirect_uri")).toBe(`${API_ORIGIN}/api/Auth/external/google/callback`);
+  } else {
+    expect(url.hostname).toBe("steamcommunity.com");
+    const returnTo = new URL(url.searchParams.get("openid.return_to"));
+    expect(returnTo.origin + returnTo.pathname).toBe(`${API_ORIGIN}/api/Auth/external/steam/callback`);
+    expect(url.searchParams.get("openid.realm")).toBe(API_ORIGIN);
+  }
+}
+
+test("public API and disposable learner journey", async () => {
+  const suffix = `${Date.now().toString(36)}${crypto.randomBytes(3).toString("hex")}`;
+  const username = `smoke_${suffix}`.slice(0, 28);
+  const email = `${username}@smoke.invalid`;
+  const password = `Smk!${crypto.randomBytes(18).toString("base64url")}`;
+  let token = "";
+  let refreshToken = "";
+  let accountCreated = false;
+  let activeCcsAttempt = "";
+  let mainError = null;
+  let cleanupError = null;
+
+  try {
+    const health = await call("/healthz");
+    expect(String(health.data).trim()).toBe("ok");
+    const ready = await call("/readyz");
+    expect(String(ready.data).trim()).toBe("ready");
+
+    const { data: runtime } = await call("/api/runtime");
+    expect(String(runtime.environment).toLowerCase()).toBe("uat");
+    if (EXPECTED_REVISION) expect(runtime.buildRevision).toBe(EXPECTED_REVISION);
+
+    const { data: legal } = await call("/api/Auth/legal");
+    expect(legal.termsUrl).toBeTruthy();
+    expect(legal.privacyUrl).toBeTruthy();
+
+    const { data: providers } = await call("/api/Auth/external/providers");
+    expect(providerIds(providers)).toEqual(expect.arrayContaining(["steam", "google"]));
+    for (const provider of ["steam", "google"]) {
+      const { data: start } = await call(`/api/Auth/external/${provider}/start`, {
+        method: "POST", body: { purpose: "login" }
+      });
+      expect(start.transactionId).toBeTruthy();
+      validateExternalUrl(provider, start.authUrl);
+    }
+
+    await call("/api/Auth/register", {
+      method: "POST", body: { username, email, password, acceptLegal: true }
+    });
+    accountCreated = true;
+
+    const { data: session } = await call("/api/Auth/login", {
+      method: "POST", body: { username, password }
+    });
+    token = session.accessToken;
+    refreshToken = session.refreshToken;
+    expect(token).toBeTruthy();
+    expect(refreshToken).toBeTruthy();
+
+    const { data: me } = await call("/api/Auth/me", { token });
+    expect(me.username).toBe(username);
+    expect(me.availableProviders).toEqual(expect.arrayContaining(["steam", "google"]));
+
+    const { data: refreshed } = await call("/api/Auth/refresh", {
+      method: "POST", body: { username, refreshToken }
+    });
+    token = refreshed.accessToken;
+    expect(token).toBeTruthy();
+
+    await call("/api/profile/me/preferences", { token });
+    await call("/api/dashboard", { token });
+    await call("/api/ratings/me", { token });
+
+    const { data: queue } = await call("/api/review-queue/next", {
+      method: "POST", token, body: { excludeExternalIds: [], supportsChoicePools: true }
+    });
+    expect(Number.isInteger(queue.pendingCount)).toBeTruthy();
+
+    const { data: questions } = await call("/api/questions/search", {
+      method: "POST",
+      token,
+      body: { limit: 1, supportsChoicePools: true, requireMedia: false, excludeExternalIds: [] }
+    });
+    expect(Array.isArray(questions)).toBeTruthy();
+    expect(questions).toHaveLength(1);
+    expect(questions[0].externalId).toBeTruthy();
+    expect(questions[0].stem).toBeTruthy();
+    expect(questions[0].choices.length).toBeGreaterThanOrEqual(5);
+    expect(questions[0].choices.map((choice) => choice.id)).toContain(questions[0].correctChoiceId);
+
+    const { data: cases } = await call("/api/ccs/cases", { token });
+    expect(Array.isArray(cases)).toBeTruthy();
+    expect(cases.length).toBeGreaterThan(0);
+    const { data: attempt } = await call("/api/ccs/attempts", {
+      method: "POST", token, body: { caseExternalId: "", mode: "practice" }
+    });
+    activeCcsAttempt = attempt.id;
+    expect(activeCcsAttempt).toBeTruthy();
+    expect(attempt.caseExternalId).toBeTruthy();
+    expect(attempt.state).toBe("active");
+    await call(`/api/ccs/attempts/${activeCcsAttempt}/abandon`, {
+      method: "POST", token, expected: [204]
+    });
+    activeCcsAttempt = "";
+  } catch (error) {
+    mainError = error;
+  } finally {
+    try {
+      if (accountCreated && !token) {
+        const login = await call("/api/Auth/login", {
+          method: "POST", body: { username, password }
+        });
+        token = login.data.accessToken;
+      }
+      if (activeCcsAttempt && token) {
+        await call(`/api/ccs/attempts/${activeCcsAttempt}/abandon`, {
+          method: "POST", token, expected: [204, 404]
+        });
+      }
+      if (accountCreated && token) {
+        await call("/api/Auth/me", {
+          method: "DELETE", token, body: { password, confirmation: "DELETE" }, expected: [204]
+        });
+      }
+    } catch (error) {
+      cleanupError = error;
+    }
+  }
+
+  if (mainError && cleanupError) throw new AggregateError([mainError, cleanupError], "Smoke journey and cleanup failed");
+  if (mainError) throw mainError;
+  if (cleanupError) throw cleanupError;
+});
+
+test("web export boots a nonblank Godot canvas", async ({ page }, testInfo) => {
+  const consoleErrors = [];
+  const requestFailures = [];
+  page.on("pageerror", (error) => consoleErrors.push(error.message));
+  page.on("console", (message) => {
+    if (message.type() === "error") consoleErrors.push(message.text());
+  });
+  page.on("requestfailed", (request) => {
+    const url = new URL(request.url());
+    const failure = request.failure()?.errorText || "failed";
+    // Godot cancels redundant range/download requests once its streamed asset
+    // loader has the complete payload. A removed loader plus a rendered canvas
+    // proves those expected ERR_ABORTED requests were not load failures.
+    if (url.origin === WEB_ORIGIN && failure !== "net::ERR_ABORTED" &&
+        /\.(js|wasm|pck|ttf|png)$/.test(url.pathname)) {
+      requestFailures.push(`${url.pathname}: ${failure}`);
+    }
+  });
+
+  for (const path of ["/healthz", "/terms.html", "/privacy.html", "/legal/legal_documents.json", "/ranaze-loader-icon-v1.png"]) {
+    const response = await page.request.get(`${WEB_ORIGIN}${path}`);
+    expect(response.ok(), `${path} returned ${response.status()}`).toBeTruthy();
+  }
+
+  const response = await page.goto(WEB_ORIGIN, { waitUntil: "domcontentloaded", timeout: 30_000 });
+  expect(response?.ok()).toBeTruthy();
+  await page.locator("#status").waitFor({ state: "detached", timeout: 90_000 });
+  const canvas = page.locator("#canvas");
+  await expect(canvas).toBeVisible();
+  const bounds = await canvas.boundingBox();
+  expect(bounds.width).toBeGreaterThan(1000);
+  expect(bounds.height).toBeGreaterThan(600);
+  await page.waitForTimeout(1_000);
+
+  const screenshot = await canvas.screenshot({ path: testInfo.outputPath("uat-canvas.png") });
+  const png = PNG.sync.read(screenshot);
+  const colors = new Set();
+  for (let y = 0; y < png.height; y += 12) {
+    for (let x = 0; x < png.width; x += 12) {
+      const offset = (png.width * y + x) * 4;
+      colors.add(`${png.data[offset]},${png.data[offset + 1]},${png.data[offset + 2]}`);
+    }
+  }
+  expect(colors.size, "Godot canvas appears blank or uniform").toBeGreaterThan(40);
+  expect(requestFailures, "critical web assets failed to load").toEqual([]);
+  expect(consoleErrors, "browser console errors were emitted").toEqual([]);
+});