/tests/e2e/a11y.spec.ts

/**
 * tests/e2e/a11y.spec.ts
 *
 * Accessibility gate using @axe-core/playwright (Task 13 — Fase 7).
 *
 * NIGHTLY advisory: this suite is scheduled in the NIGHTLY CI job, not in the
 * per-PR job, because axe analysis adds ~10–20 s per page and the results are
 * frozen baselines (see approach below).
 *
 * Approach — freeze-and-alert (not fail-on-first-violation):
 *   1. Run axe on each key page.
 *   2. Count `violations.length` per page.
 *   3. Assert the count has NOT increased since the frozen baseline.
 *   4. Report violations in the test output so they are visible in CI logs.
 *
 * This means:
 *   - Existing violations are GRANDFATHERED (baseline frozen).
 *   - A new violation (count grows) FAILS the gate — catraca `down`.
 *   - Fixing a violation (count drops) passes + you can lower the baseline.
 *
 * Graceful degradation:
 *   - If @axe-core/playwright is not installed the entire suite is skipped
 *     with a clear message instead of crashing the job.
 *   - The frozen baselines below are ADVISORY defaults (0). On the first real
 *     run update them to the actual counts (grep "axeViolationCount" in CI logs).
 *
 * Pages audited (key dashboard surfaces):
 *   /dashboard            — main overview
 *   /dashboard/providers  — provider management (most complex UI surface)
 *   /login                — public auth gate (a11y critical for users)
 *   /dashboard/settings   — settings (redirects to /dashboard/settings/general)
 *
 * Run locally (requires the app running on localhost:20128):
 *   npx playwright test tests/e2e/a11y.spec.ts --headed
 */

import { test, expect, type Page } from "@playwright/test";
import { gotoDashboardRoute } from "./helpers/dashboardAuth";

// ---------------------------------------------------------------------------
// Conditional import — skip entire suite if @axe-core/playwright is absent.
// ---------------------------------------------------------------------------

let AxeBuilder: (new (args: { page: Page }) => {
  analyze(): Promise<{ violations: Array<{ id: string; description: string; impact: string | null; nodes: unknown[] }> }>;
  withTags(tags: string[]): unknown;
  exclude(selector: string): unknown;
  disableRules(rules: string[]): unknown;
}) | null = null;

try {
  // Dynamic import so the module parse does not fail when the package is absent.
  const mod = await import("@axe-core/playwright");
  AxeBuilder = mod.default ?? (mod as unknown as { AxeBuilder: typeof AxeBuilder }).AxeBuilder ?? null;
} catch {
  // Package not installed — suite will skip gracefully below.
  AxeBuilder = null;
}

// ---------------------------------------------------------------------------
// Frozen violation baselines.
//
// Update these after the first real run by reading the "axeViolationCount"
// lines from the CI log and setting each value to the actual count.
// Format: { [pageLabel]: maxAllowedViolations }
// ---------------------------------------------------------------------------

// Frozen from the first real nightly measurement (run 27852779527, REQUIRE_AXE=1,
// wcag2a/2aa/21a/21aa). Each value is the actual `axeViolationCount` for that page —
// existing violations are grandfathered; a NEW violation (count grows) fails the gate.
// Lower a value (and re-run) whenever a violation is fixed.
const VIOLATION_BASELINES: Record<string, number> = {
  "/login": 1,
  "/dashboard": 4,
  "/dashboard/providers": 3,
  "/dashboard/settings": 5,
};

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

type AxeViolation = {
  id: string;
  description: string;
  impact: string | null;
  nodes: unknown[];
};

async function runAxe(
  page: Page,
  label: string
): Promise<AxeViolation[]> {
  if (!AxeBuilder) {
    throw new Error("@axe-core/playwright not available");
  }
  const results = await new AxeBuilder({ page })
    .withTags(["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"])
    // Exclude third-party iframes / injected widgets that we don't control.
    .exclude("[data-axe-exclude]")
    .analyze();

  // Emit machine-parseable line for CI baseline tracking.
  console.log(`axeViolationCount page=${label} count=${results.violations.length}`);

  if (results.violations.length > 0) {
    const summary = results.violations
      .map((v) => `  [${v.impact ?? "unknown"}] ${v.id}: ${v.description} (${(v.nodes as unknown[]).length} nodes)`)
      .join("\n");
    console.log(`axeViolations page=${label}:\n${summary}`);
  }

  return results.violations;
}

// ---------------------------------------------------------------------------
// Test suite
// ---------------------------------------------------------------------------

test.describe("A11y — Dashboard key surfaces (@axe-core, nightly advisory)", () => {
  test.beforeAll(() => {
    if (!AxeBuilder) {
      // Log once; individual tests will call test.skip().
      console.log(
        "[a11y.spec.ts] SKIP: @axe-core/playwright is not installed.\n" +
          "Install with: npm install --save-dev @axe-core/playwright"
      );
    }
  });

  // -------------------------------------------------------------------------
  // /login — public auth gate
  // -------------------------------------------------------------------------
  test("/login — axe wcag2a/wcag2aa violations must not exceed baseline", async ({ page }) => {
    if (!AxeBuilder || process.env.REQUIRE_AXE !== "1") {
      // Nightly-only: the real axe analysis (~10–20 s/page) runs in the nightly job
      // (REQUIRE_AXE=1), NOT in the per-PR e2e shards — a11y.spec.ts is matched by the
      // per-PR `tests/e2e/*.spec.ts` glob, so without this gate installing the package
      // would silently flip axe on for every PR (and fail at baseline 0).
      test.skip(
        true,
        AxeBuilder
          ? "axe analysis runs in the nightly job only (set REQUIRE_AXE=1)"
          : "@axe-core/playwright not installed"
      );
      return;
    }

    await page.goto("/login");
    await page.locator("body").waitFor({ state: "visible" });

    const violations = await runAxe(page, "/login");
    const baseline = VIOLATION_BASELINES["/login"] ?? 0;

    expect(violations.length).toBeLessThanOrEqual(
      baseline,
      `New a11y violations introduced on /login. ` +
        `Expected ≤${baseline}, got ${violations.length}. ` +
        `Run axe locally and update VIOLATION_BASELINES["/login"] if the new count is intentional.`
    );
  });

  // -------------------------------------------------------------------------
  // /dashboard — main overview
  // -------------------------------------------------------------------------
  test("/dashboard — axe wcag2a/wcag2aa violations must not exceed baseline", async ({ page }) => {
    if (!AxeBuilder || process.env.REQUIRE_AXE !== "1") {
      // Nightly-only: the real axe analysis (~10–20 s/page) runs in the nightly job
      // (REQUIRE_AXE=1), NOT in the per-PR e2e shards — a11y.spec.ts is matched by the
      // per-PR `tests/e2e/*.spec.ts` glob, so without this gate installing the package
      // would silently flip axe on for every PR (and fail at baseline 0).
      test.skip(
        true,
        AxeBuilder
          ? "axe analysis runs in the nightly job only (set REQUIRE_AXE=1)"
          : "@axe-core/playwright not installed"
      );
      return;
    }

    await gotoDashboardRoute(page, "/dashboard");

    const violations = await runAxe(page, "/dashboard");
    const baseline = VIOLATION_BASELINES["/dashboard"] ?? 0;

    expect(violations.length).toBeLessThanOrEqual(
      baseline,
      `New a11y violations introduced on /dashboard. ` +
        `Expected ≤${baseline}, got ${violations.length}.`
    );
  });

  // -------------------------------------------------------------------------
  // /dashboard/providers — provider management (most complex UI surface)
  // -------------------------------------------------------------------------
  test("/dashboard/providers — axe wcag2a/wcag2aa violations must not exceed baseline", async ({
    page,
  }) => {
    if (!AxeBuilder || process.env.REQUIRE_AXE !== "1") {
      // Nightly-only: the real axe analysis (~10–20 s/page) runs in the nightly job
      // (REQUIRE_AXE=1), NOT in the per-PR e2e shards — a11y.spec.ts is matched by the
      // per-PR `tests/e2e/*.spec.ts` glob, so without this gate installing the package
      // would silently flip axe on for every PR (and fail at baseline 0).
      test.skip(
        true,
        AxeBuilder
          ? "axe analysis runs in the nightly job only (set REQUIRE_AXE=1)"
          : "@axe-core/playwright not installed"
      );
      return;
    }

    await gotoDashboardRoute(page, "/dashboard/providers");

    const violations = await runAxe(page, "/dashboard/providers");
    const baseline = VIOLATION_BASELINES["/dashboard/providers"] ?? 0;

    expect(violations.length).toBeLessThanOrEqual(
      baseline,
      `New a11y violations introduced on /dashboard/providers. ` +
        `Expected ≤${baseline}, got ${violations.length}.`
    );
  });

  // -------------------------------------------------------------------------
  // /dashboard/settings — settings area
  // -------------------------------------------------------------------------
  test("/dashboard/settings — axe wcag2a/wcag2aa violations must not exceed baseline", async ({
    page,
  }) => {
    if (!AxeBuilder || process.env.REQUIRE_AXE !== "1") {
      // Nightly-only: the real axe analysis (~10–20 s/page) runs in the nightly job
      // (REQUIRE_AXE=1), NOT in the per-PR e2e shards — a11y.spec.ts is matched by the
      // per-PR `tests/e2e/*.spec.ts` glob, so without this gate installing the package
      // would silently flip axe on for every PR (and fail at baseline 0).
      test.skip(
        true,
        AxeBuilder
          ? "axe analysis runs in the nightly job only (set REQUIRE_AXE=1)"
          : "@axe-core/playwright not installed"
      );
      return;
    }

    // The settings route redirects to /dashboard/settings/general; follow it.
    await gotoDashboardRoute(page, "/dashboard/settings");

    const violations = await runAxe(page, "/dashboard/settings");
    const baseline = VIOLATION_BASELINES["/dashboard/settings"] ?? 0;

    expect(violations.length).toBeLessThanOrEqual(
      baseline,
      `New a11y violations introduced on /dashboard/settings. ` +
        `Expected ≤${baseline}, got ${violations.length}.`
    );
  });

  // -------------------------------------------------------------------------
  // Regression guard: suite is skippable but the skip reason must be explicit.
  // This test always runs (no AxeBuilder check) and verifies the skip is
  // legitimate (package absent) and not an infrastructure failure.
  // -------------------------------------------------------------------------
  test("axe package availability is declared (meta-test)", async () => {
    if (AxeBuilder !== null) {
      // Package is present — nothing to check.
      expect(AxeBuilder).toBeTruthy();
    } else {
      // Package absent — this is acceptable in PR CI; fatal in the NIGHTLY job.
      // In the nightly job, set REQUIRE_AXE=1 and the check below will fail.
      const requireAxe = process.env.REQUIRE_AXE === "1";
      if (requireAxe) {
        throw new Error(
          "REQUIRE_AXE=1 but @axe-core/playwright is not installed. " +
            "Add it as a devDependency and run npm install."
        );
      }
      // Advisory skip in PR context.
      test.skip(true, "@axe-core/playwright not installed — advisory skip in PR context");
    }
  });
});