learning_ai_common_plat/dashboards/tracker-web/e2e/fleet.spec.ts

import { test, expect, type Page, type Route } from '@playwright/test';

/**
 * E2E tests for the Fleet (Agent Gigafactory) control-plane dashboard.
 *
 * Deterministic: every backend call is mocked at the Next.js proxy boundary
 * (`/api/fleet/**`, `/api/auth/**`). No live platform-service is required. The
 * mock dispatcher is method- and URL-aware and holds a little mutable state so
 * operator actions (requeue) and budget pause/resume reflect in follow-up GETs.
 */

const PRODUCT = 'tracker-e2e';
const ISO = '2025-01-01T00:00:00.000Z';

type Job = {
  id: string;
  productId: string;
  stage: string;
  idempotencyKey: string;
  bodyMd: string;
  priority: string;
  priorityOrder: number;
  capabilities: string[];
  kind: string;
  attempts: number;
  leaseEpoch: number;
  createdAt: string;
  updatedAt: string;
  reviewPolicy?: { requiredApprovals: number; reviewers: string[] };
  reviewDecisions?: {
    reviewer: string;
    decision: 'approve' | 'reject';
    at: string;
    note?: string;
  }[];
  gate?: string;
};

function makeJob(partial: Partial<Job> & { id: string; idempotencyKey: string }): Job {
  return {
    productId: PRODUCT,
    stage: 'building',
    bodyMd: '# task',
    priority: 'high',
    priorityOrder: 1,
    capabilities: ['build'],
    kind: 'task',
    attempts: 1,
    leaseEpoch: 2,
    createdAt: ISO,
    updatedAt: ISO,
    ...partial,
  };
}

const FACTORY = {
  id: 'f1',
  productId: PRODUCT,
  factoryId: 'factory-alpha',
  capabilities: ['build', 'test'],
  health: 'ok' as const,
  load: 1,
  seatLimit: 4,
  lastHeartbeatAt: ISO,
};

/** Authenticate the dashboard: seed the token + selected product, mock /me. */
async function authenticate(page: Page): Promise<void> {
  await page.addInitScript(
    ([product]) => {
      localStorage.setItem('tracker_token', 'fake-e2e-token');
      localStorage.setItem('tracker_selected_product', product);
    },
    [PRODUCT]
  );
  await page.route('**/api/auth/me', (route: Route) =>
    route.fulfill({
      json: { id: 'u1', email: 'admin@example.com', role: 'admin', displayName: 'Admin' },
    })
  );
}

/**
 * Install the fleet API mock. Returns the mutable state so a test can inspect or
 * assert on the latest values the dispatcher served.
 */
async function mockFleet(
  page: Page,
  opts?: { jobStage?: string; budgetStatus?: 'active' | 'paused' }
): Promise<{ state: { jobStage: string; budgetStatus: 'active' | 'paused' } }> {
  const state = {
    jobStage: opts?.jobStage ?? 'building',
    budgetStatus: opts?.budgetStatus ?? ('active' as 'active' | 'paused'),
    reviewDecisions: [] as { reviewer: string; decision: 'approve' | 'reject'; at: string }[],
  };

  const jobView = () =>
    makeJob({
      id: 'job-1',
      idempotencyKey: 'feat-x',
      stage: state.jobStage,
      ...(state.jobStage === 'review'
        ? {
            reviewPolicy: { requiredApprovals: 2, reviewers: ['admin@example.com', 'bob@x.com'] },
            reviewDecisions: state.reviewDecisions,
          }
        : {}),
    });

  const budget = () => ({
    id: PRODUCT,
    productId: PRODUCT,
    ceilingUsd: 100,
    window: 'monthly',
    spentUsd: 25,
    status: state.budgetStatus,
    updatedAt: ISO,
  });

  await page.route('**/api/fleet/**', (route: Route) => {
    const req = route.request();
    const url = new URL(req.url());
    const path = url.pathname; // e.g. /api/fleet/jobs/job-1/events
    const method = req.method();

    // ── Live event stream (SSE) ──
    if (path.includes('/events/stream')) {
      const evt = {
        id: 'job-1:evt:0',
        jobId: 'job-1',
        seq: 0,
        type: 'submitted',
        at: ISO,
        data: {},
      };
      return route.fulfill({
        status: 200,
        headers: { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' },
        body: `retry: 3000\n\nid: 0\nevent: fleet-event\ndata: ${JSON.stringify(evt)}\n\n`,
      });
    }
    if (path.endsWith('/events')) {
      return route.fulfill({
        json: {
          events: [
            { id: 'job-1:evt:0', jobId: 'job-1', seq: 0, type: 'submitted', at: ISO, data: {} },
          ],
        },
      });
    }
    if (path.endsWith('/runs')) return route.fulfill({ json: { runs: [] } });
    if (path.endsWith('/artifacts')) return route.fulfill({ json: { artifacts: [] } });
    // DAG + explain: degrade to 404 so the page renders without those panels.
    // Body omits `error` so the client throws "HTTP 404" (apiFetchOptional → null).
    if (path.endsWith('/dag') || path.endsWith('/explain')) {
      return route.fulfill({ status: 404, json: {} });
    }

    // ── Operator actions (requeue / cancel / reject) ──
    const actionMatch = path.match(/\/jobs\/[^/]+\/actions\/(\w+)$/);
    if (actionMatch && method === 'POST') {
      const action = actionMatch[1];
      state.jobStage =
        action === 'requeue' ? 'queued' : action === 'reject' ? 'dead_letter' : 'failed';
      return route.fulfill({
        json: makeJob({ id: 'job-1', idempotencyKey: 'feat-x', stage: state.jobStage }),
      });
    }

    // ── Review gate (multi-reviewer) ──
    if (path.endsWith('/review/request') && method === 'POST') {
      state.jobStage = 'review';
      state.reviewDecisions = [];
      return route.fulfill({ json: jobView() });
    }
    if (path.endsWith('/review') && method === 'POST') {
      const payload = req.postDataJSON() as {
        reviewer: string;
        decision: 'approve' | 'reject';
      };
      if (payload.decision === 'reject') {
        state.jobStage = 'queued';
        return route.fulfill({ json: { ...jobView(), gate: 'rejected' } });
      }
      state.reviewDecisions = state.reviewDecisions
        .filter(d => d.reviewer !== payload.reviewer)
        .concat({ reviewer: payload.reviewer, decision: 'approve', at: ISO });
      const approvals = new Set(
        state.reviewDecisions.filter(d => d.decision === 'approve').map(d => d.reviewer)
      ).size;
      const gate = approvals >= 2 ? 'approved' : 'pending';
      if (gate === 'approved') state.jobStage = 'testing';
      return route.fulfill({ json: { ...jobView(), gate } });
    }

    // ── Budgets ──
    if (path.endsWith('/burndown')) return route.fulfill({ status: 404, json: {} });
    if (path.endsWith('/pause') && method === 'POST') {
      state.budgetStatus = 'paused';
      return route.fulfill({ json: budget() });
    }
    if (path.endsWith('/resume') && method === 'POST') {
      state.budgetStatus = 'active';
      return route.fulfill({ json: budget() });
    }
    if (path.match(/\/budgets\/[^/]+$/)) return route.fulfill({ json: budget() });

    // ── Jobs ──
    if (path.match(/\/jobs\/[^/]+$/) && method === 'GET') {
      return route.fulfill({ json: jobView() });
    }
    if (path.endsWith('/jobs') && method === 'GET') {
      return route.fulfill({ json: { jobs: [jobView()] } });
    }
    if (path.endsWith('/factories')) return route.fulfill({ json: { factories: [FACTORY] } });

    // ── Fleet metrics ──
    if (path.endsWith('/metrics')) {
      return route.fulfill({
        json: {
          productId: 'lysnrai',
          generatedAt: ISO,
          jobs: {
            total: 1,
            byStage: { queued: 1 },
            queueDepth: 1,
            blocked: 0,
            active: 0,
            oldestQueuedAgeMs: 1000,
          },
          factories: {
            total: 1,
            live: 1,
            stale: 0,
            byHealth: { ok: 1, degraded: 0, down: 0 },
            seatsUsed: 1,
            seatsTotal: 4,
            utilizationPct: 25,
          },
          alerts: [
            { level: 'warning', code: 'queue_starvation', message: 'A job has waited too long.' },
          ],
        },
      });
    }

    return route.fulfill({ json: {} });
  });

  return { state };
}

test.describe('Fleet — Overview', () => {
  test('renders factory cards and the recent-jobs table', async ({ page }) => {
    await authenticate(page);
    await mockFleet(page);
    await page.goto('/dashboard/fleet');

    await expect(page.getByRole('heading', { name: 'Fleet Control Plane' })).toBeVisible();
    await expect(page.getByTestId('fleet-metrics')).toBeVisible();
    await expect(page.getByTestId('fleet-alerts')).toBeVisible();
    await expect(page.getByLabel('Factory factory-alpha')).toBeVisible();
    await expect(page.getByRole('table', { name: 'Recent fleet jobs' })).toBeVisible();
    await expect(page.getByRole('link', { name: 'feat-x' })).toBeVisible();
  });
});

test.describe('Fleet — Jobs table', () => {
  test('lists jobs and exposes the stage filter', async ({ page }) => {
    await authenticate(page);
    await mockFleet(page);
    await page.goto('/dashboard/fleet/jobs');

    await expect(page.getByRole('heading', { name: 'Fleet Jobs' })).toBeVisible();
    await expect(page.getByRole('table', { name: 'Fleet jobs table' })).toBeVisible();
    await expect(page.getByLabel('Filter by stage')).toBeVisible();
    await expect(page.getByRole('link', { name: 'View job feat-x' })).toBeVisible();
  });
});

test.describe('Fleet — Job detail', () => {
  test('requeues a job and reflects the new stage; shows the live badge', async ({ page }) => {
    await authenticate(page);
    await mockFleet(page, { jobStage: 'building' });
    await page.goto('/dashboard/fleet/jobs/job-1');

    await expect(page.getByRole('heading', { name: 'feat-x' })).toBeVisible();
    // SSE snapshot delivers an event → the Live indicator should appear.
    await expect(page.getByTestId('live-indicator')).toBeVisible();
    // Stage card starts at 'building'.
    await expect(page.getByText('building', { exact: true })).toBeVisible();

    await page.getByRole('button', { name: 'Requeue this job' }).click();
    // After requeue the coordinator returns stage 'queued', mirrored on refresh.
    await expect(page.getByText('queued', { exact: true }).first()).toBeVisible();
  });

  test('surfaces the polling indicator when the live stream is unavailable', async ({ page }) => {
    await authenticate(page);
    await mockFleet(page, { jobStage: 'building' });
    // Override just the SSE stream to fail → the client falls back to polling.
    // Registered after mockFleet so it takes precedence over the catch-all.
    await page.route('**/api/fleet/jobs/*/events/stream', (route: Route) =>
      route.fulfill({ status: 500, body: 'stream down' })
    );
    await page.goto('/dashboard/fleet/jobs/job-1');

    await expect(page.getByRole('heading', { name: 'feat-x' })).toBeVisible();
    // The degraded transport must be visible to the operator, not silent.
    await expect(page.getByTestId('polling-indicator')).toBeVisible();
    await expect(page.getByTestId('live-indicator')).toHaveCount(0);
    // Events still render via the polling fallback (GET /events).
    await expect(page.getByText('submitted', { exact: true })).toBeVisible();
  });
});

test.describe('Fleet — Review gate', () => {
  test('routes a building job to review and approves through the gate', async ({ page }) => {
    await authenticate(page);
    await mockFleet(page, { jobStage: 'building' });
    await page.goto('/dashboard/fleet/jobs/job-1');

    await expect(page.getByRole('heading', { name: 'feat-x' })).toBeVisible();
    // Send the building job to review.
    await page.getByRole('button', { name: 'Send this job to review' }).click();
    await expect(page.getByTestId('review-gate')).toBeVisible();
    await expect(page.getByTestId('review-progress')).toHaveText('0 / 2 approvals');

    // First approval (admin@example.com) keeps the gate pending at 1/2.
    await page.getByRole('button', { name: 'Approve this job' }).click();
    await expect(page.getByTestId('review-progress')).toHaveText('1 / 2 approvals');
  });
});

test.describe('Fleet — Budget', () => {
  test('pauses and resumes the budget', async ({ page }) => {
    await authenticate(page);
    await mockFleet(page, { budgetStatus: 'active' });
    await page.goto('/dashboard/fleet/budget');

    await expect(page.getByRole('heading', { name: 'Fleet Budget' })).toBeVisible();
    await expect(page.getByText('active', { exact: true })).toBeVisible();

    await page.getByRole('button', { name: 'Pause budget' }).click();
    await expect(page.getByText('paused', { exact: true })).toBeVisible();

    await page.getByRole('button', { name: 'Resume budget' }).click();
    await expect(page.getByText('active', { exact: true })).toBeVisible();
  });
});