4ac499f301
Implements the §14 Phase 3 review gate. requestReview() routes a building job into the review stage (fencing any worker), carrying a normalized policy (requiredApprovals + reviewer allowlist) and clearing prior decisions. submitReview() records one decision per reviewer (last-write-wins, identity- normalized), advances the job to testing once distinct approvals reach the quorum, and treats any reject as a veto that returns the job to queued for rework. Adds POST /fleet/jobs/:id/review/request and POST /fleet/jobs/:id/review, a typed client, and a review-gate card on the job-detail page. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
333 lines
11 KiB
TypeScript
333 lines
11 KiB
TypeScript
import { test, expect, type Page, type Route } from '@playwright/test';
|
|
|
|
/**
|
|
* E2E tests for the Fleet (Agent Gigafactory) control-plane dashboard.
|
|
*
|
|
* Deterministic: every backend call is mocked at the Next.js proxy boundary
|
|
* (`/api/fleet/**`, `/api/auth/**`). No live platform-service is required. The
|
|
* mock dispatcher is method- and URL-aware and holds a little mutable state so
|
|
* operator actions (requeue) and budget pause/resume reflect in follow-up GETs.
|
|
*/
|
|
|
|
const PRODUCT = 'tracker-e2e';
|
|
const ISO = '2025-01-01T00:00:00.000Z';
|
|
|
|
type Job = {
|
|
id: string;
|
|
productId: string;
|
|
stage: string;
|
|
idempotencyKey: string;
|
|
bodyMd: string;
|
|
priority: string;
|
|
priorityOrder: number;
|
|
capabilities: string[];
|
|
kind: string;
|
|
attempts: number;
|
|
leaseEpoch: number;
|
|
createdAt: string;
|
|
updatedAt: string;
|
|
reviewPolicy?: { requiredApprovals: number; reviewers: string[] };
|
|
reviewDecisions?: {
|
|
reviewer: string;
|
|
decision: 'approve' | 'reject';
|
|
at: string;
|
|
note?: string;
|
|
}[];
|
|
gate?: string;
|
|
};
|
|
|
|
function makeJob(partial: Partial<Job> & { id: string; idempotencyKey: string }): Job {
|
|
return {
|
|
productId: PRODUCT,
|
|
stage: 'building',
|
|
bodyMd: '# task',
|
|
priority: 'high',
|
|
priorityOrder: 1,
|
|
capabilities: ['build'],
|
|
kind: 'task',
|
|
attempts: 1,
|
|
leaseEpoch: 2,
|
|
createdAt: ISO,
|
|
updatedAt: ISO,
|
|
...partial,
|
|
};
|
|
}
|
|
|
|
const FACTORY = {
|
|
id: 'f1',
|
|
productId: PRODUCT,
|
|
factoryId: 'factory-alpha',
|
|
capabilities: ['build', 'test'],
|
|
health: 'ok' as const,
|
|
load: 1,
|
|
seatLimit: 4,
|
|
lastHeartbeatAt: ISO,
|
|
};
|
|
|
|
/** Authenticate the dashboard: seed the token + selected product, mock /me. */
|
|
async function authenticate(page: Page): Promise<void> {
|
|
await page.addInitScript(
|
|
([product]) => {
|
|
localStorage.setItem('tracker_token', 'fake-e2e-token');
|
|
localStorage.setItem('tracker_selected_product', product);
|
|
},
|
|
[PRODUCT]
|
|
);
|
|
await page.route('**/api/auth/me', (route: Route) =>
|
|
route.fulfill({
|
|
json: { id: 'u1', email: 'admin@example.com', role: 'admin', displayName: 'Admin' },
|
|
})
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Install the fleet API mock. Returns the mutable state so a test can inspect or
|
|
* assert on the latest values the dispatcher served.
|
|
*/
|
|
async function mockFleet(
|
|
page: Page,
|
|
opts?: { jobStage?: string; budgetStatus?: 'active' | 'paused' }
|
|
): Promise<{ state: { jobStage: string; budgetStatus: 'active' | 'paused' } }> {
|
|
const state = {
|
|
jobStage: opts?.jobStage ?? 'building',
|
|
budgetStatus: opts?.budgetStatus ?? ('active' as 'active' | 'paused'),
|
|
reviewDecisions: [] as { reviewer: string; decision: 'approve' | 'reject'; at: string }[],
|
|
};
|
|
|
|
const jobView = () =>
|
|
makeJob({
|
|
id: 'job-1',
|
|
idempotencyKey: 'feat-x',
|
|
stage: state.jobStage,
|
|
...(state.jobStage === 'review'
|
|
? {
|
|
reviewPolicy: { requiredApprovals: 2, reviewers: ['admin@example.com', 'bob@x.com'] },
|
|
reviewDecisions: state.reviewDecisions,
|
|
}
|
|
: {}),
|
|
});
|
|
|
|
const budget = () => ({
|
|
id: PRODUCT,
|
|
productId: PRODUCT,
|
|
ceilingUsd: 100,
|
|
window: 'monthly',
|
|
spentUsd: 25,
|
|
status: state.budgetStatus,
|
|
updatedAt: ISO,
|
|
});
|
|
|
|
await page.route('**/api/fleet/**', (route: Route) => {
|
|
const req = route.request();
|
|
const url = new URL(req.url());
|
|
const path = url.pathname; // e.g. /api/fleet/jobs/job-1/events
|
|
const method = req.method();
|
|
|
|
// ── Live event stream (SSE) ──
|
|
if (path.includes('/events/stream')) {
|
|
const evt = {
|
|
id: 'job-1:evt:0',
|
|
jobId: 'job-1',
|
|
seq: 0,
|
|
type: 'submitted',
|
|
at: ISO,
|
|
data: {},
|
|
};
|
|
return route.fulfill({
|
|
status: 200,
|
|
headers: { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' },
|
|
body: `retry: 3000\n\nid: 0\nevent: fleet-event\ndata: ${JSON.stringify(evt)}\n\n`,
|
|
});
|
|
}
|
|
if (path.endsWith('/events')) {
|
|
return route.fulfill({
|
|
json: {
|
|
events: [
|
|
{ id: 'job-1:evt:0', jobId: 'job-1', seq: 0, type: 'submitted', at: ISO, data: {} },
|
|
],
|
|
},
|
|
});
|
|
}
|
|
if (path.endsWith('/runs')) return route.fulfill({ json: { runs: [] } });
|
|
if (path.endsWith('/artifacts')) return route.fulfill({ json: { artifacts: [] } });
|
|
// DAG + explain: degrade to 404 so the page renders without those panels.
|
|
// Body omits `error` so the client throws "HTTP 404" (apiFetchOptional → null).
|
|
if (path.endsWith('/dag') || path.endsWith('/explain')) {
|
|
return route.fulfill({ status: 404, json: {} });
|
|
}
|
|
|
|
// ── Operator actions (requeue / cancel / reject) ──
|
|
const actionMatch = path.match(/\/jobs\/[^/]+\/actions\/(\w+)$/);
|
|
if (actionMatch && method === 'POST') {
|
|
const action = actionMatch[1];
|
|
state.jobStage =
|
|
action === 'requeue' ? 'queued' : action === 'reject' ? 'dead_letter' : 'failed';
|
|
return route.fulfill({
|
|
json: makeJob({ id: 'job-1', idempotencyKey: 'feat-x', stage: state.jobStage }),
|
|
});
|
|
}
|
|
|
|
// ── Review gate (multi-reviewer) ──
|
|
if (path.endsWith('/review/request') && method === 'POST') {
|
|
state.jobStage = 'review';
|
|
state.reviewDecisions = [];
|
|
return route.fulfill({ json: jobView() });
|
|
}
|
|
if (path.endsWith('/review') && method === 'POST') {
|
|
const payload = req.postDataJSON() as {
|
|
reviewer: string;
|
|
decision: 'approve' | 'reject';
|
|
};
|
|
if (payload.decision === 'reject') {
|
|
state.jobStage = 'queued';
|
|
return route.fulfill({ json: { ...jobView(), gate: 'rejected' } });
|
|
}
|
|
state.reviewDecisions = state.reviewDecisions
|
|
.filter(d => d.reviewer !== payload.reviewer)
|
|
.concat({ reviewer: payload.reviewer, decision: 'approve', at: ISO });
|
|
const approvals = new Set(
|
|
state.reviewDecisions.filter(d => d.decision === 'approve').map(d => d.reviewer)
|
|
).size;
|
|
const gate = approvals >= 2 ? 'approved' : 'pending';
|
|
if (gate === 'approved') state.jobStage = 'testing';
|
|
return route.fulfill({ json: { ...jobView(), gate } });
|
|
}
|
|
|
|
// ── Budgets ──
|
|
if (path.endsWith('/burndown')) return route.fulfill({ status: 404, json: {} });
|
|
if (path.endsWith('/pause') && method === 'POST') {
|
|
state.budgetStatus = 'paused';
|
|
return route.fulfill({ json: budget() });
|
|
}
|
|
if (path.endsWith('/resume') && method === 'POST') {
|
|
state.budgetStatus = 'active';
|
|
return route.fulfill({ json: budget() });
|
|
}
|
|
if (path.match(/\/budgets\/[^/]+$/)) return route.fulfill({ json: budget() });
|
|
|
|
// ── Jobs ──
|
|
if (path.match(/\/jobs\/[^/]+$/) && method === 'GET') {
|
|
return route.fulfill({ json: jobView() });
|
|
}
|
|
if (path.endsWith('/jobs') && method === 'GET') {
|
|
return route.fulfill({ json: { jobs: [jobView()] } });
|
|
}
|
|
if (path.endsWith('/factories')) return route.fulfill({ json: { factories: [FACTORY] } });
|
|
|
|
// ── Fleet metrics ──
|
|
if (path.endsWith('/metrics')) {
|
|
return route.fulfill({
|
|
json: {
|
|
productId: 'lysnrai',
|
|
generatedAt: ISO,
|
|
jobs: {
|
|
total: 1,
|
|
byStage: { queued: 1 },
|
|
queueDepth: 1,
|
|
blocked: 0,
|
|
active: 0,
|
|
oldestQueuedAgeMs: 1000,
|
|
},
|
|
factories: {
|
|
total: 1,
|
|
live: 1,
|
|
stale: 0,
|
|
byHealth: { ok: 1, degraded: 0, down: 0 },
|
|
seatsUsed: 1,
|
|
seatsTotal: 4,
|
|
utilizationPct: 25,
|
|
},
|
|
alerts: [
|
|
{ level: 'warning', code: 'queue_starvation', message: 'A job has waited too long.' },
|
|
],
|
|
},
|
|
});
|
|
}
|
|
|
|
return route.fulfill({ json: {} });
|
|
});
|
|
|
|
return { state };
|
|
}
|
|
|
|
test.describe('Fleet — Overview', () => {
|
|
test('renders factory cards and the recent-jobs table', async ({ page }) => {
|
|
await authenticate(page);
|
|
await mockFleet(page);
|
|
await page.goto('/dashboard/fleet');
|
|
|
|
await expect(page.getByRole('heading', { name: 'Fleet Control Plane' })).toBeVisible();
|
|
await expect(page.getByTestId('fleet-metrics')).toBeVisible();
|
|
await expect(page.getByTestId('fleet-alerts')).toBeVisible();
|
|
await expect(page.getByLabel('Factory factory-alpha')).toBeVisible();
|
|
await expect(page.getByRole('table', { name: 'Recent fleet jobs' })).toBeVisible();
|
|
await expect(page.getByRole('link', { name: 'feat-x' })).toBeVisible();
|
|
});
|
|
});
|
|
|
|
test.describe('Fleet — Jobs table', () => {
|
|
test('lists jobs and exposes the stage filter', async ({ page }) => {
|
|
await authenticate(page);
|
|
await mockFleet(page);
|
|
await page.goto('/dashboard/fleet/jobs');
|
|
|
|
await expect(page.getByRole('heading', { name: 'Fleet Jobs' })).toBeVisible();
|
|
await expect(page.getByRole('table', { name: 'Fleet jobs table' })).toBeVisible();
|
|
await expect(page.getByLabel('Filter by stage')).toBeVisible();
|
|
await expect(page.getByRole('link', { name: 'View job feat-x' })).toBeVisible();
|
|
});
|
|
});
|
|
|
|
test.describe('Fleet — Job detail', () => {
|
|
test('requeues a job and reflects the new stage; shows the live badge', async ({ page }) => {
|
|
await authenticate(page);
|
|
await mockFleet(page, { jobStage: 'building' });
|
|
await page.goto('/dashboard/fleet/jobs/job-1');
|
|
|
|
await expect(page.getByRole('heading', { name: 'feat-x' })).toBeVisible();
|
|
// SSE snapshot delivers an event → the Live indicator should appear.
|
|
await expect(page.getByTestId('live-indicator')).toBeVisible();
|
|
// Stage card starts at 'building'.
|
|
await expect(page.getByText('building', { exact: true })).toBeVisible();
|
|
|
|
await page.getByRole('button', { name: 'Requeue this job' }).click();
|
|
// After requeue the coordinator returns stage 'queued', mirrored on refresh.
|
|
await expect(page.getByText('queued', { exact: true }).first()).toBeVisible();
|
|
});
|
|
});
|
|
|
|
test.describe('Fleet — Review gate', () => {
|
|
test('routes a building job to review and approves through the gate', async ({ page }) => {
|
|
await authenticate(page);
|
|
await mockFleet(page, { jobStage: 'building' });
|
|
await page.goto('/dashboard/fleet/jobs/job-1');
|
|
|
|
await expect(page.getByRole('heading', { name: 'feat-x' })).toBeVisible();
|
|
// Send the building job to review.
|
|
await page.getByRole('button', { name: 'Send this job to review' }).click();
|
|
await expect(page.getByTestId('review-gate')).toBeVisible();
|
|
await expect(page.getByTestId('review-progress')).toHaveText('0 / 2 approvals');
|
|
|
|
// First approval (admin@example.com) keeps the gate pending at 1/2.
|
|
await page.getByRole('button', { name: 'Approve this job' }).click();
|
|
await expect(page.getByTestId('review-progress')).toHaveText('1 / 2 approvals');
|
|
});
|
|
});
|
|
|
|
test.describe('Fleet — Budget', () => {
|
|
test('pauses and resumes the budget', async ({ page }) => {
|
|
await authenticate(page);
|
|
await mockFleet(page, { budgetStatus: 'active' });
|
|
await page.goto('/dashboard/fleet/budget');
|
|
|
|
await expect(page.getByRole('heading', { name: 'Fleet Budget' })).toBeVisible();
|
|
await expect(page.getByText('active', { exact: true })).toBeVisible();
|
|
|
|
await page.getByRole('button', { name: 'Pause budget' }).click();
|
|
await expect(page.getByText('paused', { exact: true })).toBeVisible();
|
|
|
|
await page.getByRole('button', { name: 'Resume budget' }).click();
|
|
await expect(page.getByText('active', { exact: true })).toBeVisible();
|
|
});
|
|
});
|