diff --git a/dashboards/tracker-web/src/__tests__/fleet-overview.test.tsx b/dashboards/tracker-web/src/__tests__/fleet-overview.test.tsx new file mode 100644 index 00000000..dd537848 --- /dev/null +++ b/dashboards/tracker-web/src/__tests__/fleet-overview.test.tsx @@ -0,0 +1,178 @@ +// @vitest-environment happy-dom +/** + * Fleet overview page (§1) — the consolidated operations surface. + * + * Verifies the panels that surface the §2/§3 signal: the engine circuit-breaker + * panel, the budget guardrail, and the dead-letter triage list with its Re-drive + * button (which calls the operator action). Uses the jsdom render harness + + * mocked fleet-client/auth-context (no network). + */ + +import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest'; +import { act, createElement } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; + +const { operatorAction, listFactories, getFleetMetrics, listJobs } = vi.hoisted(() => ({ + operatorAction: vi.fn(), + listFactories: vi.fn(), + getFleetMetrics: vi.fn(), + listJobs: vi.fn(), +})); + +vi.mock('@/lib/auth-context', () => ({ useAuth: () => ({ token: 'tok' }) })); +vi.mock('@/lib/fleet-client', () => ({ + listFactories, + getFleetMetrics, + listJobs, + operatorAction, +})); + +import FleetOverviewPage from '@/app/dashboard/fleet/page'; + +beforeAll(() => { + (globalThis as Record).IS_REACT_ACT_ENVIRONMENT = true; +}); + +const metricsBase = { + productId: 'p', + generatedAt: '2026-06-01T00:00:00.000Z', + jobs: { total: 1, byStage: {}, queueDepth: 0, blocked: 0, active: 0, oldestQueuedAgeMs: null }, + factories: { + total: 0, + live: 0, + stale: 0, + byHealth: { ok: 0, degraded: 0, down: 0 }, + seatsUsed: 0, + seatsTotal: 0, + utilizationPct: 0, + }, + alerts: [], +}; + +const deadLetterJob = { + id: 'j-dead', + productId: 'p', + stage: 'dead_letter', + idempotencyKey: 'broken-task', + bodyMd: '', + priority: 'medium', + priorityOrder: 2, + capabilities: [], + kind: 'leaf', + attempts: 3, + leaseEpoch: 3, + engine: 'codex', + createdAt: '2026-06-01T00:00:00.000Z', + updatedAt: '2026-06-01T00:00:00.000Z', +}; + +async function renderPage(): Promise<{ root: Root; container: HTMLDivElement }> { + const container = document.createElement('div'); + document.body.appendChild(container); + let root!: Root; + await act(async () => { + root = createRoot(container); + root.render(createElement(FleetOverviewPage)); + }); + // Flush the async refresh() (Promise.all of the mocked client calls). + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + return { root, container }; +} + +beforeEach(() => { + vi.clearAllMocks(); + listFactories.mockResolvedValue({ factories: [] }); + operatorAction.mockResolvedValue({}); + // Default: recent jobs empty, dead-letter query returns the broken job. + listJobs.mockImplementation(async (params?: { stage?: string }) => + params?.stage === 'dead_letter' ? { jobs: [deadLetterJob] } : { jobs: [] } + ); +}); + +describe('fleet overview §1 — consolidated panels', () => { + it('renders the budget guardrail with projection and per-engine breakdown', async () => { + getFleetMetrics.mockResolvedValue({ + ...metricsBase, + budget: { + ceilingUsd: 100, + spentUsd: 40, + status: 'active', + window: 'monthly', + projectedUsd: 400, + engines: [{ engine: 'codex', spentUsd: 30, ceilingUsd: 30, exhausted: true }], + }, + }); + const { root, container } = await renderPage(); + const budget = container.querySelector('[data-testid="fleet-budget"]'); + expect(budget).not.toBeNull(); + expect(budget!.textContent).toContain('$40.00 / $100.00'); + expect(budget!.textContent).toContain('Projected monthly: $400.00'); + expect(budget!.textContent).toContain('codex'); + act(() => root.unmount()); + container.remove(); + }); + + it('renders only tripped circuit breakers', async () => { + getFleetMetrics.mockResolvedValue({ + ...metricsBase, + engineBreakers: [ + { + factoryId: 'fac_1', + engine: 'codex', + state: 'OPEN', + failureCount: 3, + lastFailureAt: null, + }, + { + factoryId: 'fac_1', + engine: 'devin', + state: 'CLOSED', + failureCount: 0, + lastFailureAt: null, + }, + ], + }); + const { root, container } = await renderPage(); + const panel = container.querySelector('[data-testid="fleet-breakers"]'); + expect(panel).not.toBeNull(); + expect(panel!.textContent).toContain('fac_1 · codex'); + expect(panel!.textContent).toContain('OPEN'); + expect(panel!.textContent).not.toContain('devin'); // CLOSED pairs are hidden + act(() => root.unmount()); + container.remove(); + }); + + it('lists dead-letter jobs and re-drives one via the operator action', async () => { + getFleetMetrics.mockResolvedValue({ ...metricsBase }); + const { root, container } = await renderPage(); + const triage = container.querySelector('[data-testid="fleet-dead-letter"]'); + expect(triage).not.toBeNull(); + expect(triage!.textContent).toContain('broken-task'); + + const btn = container.querySelector( + '[aria-label="Re-drive job broken-task"]' + ) as HTMLButtonElement; + expect(btn).not.toBeNull(); + await act(async () => { + btn.click(); + await Promise.resolve(); + }); + expect(operatorAction).toHaveBeenCalledWith('j-dead', 'redrive'); + act(() => root.unmount()); + container.remove(); + }); + + it('hides the budget and breaker panels when the metrics omit them', async () => { + getFleetMetrics.mockResolvedValue({ ...metricsBase }); + listJobs.mockResolvedValue({ jobs: [] }); // no dead letters either + const { root, container } = await renderPage(); + expect(container.querySelector('[data-testid="fleet-budget"]')).toBeNull(); + expect(container.querySelector('[data-testid="fleet-breakers"]')).toBeNull(); + expect(container.querySelector('[data-testid="fleet-dead-letter"]')).toBeNull(); + act(() => root.unmount()); + container.remove(); + }); +}); diff --git a/dashboards/tracker-web/src/app/dashboard/fleet/page.tsx b/dashboards/tracker-web/src/app/dashboard/fleet/page.tsx index c39ae207..a0183021 100644 --- a/dashboards/tracker-web/src/app/dashboard/fleet/page.tsx +++ b/dashboards/tracker-web/src/app/dashboard/fleet/page.tsx @@ -8,6 +8,7 @@ import { listFactories, listJobs, getFleetMetrics, + operatorAction, type FleetFactory, type FleetJob, type FleetMetrics, @@ -56,23 +57,114 @@ function StageBadge({ stage }: { stage: string }) { ); } +const usd = (n: number) => `$${n.toFixed(2)}`; + +/** Budget guardrail: spend vs ceiling + projected end-of-window burn (§3). */ +function BudgetGuardrail({ budget }: { budget: NonNullable }) { + const pct = + budget.ceilingUsd > 0 ? Math.min(100, (budget.spentUsd / budget.ceilingUsd) * 100) : 0; + const overProjected = budget.projectedUsd !== null && budget.projectedUsd > budget.ceilingUsd; + const exhausted = budget.spentUsd >= budget.ceilingUsd; + return ( +
+

Budget

+
+
+ + {usd(budget.spentUsd)} / {usd(budget.ceilingUsd)}{' '} + ({budget.window}) + + {budget.status} +
+
+
+
+ {budget.projectedUsd !== null && ( +

+ Projected {budget.window}: {usd(budget.projectedUsd)} + {overProjected && ' — over ceiling at current burn'} +

+ )} + {budget.engines.length > 0 && ( +
    + {budget.engines.map(e => ( +
  • + {e.engine} + + {usd(e.spentUsd)} / {usd(e.ceilingUsd)} + {e.exhausted && ' (capped)'} + +
    • + ))} +
+ )} +
+
+ ); +} + +/** Circuit-breaker panel: (factory, engine) pairs currently being routed around (§2). */ +function BreakerPanel({ breakers }: { breakers: NonNullable }) { + const tripped = breakers.filter(b => b.state !== 'CLOSED'); + if (tripped.length === 0) return null; + return ( +
+

Engine circuit breakers

+
+ {tripped.map(b => ( +
+
+ + {b.factoryId} · {b.engine} + + + {b.state} + +
+

+ {b.failureCount} failure(s) + {b.lastFailureAt && ` · last ${new Date(b.lastFailureAt).toLocaleTimeString()}`} +

+
+ ))} +
+
+ ); +} + export default function FleetOverviewPage() { const { token } = useAuth(); const [factories, setFactories] = useState([]); const [jobs, setJobs] = useState([]); const [metrics, setMetrics] = useState(null); + const [deadLetters, setDeadLetters] = useState([]); + const [redriving, setRedriving] = useState(null); const [loading, setLoading] = useState(true); const refresh = useCallback(async () => { try { - const [facRes, jobRes, metricsRes] = await Promise.all([ + const [facRes, jobRes, metricsRes, dlRes] = await Promise.all([ listFactories(), listJobs({ limit: 10 }), getFleetMetrics().catch(() => null), + listJobs({ stage: 'dead_letter', limit: 50 }).catch(() => ({ jobs: [] as FleetJob[] })), ]); setFactories(facRes.factories); setJobs(jobRes.jobs); setMetrics(metricsRes); + // Filter client-side too, so the triage list is correct even if the server + // ignores the stage filter. + setDeadLetters(dlRes.jobs.filter(j => j.stage === 'dead_letter')); } catch { /* degrade gracefully */ } finally { @@ -80,6 +172,21 @@ export default function FleetOverviewPage() { } }, []); + const handleRedrive = useCallback( + async (id: string) => { + setRedriving(id); + try { + await operatorAction(id, 'redrive'); + await refresh(); + } catch { + /* surfaced on next poll */ + } finally { + setRedriving(null); + } + }, + [refresh] + ); + useEffect(() => { if (!token) return; refresh(); @@ -134,6 +241,67 @@ export default function FleetOverviewPage() { )} + {/* Budget guardrail */} + {metrics?.budget && } + + {/* Engine circuit breakers */} + {metrics?.engineBreakers && metrics.engineBreakers.length > 0 && ( + + )} + + {/* Dead-letter triage */} + {deadLetters.length > 0 && ( +
+

+ Dead-letter triage{' '} + + ({deadLetters.length}) + +

+
+ + + + + + + + + + + {deadLetters.map(j => ( + + + + + + + ))} + +
KeyEngineAttemptsAction
+ + {j.idempotencyKey} + + + {j.engine ?? j.engineClass ?? '—'} + {j.attempts} + +
+
+
+ )} + {/* Factory cards */}

Factories

diff --git a/dashboards/tracker-web/src/lib/fleet-client.ts b/dashboards/tracker-web/src/lib/fleet-client.ts index caaf6655..916cb280 100644 --- a/dashboards/tracker-web/src/lib/fleet-client.ts +++ b/dashboards/tracker-web/src/lib/fleet-client.ts @@ -31,6 +31,8 @@ export interface FleetJob { reviewDecisions?: ReviewDecision[]; /** Concrete engine to run (overrides engineClass); falls back to factory default. */ engine?: FleetEngine; + /** Abstract engine class resolved on the runner when no concrete `engine` is set. */ + engineClass?: string; repo?: string; baseBranch?: string; /** PR mode: verify command run in the checkout before the PR opens. */ @@ -346,6 +348,25 @@ export interface FleetAlert { message: string; } +/** One tripped/probing (factory, engine) circuit-breaker pair (§2). */ +export interface EngineBreakerEntry { + factoryId: string; + engine: string; + state: 'CLOSED' | 'OPEN' | 'HALF_OPEN'; + failureCount: number; + lastFailureAt: string | null; +} + +/** Budget guardrail summary surfaced on metrics (§3). */ +export interface FleetBudgetSummary { + ceilingUsd: number; + spentUsd: number; + status: string; + window: string; + projectedUsd: number | null; + engines: { engine: string; spentUsd: number; ceilingUsd: number; exhausted: boolean }[]; +} + export interface FleetMetrics { productId: string; generatedAt: string; @@ -366,6 +387,10 @@ export interface FleetMetrics { seatsTotal: number; utilizationPct: number; }; + /** Budget guardrail summary (§3) — null when no budget is configured. */ + budget?: FleetBudgetSummary | null; + /** Per-(factory, engine) circuit-breaker snapshot (§2) — process-wide. */ + engineBreakers?: EngineBreakerEntry[]; alerts: FleetAlert[]; }