/**
 * Fleet coordinator — the concurrency core (Phase 2 §4/§7/§8/§18/§25).
 *
 * Responsibilities:
 *  - submitJob       idempotent submit + submit-time dependency cycle detection
 *  - claimNextJob    atomic, single-winner claim (priority+age, deps + capability
 *                    gated) via a rev compare-and-swap, then lease acquisition
 *  - patchJobFenced  fenced state transition (rejects a stale leaseEpoch)
 *  - renewLease / releaseLease
 *  - heartbeat       factory liveness
 *  - reapExpiredLeases  reclaim dead-worker jobs (bumps leaseEpoch to fence the
 *                    zombie, returns the job to queued/blocked, preserves checkpoint)
 *
 * Concurrency model: `rev` is an optimistic-concurrency token on jobs + leases.
 * Every contended mutation goes through repository.revUpdate*, which writes only
 * if the stored `rev` still matches — so under contention EXACTLY ONE caller wins
 * and losers get a `conflict` and retry. Maps to Cosmos `_etag` / If-Match in prod.
 */

import { createHash } from 'node:crypto';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { BadRequestError, ConflictError } from '../../lib/errors.js';
import * as repo from './repository.js';
import { deleteArtifact as deleteArtifactWithBlob } from './artifacts-blob.js';
import { pruneInvalidatedTokens } from './enrollment.js';
import {
  selectJob,
  selectPreemptionVictim,
  scoreCandidate,
  capabilitiesSubset,
  engineEligible,
  type RunningJobView,
  type ScoreBreakdown,
  type SchedulerContext,
  type SchedulerFactory,
  type SchedulerWeights,
  type FleetWeightRegistry,
  resolveWeights,
  resolveJobEngine,
} from './scheduler.js';
import { engineBreaker, isEngineBreakerEnabled } from './engine-breaker.js';
import { computeEngineQuality, isCostRoutingEnabled } from './engine-stats.js';
import {
  ACTIVE_STAGES,
  FLEET_STAGES,
  DEP_DONE_HARD,
  DEP_DONE_SOFT,
  PRIORITY_ORDER,
  type FleetJobDoc,
  type FleetLeaseDoc,
  type FleetRunDoc,
  type FleetStage,
  type FleetBudgetDoc,
  type BudgetWindow,
  type SubmitJobInput,
  type UpdateDraftInput,
  type ReviewPolicy,
  type ReviewDecision,
  type ReviewDecisionKind,
} from './types.js';

const CLAIM_MAX_RETRIES = 8;

export function contentHash(bodyMd: string): string {
  return createHash('sha256').update(bodyMd).digest('hex');
}

// The capability-subset predicate + the §7 scoring/selection engine live in the
// pure, unit-tested scheduler module. Re-export the predicate here to preserve the
// coordinator's public surface (claimNextJob now ranks candidates via selectJob).
export { capabilitiesSubset } from './scheduler.js';

// ── Feature flags (Phase 3) ──────────────────────────────────────────────────

/** FLEET_PREEMPTION env gate — default OFF (byte-for-byte Phase 2 behavior). */
export function isPreemptionEnabled(): boolean {
  const v = (process.env.FLEET_PREEMPTION ?? '').trim().toLowerCase();
  return v === '1' || v === 'true' || v === 'on';
}

/** FLEET_BUDGETS env gate — default OFF (no budget enforcement). */
export function isBudgetsEnabled(): boolean {
  const v = (process.env.FLEET_BUDGETS ?? '').trim().toLowerCase();
  return v === '1' || v === 'true' || v === 'on';
}

/** Weight registry — in Phase 3 loaded from env/config; in-memory for now. */
let weightRegistry: FleetWeightRegistry = {};

export function setWeightRegistry(reg: FleetWeightRegistry): void {
  weightRegistry = reg;
}

export function getWeightRegistry(): FleetWeightRegistry {
  return weightRegistry;
}

// ── Dependency evaluation (§5) ────────────────────────────────────────────────

/** Unmet dependency keys for a job given the current store state. */
export async function unmetDeps(job: FleetJobDoc): Promise<string[]> {
  if (!job.deps || job.deps.length === 0) return [];
  const done = job.depsMode === 'soft' ? DEP_DONE_SOFT : DEP_DONE_HARD;
  const unmet: string[] = [];
  for (const depKey of job.deps) {
    const matches = await repo.findJobsByIdempotencyKey(job.productId, depKey);
    const satisfied = matches.some(m => done.includes(m.stage));
    if (!satisfied) unmet.push(depKey);
  }
  return unmet;
}

async function stageForDeps(job: FleetJobDoc): Promise<{ stage: FleetStage; unmet: string[] }> {
  const unmet = await unmetDeps(job);
  return { stage: unmet.length > 0 ? 'blocked' : 'queued', unmet };
}

/**
 * Submit-time cycle detection: would a job with `newKey` depending on `newDeps`
 * create a cycle in the idempotency-key dependency graph (existing jobs + new)?
 */
export async function wouldCreateCycle(
  productId: string,
  newKey: string,
  newDeps: string[]
): Promise<boolean> {
  if (newDeps.includes(newKey)) return true; // self-dependency
  const visited = new Set<string>();
  let frontier = [...newDeps];
  while (frontier.length > 0) {
    const next: string[] = [];
    for (const key of frontier) {
      if (key === newKey) return true; // reached back to the new node
      if (visited.has(key)) continue;
      visited.add(key);
      const matches = await repo.findJobsByIdempotencyKey(productId, key);
      for (const m of matches) next.push(...m.deps);
    }
    frontier = next;
  }
  return false;
}

// ── Submit (idempotent) ───────────────────────────────────────────────────────

export interface SubmitResult {
  job: FleetJobDoc;
  outcome: 'created' | 'deduplicated' | 'superseded';
}

export async function submitJob(
  productId: string,
  input: SubmitJobInput,
  opts?: { correlationId?: string }
): Promise<SubmitResult> {
  const correlationId = opts?.correlationId;
  const hash = contentHash(input.bodyMd);
  const existingForKey = await repo.findJobsByIdempotencyKey(productId, input.idempotencyKey);

  // Idempotency (§4): same key + identical content => return existing (no dup).
  const identical = existingForKey.find(j => j.contentHash === hash);
  if (identical) return { job: identical, outcome: 'deduplicated' };

  if (existingForKey.length > 0) {
    // same key, different content
    const supersedable = existingForKey.find(
      j => j.stage === 'draft' || j.stage === 'queued' || j.stage === 'blocked'
    );
    if (!supersedable) {
      throw new ConflictError(
        `idempotency-key '${input.idempotencyKey}' already in use by an in-flight/terminal job with different content`
      );
    }
    // supersede the still-queued/blocked job in place
    if (await wouldCreateCycle(productId, input.idempotencyKey, input.deps)) {
      throw new BadRequestError('dependency cycle detected — submission rejected');
    }
    const refreshed = applyInputToJob(supersedable, input, hash);
    // A draft resubmit stays a draft; otherwise re-derive queued/blocked from deps
    // (so resubmitting a draft without the flag promotes it).
    const stage = input.draft ? 'draft' : (await stageForDeps(refreshed)).stage;
    const updated = await repo.updateJob(supersedable.id, productId, {
      ...stripIdentity(refreshed),
      stage,
    });
    await repo.appendEvent({
      jobId: supersedable.id,
      productId,
      type: 'superseded',
      data: { idempotencyKey: input.idempotencyKey },
    });
    return { job: updated ?? supersedable, outcome: 'superseded' };
  }

  // brand-new job
  if (await wouldCreateCycle(productId, input.idempotencyKey, input.deps)) {
    throw new BadRequestError('dependency cycle detected — submission rejected');
  }
  const now = new Date().toISOString();
  const id = `fjob_${crypto.randomUUID()}`;
  const base: FleetJobDoc = {
    id,
    productId,
    stage: 'queued',
    idempotencyKey: input.idempotencyKey,
    contentHash: hash,
    bodyMd: input.bodyMd,
    manifestSnapshot: {
      priority: input.priority,
      capabilities: input.capabilities,
      engineClass: input.engineClass,
      profile: input.profile,
      prefersEngine: input.prefersEngine,
      allowedScope: input.allowedScope,
      deps: input.deps,
      depsMode: input.depsMode,
      budget: input.budget,
      retry: input.retry,
    },
    priority: input.priority,
    priorityOrder: PRIORITY_ORDER[input.priority],
    capabilities: input.capabilities,
    engine: input.engine,
    engineClass: input.engineClass,
    profile: input.profile,
    deps: input.deps,
    depsMode: input.depsMode,
    budget: input.budget,
    retry: input.retry,
    kind: input.kind,
    parentId: input.parentId,
    trackerItemId: input.trackerItemId,
    repo: input.repo,
    baseBranch: input.baseBranch,
    verify: input.verify,
    autoMerge: input.autoMerge,
    attempts: 0,
    attemptsBase: 0,
    leaseEpoch: 0,
    rev: 0,
    correlationId,
    createdAt: now,
    updatedAt: now,
  };
  const { stage } = await stageForDeps(base);
  base.stage = stage;

  // Phase 3 DAG: if children are specified, create them atomically and block parent
  const children = (input as { children?: SubmitJobInput['children'] }).children ?? [];
  if (children.length > 0) {
    base.kind = 'composite';
    // Parent is blocked until all children complete
    const childKeys = children.map(c => c.idempotencyKey);
    base.deps = [...base.deps, ...childKeys];
    base.stage = 'blocked';
    base.blockedReason = `waiting on children: ${childKeys.join(', ')}`;
  }

  // Save-as-draft: a brand-new (non-composite) job can be parked as a
  // non-claimable, editable draft instead of going straight to queued.
  if (input.draft && children.length === 0) base.stage = 'draft';

  const created = await repo.createJob(base);

  // Create child jobs
  for (const child of children) {
    const childId = `fjob_${crypto.randomUUID()}`;
    const childNow = new Date().toISOString();
    const childHash = contentHash(child.bodyMd);
    const childPriority = child.priority ?? input.priority;
    const childJob: FleetJobDoc = {
      id: childId,
      productId,
      stage: 'queued',
      idempotencyKey: child.idempotencyKey,
      contentHash: childHash,
      bodyMd: child.bodyMd,
      manifestSnapshot: {
        priority: childPriority,
        capabilities: child.capabilities,
        engineClass: child.engineClass,
        profile: child.profile,
        prefersEngine: child.prefersEngine,
        allowedScope: child.allowedScope,
        deps: child.deps,
        depsMode: child.depsMode,
        budget: child.budget,
        retry: child.retry,
      },
      priority: childPriority,
      priorityOrder: PRIORITY_ORDER[childPriority],
      capabilities: child.capabilities,
      engineClass: child.engineClass,
      profile: child.profile,
      deps: child.deps,
      depsMode: child.depsMode,
      budget: child.budget,
      retry: child.retry,
      kind: 'leaf',
      parentId: id,
      attempts: 0,
      attemptsBase: 0,
      leaseEpoch: 0,
      rev: 0,
      // Children share the parent's trace so a composite job is one work-unit.
      correlationId,
      createdAt: childNow,
      updatedAt: childNow,
    };
    const { stage: childStage } = await stageForDeps(childJob);
    childJob.stage = childStage;
    await repo.createJob(childJob);
    await repo.appendEvent({
      jobId: childId,
      productId,
      correlationId,
      type: 'submitted',
      data: { stage: childStage, idempotencyKey: child.idempotencyKey, parentId: id },
    });
  }

  await repo.appendEvent({
    jobId: id,
    productId,
    correlationId,
    type: 'submitted',
    data: { stage: base.stage, idempotencyKey: input.idempotencyKey, childCount: children.length },
  });
  return { job: created, outcome: 'created' };
}

/** Stages in which a job's prompt/config may still be edited (not yet picked up). */
const EDITABLE_STAGES: readonly FleetStage[] = ['draft', 'queued', 'blocked'];

/**
 * Edit a not-yet-picked-up job's prompt/config in place. Allowed only while the
 * job is `draft`/`queued`/`blocked`; once a factory has claimed it (assigned+) the
 * prompt is locked and this returns `invalid_state`. Recomputes contentHash so the
 * idempotency/supersede logic stays consistent.
 */
export async function updateDraft(
  jobId: string,
  productId: string,
  patch: UpdateDraftInput
): Promise<repo.RevResult<FleetJobDoc>> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return { ok: false, reason: 'not_found' };
  if (!EDITABLE_STAGES.includes(job.stage)) return { ok: false, reason: 'conflict' };

  const updates: Partial<FleetJobDoc> = {};
  if (patch.bodyMd !== undefined) {
    updates.bodyMd = patch.bodyMd;
    updates.contentHash = contentHash(patch.bodyMd);
  }
  if (patch.priority !== undefined) {
    updates.priority = patch.priority;
    updates.priorityOrder = PRIORITY_ORDER[patch.priority];
  }
  if (patch.capabilities !== undefined) updates.capabilities = patch.capabilities;
  if (patch.engine !== undefined) updates.engine = patch.engine;
  if (patch.repo !== undefined) updates.repo = patch.repo;
  if (patch.baseBranch !== undefined) updates.baseBranch = patch.baseBranch;
  if (patch.verify !== undefined) updates.verify = patch.verify;
  if (patch.autoMerge !== undefined) updates.autoMerge = patch.autoMerge;

  const updated = await repo.updateJob(jobId, productId, updates);
  if (!updated) return { ok: false, reason: 'not_found' };
  // Editing caps/deps/priority changes what's claimable, so bump the M0 gate to
  // wake gated factories now instead of waiting for the safety interval.
  await repo.bumpQueueVersion(productId);
  await repo.appendEvent({
    jobId,
    productId,
    type: 'edited',
    data: { fields: Object.keys(updates) },
  });
  return { ok: true, doc: updated };
}

/**
 * Submit a draft → `queued` (or `blocked` if it has unmet deps). Idempotent: a job
 * already past draft is returned unchanged. Only a `draft` may be submitted.
 */
export async function submitDraft(
  jobId: string,
  productId: string
): Promise<repo.RevResult<FleetJobDoc>> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return { ok: false, reason: 'not_found' };
  if (job.stage !== 'draft') return { ok: true, doc: job }; // already submitted — idempotent
  const { stage } = await stageForDeps(job);
  const updated = await repo.updateJob(jobId, productId, { stage });
  if (!updated) return { ok: false, reason: 'not_found' };
  await repo.appendEvent({ jobId, productId, type: 'submitted', data: { stage, from: 'draft' } });
  return { ok: true, doc: updated };
}

function applyInputToJob(job: FleetJobDoc, input: SubmitJobInput, hash: string): FleetJobDoc {
  return {
    ...job,
    contentHash: hash,
    bodyMd: input.bodyMd,
    priority: input.priority,
    priorityOrder: PRIORITY_ORDER[input.priority],
    capabilities: input.capabilities,
    engine: input.engine,
    engineClass: input.engineClass,
    profile: input.profile,
    deps: input.deps,
    depsMode: input.depsMode,
    budget: input.budget,
    retry: input.retry,
    manifestSnapshot: {
      priority: input.priority,
      capabilities: input.capabilities,
      engineClass: input.engineClass,
      profile: input.profile,
      prefersEngine: input.prefersEngine,
      allowedScope: input.allowedScope,
      deps: input.deps,
      depsMode: input.depsMode,
      budget: input.budget,
      retry: input.retry,
    },
  };
}

function stripIdentity(job: FleetJobDoc): Partial<FleetJobDoc> {
  const { id: _id, productId: _pid, createdAt: _c, rev: _r, ...rest } = job;
  return rest;
}

// ── Atomic claim (§4/§7) ────────────────────────────────────────────────────

export interface ClaimContext {
  productId: string;
  factoryId: string;
  capabilities: string[];
  leaseSeconds: number;
  // ── §7 scheduler inputs (additive, optional — sane defaults below) ──
  /** Factory health; below the floor (`down`) the factory claims nothing. */
  health?: 'ok' | 'degraded' | 'down';
  /** Current factory load (busier ⇒ lower score). */
  load?: number;
  /** Per-engine seat limit (carried for scoring/future seat-aware routing). */
  seatLimit?: number;
  /** Engines this factory runs (prefers-engine affinity). */
  factoryEngines?: string[];
  /** Scopes (repos/locks) the factory has warm (repo-stickiness affinity). */
  warmScopes?: string[];
  /** Factory/budget cost ceiling in USD (cost-fit). */
  costCeilingUsd?: number;
  // ── Phase 3 additions ──
  /** Per-request weight overrides (Phase 3 tunability). */
  weights?: Partial<SchedulerWeights>;
}

export interface ClaimResult {
  job: FleetJobDoc;
  lease: FleetLeaseDoc;
  run: FleetRunDoc;
}

/**
 * Resolve which stage-eligible (queued/blocked) jobs currently have their deps
 * satisfied. This is the store-backed (async) half of eligibility; the pure
 * capability-subset filter + §7 scoring + tie-break are applied by `selectJob`.
 */
async function depsSatisfiedIds(jobs: FleetJobDoc[]): Promise<Set<string>> {
  const satisfied = new Set<string>();
  for (const job of jobs) {
    if (job.stage !== 'queued' && job.stage !== 'blocked') continue;
    if ((await unmetDeps(job)).length === 0) satisfied.add(job.id);
  }
  return satisfied;
}

/**
 * Try to claim ONE specific job version. The rev compare-and-swap is the single
 * point at which exactly one contender wins; a stale `rev` => conflict (no write,
 * no double-assignment).
 */
export async function tryClaimJob(
  job: FleetJobDoc,
  ctx: ClaimContext
): Promise<repo.RevResult<ClaimResult>> {
  const newEpoch = job.leaseEpoch + 1;
  const attempt = job.attempts + 1;
  const claimed = await repo.revUpdateJob(job.id, ctx.productId, job.rev, {
    stage: 'assigned',
    leaseEpoch: newEpoch,
    attempts: attempt,
  });
  if (!claimed.ok) return claimed;

  const now = Date.now();
  const expiresAt = new Date(now + ctx.leaseSeconds * 1000).toISOString();
  const nowIso = new Date(now).toISOString();

  // The run/lease inherit the job's trace so the whole work-unit shares one id.
  const correlationId = job.correlationId;

  const existingLease = await repo.getLease(job.id);
  let lease: FleetLeaseDoc;
  if (existingLease) {
    const updated = await repo.revUpdateLease(job.id, existingLease.rev, {
      holderFactoryId: ctx.factoryId,
      expiresAt,
      leaseEpoch: newEpoch,
      renewals: 0,
      status: 'held',
      correlationId,
    });
    lease = updated.ok ? updated.doc : existingLease;
  } else {
    lease = await repo.createLease({
      id: job.id,
      productId: ctx.productId,
      jobId: job.id,
      holderFactoryId: ctx.factoryId,
      expiresAt,
      leaseEpoch: newEpoch,
      renewals: 0,
      status: 'held',
      rev: 0,
      correlationId,
      updatedAt: nowIso,
    });
  }

  const run = await repo.createRun({
    id: `${job.id}:run:${attempt}`,
    productId: ctx.productId,
    jobId: job.id,
    attempt,
    factoryId: ctx.factoryId,
    // Prefer the job's concrete engine, then its abstract class; the worker later
    // reports the actually-resolved engine (insights.engine) on release.
    engine: job.engine ?? job.engineClass ?? 'unknown',
    startedAt: nowIso,
    insights: {},
    correlationId,
  });

  await repo.appendEvent({
    jobId: job.id,
    productId: ctx.productId,
    type: 'assigned',
    actor: ctx.factoryId,
    correlationId,
    data: { leaseEpoch: newEpoch, attempt },
  });

  return { ok: true, doc: { job: claimed.doc, lease, run } };
}

/**
 * Select the best eligible job via the §7 scoring engine and atomically claim it.
 *
 * The coordinator owns all I/O: it lists candidates, resolves deps (store-backed)
 * into a pure predicate, and builds the factory view + authoritative `now`. The
 * pure `selectJob` then applies the capability hard-filter + fixed-weight scoring
 * + deterministic tie-break (priority → age → cost class). The atomic
 * single-winner guarantee remains entirely in `tryClaimJob`'s rev compare-and-swap,
 * which is unchanged — on conflict we re-select and retry.
 *
 * Phase 3: weights are resolved via resolveWeights (per-product tunable).
 * Phase 3: when FLEET_PREEMPTION=1 and selectJob returns null, attempt preemption
 * of a strictly-lower-priority running job for any CRITICAL queued job.
 */
export async function claimNextJob(ctx: ClaimContext): Promise<ClaimResult | null> {
  const factory: SchedulerFactory = {
    capabilities: ctx.capabilities,
    health: ctx.health ?? 'ok',
    load: ctx.load ?? 0,
    seatLimit: ctx.seatLimit ?? 1,
  };
  const weights = resolveWeights(weightRegistry, ctx.productId, ctx.weights);
  const seatLimit = ctx.seatLimit ?? 1;

  // §2 cost/latency-aware routing: derive a per-engine quality lookup from run
  // history once (flag-gated; off ⇒ the soft engineQuality term stays inert).
  let engineQuality: ((engine: string | undefined) => number) | undefined;
  if (isCostRoutingEnabled()) {
    const runs = await repo.listRunsByProduct(ctx.productId);
    engineQuality = computeEngineQuality(runs).lookup;
  }
  // §2/§3 per-engine availability gate — composes two RESTRICTING guards (each
  // flag-gated; neither can force a route):
  //  • circuit breaker: stop routing an engine that keeps failing on THIS factory.
  //  • per-engine budget: stop routing an engine that has hit its sub-ceiling.
  // The product budget itself (read fresh each retry below) is unchanged during a
  // claim, so its per-engine ceilings are read once here.
  const breakerOn = isEngineBreakerEnabled();
  const budgetForEngines = isBudgetsEnabled() ? await repo.getBudget(ctx.productId) : null;
  const engineCeilings = budgetForEngines?.engineCeilingsUsd;
  const engineBudgetExhausted = (engine: string): boolean => {
    const ceiling = engineCeilings?.[engine];
    if (ceiling === undefined) return false;
    return (budgetForEngines?.spentByEngineUsd?.[engine] ?? 0) >= ceiling;
  };
  const isEngineAvailable =
    breakerOn || engineCeilings
      ? (engine: string): boolean =>
          (!breakerOn || engineBreaker.allow(ctx.factoryId, engine)) &&
          !engineBudgetExhausted(engine)
      : undefined;

  for (let i = 0; i < CLAIM_MAX_RETRIES; i++) {
    // Phase 3: budget enforcement (FLEET_BUDGETS flag)
    if (isBudgetsEnabled()) {
      const budget = await repo.getBudget(ctx.productId);
      if (budget) {
        if (budget.status === 'paused') return null;
        if (budget.spentUsd >= budget.ceilingUsd) return null;
      }
    }

    const candidates = await repo.listJobs({ productId: ctx.productId });
    const satisfied = await depsSatisfiedIds(candidates);

    // Phase 3: seat-limit enforcement (when preemption is ON).
    // If this factory is at its seat limit, skip normal selection and try preemption.
    let atSeatLimit = false;
    if (isPreemptionEnabled()) {
      const activeJobs = candidates.filter(j => ACTIVE_STAGES.includes(j.stage));
      let heldCount = 0;
      for (const aj of activeJobs) {
        const lease = await repo.getLease(aj.id);
        if (lease && lease.holderFactoryId === ctx.factoryId && lease.status === 'held') {
          heldCount++;
        }
      }
      atSeatLimit = heldCount >= seatLimit;
    }

    let pick: FleetJobDoc | null = null;
    if (!atSeatLimit) {
      const schedulerCtx: SchedulerContext = {
        now: Date.now(),
        isDepsSatisfied: job => satisfied.has(job.id),
        factoryEngines: ctx.factoryEngines,
        warmScopes: ctx.warmScopes,
        costCeilingUsd: ctx.costCeilingUsd,
        engineQuality,
        isEngineAvailable,
      };
      pick = selectJob(candidates, factory, schedulerCtx, weights);
    }

    if (pick) {
      const result = await tryClaimJob(pick, ctx);
      if (result.ok) return result.doc;
      if (result.reason === 'not_found') continue;
      continue;
    }

    // No eligible job found (or at seat limit) — attempt preemption if enabled
    if (!isPreemptionEnabled()) return null;

    // Find critical queued/blocked jobs that are deps-satisfied
    const criticalCandidates = candidates.filter(
      j =>
        (j.stage === 'queued' || j.stage === 'blocked') &&
        j.priorityOrder === 0 && // critical
        satisfied.has(j.id)
    );
    if (criticalCandidates.length === 0) return null;

    // Get running jobs on this factory
    const allJobs = candidates.filter(j => ACTIVE_STAGES.includes(j.stage));
    const runningOnFactory: RunningJobView[] = [];
    for (const rj of allJobs) {
      const lease = await repo.getLease(rj.id);
      if (lease && lease.holderFactoryId === ctx.factoryId && lease.status === 'held') {
        runningOnFactory.push({
          id: rj.id,
          productId: rj.productId,
          priority: rj.priority,
          priorityOrder: rj.priorityOrder,
          leaseEpoch: rj.leaseEpoch,
          checkpoint: rj.checkpoint,
        });
      }
    }
    if (runningOnFactory.length === 0) return null;

    // Try to preempt for the highest-priority critical candidate
    for (const critJob of criticalCandidates) {
      const decision = selectPreemptionVictim(critJob, runningOnFactory, factory);
      if (!decision) continue;

      // Execute eviction: bump epoch (fence zombie), requeue with preserved checkpoint
      const victimJob = await repo.getJob(decision.evict, ctx.productId);
      if (!victimJob) continue;
      const newEpoch = victimJob.leaseEpoch + 1;
      const victimUnmet = await unmetDeps(victimJob);
      const returnStage: FleetStage = victimUnmet.length > 0 ? 'blocked' : 'queued';
      const evictRes = await repo.revUpdateJob(decision.evict, ctx.productId, victimJob.rev, {
        stage: returnStage,
        leaseEpoch: newEpoch,
        blockedReason: victimUnmet.length > 0 ? `waiting on: ${victimUnmet.join(', ')}` : undefined,
      });
      if (!evictRes.ok) continue;

      // Mark lease as expired (fence the zombie's stale reports)
      const victimLease = await repo.getLease(decision.evict);
      if (victimLease) {
        await repo.revUpdateLease(decision.evict, victimLease.rev, {
          status: 'expired',
          leaseEpoch: newEpoch,
          holderFactoryId: undefined,
        });
      }

      await repo.appendEvent({
        jobId: decision.evict,
        productId: ctx.productId,
        type: 'preempted',
        actor: ctx.factoryId,
        data: { reason: decision.reason, preemptedBy: critJob.id, returnedTo: returnStage },
      });

      // Now claim the critical job
      const claimResult = await tryClaimJob(critJob, ctx);
      if (claimResult.ok) return claimResult.doc;
      break;
    }
  }
  return null;
}

// ── Fenced transitions + leases (§4/§8) ──────────────────────────────────────

export type FenceResult<T> =
  | { ok: true; doc: T }
  | { ok: false; reason: 'not_found' | 'fenced' | 'conflict' };

function fenced(job: FleetJobDoc, leaseEpoch: number): boolean {
  // a call older than the current epoch is a stale/zombie worker — reject it
  return leaseEpoch < job.leaseEpoch;
}

export interface PatchJobInputInternal {
  leaseEpoch: number;
  stage?: FleetStage;
  checkpoint?: FleetJobDoc['checkpoint'];
  blockedReason?: string;
}

/**
 * Mark a job's latest run as shipped so the run-level `result` mirrors the
 * terminal job stage (a job can reach `shipped` via autoship PATCH, the `ship`
 * operator action, or a lease release). Idempotent; returns the latest run.
 */
async function markLatestRunShipped(jobId: string): Promise<FleetRunDoc | undefined> {
  const runs = await repo.listRunsByJob(jobId);
  const latest = runs.reduce<FleetRunDoc | undefined>(
    (acc, r) => (!acc || r.attempt > acc.attempt ? r : acc),
    undefined
  );
  if (latest && latest.result !== 'shipped') {
    await repo.updateRun(latest.id, jobId, {
      result: 'shipped',
      ...(latest.endedAt ? {} : { endedAt: new Date().toISOString() }),
    });
  }
  return latest;
}

const execFileAsync = promisify(execFile);

/**
 * On ship, optionally merge the linked PR via the GitHub CLI (so "Ship" also
 * lands the code). Opt-in via FLEET_SHIP_MERGES_PR=1 and only where `gh` is
 * authenticated (the coordinator host). Best-effort: a failure leaves the PR open
 * and never fails the ship. Marks the run prState=merged on success.
 */
async function ghMergePr(prUrl: string): Promise<boolean> {
  try {
    await execFileAsync('gh', ['pr', 'merge', prUrl, '--squash', '--delete-branch'], {
      timeout: 60_000,
    });
    return true;
  } catch {
    // Network/proxy hiccup (e.g. intermittent 407), checks pending, or conflict.
    return false;
  }
}

/**
 * Whether a shipped job's PR should be auto-merged: when the job opted in
 * (`job.autoMerge`) OR the host enables it globally (`FLEET_SHIP_MERGES_PR=1`).
 * Pure (env + job only) so the gating is unit-testable without invoking `gh`.
 */
export function shouldMergePrOnShip(job: Pick<FleetJobDoc, 'autoMerge'>): boolean {
  return job.autoMerge === true || process.env.FLEET_SHIP_MERGES_PR === '1';
}

async function mergeRunPrOnShip(job: FleetJobDoc, latest: FleetRunDoc | undefined): Promise<void> {
  if (!shouldMergePrOnShip(job)) return;
  if (!latest?.prUrl || latest.prState === 'merged') return;
  const { prUrl, id: runId } = latest;
  const { id: jobId, productId } = job;
  const recordMerged = async () => {
    try {
      await repo.updateRun(runId, jobId, { prState: 'merged' });
      await repo.appendEvent({ jobId, productId, type: 'pr_merged', data: { prUrl } });
    } catch {
      /* run gone — ignore */
    }
  };
  // Fast attempt inline (so a healthy proxy merges immediately without delaying ship).
  if (await ghMergePr(prUrl)) {
    await recordMerged();
    return;
  }
  // The corporate proxy intermittently 407s GitHub's API. Retry in the BACKGROUND
  // with backoff so the ship response is never blocked; mark merged when one lands,
  // and SURFACE a `pr_merge_failed` event if every attempt fails (so a stuck PR is
  // visible on the job timeline instead of silently left open).
  void (async () => {
    for (const delayMs of [3_000, 8_000, 20_000, 45_000]) {
      await new Promise(r => setTimeout(r, delayMs));
      if (await ghMergePr(prUrl)) {
        await recordMerged();
        return;
      }
    }
    try {
      await repo.appendEvent({
        jobId,
        productId,
        type: 'pr_merge_failed',
        data: { prUrl, reason: 'gh pr merge failed after inline + 4 background retries' },
      });
    } catch {
      /* job gone — ignore */
    }
  })().catch(() => {
    /* detached best-effort — never throws */
  });
}

// ── PR-state reconcile (§PR mode — detect an externally-merged PR) ────────────
//
// The platform only learns a PR is merged when IT merges it (ghMergePr on ship)
// or a runner reports it. A PR merged manually in the GitHub UI is otherwise
// invisible. `reconcileJobPrState` closes that gap on demand: it asks `gh` for
// the PR's current state and flips the run's `prState` to `merged` when GitHub
// says so. Pull/best-effort — no inbound webhook or public ingress required.

/** Map a GitHub PR state (`gh pr view --json state` → MERGED/OPEN/CLOSED) onto
 *  our run `prState`. Unknown/empty ⇒ null (leave the run untouched). Pure. */
export function mapGhPrState(raw: string | undefined): FleetRunDoc['prState'] | null {
  switch ((raw ?? '').trim().toUpperCase()) {
    case 'MERGED':
      return 'merged';
    case 'OPEN':
      return 'open';
    default:
      return null; // CLOSED (not merged) or unknown — no prState change
  }
}

/** Query the live PR state via the GitHub CLI. Best-effort: any failure ⇒ null. */
async function fetchGhPrState(prUrl: string): Promise<FleetRunDoc['prState'] | null> {
  try {
    const { stdout } = await execFileAsync('gh', ['pr', 'view', prUrl, '--json', 'state'], {
      timeout: 30_000,
    });
    const parsed = JSON.parse(stdout) as { state?: string };
    return mapGhPrState(parsed.state);
  } catch {
    return null;
  }
}

export interface PrReconcileResult {
  /** false when the job/PR could not be found. */
  ok: boolean;
  /** Whether a run was updated (prState flipped to `merged`). */
  updated: boolean;
  /** The PR state after reconcile (current stored value when unchanged). */
  prState?: FleetRunDoc['prState'];
  reason?: 'not_found' | 'no_pr' | 'already_merged' | 'unchanged';
}

/**
 * Reconcile a job's PR state against GitHub. Finds the latest run carrying a
 * `prUrl`; if it is not already `merged` and `gh` reports the PR MERGED, flips
 * the run's `prState` to `merged` and appends a `pr_merged` event. The GitHub
 * lookup is injectable for tests.
 */
export async function reconcileJobPrState(
  jobId: string,
  productId: string,
  fetcher: (prUrl: string) => Promise<FleetRunDoc['prState'] | null> = fetchGhPrState
): Promise<PrReconcileResult> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return { ok: false, updated: false, reason: 'not_found' };

  const runs = await repo.listRunsByJob(jobId);
  const prRun = runs
    .filter(r => !!r.prUrl)
    .reduce<
      FleetRunDoc | undefined
    >((acc, r) => (!acc || r.attempt > acc.attempt ? r : acc), undefined);
  if (!prRun?.prUrl) return { ok: true, updated: false, reason: 'no_pr' };
  if (prRun.prState === 'merged') {
    return { ok: true, updated: false, prState: 'merged', reason: 'already_merged' };
  }

  const live = await fetcher(prRun.prUrl);
  if (live !== 'merged') {
    return { ok: true, updated: false, prState: prRun.prState, reason: 'unchanged' };
  }

  await repo.updateRun(prRun.id, jobId, { prState: 'merged' });
  await repo.appendEvent({
    jobId,
    productId,
    type: 'pr_merged',
    correlationId: job.correlationId,
    data: { prUrl: prRun.prUrl, via: 'reconcile' },
  });
  return { ok: true, updated: true, prState: 'merged' };
}

export async function patchJobFenced(
  jobId: string,
  productId: string,
  patch: PatchJobInputInternal
): Promise<FenceResult<FleetJobDoc>> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return { ok: false, reason: 'not_found' };
  if (fenced(job, patch.leaseEpoch)) return { ok: false, reason: 'fenced' };

  const updates: Partial<FleetJobDoc> = {};
  if (patch.stage) updates.stage = patch.stage;
  if (patch.checkpoint) updates.checkpoint = patch.checkpoint;
  if (patch.blockedReason !== undefined) updates.blockedReason = patch.blockedReason;

  const res = await repo.revUpdateJob(jobId, productId, job.rev, updates);
  if (!res.ok) return { ok: false, reason: res.reason === 'not_found' ? 'not_found' : 'conflict' };
  await repo.appendEvent({
    jobId,
    productId,
    type: 'transition',
    correlationId: job.correlationId,
    data: { stage: patch.stage ?? job.stage, leaseEpoch: patch.leaseEpoch },
  });

  // Phase 3 DAG: when a child reaches a terminal dep-satisfying stage,
  // check if its parent should be unblocked.
  if (patch.stage && job.parentId) {
    await maybeUnblockParent(job.parentId, productId);
  }

  // On ship (terminal success): mirror the outcome onto the run, and accrue spend.
  // Best-effort — never fail the transition. Budget accrual is flag-gated and
  // idempotent per run via `<jobId>:<leaseEpoch>`, using the run's ACTUAL cost
  // (insights.costUsd) so spentUsd and the costBurndown agree.
  if (patch.stage === 'shipped') {
    try {
      // Run-level result mirrors the terminal stage (ungated).
      const latest = await markLatestRunShipped(jobId);
      // Optionally merge the linked PR (per-job autoMerge or global flag, best-effort).
      await mergeRunPrOnShip(job, latest);
      // Budgets (flag-gated): accrue the run's actual cost, idempotent per run.
      if (isBudgetsEnabled()) {
        await accrueSpend(
          productId,
          latest?.insights?.costUsd ?? 0,
          `${jobId}:${job.leaseEpoch}`,
          latest?.insights?.engine ?? latest?.engine
        );
      }
    } catch {
      // swallow — run/budget bookkeeping is downstream of the job lifecycle
    }
  }

  return { ok: true, doc: res.doc };
}

// ── Phase 3 DAG: parent unblocking + child submission + subtree query ─────────

/**
 * Check if a parent job's deps are all satisfied and unblock it if so.
 * Called after a child job transitions to a terminal stage.
 */
export async function maybeUnblockParent(parentId: string, productId: string): Promise<void> {
  const parent = await repo.getJob(parentId, productId);
  if (!parent) return;
  if (parent.stage !== 'blocked') return;
  const unmet = await unmetDeps(parent);
  if (unmet.length === 0) {
    await repo.revUpdateJob(parentId, productId, parent.rev, {
      stage: 'queued',
      blockedReason: undefined,
    });
    await repo.appendEvent({
      jobId: parentId,
      productId,
      type: 'unblocked',
      data: { reason: 'all children/deps completed' },
    });
  }
}

/** Submit children for an existing parent (POST /fleet/jobs/:id/children). */
export async function submitChildren(
  parentId: string,
  productId: string,
  children: Array<{
    idempotencyKey: string;
    bodyMd: string;
    priority?: 'critical' | 'high' | 'medium' | 'low';
    capabilities?: string[];
    engineClass?: string;
    profile?: string;
    prefersEngine?: string[];
    allowedScope?: string[];
    deps?: string[];
    depsMode?: 'hard' | 'soft';
    budget?: { usd?: number; tokens?: number; wall?: number };
    retry?: { max: number; backoff?: string; on?: string[] };
  }>
): Promise<{ parent: FleetJobDoc; childJobs: FleetJobDoc[] }> {
  const parent = await repo.getJob(parentId, productId);
  if (!parent) throw new BadRequestError('Parent job not found');

  // Cycle check (batch-aware): after submit the parent depends on EVERY child key,
  // so a cycle exists if any child — through its own declared deps, resolved across
  // both this unpersisted batch and existing jobs — can reach the parent or itself.
  // `wouldCreateCycle` alone only walks persisted jobs, so it misses sibling/declared
  // edges within the batch; this resolver covers both.
  const childKeys = children.map(c => c.idempotencyKey);
  const seenKeys = new Set<string>();
  for (const ck of childKeys) {
    if (seenKeys.has(ck)) throw new BadRequestError(`duplicate child idempotency-key '${ck}'`);
    seenKeys.add(ck);
  }
  const batchDeps = new Map<string, string[]>();
  for (const c of children) batchDeps.set(c.idempotencyKey, c.deps ?? []);
  const depsOf = async (key: string): Promise<string[]> => {
    if (batchDeps.has(key)) return batchDeps.get(key) ?? [];
    const matches = await repo.findJobsByIdempotencyKey(productId, key);
    return matches.flatMap(m => m.deps);
  };
  for (const ck of childKeys) {
    const visited = new Set<string>();
    let frontier = await depsOf(ck);
    while (frontier.length > 0) {
      const nextFrontier: string[] = [];
      for (const k of frontier) {
        if (k === parent.idempotencyKey || k === ck) {
          throw new BadRequestError(`dependency cycle detected for child '${ck}'`);
        }
        if (visited.has(k)) continue;
        visited.add(k);
        nextFrontier.push(...(await depsOf(k)));
      }
      frontier = nextFrontier;
    }
  }

  const createdChildren: FleetJobDoc[] = [];
  for (const child of children) {
    const childId = `fjob_${crypto.randomUUID()}`;
    const now = new Date().toISOString();
    const childPriority = child.priority ?? parent.priority;
    const childJob: FleetJobDoc = {
      id: childId,
      productId,
      stage: 'queued',
      idempotencyKey: child.idempotencyKey,
      contentHash: contentHash(child.bodyMd),
      bodyMd: child.bodyMd,
      manifestSnapshot: {
        priority: childPriority,
        capabilities: child.capabilities ?? [],
        engineClass: child.engineClass as FleetJobDoc['engineClass'],
        profile: child.profile,
        prefersEngine: child.prefersEngine ?? [],
        allowedScope: child.allowedScope ?? [],
        deps: child.deps ?? [],
        depsMode: child.depsMode as FleetJobDoc['depsMode'],
        budget: child.budget,
        retry: child.retry as FleetJobDoc['retry'],
      },
      priority: childPriority,
      priorityOrder: PRIORITY_ORDER[childPriority],
      capabilities: child.capabilities ?? [],
      engineClass: child.engineClass as FleetJobDoc['engineClass'],
      profile: child.profile,
      deps: child.deps ?? [],
      depsMode: child.depsMode as FleetJobDoc['depsMode'],
      budget: child.budget,
      retry: child.retry as FleetJobDoc['retry'],
      kind: 'leaf',
      parentId,
      attempts: 0,
      attemptsBase: 0,
      leaseEpoch: 0,
      rev: 0,
      createdAt: now,
      updatedAt: now,
    };
    const { stage } = await stageForDeps(childJob);
    childJob.stage = stage;
    const created = await repo.createJob(childJob);
    createdChildren.push(created);
    await repo.appendEvent({
      jobId: childId,
      productId,
      type: 'submitted',
      data: { stage, idempotencyKey: child.idempotencyKey, parentId },
    });
  }

  // Block parent on new children
  const existingDeps = parent.deps ?? [];
  const newDeps = [...existingDeps, ...childKeys];
  const updatedParent = await repo.revUpdateJob(parentId, productId, parent.rev, {
    kind: 'composite',
    deps: newDeps,
    stage: 'blocked',
    blockedReason: `waiting on children: ${childKeys.join(', ')}`,
  });

  return { parent: updatedParent.ok ? updatedParent.doc : parent, childJobs: createdChildren };
}

/** DAG node representation for the subtree endpoint. */
export interface DagNode {
  id: string;
  idempotencyKey: string;
  stage: string;
  priority: string;
  kind: string;
  parentId?: string;
  children: DagNode[];
}

/** Get the DAG subtree rooted at jobId. */
export async function getDagSubtree(jobId: string, productId: string): Promise<DagNode | null> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return null;

  async function buildNode(j: FleetJobDoc): Promise<DagNode> {
    const children = await repo.listChildrenByParent(productId, j.id);
    const childNodes = await Promise.all(children.map(buildNode));
    return {
      id: j.id,
      idempotencyKey: j.idempotencyKey,
      stage: j.stage,
      priority: j.priority,
      kind: j.kind,
      parentId: j.parentId,
      children: childNodes,
    };
  }

  return buildNode(job);
}

export async function renewLease(
  jobId: string,
  productId: string,
  leaseEpoch: number,
  leaseSeconds: number
): Promise<FenceResult<FleetLeaseDoc>> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return { ok: false, reason: 'not_found' };
  if (fenced(job, leaseEpoch)) return { ok: false, reason: 'fenced' };
  const lease = await repo.getLease(jobId);
  if (!lease) return { ok: false, reason: 'not_found' };
  const expiresAt = new Date(Date.now() + leaseSeconds * 1000).toISOString();
  const res = await repo.revUpdateLease(jobId, lease.rev, {
    expiresAt,
    renewals: lease.renewals + 1,
    status: 'held',
  });
  if (!res.ok) return { ok: false, reason: 'conflict' };
  await repo.appendEvent({
    jobId,
    productId,
    type: 'lease_renewed',
    correlationId: job.correlationId,
    data: { leaseEpoch },
  });
  return { ok: true, doc: res.doc };
}

/** Run results that denote a failed attempt (vs. a successful review/testing/shipped). */
const FAILURE_RESULTS: readonly string[] = [
  'failed',
  'timeout',
  'verify_failed',
  'capability_mismatch',
  'no_engine',
  'retries_exhausted',
];

/** Map a run result onto the retry classes (`retry.on`) it can satisfy. Deterministic
 *  config failures (capability_mismatch/no_engine) map to nothing — retrying can't help. */
function retryClassesForResult(result?: string): string[] {
  switch (result) {
    case 'timeout':
      return ['timeout'];
    case 'verify_failed':
      return ['verify_failed'];
    case 'failed':
      return ['agent_error', 'crash'];
    default:
      return [];
  }
}

/** Parse a `retry.backoff` hint into milliseconds for the given (1-based) attempt.
 *  Supports `"<n>"` (seconds), `"<n>s|m|h"`, and `"exp"`/`"exponential"` (30s·2^(n-1),
 *  capped at 1h). Anything unparseable ⇒ 0 (immediate requeue). */
export function parseBackoffMs(backoff: string | undefined, attempt: number): number {
  if (!backoff) return 0;
  const b = backoff.trim().toLowerCase();
  if (b === 'exp' || b === 'exponential') {
    const base = 30_000;
    return Math.min(base * 2 ** Math.max(0, attempt - 1), 60 * 60 * 1000);
  }
  const m = b.match(/^(\d+)\s*(ms|s|m|h)?$/);
  if (!m) return 0;
  const n = Number(m[1]);
  switch (m[2]) {
    case 'ms':
      return n;
    case 'm':
      return n * 60_000;
    case 'h':
      return n * 60 * 60_000;
    case 's':
    default:
      return n * 1_000; // bare number ⇒ seconds
  }
}

export type FailureDecision =
  | { kind: 'retry'; backoffMs: number }
  | { kind: 'dead_letter' }
  | { kind: 'failed' };

/**
 * Decide what becomes of a job whose attempt failed with `result`, per its
 * `retry` policy: retry (when the result matches a `retry.on` class and attempts
 * remain), dead-letter (matching class but attempts exhausted), or plain failed
 * (no policy / non-retryable result — preserves the prior behavior). Pure.
 */
export function decideFailureOutcome(job: FleetJobDoc, result?: string): FailureDecision {
  const policy = job.retry;
  const classes = retryClassesForResult(result);
  const on = (policy?.on ?? []) as readonly string[];
  const retryable =
    !!policy && policy.max > 0 && classes.length > 0 && classes.some(c => on.includes(c));
  if (!retryable) return { kind: 'failed' };
  // Budget is consumed since the last redrive: attempts - attemptsBase.
  const used = job.attempts - (job.attemptsBase ?? 0);
  if (used >= policy!.max) return { kind: 'dead_letter' };
  return { kind: 'retry', backoffMs: parseBackoffMs(policy!.backoff, used) };
}

export async function releaseLease(
  jobId: string,
  productId: string,
  leaseEpoch: number,
  stage?: FleetStage,
  report?: {
    insights?: FleetRunDoc['insights'];
    result?: FleetRunDoc['result'];
    prUrl?: string;
    branch?: string;
    prState?: FleetRunDoc['prState'];
  }
): Promise<FenceResult<FleetLeaseDoc>> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return { ok: false, reason: 'not_found' };
  if (fenced(job, leaseEpoch)) return { ok: false, reason: 'fenced' };
  const lease = await repo.getLease(jobId);
  if (!lease) return { ok: false, reason: 'not_found' };
  const res = await repo.revUpdateLease(jobId, lease.rev, { status: 'released' });
  if (!res.ok) return { ok: false, reason: 'conflict' };

  // A failed attempt (by reported result, or an explicit `failed` stage) is routed
  // through the retry policy: auto-requeue with backoff, dead-letter when attempts
  // are exhausted, else plain `failed` (no policy / non-retryable). Success stages
  // (review/testing/shipped) pass through unchanged.
  const isFailure = report?.result ? FAILURE_RESULTS.includes(report.result) : stage === 'failed';
  if (stage && isFailure) {
    const decision = decideFailureOutcome(job, report?.result ?? 'failed');
    if (decision.kind === 'retry') {
      const unmet = await unmetDeps(job);
      const nextStage: FleetStage = unmet.length > 0 ? 'blocked' : 'queued';
      const retryNotBefore = new Date(Date.now() + decision.backoffMs).toISOString();
      await repo.revUpdateJob(jobId, productId, job.rev, {
        stage: nextStage,
        retryNotBefore,
        blockedReason: unmet.length > 0 ? `waiting on: ${unmet.join(', ')}` : undefined,
      });
      await repo.appendEvent({
        jobId,
        productId,
        type: 'retry_scheduled',
        correlationId: job.correlationId,
        data: {
          attempt: job.attempts,
          max: job.retry?.max,
          result: report?.result ?? 'failed',
          retryNotBefore,
          returnedTo: nextStage,
        },
      });
    } else if (decision.kind === 'dead_letter') {
      await repo.revUpdateJob(jobId, productId, job.rev, { stage: 'dead_letter' });
      await repo.appendEvent({
        jobId,
        productId,
        type: 'dead_letter',
        correlationId: job.correlationId,
        data: { attempts: job.attempts, max: job.retry?.max, result: report?.result ?? 'failed' },
      });
    } else {
      await repo.revUpdateJob(jobId, productId, job.rev, { stage });
    }
  } else if (stage) {
    await repo.revUpdateJob(jobId, productId, job.rev, { stage });
  }
  // Record the factory's reported metrics + outcome + PR deliverable on the run.
  if (report?.insights || report?.result || report?.prUrl || report?.branch || report?.prState) {
    const runId = `${jobId}:run:${job.attempts}`;
    await repo.updateRun(runId, jobId, {
      ...(report.insights ? { insights: report.insights } : {}),
      // Promote the concrete engine the factory reported onto the run (the run was
      // created with the job's abstract engineClass, often 'unknown').
      ...(report.insights?.engine ? { engine: report.insights.engine } : {}),
      ...(report.result ? { result: report.result } : {}),
      ...(report.prUrl ? { prUrl: report.prUrl } : {}),
      ...(report.branch ? { branch: report.branch } : {}),
      ...(report.prState ? { prState: report.prState } : {}),
      endedAt: new Date().toISOString(),
    });
  }
  // §2 per-engine circuit breaker bookkeeping: attribute this attempt's outcome to
  // the (factory, engine) pair so a repeatedly-failing engine on a box can be routed
  // around. We RECORD regardless of the enforcement flag (so breaker state is
  // observable in metrics even before enforcement is enabled); only the claim-time
  // restriction is gated. The concrete engine the factory ran wins, else the job's.
  const breakerFactory = lease.holderFactoryId;
  const breakerEngine = report?.insights?.engine ?? resolveJobEngine(job);
  if (breakerFactory && breakerEngine && breakerEngine !== 'unknown') {
    if (isFailure) {
      engineBreaker.recordFailure(breakerFactory, breakerEngine);
    } else if (stage === 'review' || stage === 'testing' || stage === 'shipped') {
      engineBreaker.recordSuccess(breakerFactory, breakerEngine);
    }
  }
  await repo.appendEvent({
    jobId,
    productId,
    type: 'lease_released',
    correlationId: job.correlationId,
    data: { leaseEpoch, stage },
  });
  return { ok: true, doc: res.doc };
}

// ── Operator actions (§14 Phase 3 — approve/ship/reject/requeue) ──────────────

export const OPERATOR_ACTIONS = ['requeue', 'reject', 'cancel', 'ship', 'redrive'] as const;
export type OperatorActionKind = (typeof OPERATOR_ACTIONS)[number];

/** Bounded retries for the operator-action optimistic-concurrency loop. */
const OPERATOR_ACTION_MAX_RETRIES = 5;

export type OperatorActionResult =
  | { ok: true; doc: FleetJobDoc }
  | { ok: false; reason: 'not_found' | 'conflict' | 'invalid_state' };

/**
 * Operator-initiated lifecycle action from the control plane. Unlike
 * `patchJobFenced`, the operator does NOT hold a lease — so this bumps the
 * `leaseEpoch` to FENCE any current factory holder (a late report from the
 * displaced worker is rejected by `fenced()`), mirroring the reaper (§25.3).
 *
 * - `requeue`: return the job to `queued` (or `blocked` if deps unmet) for a
 *    fresh claim. The checkpoint pointer is preserved (resume-friendly).
 * - `redrive`: like `requeue`, but ALSO grants a FRESH retry budget by anchoring
 *    `attemptsBase` to the current `attempts` and clearing any `retryNotBefore`
 *    backoff. (`attempts` stays monotonic so run ids never collide.) This is how
 *    an operator re-drives a `dead_letter`/`failed` job — a plain requeue leaves
 *    the budget exhausted, so the job would immediately dead-letter again.
 * - `reject`:  move the job to `dead_letter` (operator gives up on it).
 * - `cancel`:  move the job to `failed` (stop work, not a dead letter).
 *
 * A successfully shipped job is terminal and cannot be acted on.
 */
export async function operatorAction(
  jobId: string,
  productId: string,
  action: OperatorActionKind
): Promise<OperatorActionResult> {
  // Retry on optimistic-concurrency conflict (real Cosmos read-after-write lag),
  // mirroring submitReview — a single attempt can spuriously 409 right after a
  // prior transition (e.g. review->testing) has not yet settled.
  for (let attempt = 0; attempt < OPERATOR_ACTION_MAX_RETRIES; attempt++) {
    const job = await repo.getJob(jobId, productId);
    if (!job) return { ok: false, reason: 'not_found' };

    // Idempotent terminal actions: re-shipping/rejecting/cancelling a job already
    // in that terminal state is a no-op (don't bump the epoch or append a dup event).
    if (action === 'ship' && job.stage === 'shipped') return { ok: true, doc: job };
    if (job.stage === 'shipped') return { ok: false, reason: 'invalid_state' };
    if (action === 'reject' && job.stage === 'dead_letter') return { ok: true, doc: job };
    if (action === 'cancel' && job.stage === 'failed') return { ok: true, doc: job };

    const newEpoch = job.leaseEpoch + 1; // fence any current holder

    let stage: FleetStage;
    let blockedReason: string | undefined;
    // `redrive` additionally resets the retry budget so the job gets a full set of
    // attempts again (a plain requeue keeps attempts at max → instant re-dead-letter).
    const resetRetry = action === 'redrive';
    if (action === 'requeue' || action === 'redrive') {
      const unmet = await unmetDeps(job);
      stage = unmet.length > 0 ? 'blocked' : 'queued';
      blockedReason = unmet.length > 0 ? `waiting on: ${unmet.join(', ')}` : undefined;
    } else if (action === 'reject') {
      stage = 'dead_letter';
    } else if (action === 'ship') {
      // Operator/controller marks the job done (e.g. after the review gate routes
      // it to `testing`, where no factory holds a lease to release it).
      stage = 'shipped';
    } else {
      stage = 'failed';
    }

    const res = await repo.revUpdateJob(jobId, productId, job.rev, {
      stage,
      leaseEpoch: newEpoch,
      blockedReason,
      ...(resetRetry ? { attemptsBase: job.attempts, retryNotBefore: undefined } : {}),
    });
    if (!res.ok) {
      if (res.reason === 'not_found') return { ok: false, reason: 'not_found' };
      // conflict -> brief backoff so a read-after-write (real Cosmos) settles, then re-read
      await new Promise(r => globalThis.setTimeout(r, 40 * (attempt + 1)));
      continue;
    }

    // Free the seat: release any still-held lease, fencing it with the new epoch so
    // a stale renewal cannot resurrect it (mirrors the reaper, §25.3).
    const lease = await repo.getLease(jobId);
    if (lease && lease.status === 'held') {
      await repo.revUpdateLease(jobId, lease.rev, {
        status: 'released',
        leaseEpoch: newEpoch,
        holderFactoryId: undefined,
      });
    }

    await repo.appendEvent({
      jobId,
      productId,
      type: 'operator_action',
      actor: 'operator',
      data: { action, leaseEpoch: newEpoch, returnedTo: stage },
    });

    // Operator `ship` is a terminal success — mirror it onto the run result and
    // optionally merge the linked PR (flag-gated, best-effort).
    if (stage === 'shipped') {
      try {
        const latest = await markLatestRunShipped(jobId);
        await mergeRunPrOnShip(job, latest);
      } catch {
        // best-effort — run bookkeeping never fails the operator action
      }
    }

    return { ok: true, doc: res.doc };
  }
  return { ok: false, reason: 'conflict' };
}

// ── Multi-reviewer human gate (§14 Phase 3 — review-policy routing) ────────────

/** Canonicalize a reviewer identity so allowlist + dedupe are case/space stable. */
function normalizeReviewer(id: string): string {
  return id.trim().toLowerCase();
}

export type RequestReviewResult =
  | { ok: true; doc: FleetJobDoc }
  | { ok: false; reason: 'not_found' | 'conflict' | 'invalid_state' | 'invalid_policy' };

export type SubmitReviewResult =
  | { ok: true; doc: FleetJobDoc; gate: 'pending' | 'approved' | 'rejected' }
  | {
      ok: false;
      reason: 'not_found' | 'conflict' | 'invalid_state' | 'not_authorized';
    };

/**
 * Route a job into the `review` human gate (§14). Only a `building` job can enter
 * review (its work product is ready to inspect). Like `operatorAction`, the
 * control plane holds no lease, so this bumps `leaseEpoch` to fence the building
 * worker and releases its lease. The policy is normalized + validated, and any
 * prior decisions are cleared so each review round starts fresh.
 */
export async function requestReview(
  jobId: string,
  productId: string,
  policy?: { requiredApprovals?: number; reviewers?: string[] }
): Promise<RequestReviewResult> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return { ok: false, reason: 'not_found' };
  if (job.stage !== 'building') return { ok: false, reason: 'invalid_state' };

  const requiredApprovals = Math.max(1, Math.floor(policy?.requiredApprovals ?? 1));
  const reviewers = Array.from(
    new Set((policy?.reviewers ?? []).map(normalizeReviewer).filter(Boolean))
  );
  // An allowlist smaller than the approval quorum can never be satisfied.
  if (reviewers.length > 0 && requiredApprovals > reviewers.length) {
    return { ok: false, reason: 'invalid_policy' };
  }
  const reviewPolicy: ReviewPolicy = { requiredApprovals, reviewers };

  const newEpoch = job.leaseEpoch + 1; // fence the building worker
  const res = await repo.revUpdateJob(jobId, productId, job.rev, {
    stage: 'review',
    leaseEpoch: newEpoch,
    reviewPolicy,
    reviewDecisions: [],
    blockedReason: undefined,
  });
  if (!res.ok) {
    return { ok: false, reason: res.reason === 'not_found' ? 'not_found' : 'conflict' };
  }

  const lease = await repo.getLease(jobId);
  if (lease && lease.status === 'held') {
    await repo.revUpdateLease(jobId, lease.rev, {
      status: 'released',
      leaseEpoch: newEpoch,
      holderFactoryId: undefined,
    });
  }

  await repo.appendEvent({
    jobId,
    productId,
    type: 'review_requested',
    actor: 'operator',
    data: { requiredApprovals, reviewers, leaseEpoch: newEpoch },
  });

  return { ok: true, doc: res.doc };
}

const REVIEW_MAX_RETRIES = 4;

/**
 * Record one reviewer's decision against a job in the `review` gate (§14):
 *  - any `reject` is a veto → the job returns to `queued` for rework (fenced),
 *  - once distinct approvals reach `requiredApprovals` → the job advances to
 *    `testing`,
 *  - otherwise the decision is recorded and the job stays in `review`.
 *
 * Decisions are upserted per reviewer (last-write-wins). The read-modify-write is
 * guarded by the job `rev` CAS; on a concurrent-writer conflict we re-read and
 * retry a bounded number of times before surfacing `conflict`.
 */
export async function submitReview(
  jobId: string,
  productId: string,
  input: { reviewer: string; decision: ReviewDecisionKind; note?: string }
): Promise<SubmitReviewResult> {
  const reviewer = normalizeReviewer(input.reviewer);

  for (let attempt = 0; attempt < REVIEW_MAX_RETRIES; attempt++) {
    const job = await repo.getJob(jobId, productId);
    if (!job) return { ok: false, reason: 'not_found' };
    if (job.stage !== 'review') return { ok: false, reason: 'invalid_state' };

    const allowlist = job.reviewPolicy?.reviewers ?? [];
    if (allowlist.length > 0 && !allowlist.includes(reviewer)) {
      return { ok: false, reason: 'not_authorized' };
    }

    const decision: ReviewDecision = {
      reviewer,
      decision: input.decision,
      at: new Date().toISOString(),
      ...(input.note ? { note: input.note } : {}),
    };
    // Upsert: one current decision per reviewer (last-write-wins).
    const decisions = (job.reviewDecisions ?? [])
      .filter(d => d.reviewer !== reviewer)
      .concat(decision);

    if (input.decision === 'reject') {
      const newEpoch = job.leaseEpoch + 1; // fence any holder before rework
      const res = await repo.revUpdateJob(jobId, productId, job.rev, {
        stage: 'queued',
        leaseEpoch: newEpoch,
        reviewDecisions: decisions,
        blockedReason: `review rejected by ${reviewer}${input.note ? `: ${input.note}` : ''}`,
      });
      if (!res.ok) {
        if (res.reason === 'not_found') return { ok: false, reason: 'not_found' };
        continue; // conflict → re-read + retry
      }
      const lease = await repo.getLease(jobId);
      if (lease && lease.status === 'held') {
        await repo.revUpdateLease(jobId, lease.rev, {
          status: 'released',
          leaseEpoch: newEpoch,
          holderFactoryId: undefined,
        });
      }
      await repo.appendEvent({
        jobId,
        productId,
        type: 'review_rejected',
        actor: reviewer,
        data: { decisions, leaseEpoch: newEpoch, note: input.note },
      });
      return { ok: true, doc: res.doc, gate: 'rejected' };
    }

    // Approve: count DISTINCT approving reviewers against the quorum.
    const approvals = new Set(decisions.filter(d => d.decision === 'approve').map(d => d.reviewer))
      .size;
    const required = job.reviewPolicy?.requiredApprovals ?? 1;
    const satisfied = approvals >= required;

    const res = await repo.revUpdateJob(jobId, productId, job.rev, {
      stage: satisfied ? 'testing' : 'review',
      reviewDecisions: decisions,
    });
    if (!res.ok) {
      if (res.reason === 'not_found') return { ok: false, reason: 'not_found' };
      continue; // conflict → re-read + retry
    }
    await repo.appendEvent({
      jobId,
      productId,
      type: satisfied ? 'review_approved' : 'review_decision',
      actor: reviewer,
      data: { decision: 'approve', approvals, required, decisions },
    });
    return { ok: true, doc: res.doc, gate: satisfied ? 'approved' : 'pending' };
  }

  return { ok: false, reason: 'conflict' };
}

/** One factory's scored explanation for a job (already-weighted breakdown). */
export interface FactoryScoreExplain {
  factoryId: string;
  eligible: boolean;
  ineligibleReasons: string[];
  score: number;
  breakdown: ScoreBreakdown;
}

/** Full explainability payload for a job against the current fleet. */
export interface JobExplain {
  jobId: string;
  stage: FleetStage;
  weights: SchedulerWeights;
  depsSatisfied: boolean;
  unmetDeps: string[];
  factories: FactoryScoreExplain[];
  bestFactoryId: string | null;
}

/**
 * Explain how a job would be scored against every live factory for its product
 * (§7 scoring surfaced for the control plane). Read-only and side-effect free:
 * it re-runs the same `scoreCandidate` the scheduler uses, against persisted
 * factory state, so operators can see WHY a job routes (or fails to route).
 *
 * Affinity (prefers-engine) and cost-fit depend on claim-time hints that are not
 * persisted on the factory doc, so they score as neutral here — the breakdown
 * reflects the structural signals available from stored state.
 */
export async function explainJob(jobId: string, productId: string): Promise<JobExplain | null> {
  const job = await repo.getJob(jobId, productId);
  if (!job) return null;

  const weights = resolveWeights(weightRegistry, productId);
  const unmet = await unmetDeps(job);
  const depsSatisfied = unmet.length === 0;
  const factories = await repo.listFactories(productId);
  const ctx: SchedulerContext = { now: Date.now() };

  const scored: FactoryScoreExplain[] = factories.map(f => {
    const sf: SchedulerFactory = {
      capabilities: f.capabilities,
      health: f.health,
      load: f.load,
      seatLimit: f.seatLimit,
    };
    const { score, breakdown } = scoreCandidate(job, sf, ctx, weights);
    const reasons: string[] = [];
    if ((f.health ?? 'ok') === 'down') reasons.push('factory health is down');
    if (!capabilitiesSubset(job.capabilities ?? [], f.capabilities)) {
      reasons.push('missing required capabilities');
    }
    if (!engineEligible(job, f.capabilities)) {
      reasons.push(`engine '${job.engine}' not advertised by factory`);
    }
    if (!depsSatisfied) reasons.push(`unmet deps: ${unmet.join(', ')}`);
    return {
      factoryId: f.factoryId,
      eligible: reasons.length === 0,
      ineligibleReasons: reasons,
      score,
      breakdown,
    };
  });
  scored.sort((a, b) => b.score - a.score);
  const best = scored.find(s => s.eligible) ?? null;

  return {
    jobId,
    stage: job.stage,
    weights,
    depsSatisfied,
    unmetDeps: unmet,
    factories: scored,
    bestFactoryId: best?.factoryId ?? null,
  };
}

export interface HeartbeatContext {
  productId: string;
  factoryId: string;
  descriptor?: Record<string, unknown>;
  capabilities?: string[];
  health?: FleetFactoryHealthInput;
  load?: number;
  seatLimit?: number;
}
type FleetFactoryHealthInput = 'ok' | 'degraded' | 'down';

export async function heartbeat(ctx: HeartbeatContext): Promise<repo.RevResult<true>> {
  const now = new Date().toISOString();
  const existing = await repo.getFactory(ctx.factoryId, ctx.productId);
  await repo.upsertFactory({
    id: ctx.factoryId,
    productId: ctx.productId,
    factoryId: ctx.factoryId,
    descriptor: ctx.descriptor ?? existing?.descriptor ?? {},
    capabilities: ctx.capabilities ?? existing?.capabilities ?? [],
    health: ctx.health ?? existing?.health ?? 'ok',
    load: ctx.load ?? existing?.load ?? 0,
    seatLimit: ctx.seatLimit ?? existing?.seatLimit ?? 1,
    lastHeartbeatAt: now,
  });
  return { ok: true, doc: true };
}

/** A factory is stale if its last heartbeat is older than `maxAgeMs`. */
export function isFactoryStale(
  factory: { lastHeartbeatAt: string },
  nowMs: number,
  maxAgeMs: number
): boolean {
  return nowMs - new Date(factory.lastHeartbeatAt).getTime() > maxAgeMs;
}

// ── Phase 3: Per-product budgets (§11/§13) ────────────────────────────────────

/**
 * Get or create a product's budget. Returns existing budget or null if none set.
 */
export async function getBudget(productId: string): Promise<FleetBudgetDoc | null> {
  return repo.getBudget(productId);
}

/** Upsert a product's budget (ceiling + window, plus optional per-engine ceilings). */
export async function upsertBudget(
  productId: string,
  ceilingUsd: number,
  window: BudgetWindow = 'monthly',
  engineCeilingsUsd?: Record<string, number>
): Promise<FleetBudgetDoc> {
  const now = new Date().toISOString();
  const existing = await repo.getBudget(productId);
  const doc: FleetBudgetDoc = {
    id: productId,
    productId,
    ceilingUsd,
    window,
    spentUsd: existing?.spentUsd ?? 0,
    status: existing?.status ?? 'active',
    windowStart: existing?.windowStart ?? now,
    accruedRunIds: existing?.accruedRunIds ?? [],
    // Per-engine config is replaced when provided, else preserved; spend is preserved.
    ...(engineCeilingsUsd !== undefined
      ? { engineCeilingsUsd }
      : existing?.engineCeilingsUsd
        ? { engineCeilingsUsd: existing.engineCeilingsUsd }
        : {}),
    ...(existing?.spentByEngineUsd ? { spentByEngineUsd: existing.spentByEngineUsd } : {}),
    updatedAt: now,
  };
  return repo.upsertBudget(doc);
}

/** Pause a product's budget — claims for this product are blocked. */
export async function pauseBudget(productId: string): Promise<FleetBudgetDoc | null> {
  return repo.updateBudget(productId, { status: 'paused' });
}

/** Resume a product's budget — claims for this product proceed again. */
export async function resumeBudget(productId: string): Promise<FleetBudgetDoc | null> {
  return repo.updateBudget(productId, { status: 'active' });
}

/**
 * Accrue spend for a product after a run completes.
 *
 * Idempotent per run: `runId` identifies the completed run (e.g. `<jobId>:<leaseEpoch>`).
 * If that runId was already accrued, this is a no-op — re-delivering the same completion
 * (retries, duplicate transitions) never double-counts. Spend is monotonic. Auto-pauses
 * the product when the ceiling is reached.
 */
export async function accrueSpend(
  productId: string,
  costUsd: number,
  runId?: string,
  engine?: string
): Promise<FleetBudgetDoc | null> {
  const budget = await repo.getBudget(productId);
  if (!budget) return null;
  // Idempotency guard: skip if this run was already accrued (retries / duplicate
  // ship transitions never double-count; spend stays monotonic).
  if (runId && (budget.accruedRunIds ?? []).includes(runId)) return budget;
  const updates: Partial<FleetBudgetDoc> = {};
  if (runId) updates.accruedRunIds = [...(budget.accruedRunIds ?? []), runId];
  if (costUsd > 0) {
    const newSpent = budget.spentUsd + costUsd;
    updates.spentUsd = newSpent;
    // Track per-engine spend alongside the total (drives per-engine ceilings + the
    // dashboard breakdown). Same idempotency guard as the total above.
    if (engine && engine !== 'unknown') {
      const prev = budget.spentByEngineUsd ?? {};
      updates.spentByEngineUsd = { ...prev, [engine]: (prev[engine] ?? 0) + costUsd };
    }
    // Auto-pause when the PRODUCT ceiling is exceeded (per-engine ceilings only
    // route around that engine at claim time — they never pause the whole product).
    if (newSpent >= budget.ceilingUsd && budget.status === 'active') {
      updates.status = 'paused';
    }
  }
  if (Object.keys(updates).length === 0) return budget;
  return repo.updateBudget(productId, updates);
}

/** Nominal length of each budget window, in ms (monthly ≈ 30d). */
const BUDGET_WINDOW_MS: Record<BudgetWindow, number> = {
  daily: 86_400_000,
  weekly: 7 * 86_400_000,
  monthly: 30 * 86_400_000,
};

/** A budget's projected end-of-window spend at the current burn rate. */
export interface BudgetProjection {
  /** Fraction of the window elapsed, in (0,1]. */
  fraction: number;
  /** spentUsd extrapolated to the full window (spentUsd / fraction). */
  projectedUsd: number;
}

/**
 * Project a budget's full-window spend from its burn rate so far (PURE). Returns
 * null when we shouldn't project yet: no `windowStart`, or too little of the
 * window has elapsed (`minFraction`, default 10%) — early in a window a couple of
 * runs extrapolate to wildly inflated totals and would cry wolf. The fraction is
 * capped at 1 so a window that has fully elapsed reports actual spend, not more.
 */
export function projectBudgetSpend(
  budget: Pick<FleetBudgetDoc, 'spentUsd' | 'window' | 'windowStart'>,
  nowMs: number,
  minFraction = 0.1
): BudgetProjection | null {
  if (!budget.windowStart) return null;
  const start = Date.parse(budget.windowStart);
  if (Number.isNaN(start)) return null;
  const windowMs = BUDGET_WINDOW_MS[budget.window];
  const fraction = Math.min(1, Math.max(0, nowMs - start) / windowMs);
  if (fraction < minFraction) return null;
  return { fraction, projectedUsd: budget.spentUsd / fraction };
}

// ── Cost burndown (§14 Phase 3 — spend-over-time vs ceiling) ──────────────────

/** One day of the burndown series. */
export interface BurndownPoint {
  date: string; // YYYY-MM-DD (UTC)
  costUsd: number; // spend attributed to that day
  cumulativeUsd: number; // running total up to and including that day
}

export interface CostBurndown {
  productId: string;
  ceilingUsd: number | null; // budget ceiling, if configured
  window: BudgetWindow | null;
  totalUsd: number;
  days: BurndownPoint[];
}

/**
 * Aggregate completed run cost by UTC day for the last `days` days (default 30),
 * returning a cumulative burndown series plus the budget ceiling for overlay.
 * Read-only. Costs come from each run's `insights.costUsd`; a run is attributed
 * to the day it ended (or started, if still in flight without an end time).
 */
export async function costBurndown(productId: string, days = 30): Promise<CostBurndown> {
  const budget = await repo.getBudget(productId);
  const runs = await repo.listRunsByProduct(productId);

  const windowDays = Math.max(1, Math.min(days, 365));
  const dayMs = 86_400_000;
  const todayUtc = Math.floor(Date.now() / dayMs) * dayMs;
  const startUtc = todayUtc - (windowDays - 1) * dayMs;

  // Initialise every day in the window to zero so the series has no gaps.
  const byDay = new Map<string, number>();
  for (let t = startUtc; t <= todayUtc; t += dayMs) {
    byDay.set(new Date(t).toISOString().slice(0, 10), 0);
  }

  for (const run of runs) {
    const cost = run.insights?.costUsd;
    if (!cost || cost <= 0) continue;
    const when = Date.parse(run.endedAt ?? run.startedAt);
    if (Number.isNaN(when) || when < startUtc) continue;
    const key = new Date(when).toISOString().slice(0, 10);
    byDay.set(key, (byDay.get(key) ?? 0) + cost);
  }

  const sortedKeys = [...byDay.keys()].sort();
  let cumulative = 0;
  const series: BurndownPoint[] = sortedKeys.map(date => {
    const costUsd = byDay.get(date) ?? 0;
    cumulative += costUsd;
    return { date, costUsd, cumulativeUsd: cumulative };
  });

  return {
    productId,
    ceilingUsd: budget?.ceilingUsd ?? null,
    window: budget?.window ?? null,
    totalUsd: cumulative,
    days: series,
  };
}

// ── Phase 3: Fleet metrics + alerting (§17) ───────────────────────────────────

export interface FleetAlert {
  level: 'warning' | 'critical';
  code: string;
  message: string;
}

export interface FleetMetrics {
  productId: string;
  generatedAt: string;
  jobs: {
    total: number;
    byStage: Record<FleetStage, number>;
    queueDepth: number;
    blocked: number;
    active: number;
    oldestQueuedAgeMs: number | null;
  };
  factories: {
    total: number;
    live: number;
    stale: number;
    byHealth: { ok: number; degraded: number; down: number };
    seatsUsed: number;
    seatsTotal: number;
    utilizationPct: number;
  };
  /** Budget summary for the dashboard (§3) — null when no budget is configured. */
  budget: FleetBudgetSummary | null;
  alerts: FleetAlert[];
}

/** Compact budget snapshot surfaced on metrics for the dashboard burndown/guardrail. */
export interface FleetBudgetSummary {
  ceilingUsd: number;
  spentUsd: number;
  status: string;
  window: BudgetWindow;
  /** Projected end-of-window spend at the current burn rate (null if too early). */
  projectedUsd: number | null;
  /** Per-engine spend vs sub-ceiling, only for engines that HAVE a ceiling. */
  engines: { engine: string; spentUsd: number; ceilingUsd: number; exhausted: boolean }[];
}

/** A factory is considered stale after this long without a heartbeat. */
const DEFAULT_STALE_FACTORY_MS = 90_000;
/** A queued job waiting longer than this raises a starvation alert. */
const DEFAULT_STARVATION_MS = 900_000;

/**
 * Compute point-in-time operational metrics for a product's fleet (§17):
 * queue depth, stage histogram, oldest-queued age (starvation), factory health
 * and seat utilization — plus derived alerts. Read-only: a single per-partition
 * job + factory scan, no writes. Thresholds are injectable for deterministic tests.
 */
export async function fleetMetrics(
  productId: string,
  opts?: { nowMs?: number; staleMaxAgeMs?: number; starvationMs?: number }
): Promise<FleetMetrics> {
  const nowMs = opts?.nowMs ?? Date.now();
  const staleMaxAgeMs = opts?.staleMaxAgeMs ?? DEFAULT_STALE_FACTORY_MS;
  const starvationMs = opts?.starvationMs ?? DEFAULT_STARVATION_MS;

  const [allJobs, factories, budgetDoc] = await Promise.all([
    repo.listJobs({ productId }),
    repo.listFactories(productId),
    repo.getBudget(productId),
  ]);

  const byStage = Object.fromEntries(FLEET_STAGES.map(s => [s, 0])) as Record<FleetStage, number>;
  let oldestQueuedAgeMs: number | null = null;
  for (const j of allJobs) {
    byStage[j.stage] = (byStage[j.stage] ?? 0) + 1;
    if (j.stage === 'queued') {
      const age = nowMs - Date.parse(j.createdAt);
      if (Number.isFinite(age) && (oldestQueuedAgeMs === null || age > oldestQueuedAgeMs)) {
        oldestQueuedAgeMs = age;
      }
    }
  }
  const queueDepth = byStage.queued ?? 0;
  const blocked = byStage.blocked ?? 0;
  const active = ACTIVE_STAGES.reduce((n, s) => n + (byStage[s] ?? 0), 0);

  const byHealth = { ok: 0, degraded: 0, down: 0 };
  let live = 0;
  let stale = 0;
  let seatsUsed = 0;
  let seatsTotal = 0;
  for (const f of factories) {
    if (isFactoryStale(f, nowMs, staleMaxAgeMs)) stale++;
    else live++;
    if (f.health === 'ok' || f.health === 'degraded' || f.health === 'down') byHealth[f.health]++;
    seatsUsed += f.load ?? 0;
    seatsTotal += f.seatLimit ?? 0;
  }
  const utilizationPct = seatsTotal > 0 ? Math.round((seatsUsed / seatsTotal) * 1000) / 10 : 0;

  const alerts: FleetAlert[] = [];
  const liveHealthy = factories.filter(
    f => !isFactoryStale(f, nowMs, staleMaxAgeMs) && f.health !== 'down'
  ).length;
  if (queueDepth > 0 && liveHealthy === 0) {
    alerts.push({
      level: 'critical',
      code: 'no_live_capacity',
      message: `${queueDepth} job(s) queued but no live, healthy factory is available.`,
    });
  }
  if (factories.length > 0 && byHealth.down === factories.length) {
    alerts.push({
      level: 'critical',
      code: 'all_factories_down',
      message: 'All registered factories report health "down".',
    });
  }
  if (oldestQueuedAgeMs !== null && oldestQueuedAgeMs > starvationMs) {
    alerts.push({
      level: 'warning',
      code: 'queue_starvation',
      message: `Oldest queued job has waited ${Math.round(oldestQueuedAgeMs / 1000)}s (threshold ${Math.round(starvationMs / 1000)}s).`,
    });
  }
  if (seatsTotal > 0 && seatsUsed >= seatsTotal && queueDepth > 0) {
    alerts.push({
      level: 'warning',
      code: 'saturated',
      message: `All ${seatsTotal} factory seat(s) are in use with ${queueDepth} job(s) still queued.`,
    });
  }
  if (stale > 0) {
    alerts.push({
      level: 'warning',
      code: 'stale_factories',
      message: `${stale} factory(ies) have not sent a heartbeat within ${Math.round(staleMaxAgeMs / 1000)}s.`,
    });
  }
  const deadLetter = byStage.dead_letter ?? 0;
  if (deadLetter > 0) {
    alerts.push({
      level: 'warning',
      code: 'dead_letter',
      message: `${deadLetter} job(s) in dead_letter — exhausted retries or rejected; needs operator triage.`,
    });
  }

  // ── Budget guardrails (§3): exhaustion + projected-overspend warnings ──
  // Surfaced whenever a budget is configured (read-only; independent of the
  // FLEET_BUDGETS enforcement flag, so operators see the burn even in dry-run).
  let budget: FleetBudgetSummary | null = null;
  if (budgetDoc && budgetDoc.ceilingUsd > 0) {
    const projection = projectBudgetSpend(budgetDoc, nowMs);
    const engines = Object.entries(budgetDoc.engineCeilingsUsd ?? {}).map(
      ([engine, ceilingUsd]) => {
        const spentUsd = budgetDoc.spentByEngineUsd?.[engine] ?? 0;
        return { engine, spentUsd, ceilingUsd, exhausted: spentUsd >= ceilingUsd };
      }
    );
    budget = {
      ceilingUsd: budgetDoc.ceilingUsd,
      spentUsd: budgetDoc.spentUsd,
      status: budgetDoc.status,
      window: budgetDoc.window,
      projectedUsd: projection?.projectedUsd ?? null,
      engines,
    };

    if (budgetDoc.spentUsd >= budgetDoc.ceilingUsd) {
      alerts.push({
        level: 'critical',
        code: 'budget_exhausted',
        message: `Budget exhausted: $${budgetDoc.spentUsd.toFixed(2)} of the $${budgetDoc.ceilingUsd.toFixed(2)} ${budgetDoc.window} ceiling spent — new claims are blocked.`,
      });
    } else if (projection && projection.projectedUsd > budgetDoc.ceilingUsd) {
      alerts.push({
        level: 'warning',
        code: 'budget_overspend_projected',
        message: `Projected ${budgetDoc.window} spend $${projection.projectedUsd.toFixed(2)} exceeds the $${budgetDoc.ceilingUsd.toFixed(2)} ceiling at the current burn rate (${Math.round(projection.fraction * 100)}% of the window elapsed, $${budgetDoc.spentUsd.toFixed(2)} spent).`,
      });
    }
    for (const e of engines) {
      if (e.exhausted) {
        alerts.push({
          level: 'warning',
          code: 'engine_budget_exhausted',
          message: `Engine '${e.engine}' reached its $${e.ceilingUsd.toFixed(2)} sub-ceiling ($${e.spentUsd.toFixed(2)} spent) — it is being routed around.`,
        });
      }
    }
  }

  return {
    productId,
    generatedAt: new Date(nowMs).toISOString(),
    jobs: { total: allJobs.length, byStage, queueDepth, blocked, active, oldestQueuedAgeMs },
    factories: {
      total: factories.length,
      live,
      stale,
      byHealth,
      seatsUsed,
      seatsTotal,
      utilizationPct,
    },
    budget,
    alerts,
  };
}

export interface ReapResult {
  reaped: number;
  jobIds: string[];
}

/**
 * Reclaim ONE held lease's job: fence the zombie holder (bump leaseEpoch), return
 * the job to queued/blocked preserving its checkpoint, mark the lease terminal, and
 * emit an audit event. Returns the reclaimed jobId, or null when there was nothing
 * to do (job gone, already resting, or a lost CAS race — all idempotent no-ops).
 * Shared by the expiry reaper and the stale-factory reclaim so both behave identically.
 */
async function reclaimLeaseJob(
  lease: FleetLeaseDoc,
  eventType: 'lease_expired' | 'factory_stale'
): Promise<string | null> {
  const job = await repo.getJob(lease.jobId, lease.productId);
  if (!job) return null;
  if (!ACTIVE_STAGES.includes(job.stage)) return null; // already resting; nothing to reclaim
  const newEpoch = job.leaseEpoch + 1; // fence the zombie holder
  const unmet = await unmetDeps(job);
  const stage: FleetStage = unmet.length > 0 ? 'blocked' : 'queued';
  // checkpoint pointer is intentionally preserved on the job (resume-friendly)
  const jobRes = await repo.revUpdateJob(job.id, lease.productId, job.rev, {
    stage,
    leaseEpoch: newEpoch,
    blockedReason: unmet.length > 0 ? `waiting on: ${unmet.join(', ')}` : undefined,
  });
  if (!jobRes.ok) return null;
  await repo.revUpdateLease(lease.jobId, lease.rev, {
    status: 'expired',
    leaseEpoch: newEpoch,
    holderFactoryId: undefined,
  });
  await repo.appendEvent({
    jobId: job.id,
    productId: lease.productId,
    type: eventType,
    data: {
      leaseEpoch: newEpoch,
      returnedTo: stage,
      ...(lease.holderFactoryId ? { holderFactoryId: lease.holderFactoryId } : {}),
    },
  });
  return job.id;
}

/**
 * Reclaim jobs whose lease has expired. Cosmos TTL would only DELETE the lease
 * doc — it cannot return the job to `queued`, bump the fencing epoch, or preserve
 * the checkpoint — so the reaper (not TTL) owns recovery. Idempotent: a reaped
 * lease becomes `expired` and is no longer selected by listExpiredLeases.
 */
export async function reapExpiredLeases(nowIso: string): Promise<ReapResult> {
  const expired = await repo.listExpiredLeases(nowIso);
  const jobIds: string[] = [];
  for (const lease of expired) {
    const id = await reclaimLeaseJob(lease, 'lease_expired');
    if (id) jobIds.push(id);
  }
  return { reaped: jobIds.length, jobIds };
}

/**
 * Reclaim held leases whose holder factory has gone stale (missed heartbeats past
 * `staleMaxAgeMs`) or vanished entirely, without waiting out the (much longer)
 * lease TTL. A crashed/network-partitioned factory stops heartbeating long before
 * its 900s lease expires; this recovers its job within one stale window instead.
 * Idempotent and safe alongside the expiry reaper (both go through reclaimLeaseJob).
 */
export async function reclaimStaleFactoryLeases(
  nowMs: number,
  staleMaxAgeMs = DEFAULT_STALE_FACTORY_MS
): Promise<ReapResult> {
  const [held, factories] = await Promise.all([repo.listHeldLeases(), repo.listAllFactories()]);
  // factoryId → most-recent heartbeat (a factory can appear once per product).
  const lastBeat = new Map<string, number>();
  for (const f of factories) {
    const t = Date.parse(f.lastHeartbeatAt);
    if (Number.isNaN(t)) continue;
    lastBeat.set(f.factoryId, Math.max(lastBeat.get(f.factoryId) ?? 0, t));
  }
  const jobIds: string[] = [];
  for (const lease of held) {
    const holder = lease.holderFactoryId;
    if (!holder) continue;
    const beat = lastBeat.get(holder);
    // Reclaim when the holder is unknown (vanished) or hasn't heartbeat recently.
    const isStale = beat === undefined || nowMs - beat > staleMaxAgeMs;
    if (!isStale) continue;
    const id = await reclaimLeaseJob(lease, 'factory_stale');
    if (id) jobIds.push(id);
  }
  return { reaped: jobIds.length, jobIds };
}

/** Terminal job stages eligible for retention GC (resting, never to run again). */
export const GC_TERMINAL_STAGES: readonly FleetStage[] = ['shipped', 'failed', 'dead_letter'];

export interface SweepOptions {
  /** Authoritative now (ms epoch). */
  now: number;
  /** Delete finished (expired/released) lease docs older than this. Default 24h. */
  finishedLeaseTtlMs?: number;
  /** Delete factory docs with no heartbeat for longer than this. Default 7d. */
  deadFactoryTtlMs?: number;
  /** When > 0, delete terminal jobs (and cascade runs/events/artifacts/lease) last
   *  updated longer ago than this. 0/undefined ⇒ DISABLED (never deletes history). */
  jobRetentionMs?: number;
}

export interface SweepResult {
  leasesDeleted: number;
  factoriesDeleted: number;
  tokensDeleted: number;
  jobsDeleted: number;
  runsDeleted: number;
  eventsDeleted: number;
  artifactsDeleted: number;
}

const DEFAULT_FINISHED_LEASE_TTL_MS = 24 * 60 * 60 * 1000; // 1 day
const DEFAULT_DEAD_FACTORY_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days

/**
 * Garbage-collect fleet bookkeeping so the containers don't grow without bound.
 *
 * On by default (safe — these are coordination ephemera, not history):
 *  • finished leases (expired/released) older than `finishedLeaseTtlMs`
 *  • factory docs with no heartbeat for longer than `deadFactoryTtlMs` (a live
 *    host simply re-registers on its next heartbeat)
 *
 * Opt-in only (destroys audit history — `jobRetentionMs` must be explicitly > 0):
 *  • terminal jobs older than `jobRetentionMs`, cascading to their runs, events,
 *    artifact pointers (+ backing blobs, best-effort), and lease.
 *
 * Every deletion is best-effort: one failure is swallowed so the sweep makes
 * forward progress and retries on the next pass.
 */
export async function sweepFleetGarbage(opts: SweepOptions): Promise<SweepResult> {
  const now = opts.now;
  const finishedLeaseTtlMs = opts.finishedLeaseTtlMs ?? DEFAULT_FINISHED_LEASE_TTL_MS;
  const deadFactoryTtlMs = opts.deadFactoryTtlMs ?? DEFAULT_DEAD_FACTORY_TTL_MS;
  const jobRetentionMs = opts.jobRetentionMs ?? 0;
  const result: SweepResult = {
    leasesDeleted: 0,
    factoriesDeleted: 0,
    tokensDeleted: 0,
    jobsDeleted: 0,
    runsDeleted: 0,
    eventsDeleted: 0,
    artifactsDeleted: 0,
  };

  // 1) Finished leases (ephemeral coordination state) past their TTL.
  const leaseCutoff = new Date(now - finishedLeaseTtlMs).toISOString();
  for (const lease of await repo.listFinishedLeasesOlderThan(leaseCutoff)) {
    try {
      await repo.deleteLease(lease.jobId);
      result.leasesDeleted++;
    } catch {
      /* best-effort */
    }
  }

  // 2) Long-dead factory docs (a live host re-registers on next heartbeat).
  for (const f of await repo.listAllFactories()) {
    const beat = Date.parse(f.lastHeartbeatAt);
    if (Number.isNaN(beat) || now - beat <= deadFactoryTtlMs) continue;
    try {
      await repo.deleteFactory(f.factoryId, f.productId);
      result.factoriesDeleted++;
    } catch {
      /* best-effort */
    }
  }

  // 2b) Invalidated factory tokens (revoked / expired / rotating past grace).
  result.tokensDeleted = await pruneInvalidatedTokens(now);

  // 3) Terminal-job retention cascade — OPT-IN only (destroys history).
  if (jobRetentionMs > 0) {
    const jobCutoff = new Date(now - jobRetentionMs).toISOString();
    for (const job of await repo.listTerminalJobsOlderThan(jobCutoff, GC_TERMINAL_STAGES)) {
      try {
        for (const run of await repo.listRunsByJob(job.id)) {
          await repo.deleteRun(run.id, job.id);
          result.runsDeleted++;
        }
        for (const ev of await repo.listEvents(job.id)) {
          await repo.deleteEvent(ev.id, job.id);
          result.eventsDeleted++;
        }
        for (const art of await repo.listArtifactsByJob(job.id, job.productId)) {
          try {
            await deleteArtifactWithBlob(art.id, job.productId);
          } catch {
            await repo.deleteArtifact(art.id, job.productId); // pointer at least
          }
          result.artifactsDeleted++;
        }
        await repo.deleteLease(job.id).catch(() => {});
        await repo.deleteJob(job.id, job.productId);
        result.jobsDeleted++;
      } catch {
        /* best-effort: leave this job for the next pass */
      }
    }
  }

  return result;
}
