feat(agent-queue): evolved manifest, priority, capabilities, engine-class, idempotency (P1-S1)

Implements Gigafactory Phase 1 - Slice 1 in the bash runner (backward-compatible;
a legacy engine/cwd/yolo-only .md behaves exactly as before):

- Parse all new §5 manifest keys via fm_get with safe defaults; record them in
  <job>.meta and surface priority/profile/capabilities/tracker-item in `status`.
  Only priority, capabilities, engine-class and idempotency-key are functional
  this slice; the rest (profile, prefers, budget, deps, deps-mode, retry,
  review-policy, artifacts, tracker-item) are stored but inert.
- priority ordering: inbox_sorted picks critical>high>medium>low, ties by oldest;
  per-lock serialization preserved.
- capability grammar + match: detect_capabilities advertises os/engine/node/has
  tokens; caps_match honors key, key:value, key<op>version and os:any. A job whose
  declared capabilities the host cannot satisfy is moved to failed/ with
  result=capability_mismatch and the agent is never launched.
- engine-class resolution: explicit engine wins; else engine-class picks the first
  available engine honoring prefers-engine (agentic-coder->devin,claude,codex;
  chat-coder->copilot). No available engine -> result=no_engine. Adds copilot to
  the engine driver + COPILOT_BIN.
- idempotency-key dedupe on add: same key+body -> no-op; same key+different body
  supersedes an inbox prior, else is rejected with a clear error.

No change to queue/ data or the run/ship lifecycle. macOS + Linux safe.
This commit is contained in:
saravanakumardb1 2026-05-29 17:44:19 -07:00
parent 3ad9500623
commit 0be5b34123

View File

@ -62,6 +62,7 @@ TIMEOUT_BIN="${TIMEOUT_BIN:-$(command -v timeout || command -v gtimeout || true)
DEVIN_BIN="${DEVIN_BIN:-$(command -v devin || echo "$HOME/.local/bin/devin")}"
CLAUDE_BIN="${CLAUDE_BIN:-$(command -v claude || echo claude)}"
CODEX_BIN="${CODEX_BIN:-$(command -v codex || echo codex)}"
COPILOT_BIN="${COPILOT_BIN:-$(command -v copilot || echo copilot)}"
# ── Colors ──────────────────────────────────────────────────────────
if [[ -t 1 ]]; then
@ -194,6 +195,188 @@ busy_keys() {
done
}
# ── Manifest helpers (Phase 1 — §5/§6/§7/§8 single-host subset) ──────
#
# parse_list <raw> -> one token per line. Accepts a YAML-ish inline list
# ("[a, b]"), comma- or space-separated values, with surrounding quotes
# stripped. Empty tokens are dropped. Used for `capabilities`, `prefers`,
# `prefers-engine`, `deps`, etc.
parse_list() {
local raw cleaned t
cleaned=$(printf '%s' "${1:-}" | tr '[],' ' ')
for t in $cleaned; do
t=${t#[\"\']}; t=${t%[\"\']}
[[ -n "$t" ]] && printf '%s\n' "$t"
done
}
# _body_hash <file> -> stable content hash of the frontmatter-STRIPPED body.
# Drives idempotency-key dedupe on `add`. Prefers shasum/sha1sum, falls back
# to cksum so it works on a bare macOS/Linux host with no extra tools.
_body_hash() {
local f=$1
if command -v shasum >/dev/null 2>&1; then
strip_frontmatter "$f" | shasum | awk '{print $1}'
elif command -v sha1sum >/dev/null 2>&1; then
strip_frontmatter "$f" | sha1sum | awk '{print $1}'
else
strip_frontmatter "$f" | cksum | awk '{print $1"-"$2}'
fi
}
# priority_rank <priority> -> sort rank (lower = picked first). Unknown/empty
# defaults to medium. Used by inbox_sorted for priority-then-age selection.
priority_rank() {
case "$1" in
critical) echo 0;; high) echo 1;; medium) echo 2;; low) echo 3;; *) echo 2;;
esac
}
# inbox_sorted -> inbox/*.md ordered by priority (critical first) then oldest
# (filename embeds the queue timestamp, so a plain string sort = age order).
# Replaces the pure-FIFO `ls | sort`; per-lock serialization is unchanged
# (the caller still skips files whose lock key is busy).
inbox_sorted() {
local f p r
for f in "$INBOX"/*.md; do
[[ -e "$f" ]] || continue
p=$(fm_get "$f" priority medium); r=$(priority_rank "$p")
printf '%s\t%s\n' "$r" "$f"
done | sort -t"$(printf '\t')" -k1,1n -k2,2 | cut -f2-
}
# engine_available <engine> -> 0 if a runnable binary exists for the engine
# (executable path or a name resolvable on PATH). Honors the *_BIN overrides
# (so the self-test stub counts as "available").
engine_available() {
local eng=$1 bin=""
case "$eng" in
devin) bin=$DEVIN_BIN;;
claude) bin=$CLAUDE_BIN;;
codex) bin=$CODEX_BIN;;
copilot) bin=$COPILOT_BIN;;
*) return 1;;
esac
[[ -n "$bin" ]] || return 1
[[ -x "$bin" ]] && return 0
command -v "$bin" >/dev/null 2>&1
}
# engine_class_engines <class> -> candidate engines (priority order) for an
# engine-class (§5 taxonomy). `review-only` has no concrete mapping yet
# (reserved). Unknown class -> empty.
engine_class_engines() {
case "$1" in
agentic-coder) echo "devin claude codex";;
chat-coder) echo "copilot";;
review-only) echo "";;
*) echo "";;
esac
}
# detect_capabilities -> one capability token per line describing THIS host:
# os:<mac|linux|other> (os:any required-side wildcard matches this)
# engine:<devin|claude|codex|copilot> for each available engine
# node:<major> the host node major (compared by node<op>N)
# has:<git|pnpm|docker> tool probes (only emitted when present)
detect_capabilities() {
case "$(uname -s 2>/dev/null)" in
Darwin) echo "os:mac";;
Linux) echo "os:linux";;
*) echo "os:other";;
esac
local e
for e in devin claude codex copilot; do
engine_available "$e" && echo "engine:$e"
done
if command -v node >/dev/null 2>&1; then
local nv; nv=$(node -v 2>/dev/null | sed 's/^v//' | cut -d. -f1)
[[ "$nv" =~ ^[0-9]+$ ]] && echo "node:$nv"
fi
local t
for t in git pnpm docker; do
command -v "$t" >/dev/null 2>&1 && echo "has:$t"
done
}
# _cap_token_ok <required-token> <available-tokens> -> 0 if satisfied (§5 grammar):
# key:any wildcard — host advertises any "key:*" token (os:any matches all)
# key<op>ver numeric/semver-major compare against the host's "key:<val>" token
# (op in >= > = <= <)
# key:value exact token match
# key bare presence — exact, or advertised in any "key:"/"key<op>" form
_cap_token_ok() {
local tok=$1 avail=$2 a
if [[ "$tok" == *:any ]]; then
local wkey=${tok%:any}
for a in $avail; do [[ "$a" == "$wkey":* ]] && return 0; done
return 1
fi
if [[ "$tok" =~ ^([A-Za-z0-9_.:-]+)(\>=|\<=|=|\>|\<)([0-9][0-9.]*)$ ]]; then
local key=${BASH_REMATCH[1]} op=${BASH_REMATCH[2]} want=${BASH_REMATCH[3]} hostval=""
for a in $avail; do [[ "$a" == "$key":* ]] && { hostval=${a#*:}; break; }; done
[[ -n "$hostval" ]] || return 1
local hv=${hostval%%.*} wv=${want%%.*}
[[ "$hv" =~ ^[0-9]+$ && "$wv" =~ ^[0-9]+$ ]] || return 1
case "$op" in
">=") [[ "$hv" -ge "$wv" ]];;
">") [[ "$hv" -gt "$wv" ]];;
"=") [[ "$hv" -eq "$wv" ]];;
"<=") [[ "$hv" -le "$wv" ]];;
"<") [[ "$hv" -lt "$wv" ]];;
esac
return $?
fi
if [[ "$tok" == *:* ]]; then
for a in $avail; do [[ "$a" == "$tok" ]] && return 0; done
return 1
fi
for a in $avail; do
case "$a" in "$tok"|"$tok":*|"$tok"\>=*|"$tok"\>*|"$tok"=*|"$tok"\<=*|"$tok"\<*) return 0;; esac
done
return 1
}
# caps_match <required-tokens> <available-tokens> -> 0 iff EVERY required token
# is satisfied by the available set (both passed as whitespace-separated lists).
caps_match() {
local required=$1 available=$2 r
for r in $required; do
_cap_token_ok "$r" "$available" || return 1
done
return 0
}
# resolve_engine <file> -> the concrete engine to run:
# explicit `engine:` always wins; else `engine-class` picks the first
# available engine honoring `prefers-engine` then the class default order;
# else the global default engine. Prints "" only when an engine-class was
# requested but no engine in it is available (caller fails with no_engine).
resolve_engine() {
local f=$1 eng cls prefers
eng=$(fm_get "$f" engine "")
if [[ -n "$eng" ]]; then printf '%s' "$eng"; return 0; fi
cls=$(fm_get "$f" engine-class "")
if [[ -z "$cls" ]]; then printf '%s' "$DEFAULT_ENGINE"; return 0; fi
local class_engines; class_engines=$(engine_class_engines "$cls")
prefers=$(fm_get "$f" prefers-engine "")
local ordered=() seen=" " p c
if [[ -n "$prefers" ]]; then
while IFS= read -r p; do
[[ -n "$p" ]] || continue
case " $class_engines " in *" $p "*) ordered+=("$p"); seen+="$p ";; esac
done < <(parse_list "$prefers")
fi
for c in $class_engines; do
case "$seen" in *" $c "*) ;; *) ordered+=("$c"); seen+="$c ";; esac
done
local cand
for cand in ${ordered[@]+"${ordered[@]}"}; do
engine_available "$cand" && { printf '%s' "$cand"; return 0; }
done
printf '%s' ""
}
# ── Engine driver: builds argv into AGENT_CMD[]; sets AGENT_STDIN if the ──
# prompt should be fed on stdin (claude/codex) rather than a flag. $pf is the
# frontmatter-STRIPPED body file, so a body starting with '--' is never
@ -216,7 +399,14 @@ build_agent_cmd() {
[[ "$yolo" == "true" ]] && AGENT_CMD+=( --dangerously-bypass-approvals-and-sandbox )
AGENT_STDIN="$pf"
;;
*) die "unknown engine '$engine' (use: devin | claude | codex)";;
copilot)
# Best-effort GitHub Copilot CLI mapping for the chat-coder engine-class.
# Flags drift between CLI versions — this is the single place to edit.
AGENT_CMD=( "$COPILOT_BIN" -p )
[[ "$yolo" == "true" ]] && AGENT_CMD+=( --allow-all-tools )
AGENT_STDIN="$pf"
;;
*) die "unknown engine '$engine' (use: devin | claude | codex | copilot)";;
esac
}
@ -226,7 +416,6 @@ run_worker() {
local job; job=$(basename "$doing_file")
job=${job%.md}
local engine cwd yolo logf metaf
engine=$(fm_get "$doing_file" engine "$DEFAULT_ENGINE")
cwd=$(fm_get "$doing_file" cwd "$PWD")
yolo=$(fm_get "$doing_file" yolo "true")
logf="$LOGS/$job.log"
@ -234,6 +423,38 @@ run_worker() {
# NOTE: the parent (cmd_run) creates $metaf with job/engine/cwd/started/pid.
# The worker only ever APPENDS (ended/exit/result) to avoid a truncation race.
# ── Capability gate (§5/§8 single-host): if the job declares `capabilities`
# this host does not satisfy, route to failed/ WITHOUT launching the agent. ──
local req_caps; req_caps=$(parse_list "$(fm_get "$doing_file" capabilities "")" | tr '\n' ' ')
if [[ -n "${req_caps// /}" ]]; then
local avail; avail=$(detect_capabilities)
if ! caps_match "$req_caps" "$avail"; then
{
echo "===== agent-queue job: $job ====="
echo "CAPABILITY MISMATCH — host cannot satisfy required: $req_caps"
echo "host advertises: $(printf '%s' "$avail" | tr '\n' ' ')"
echo "agent NOT launched — routed to failed/ ($(date))"
} >> "$logf"
mv "$doing_file" "$FAILED/" 2>/dev/null
{ echo "result=capability_mismatch"; echo "ended=$(date +%s)"; } >> "$metaf"
return 0
fi
fi
# ── Engine resolution (§5): explicit `engine` wins, else `engine-class`.
# No available engine for a requested class -> fail (no_engine), no launch. ──
engine=$(resolve_engine "$doing_file")
if [[ -z "$engine" ]]; then
{
echo "===== agent-queue job: $job ====="
echo "NO ENGINE — engine-class '$(fm_get "$doing_file" engine-class "")' has no available engine on this host"
echo "agent NOT launched — routed to failed/ ($(date))"
} >> "$logf"
mv "$doing_file" "$FAILED/" 2>/dev/null
{ echo "result=no_engine"; echo "ended=$(date +%s)"; } >> "$metaf"
return 0
fi
{
echo "===== agent-queue job: $job ====="
echo "engine=$engine cwd=$cwd yolo=$yolo"
@ -363,6 +584,50 @@ cmd_add() {
esac
done
[[ -n "$file" && -f "$file" ]] || die "usage: add <file.md> [--engine devin|claude|codex] [--cwd PATH] [--yolo|--no-yolo]"
# ── idempotency-key dedupe (§5) ──
# If the file declares an idempotency-key, compare a content hash of its
# stripped body against existing jobs in any active stage:
# same key + same body -> no-op (skip; "duplicate")
# same key + different body, prior in inbox/ -> supersede (replace it)
# same key + different body, prior elsewhere -> reject (clear error)
local idem; idem=$(fm_get "$file" idempotency-key "")
if [[ -n "$idem" ]]; then
local newhash; newhash=$(_body_hash "$file")
local d ef ekey ehash stage
# pass 1: exact duplicate anywhere active -> no-op
for d in "$INBOX" "$BUILDING" "$REVIEW" "$TESTING" "$SHIPPED"; do
for ef in "$d"/*.md; do
[[ -e "$ef" ]] || continue
ekey=$(fm_get "$ef" idempotency-key "")
[[ "$ekey" == "$idem" ]] || continue
ehash=$(_body_hash "$ef")
if [[ "$ehash" == "$newhash" ]]; then
log "duplicate, skipped (idempotency-key=$C_BOLD$idem$C_RESET matches $(basename "$ef"))"
return 0
fi
done
done
# pass 2: same key, different body, prior is past inbox -> reject
for d in "$BUILDING" "$REVIEW" "$TESTING" "$SHIPPED"; do
for ef in "$d"/*.md; do
[[ -e "$ef" ]] || continue
ekey=$(fm_get "$ef" idempotency-key "")
[[ "$ekey" == "$idem" ]] || continue
stage=$(basename "$d")
die "idempotency-key '$idem' already in use by $(basename "$ef") (stage: $stage) with different content — refusing. Use a new key, or requeue the existing job."
done
done
# pass 3: same key, different body, prior still queued -> supersede
for ef in "$INBOX"/*.md; do
[[ -e "$ef" ]] || continue
ekey=$(fm_get "$ef" idempotency-key "")
[[ "$ekey" == "$idem" ]] || continue
log "superseding inbox job $(basename "$ef") (same idempotency-key=$idem, changed content)"
rm -f "$ef"
done
fi
local base; base=$(basename "$file")
local stamp; stamp=$(date +%Y%m%d-%H%M%S)
local dest="$INBOX/${stamp}__${base}"
@ -411,8 +676,9 @@ cmd_run() {
local running; running=$(active_workers)
# launch jobs while we have capacity and an eligible inbox file
while [[ "$running" -lt "$MAX_CONCURRENCY" ]]; do
# pick the oldest inbox file whose lock key is not currently busy, so two
# jobs sharing a cwd (or `lock:` key) never run at once, regardless of --max.
# pick by priority (critical→low) then age, skipping files whose lock key
# is currently busy, so two jobs sharing a cwd (or `lock:` key) never run
# at once regardless of --max. inbox_sorted replaces the old pure-FIFO sort.
local busy; busy=$(busy_keys)
local next="" cand cand_key
while IFS= read -r cand; do
@ -420,25 +686,45 @@ cmd_run() {
cand_key=$(lock_key_for "$cand")
if printf '%s\n' "$busy" | grep -qxF -- "$cand_key"; then continue; fi
next="$cand"; break
done < <(ls -1 "$INBOX"/*.md 2>/dev/null | sort)
done < <(inbox_sorted)
[[ -z "$next" ]] && break
local job; job=$(basename "$next"); job=${job%.md}
local doing_file="$BUILDING/$(basename "$next")"
mv "$next" "$doing_file"
local w_eng w_cwd w_yolo w_key
w_eng=$(fm_get "$doing_file" engine "$DEFAULT_ENGINE")
# resolve the concrete engine now (explicit engine / engine-class) so the
# meta + status reflect what will actually run; run_worker re-resolves and
# is the authority on capability/engine gates.
w_eng=$(resolve_engine "$doing_file")
w_cwd=$(fm_get "$doing_file" cwd "$PWD")
w_yolo=$(fm_get "$doing_file" yolo "true")
w_key=$(lock_key_for "$doing_file")
# write meta BEFORE launch (no pid yet), then append the worker pid from $!
# write meta BEFORE launch (no pid yet), then append the worker pid from $!.
# The new manifest fields (§5) are recorded here; only priority,
# capabilities, engine-class and idempotency-key are functional this
# phase — the rest are stored for `status`/audit but otherwise inert.
{
echo "job=$job"
echo "engine=$w_eng"
echo "engine=${w_eng:-<none>}"
echo "cwd=$w_cwd"
echo "yolo=$w_yolo"
echo "lock=$w_key"
echo "started=$(date +%s)"
echo "priority=$(fm_get "$doing_file" priority medium)"
echo "profile=$(fm_get "$doing_file" profile "")"
echo "engine_class=$(fm_get "$doing_file" engine-class "")"
echo "capabilities=$(fm_get "$doing_file" capabilities "")"
echo "prefers=$(fm_get "$doing_file" prefers "")"
echo "prefers_engine=$(fm_get "$doing_file" prefers-engine "")"
echo "budget=$(fm_get "$doing_file" budget "")"
echo "deps=$(fm_get "$doing_file" deps "")"
echo "deps_mode=$(fm_get "$doing_file" deps-mode "")"
echo "idempotency_key=$(fm_get "$doing_file" idempotency-key "")"
echo "retry=$(fm_get "$doing_file" retry "")"
echo "review_policy=$(fm_get "$doing_file" review-policy "")"
echo "artifacts=$(fm_get "$doing_file" artifacts "")"
echo "tracker_item=$(fm_get "$doing_file" tracker-item "")"
} > "$STATE/$job.meta"
run_worker "$doing_file" &
{ echo "pid=$!"; echo "pidstart=$(_pidstart "$!")"; } >> "$STATE/$job.meta"
@ -490,6 +776,15 @@ cmd_status() {
[[ "$age" -gt $(( STALL_MIN * 60 )) ]] && stall="${C_RED}⚠ stalled${C_RESET} "
printf ' %s%-26s%s %-7s %3dm%02ds pid %-6s %s%s%s%s\n' \
"$C_BOLD" "$job" "$C_RESET" "$eng" $((el/60)) $((el%60)) "$pid" "$stall" "$C_DIM" "$last" "$C_RESET"
# manifest sub-line (Phase 1): priority + profile + capabilities + tracker-item
local m_prio m_prof m_caps m_trk extra=""
m_prio=$(grep '^priority=' "$f" | cut -d= -f2-); m_prof=$(grep '^profile=' "$f" | cut -d= -f2-)
m_caps=$(grep '^capabilities=' "$f" | cut -d= -f2-); m_trk=$(grep '^tracker_item=' "$f" | cut -d= -f2-)
[[ -n "$m_prio" ]] && extra+="prio=$m_prio "
[[ -n "$m_prof" ]] && extra+="profile=$m_prof "
[[ -n "$m_caps" ]] && extra+="caps=$m_caps "
[[ -n "$m_trk" ]] && extra+="tracker=$m_trk "
[[ -n "$extra" ]] && printf ' %s%s%s\n' "$C_DIM" "$extra" "$C_RESET"
done
$printed || printf ' %sno workers running%s\n' "$C_DIM" "$C_RESET"
echo
@ -671,18 +966,26 @@ ${C_BOLD}KANBAN${C_RESET} inbox → building → review → testing → shipped
${C_BOLD}TASK FRONTMATTER${C_RESET} (top of each .md)
---
engine: devin
engine: devin # devin|claude|codex|copilot. Explicit engine always wins
cwd: /Users/you/code/repo
yolo: true
lock: my-repo # optional; defaults to cwd. Jobs sharing a key run serially
timeout: 45m # optional; 90s|45m|2h|1d. On expiry -> failed (result=timeout)
verify: pnpm -s test # optional; auto-QA gate. pass -> testing, fail -> failed
# --- Phase 1 manifest (active) ---
priority: high # critical|high|medium|low (default medium). Picked highest-first, then oldest
engine-class: agentic-coder # used only if `engine` unset: agentic-coder->devin,claude,codex; chat-coder->copilot
prefers-engine: [claude] # optional order hint for engine-class resolution
capabilities: [os:any, node>=20, has:git] # hard host requirements; unmet -> failed (capability_mismatch)
idempotency-key: my-task-1 # re-adding same key+body = no-op; same key+different body = reject/supersede
# --- reserved (parsed + shown in status, but no-op until a later phase) ---
profile: prefers: budget: deps: deps-mode: retry: review-policy: artifacts: tracker-item:
---
${C_BOLD}ENV${C_RESET}
AGENT_QUEUE_ROOT (=$QUEUE_ROOT) AGENT_QUEUE_MAX (=$MAX_CONCURRENCY)
AGENT_QUEUE_ENGINE (=$DEFAULT_ENGINE) AGENT_QUEUE_VERIFY (default verify cmd)
DEVIN_BIN / CLAUDE_BIN / CODEX_BIN
DEVIN_BIN / CLAUDE_BIN / CODEX_BIN / COPILOT_BIN
EOF
}