From 0be5b34123e916fbf39fce347c10edfd3d6f20a2 Mon Sep 17 00:00:00 2001 From: saravanakumardb1 Date: Fri, 29 May 2026 17:44:19 -0700 Subject: [PATCH] feat(agent-queue): evolved manifest, priority, capabilities, engine-class, idempotency (P1-S1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements Gigafactory Phase 1 - Slice 1 in the bash runner (backward-compatible; a legacy engine/cwd/yolo-only .md behaves exactly as before): - Parse all new §5 manifest keys via fm_get with safe defaults; record them in .meta and surface priority/profile/capabilities/tracker-item in `status`. Only priority, capabilities, engine-class and idempotency-key are functional this slice; the rest (profile, prefers, budget, deps, deps-mode, retry, review-policy, artifacts, tracker-item) are stored but inert. - priority ordering: inbox_sorted picks critical>high>medium>low, ties by oldest; per-lock serialization preserved. - capability grammar + match: detect_capabilities advertises os/engine/node/has tokens; caps_match honors key, key:value, keyversion and os:any. A job whose declared capabilities the host cannot satisfy is moved to failed/ with result=capability_mismatch and the agent is never launched. - engine-class resolution: explicit engine wins; else engine-class picks the first available engine honoring prefers-engine (agentic-coder->devin,claude,codex; chat-coder->copilot). No available engine -> result=no_engine. Adds copilot to the engine driver + COPILOT_BIN. - idempotency-key dedupe on add: same key+body -> no-op; same key+different body supersedes an inbox prior, else is rejected with a clear error. No change to queue/ data or the run/ship lifecycle. macOS + Linux safe. --- agent-queue/agent-queue.sh | 323 +++++++++++++++++++++++++++++++++++-- 1 file changed, 313 insertions(+), 10 deletions(-) diff --git a/agent-queue/agent-queue.sh b/agent-queue/agent-queue.sh index 58287db..ee5bba3 100755 --- a/agent-queue/agent-queue.sh +++ b/agent-queue/agent-queue.sh @@ -62,6 +62,7 @@ TIMEOUT_BIN="${TIMEOUT_BIN:-$(command -v timeout || command -v gtimeout || true) DEVIN_BIN="${DEVIN_BIN:-$(command -v devin || echo "$HOME/.local/bin/devin")}" CLAUDE_BIN="${CLAUDE_BIN:-$(command -v claude || echo claude)}" CODEX_BIN="${CODEX_BIN:-$(command -v codex || echo codex)}" +COPILOT_BIN="${COPILOT_BIN:-$(command -v copilot || echo copilot)}" # ── Colors ────────────────────────────────────────────────────────── if [[ -t 1 ]]; then @@ -194,6 +195,188 @@ busy_keys() { done } +# ── Manifest helpers (Phase 1 — §5/§6/§7/§8 single-host subset) ────── +# +# parse_list -> one token per line. Accepts a YAML-ish inline list +# ("[a, b]"), comma- or space-separated values, with surrounding quotes +# stripped. Empty tokens are dropped. Used for `capabilities`, `prefers`, +# `prefers-engine`, `deps`, etc. +parse_list() { + local raw cleaned t + cleaned=$(printf '%s' "${1:-}" | tr '[],' ' ') + for t in $cleaned; do + t=${t#[\"\']}; t=${t%[\"\']} + [[ -n "$t" ]] && printf '%s\n' "$t" + done +} + +# _body_hash -> stable content hash of the frontmatter-STRIPPED body. +# Drives idempotency-key dedupe on `add`. Prefers shasum/sha1sum, falls back +# to cksum so it works on a bare macOS/Linux host with no extra tools. +_body_hash() { + local f=$1 + if command -v shasum >/dev/null 2>&1; then + strip_frontmatter "$f" | shasum | awk '{print $1}' + elif command -v sha1sum >/dev/null 2>&1; then + strip_frontmatter "$f" | sha1sum | awk '{print $1}' + else + strip_frontmatter "$f" | cksum | awk '{print $1"-"$2}' + fi +} + +# priority_rank -> sort rank (lower = picked first). Unknown/empty +# defaults to medium. Used by inbox_sorted for priority-then-age selection. +priority_rank() { + case "$1" in + critical) echo 0;; high) echo 1;; medium) echo 2;; low) echo 3;; *) echo 2;; + esac +} + +# inbox_sorted -> inbox/*.md ordered by priority (critical first) then oldest +# (filename embeds the queue timestamp, so a plain string sort = age order). +# Replaces the pure-FIFO `ls | sort`; per-lock serialization is unchanged +# (the caller still skips files whose lock key is busy). +inbox_sorted() { + local f p r + for f in "$INBOX"/*.md; do + [[ -e "$f" ]] || continue + p=$(fm_get "$f" priority medium); r=$(priority_rank "$p") + printf '%s\t%s\n' "$r" "$f" + done | sort -t"$(printf '\t')" -k1,1n -k2,2 | cut -f2- +} + +# engine_available -> 0 if a runnable binary exists for the engine +# (executable path or a name resolvable on PATH). Honors the *_BIN overrides +# (so the self-test stub counts as "available"). +engine_available() { + local eng=$1 bin="" + case "$eng" in + devin) bin=$DEVIN_BIN;; + claude) bin=$CLAUDE_BIN;; + codex) bin=$CODEX_BIN;; + copilot) bin=$COPILOT_BIN;; + *) return 1;; + esac + [[ -n "$bin" ]] || return 1 + [[ -x "$bin" ]] && return 0 + command -v "$bin" >/dev/null 2>&1 +} + +# engine_class_engines -> candidate engines (priority order) for an +# engine-class (§5 taxonomy). `review-only` has no concrete mapping yet +# (reserved). Unknown class -> empty. +engine_class_engines() { + case "$1" in + agentic-coder) echo "devin claude codex";; + chat-coder) echo "copilot";; + review-only) echo "";; + *) echo "";; + esac +} + +# detect_capabilities -> one capability token per line describing THIS host: +# os: (os:any required-side wildcard matches this) +# engine: for each available engine +# node: the host node major (compared by nodeN) +# has: tool probes (only emitted when present) +detect_capabilities() { + case "$(uname -s 2>/dev/null)" in + Darwin) echo "os:mac";; + Linux) echo "os:linux";; + *) echo "os:other";; + esac + local e + for e in devin claude codex copilot; do + engine_available "$e" && echo "engine:$e" + done + if command -v node >/dev/null 2>&1; then + local nv; nv=$(node -v 2>/dev/null | sed 's/^v//' | cut -d. -f1) + [[ "$nv" =~ ^[0-9]+$ ]] && echo "node:$nv" + fi + local t + for t in git pnpm docker; do + command -v "$t" >/dev/null 2>&1 && echo "has:$t" + done +} + +# _cap_token_ok -> 0 if satisfied (§5 grammar): +# key:any wildcard — host advertises any "key:*" token (os:any matches all) +# keyver numeric/semver-major compare against the host's "key:" token +# (op in >= > = <= <) +# key:value exact token match +# key bare presence — exact, or advertised in any "key:"/"key" form +_cap_token_ok() { + local tok=$1 avail=$2 a + if [[ "$tok" == *:any ]]; then + local wkey=${tok%:any} + for a in $avail; do [[ "$a" == "$wkey":* ]] && return 0; done + return 1 + fi + if [[ "$tok" =~ ^([A-Za-z0-9_.:-]+)(\>=|\<=|=|\>|\<)([0-9][0-9.]*)$ ]]; then + local key=${BASH_REMATCH[1]} op=${BASH_REMATCH[2]} want=${BASH_REMATCH[3]} hostval="" + for a in $avail; do [[ "$a" == "$key":* ]] && { hostval=${a#*:}; break; }; done + [[ -n "$hostval" ]] || return 1 + local hv=${hostval%%.*} wv=${want%%.*} + [[ "$hv" =~ ^[0-9]+$ && "$wv" =~ ^[0-9]+$ ]] || return 1 + case "$op" in + ">=") [[ "$hv" -ge "$wv" ]];; + ">") [[ "$hv" -gt "$wv" ]];; + "=") [[ "$hv" -eq "$wv" ]];; + "<=") [[ "$hv" -le "$wv" ]];; + "<") [[ "$hv" -lt "$wv" ]];; + esac + return $? + fi + if [[ "$tok" == *:* ]]; then + for a in $avail; do [[ "$a" == "$tok" ]] && return 0; done + return 1 + fi + for a in $avail; do + case "$a" in "$tok"|"$tok":*|"$tok"\>=*|"$tok"\>*|"$tok"=*|"$tok"\<=*|"$tok"\<*) return 0;; esac + done + return 1 +} + +# caps_match -> 0 iff EVERY required token +# is satisfied by the available set (both passed as whitespace-separated lists). +caps_match() { + local required=$1 available=$2 r + for r in $required; do + _cap_token_ok "$r" "$available" || return 1 + done + return 0 +} + +# resolve_engine -> the concrete engine to run: +# explicit `engine:` always wins; else `engine-class` picks the first +# available engine honoring `prefers-engine` then the class default order; +# else the global default engine. Prints "" only when an engine-class was +# requested but no engine in it is available (caller fails with no_engine). +resolve_engine() { + local f=$1 eng cls prefers + eng=$(fm_get "$f" engine "") + if [[ -n "$eng" ]]; then printf '%s' "$eng"; return 0; fi + cls=$(fm_get "$f" engine-class "") + if [[ -z "$cls" ]]; then printf '%s' "$DEFAULT_ENGINE"; return 0; fi + local class_engines; class_engines=$(engine_class_engines "$cls") + prefers=$(fm_get "$f" prefers-engine "") + local ordered=() seen=" " p c + if [[ -n "$prefers" ]]; then + while IFS= read -r p; do + [[ -n "$p" ]] || continue + case " $class_engines " in *" $p "*) ordered+=("$p"); seen+="$p ";; esac + done < <(parse_list "$prefers") + fi + for c in $class_engines; do + case "$seen" in *" $c "*) ;; *) ordered+=("$c"); seen+="$c ";; esac + done + local cand + for cand in ${ordered[@]+"${ordered[@]}"}; do + engine_available "$cand" && { printf '%s' "$cand"; return 0; } + done + printf '%s' "" +} + # ── Engine driver: builds argv into AGENT_CMD[]; sets AGENT_STDIN if the ── # prompt should be fed on stdin (claude/codex) rather than a flag. $pf is the # frontmatter-STRIPPED body file, so a body starting with '--' is never @@ -216,7 +399,14 @@ build_agent_cmd() { [[ "$yolo" == "true" ]] && AGENT_CMD+=( --dangerously-bypass-approvals-and-sandbox ) AGENT_STDIN="$pf" ;; - *) die "unknown engine '$engine' (use: devin | claude | codex)";; + copilot) + # Best-effort GitHub Copilot CLI mapping for the chat-coder engine-class. + # Flags drift between CLI versions — this is the single place to edit. + AGENT_CMD=( "$COPILOT_BIN" -p ) + [[ "$yolo" == "true" ]] && AGENT_CMD+=( --allow-all-tools ) + AGENT_STDIN="$pf" + ;; + *) die "unknown engine '$engine' (use: devin | claude | codex | copilot)";; esac } @@ -226,7 +416,6 @@ run_worker() { local job; job=$(basename "$doing_file") job=${job%.md} local engine cwd yolo logf metaf - engine=$(fm_get "$doing_file" engine "$DEFAULT_ENGINE") cwd=$(fm_get "$doing_file" cwd "$PWD") yolo=$(fm_get "$doing_file" yolo "true") logf="$LOGS/$job.log" @@ -234,6 +423,38 @@ run_worker() { # NOTE: the parent (cmd_run) creates $metaf with job/engine/cwd/started/pid. # The worker only ever APPENDS (ended/exit/result) to avoid a truncation race. + # ── Capability gate (§5/§8 single-host): if the job declares `capabilities` + # this host does not satisfy, route to failed/ WITHOUT launching the agent. ── + local req_caps; req_caps=$(parse_list "$(fm_get "$doing_file" capabilities "")" | tr '\n' ' ') + if [[ -n "${req_caps// /}" ]]; then + local avail; avail=$(detect_capabilities) + if ! caps_match "$req_caps" "$avail"; then + { + echo "===== agent-queue job: $job =====" + echo "CAPABILITY MISMATCH — host cannot satisfy required: $req_caps" + echo "host advertises: $(printf '%s' "$avail" | tr '\n' ' ')" + echo "agent NOT launched — routed to failed/ ($(date))" + } >> "$logf" + mv "$doing_file" "$FAILED/" 2>/dev/null + { echo "result=capability_mismatch"; echo "ended=$(date +%s)"; } >> "$metaf" + return 0 + fi + fi + + # ── Engine resolution (§5): explicit `engine` wins, else `engine-class`. + # No available engine for a requested class -> fail (no_engine), no launch. ── + engine=$(resolve_engine "$doing_file") + if [[ -z "$engine" ]]; then + { + echo "===== agent-queue job: $job =====" + echo "NO ENGINE — engine-class '$(fm_get "$doing_file" engine-class "")' has no available engine on this host" + echo "agent NOT launched — routed to failed/ ($(date))" + } >> "$logf" + mv "$doing_file" "$FAILED/" 2>/dev/null + { echo "result=no_engine"; echo "ended=$(date +%s)"; } >> "$metaf" + return 0 + fi + { echo "===== agent-queue job: $job =====" echo "engine=$engine cwd=$cwd yolo=$yolo" @@ -363,6 +584,50 @@ cmd_add() { esac done [[ -n "$file" && -f "$file" ]] || die "usage: add [--engine devin|claude|codex] [--cwd PATH] [--yolo|--no-yolo]" + + # ── idempotency-key dedupe (§5) ── + # If the file declares an idempotency-key, compare a content hash of its + # stripped body against existing jobs in any active stage: + # same key + same body -> no-op (skip; "duplicate") + # same key + different body, prior in inbox/ -> supersede (replace it) + # same key + different body, prior elsewhere -> reject (clear error) + local idem; idem=$(fm_get "$file" idempotency-key "") + if [[ -n "$idem" ]]; then + local newhash; newhash=$(_body_hash "$file") + local d ef ekey ehash stage + # pass 1: exact duplicate anywhere active -> no-op + for d in "$INBOX" "$BUILDING" "$REVIEW" "$TESTING" "$SHIPPED"; do + for ef in "$d"/*.md; do + [[ -e "$ef" ]] || continue + ekey=$(fm_get "$ef" idempotency-key "") + [[ "$ekey" == "$idem" ]] || continue + ehash=$(_body_hash "$ef") + if [[ "$ehash" == "$newhash" ]]; then + log "duplicate, skipped (idempotency-key=$C_BOLD$idem$C_RESET matches $(basename "$ef"))" + return 0 + fi + done + done + # pass 2: same key, different body, prior is past inbox -> reject + for d in "$BUILDING" "$REVIEW" "$TESTING" "$SHIPPED"; do + for ef in "$d"/*.md; do + [[ -e "$ef" ]] || continue + ekey=$(fm_get "$ef" idempotency-key "") + [[ "$ekey" == "$idem" ]] || continue + stage=$(basename "$d") + die "idempotency-key '$idem' already in use by $(basename "$ef") (stage: $stage) with different content — refusing. Use a new key, or requeue the existing job." + done + done + # pass 3: same key, different body, prior still queued -> supersede + for ef in "$INBOX"/*.md; do + [[ -e "$ef" ]] || continue + ekey=$(fm_get "$ef" idempotency-key "") + [[ "$ekey" == "$idem" ]] || continue + log "superseding inbox job $(basename "$ef") (same idempotency-key=$idem, changed content)" + rm -f "$ef" + done + fi + local base; base=$(basename "$file") local stamp; stamp=$(date +%Y%m%d-%H%M%S) local dest="$INBOX/${stamp}__${base}" @@ -411,8 +676,9 @@ cmd_run() { local running; running=$(active_workers) # launch jobs while we have capacity and an eligible inbox file while [[ "$running" -lt "$MAX_CONCURRENCY" ]]; do - # pick the oldest inbox file whose lock key is not currently busy, so two - # jobs sharing a cwd (or `lock:` key) never run at once, regardless of --max. + # pick by priority (critical→low) then age, skipping files whose lock key + # is currently busy, so two jobs sharing a cwd (or `lock:` key) never run + # at once regardless of --max. inbox_sorted replaces the old pure-FIFO sort. local busy; busy=$(busy_keys) local next="" cand cand_key while IFS= read -r cand; do @@ -420,25 +686,45 @@ cmd_run() { cand_key=$(lock_key_for "$cand") if printf '%s\n' "$busy" | grep -qxF -- "$cand_key"; then continue; fi next="$cand"; break - done < <(ls -1 "$INBOX"/*.md 2>/dev/null | sort) + done < <(inbox_sorted) [[ -z "$next" ]] && break local job; job=$(basename "$next"); job=${job%.md} local doing_file="$BUILDING/$(basename "$next")" mv "$next" "$doing_file" local w_eng w_cwd w_yolo w_key - w_eng=$(fm_get "$doing_file" engine "$DEFAULT_ENGINE") + # resolve the concrete engine now (explicit engine / engine-class) so the + # meta + status reflect what will actually run; run_worker re-resolves and + # is the authority on capability/engine gates. + w_eng=$(resolve_engine "$doing_file") w_cwd=$(fm_get "$doing_file" cwd "$PWD") w_yolo=$(fm_get "$doing_file" yolo "true") w_key=$(lock_key_for "$doing_file") - # write meta BEFORE launch (no pid yet), then append the worker pid from $! + # write meta BEFORE launch (no pid yet), then append the worker pid from $!. + # The new manifest fields (§5) are recorded here; only priority, + # capabilities, engine-class and idempotency-key are functional this + # phase — the rest are stored for `status`/audit but otherwise inert. { echo "job=$job" - echo "engine=$w_eng" + echo "engine=${w_eng:-}" echo "cwd=$w_cwd" echo "yolo=$w_yolo" echo "lock=$w_key" echo "started=$(date +%s)" + echo "priority=$(fm_get "$doing_file" priority medium)" + echo "profile=$(fm_get "$doing_file" profile "")" + echo "engine_class=$(fm_get "$doing_file" engine-class "")" + echo "capabilities=$(fm_get "$doing_file" capabilities "")" + echo "prefers=$(fm_get "$doing_file" prefers "")" + echo "prefers_engine=$(fm_get "$doing_file" prefers-engine "")" + echo "budget=$(fm_get "$doing_file" budget "")" + echo "deps=$(fm_get "$doing_file" deps "")" + echo "deps_mode=$(fm_get "$doing_file" deps-mode "")" + echo "idempotency_key=$(fm_get "$doing_file" idempotency-key "")" + echo "retry=$(fm_get "$doing_file" retry "")" + echo "review_policy=$(fm_get "$doing_file" review-policy "")" + echo "artifacts=$(fm_get "$doing_file" artifacts "")" + echo "tracker_item=$(fm_get "$doing_file" tracker-item "")" } > "$STATE/$job.meta" run_worker "$doing_file" & { echo "pid=$!"; echo "pidstart=$(_pidstart "$!")"; } >> "$STATE/$job.meta" @@ -490,6 +776,15 @@ cmd_status() { [[ "$age" -gt $(( STALL_MIN * 60 )) ]] && stall="${C_RED}⚠ stalled${C_RESET} " printf ' %s%-26s%s %-7s %3dm%02ds pid %-6s %s%s%s%s\n' \ "$C_BOLD" "$job" "$C_RESET" "$eng" $((el/60)) $((el%60)) "$pid" "$stall" "$C_DIM" "$last" "$C_RESET" + # manifest sub-line (Phase 1): priority + profile + capabilities + tracker-item + local m_prio m_prof m_caps m_trk extra="" + m_prio=$(grep '^priority=' "$f" | cut -d= -f2-); m_prof=$(grep '^profile=' "$f" | cut -d= -f2-) + m_caps=$(grep '^capabilities=' "$f" | cut -d= -f2-); m_trk=$(grep '^tracker_item=' "$f" | cut -d= -f2-) + [[ -n "$m_prio" ]] && extra+="prio=$m_prio " + [[ -n "$m_prof" ]] && extra+="profile=$m_prof " + [[ -n "$m_caps" ]] && extra+="caps=$m_caps " + [[ -n "$m_trk" ]] && extra+="tracker=$m_trk " + [[ -n "$extra" ]] && printf ' %s%s%s\n' "$C_DIM" "$extra" "$C_RESET" done $printed || printf ' %sno workers running%s\n' "$C_DIM" "$C_RESET" echo @@ -671,18 +966,26 @@ ${C_BOLD}KANBAN${C_RESET} inbox → building → review → testing → shipped ${C_BOLD}TASK FRONTMATTER${C_RESET} (top of each .md) --- - engine: devin + engine: devin # devin|claude|codex|copilot. Explicit engine always wins cwd: /Users/you/code/repo yolo: true lock: my-repo # optional; defaults to cwd. Jobs sharing a key run serially timeout: 45m # optional; 90s|45m|2h|1d. On expiry -> failed (result=timeout) verify: pnpm -s test # optional; auto-QA gate. pass -> testing, fail -> failed + # --- Phase 1 manifest (active) --- + priority: high # critical|high|medium|low (default medium). Picked highest-first, then oldest + engine-class: agentic-coder # used only if `engine` unset: agentic-coder->devin,claude,codex; chat-coder->copilot + prefers-engine: [claude] # optional order hint for engine-class resolution + capabilities: [os:any, node>=20, has:git] # hard host requirements; unmet -> failed (capability_mismatch) + idempotency-key: my-task-1 # re-adding same key+body = no-op; same key+different body = reject/supersede + # --- reserved (parsed + shown in status, but no-op until a later phase) --- + profile: prefers: budget: deps: deps-mode: retry: review-policy: artifacts: tracker-item: --- ${C_BOLD}ENV${C_RESET} AGENT_QUEUE_ROOT (=$QUEUE_ROOT) AGENT_QUEUE_MAX (=$MAX_CONCURRENCY) AGENT_QUEUE_ENGINE (=$DEFAULT_ENGINE) AGENT_QUEUE_VERIFY (default verify cmd) - DEVIN_BIN / CLAUDE_BIN / CODEX_BIN + DEVIN_BIN / CLAUDE_BIN / CODEX_BIN / COPILOT_BIN EOF }