From 3d99f0442754ae837d32b3766cbd02e19beeb4f0 Mon Sep 17 00:00:00 2001 From: saravanakumardb1 Date: Fri, 29 May 2026 19:26:16 -0700 Subject: [PATCH] feat(agent-queue): profiles (persona + presets) and single-host deps/DAG (P1-S2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements roadmap §6 (profiles) and §5 deps on the bash runner, backward-compatible (jobs without profile/deps behave exactly as before). Profiles (§6): - profile_get / profile_persona / fm_eff helpers + PROFILES_DIR (AGENT_QUEUE_PROFILES override). A job's `profile:` inherits verify (<- default-verify), capabilities, engine-class, prefers-engine, allowed-scope, review-policy when the job omits them; job fields always override (precedence job > profile > default). Resolution runs via fm_eff inside the capability gate and resolve_engine, so inherited caps/engine-class take effect before launch. - persona injection: the profile's persona block is prepended to the stripped body fed to the engine (job .md unchanged on disk; nothing secret logged). - allowed-scope guardrail (WARN-ONLY): scope_check logs a non-blocking WARNING + records scope_warning= for changed paths outside the globs; path_in_scope is a pure, unit-testable matcher (`dir/**` = subtree). deps / DAG, single host (§5): - deps reference other jobs by idempotency-key. dep_satisfied: shipped/ (hard) or shipped/+testing/ (deps-mode: soft). deps_unmet drives a block-with-reason skip in inbox selection (never launched/failed); cmd_status surfaces "blocked (waiting on )". deps_would_cycle rejects cyclic submits on `add`. - _drain_pending: `--once` drains past dep-blocked jobs (idle can't satisfy them) while still waiting on retry/recovery backoff timers. Meta now records effective (inherited) capabilities/engine-class/prefers-engine/ review-policy/allowed-scope so `status` reflects resolved config. --- agent-queue/agent-queue.sh | 249 +++++++++++++++++++++++++++++++++++-- 1 file changed, 237 insertions(+), 12 deletions(-) diff --git a/agent-queue/agent-queue.sh b/agent-queue/agent-queue.sh index 97eae1c..f76b508 100755 --- a/agent-queue/agent-queue.sh +++ b/agent-queue/agent-queue.sh @@ -29,6 +29,8 @@ set -uo pipefail # ── Resolve paths ─────────────────────────────────────────────────── SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" QUEUE_ROOT="${AGENT_QUEUE_ROOT:-$SCRIPT_DIR/queue}" +# Profile catalog dir (persona + capability presets). Override for tests. +PROFILES_DIR="${AGENT_QUEUE_PROFILES:-$SCRIPT_DIR/profiles}" INBOX="$QUEUE_ROOT/inbox" BUILDING="$QUEUE_ROOT/building" REVIEW="$QUEUE_ROOT/review" @@ -120,6 +122,51 @@ lock_key_for() { # _keyhash -> stable filename-safe token for a lock key _keyhash() { printf '%s' "$1" | cksum | awk '{print $1}'; } +# ── Profiles (§6): persona + capability/engine/scope presets ───────── +# +# profile_get [default] -> a single-line value from +# profiles/.md, else the default. +profile_get() { + local pf="$PROFILES_DIR/$1.md" + [[ -f "$pf" ]] || { printf '%s' "${3:-}"; return; } + fm_get "$pf" "$2" "${3:-}" +} + +# profile_persona -> the multi-line `persona: |` block (2-space +# indent stripped), or empty. Used to prepend a persona overlay to the job body. +profile_persona() { + local pf="$PROFILES_DIR/$1.md" + [[ -f "$pf" ]] || return 0 + awk ' + NR==1 && $0!="---" { exit } + NR==1 { infm=1; next } + infm && $0=="---" { exit } + !infm { next } + inpersona { + if ($0 ~ /^[A-Za-z0-9_-]+[ \t]*:/) { inpersona=0 } + else { line=$0; sub(/^ /,"",line); print line; next } + } + $0 ~ /^persona[ \t]*:[ \t]*\|[ \t]*$/ { inpersona=1; next } + ' "$pf" +} + +# fm_eff [default] [profile-key] -> effective value with +# precedence job > profile > built-in default (§6 resolution). The profile is the +# job's `profile:` frontmatter; `profile-key` defaults to `job-key` (e.g. `verify` +# inherits the profile's `default-verify`). Inheritable: verify, capabilities, +# engine-class, prefers-engine, allowed-scope, review-policy. +fm_eff() { + local file=$1 jkey=$2 def=${3:-} pkey=${4:-$2} v prof pv + v=$(fm_get "$file" "$jkey" "") + if [[ -n "$v" ]]; then printf '%s' "$v"; return; fi + prof=$(fm_get "$file" profile "") + if [[ -n "$prof" ]]; then + pv=$(profile_get "$prof" "$pkey" "") + [[ -n "$pv" ]] && { printf '%s' "$pv"; return; } + fi + printf '%s' "$def" +} + # _mtime -> file modification time in epoch seconds (BSD or GNU stat); empty if missing _mtime() { [[ -e "$1" ]] || { echo ""; return; } @@ -356,10 +403,10 @@ resolve_engine() { local f=$1 eng cls prefers eng=$(fm_get "$f" engine "") if [[ -n "$eng" ]]; then printf '%s' "$eng"; return 0; fi - cls=$(fm_get "$f" engine-class "") + cls=$(fm_eff "$f" engine-class "") # inherit engine-class from the job's profile if [[ -z "$cls" ]]; then printf '%s' "$DEFAULT_ENGINE"; return 0; fi local class_engines; class_engines=$(engine_class_engines "$cls") - prefers=$(fm_get "$f" prefers-engine "") + prefers=$(fm_eff "$f" prefers-engine "") local ordered=() seen=" " p c if [[ -n "$prefers" ]]; then while IFS= read -r p; do @@ -377,6 +424,134 @@ resolve_engine() { printf '%s' "" } +# ── deps / DAG, single host (§5) ───────────────────────────────────── +# deps reference other jobs by their (author-controlled) `idempotency-key`. +# +# _key_in_dir -> 0 if some .md in has idempotency-key == key. +_key_in_dir() { + local key=$1 d=$2 ef + for ef in "$d"/*.md; do + [[ -e "$ef" ]] || continue + [[ "$(fm_get "$ef" idempotency-key "")" == "$key" ]] && return 0 + done + return 1 +} + +# dep_satisfied -> 0 when the dep is met: a job with is in +# shipped/ (default), or shipped/ OR testing/ when mode is `soft`. +dep_satisfied() { + local key=$1 mode=$2 + _key_in_dir "$key" "$SHIPPED" && return 0 + [[ "$mode" == soft ]] && _key_in_dir "$key" "$TESTING" && return 0 + return 1 +} + +# deps_unmet -> space-separated list of this job's UNMET dep keys (empty if +# none / no deps). `deps-mode` (hard|soft) is job-level. +deps_unmet() { + local f=$1 keys mode k unmet="" + keys=$(parse_list "$(fm_get "$f" deps "")" | tr '\n' ' ') + [[ -n "${keys// /}" ]] || { printf ''; return 0; } + mode=$(fm_get "$f" deps-mode "hard") + for k in $keys; do + [[ -n "$k" ]] || continue + dep_satisfied "$k" "$mode" || unmet+="$k " + done + printf '%s' "${unmet% }" +} + +# _deps_of_key -> dep keys (space-separated) of the job carrying , +# scanned across inbox + active stages. +_deps_of_key() { + local key=$1 d ef + for d in "$INBOX" "$BUILDING" "$REVIEW" "$TESTING" "$SHIPPED"; do + for ef in "$d"/*.md; do + [[ -e "$ef" ]] || continue + [[ "$(fm_get "$ef" idempotency-key "")" == "$key" ]] || continue + parse_list "$(fm_get "$ef" deps "")" | tr '\n' ' ' + return 0 + done + done +} + +# deps_would_cycle -> 0 if adding a job with +# depending on would create a cycle (BFS over existing key->deps edges +# back to new-key; also catches self-dependency). +deps_would_cycle() { + local newkey=$1 newdeps=$2 visited=" " frontier next k d kd + [[ -n "$newkey" ]] || return 1 + _in_list "$newkey" "$newdeps" && return 0 + frontier="$newdeps" + while [[ -n "${frontier// /}" ]]; do + next="" + for k in $frontier; do + [[ -n "$k" ]] || continue + [[ "$k" == "$newkey" ]] && return 0 + case "$visited" in *" $k "*) continue;; esac + visited+="$k " + kd=$(_deps_of_key "$k") + for d in $kd; do next+="$d "; done + done + frontier="$next" + done + return 1 +} + +# _drain_pending -> 0 if some inbox job can still make progress on its own: it is +# runnable now, or it is waiting on a retry/recovery backoff (which elapses with +# time). A job blocked ONLY by unmet deps is NOT pending while the loop is idle +# (no running job can satisfy its deps), so `--once` can drain past it. +_drain_pending() { + local cand cj ne now; now=$(date +%s) + for cand in "$INBOX"/*.md; do + [[ -e "$cand" ]] || continue + cj=$(basename "$cand"); cj=${cj%.md} + ne=$(grep '^next_eligible=' "$STATE/$cj.meta" 2>/dev/null | tail -1 | cut -d= -f2) + if [[ "$ne" =~ ^[0-9]+$ ]] && [[ "$ne" -gt "$now" ]]; then return 0; fi + [[ -n "$(deps_unmet "$cand")" ]] && continue + return 0 + done + return 1 +} + +# ── allowed-scope guardrail (§6/§12) — WARN-ONLY this phase ─────────── +# path_in_scope -> 0 if matches any allowed-scope glob +# (`dir/**` matches the whole subtree; `*` matches across `/`). Pure + testable. +path_in_scope() { + local path=$1 globs=$2 g pat + for g in $globs; do + [[ -n "$g" ]] || continue + pat=${g//\*\*/\*} + # shellcheck disable=SC2053 + [[ "$path" == $pat ]] && return 0 + [[ "$path" == "$g"/* ]] && return 0 + done + return 1 +} + +# scope_check -> log a WARNING (non-blocking) +# for changed paths outside allowed-scope. Records scope_warning= in the meta. +scope_check() { + local cwd=$1 base=$2 scope=$3 logf=$4 metaf=$5 changed globs p out="" + _is_git_repo "$cwd" || return 0 + if [[ -n "$base" ]]; then + changed=$(git -C "$cwd" diff --name-only "$base" HEAD 2>/dev/null) + else + changed=$(git -C "$cwd" diff --name-only HEAD 2>/dev/null) + fi + [[ -n "$changed" ]] || return 0 + globs=$(parse_list "$scope" | tr '\n' ' ') + [[ -n "${globs// /}" ]] || return 0 + while IFS= read -r p; do + [[ -n "$p" ]] || continue + path_in_scope "$p" "$globs" || out+="$p " + done <<< "$changed" + if [[ -n "$out" ]]; then + echo "WARNING: allowed-scope violation (warn-only) — changed outside [$globs]: ${out% }" >> "$logf" + echo "scope_warning=${out% }" >> "$metaf" + fi +} + # ── Engine driver: builds argv into AGENT_CMD[]; sets AGENT_STDIN if the ── # prompt should be fed on stdin (claude/codex) rather than a flag. $pf is the # frontmatter-STRIPPED body file, so a body starting with '--' is never @@ -424,8 +599,9 @@ run_worker() { # The worker only ever APPENDS (ended/exit/result) to avoid a truncation race. # ── Capability gate (§5/§8 single-host): if the job declares `capabilities` - # this host does not satisfy, route to failed/ WITHOUT launching the agent. ── - local req_caps; req_caps=$(parse_list "$(fm_get "$doing_file" capabilities "")" | tr '\n' ' ') + # (own or inherited from its profile) this host does not satisfy, route to + # failed/ WITHOUT launching the agent. ── + local req_caps; req_caps=$(parse_list "$(fm_eff "$doing_file" capabilities "")" | tr '\n' ' ') if [[ -n "${req_caps// /}" ]]; then local avail; avail=$(detect_capabilities) if ! caps_match "$req_caps" "$avail"; then @@ -474,6 +650,16 @@ run_worker() { # Strip our frontmatter so the agent only sees the task body. local bodyf="$STATE/$job.body.md" strip_frontmatter "$doing_file" > "$bodyf" + # ── Persona injection (§6): prepend the profile's persona to the body fed to + # the engine (job body unchanged on disk). Secrets are never logged. ── + local prof; prof=$(fm_get "$doing_file" profile "") + if [[ -n "$prof" ]]; then + local persona; persona=$(profile_persona "$prof") + if [[ -n "$persona" ]]; then + { printf '%s\n\n' "$persona"; cat "$bodyf"; } > "$bodyf.tmp" && mv "$bodyf.tmp" "$bodyf" + echo "profile: injected persona overlay from '$prof'" >> "$logf" + fi + fi build_agent_cmd "$engine" "$bodyf" "$yolo" # ── WIP checkpoint setup (§25.2): on a git cwd, create/checkout aq/wip/ @@ -547,6 +733,11 @@ run_worker() { _numstat_into_meta "$cwd" "$WIP_BASE" "$metaf" parse_usage "$engine" "$logf" >> "$metaf" + # ── allowed-scope guardrail (§6) — WARN-ONLY: flag out-of-scope changes but + # never block the job this phase. Scope may be inherited from the profile. ── + local scope; scope=$(fm_eff "$doing_file" allowed-scope "" allowed-scope) + [[ -n "$scope" ]] && scope_check "$cwd" "$WIP_BASE" "$scope" "$logf" "$metaf" + if $timed_out; then echo "TIMED OUT after ${tmo}s (rc=$rc): $(date)" >> "$logf" _finish_failure "$job" "$doing_file" "$metaf" "$logf" "timeout" "$rc" "$started" @@ -558,7 +749,8 @@ run_worker() { local review_file="$REVIEW/$job.md" echo "exit=$rc" >> "$metaf" echo "completed OK (rc=0): landed in review — $(date)" >> "$logf" - local verify; verify=$(fm_get "$review_file" verify "$DEFAULT_VERIFY") + # verify is job-level, else inherited from the profile's default-verify. + local verify; verify=$(fm_eff "$review_file" verify "$DEFAULT_VERIFY" default-verify) if [[ -z "$verify" ]]; then _meta_end "$metaf" "review" "$started" echo "no verify command — parked in review for manual promote: $(date)" >> "$logf" @@ -896,6 +1088,13 @@ cmd_add() { done fi + # ── dep cycle detection (§5): reject a submit that would create a cycle in the + # idempotency-key dependency graph (inbox + active stages). ── + local newdeps; newdeps=$(parse_list "$(fm_get "$file" deps "")" | tr '\n' ' ') + if [[ -n "${newdeps// /}" ]] && deps_would_cycle "$idem" "$newdeps"; then + die "dependency cycle detected: job (key '${idem:-}') with deps [${newdeps% }] would create a cycle — refusing." + fi + local base; base=$(basename "$file") local stamp; stamp=$(date +%Y%m%d-%H%M%S) local dest="$INBOX/${stamp}__${base}" @@ -963,6 +1162,8 @@ cmd_run() { cand_job=$(basename "$cand"); cand_job=${cand_job%.md} cand_ne=$(grep '^next_eligible=' "$STATE/$cand_job.meta" 2>/dev/null | tail -1 | cut -d= -f2) if [[ "$cand_ne" =~ ^[0-9]+$ ]] && [[ "$cand_ne" -gt "$now_s" ]]; then continue; fi + # skip jobs whose deps (§5 DAG) are unmet — blocked, re-evaluated next loop + if [[ -n "$(deps_unmet "$cand")" ]]; then continue; fi next="$cand"; break done < <(inbox_sorted) [[ -z "$next" ]] && break @@ -997,16 +1198,17 @@ cmd_run() { echo "attempts=$w_attempts" echo "priority=$(fm_get "$doing_file" priority medium)" echo "profile=$(fm_get "$doing_file" profile "")" - echo "engine_class=$(fm_get "$doing_file" engine-class "")" - echo "capabilities=$(fm_get "$doing_file" capabilities "")" + echo "engine_class=$(fm_eff "$doing_file" engine-class "")" + echo "capabilities=$(fm_eff "$doing_file" capabilities "")" echo "prefers=$(fm_get "$doing_file" prefers "")" - echo "prefers_engine=$(fm_get "$doing_file" prefers-engine "")" + echo "prefers_engine=$(fm_eff "$doing_file" prefers-engine "")" + echo "allowed_scope=$(fm_eff "$doing_file" allowed-scope "" allowed-scope)" echo "budget=$(fm_get "$doing_file" budget "")" echo "deps=$(fm_get "$doing_file" deps "")" echo "deps_mode=$(fm_get "$doing_file" deps-mode "")" echo "idempotency_key=$(fm_get "$doing_file" idempotency-key "")" echo "retry=$(fm_get "$doing_file" retry "")" - echo "review_policy=$(fm_get "$doing_file" review-policy "")" + echo "review_policy=$(fm_eff "$doing_file" review-policy "" review-policy)" echo "artifacts=$(fm_get "$doing_file" artifacts "")" echo "tracker_item=$(fm_get "$doing_file" tracker-item "")" } > "$STATE/$job.meta" @@ -1018,8 +1220,11 @@ cmd_run() { done if $once; then - [[ "$(active_workers)" -eq 0 && -z "$(ls -1 "$INBOX"/*.md 2>/dev/null)" ]] && { - log "drain complete — inbox empty, no workers running"; rm -f "$STATE/daemon.pid"; exit 0; } + # drain when no worker is running and nothing in inbox can still progress on + # its own (backoff jobs still count as pending; dep-blocked jobs do not). + if [[ "$(active_workers)" -eq 0 ]] && ! _drain_pending; then + log "drain complete — no runnable work, no workers running"; rm -f "$STATE/daemon.pid"; exit 0 + fi fi sleep "$POLL_SECONDS" done @@ -1072,6 +1277,18 @@ cmd_status() { printf ' %s%s%s\n' "$C_DIM" "$(_insights_line "$f")" "$C_RESET" done $printed || printf ' %sno workers running%s\n' "$C_DIM" "$C_RESET" + + # blocked jobs (unmet deps, §5) — waiting in inbox/, re-evaluated each loop + local bf bj unmet bprinted=false + for bf in "$INBOX"/*.md; do + [[ -e "$bf" ]] || continue + unmet=$(deps_unmet "$bf") + [[ -n "$unmet" ]] || continue + if ! $bprinted; then echo; printf ' %sBLOCKED%s\n' "$C_BOLD" "$C_RESET"; bprinted=true; fi + bj=$(basename "$bf"); bj=${bj%.md} + printf ' %s%-26s%s %sblocked (waiting on: %s)%s\n' \ + "$C_BOLD" "$bj" "$C_RESET" "$C_YEL" "$unmet" "$C_RESET" + done echo } @@ -1328,10 +1545,18 @@ ${C_BOLD}TASK FRONTMATTER${C_RESET} (top of each .md) capabilities: [os:any, node>=20, has:git] # hard host requirements; unmet -> failed (capability_mismatch) idempotency-key: my-task-1 # re-adding same key+body = no-op; same key+different body = reject/supersede retry: { max: 2, backoff: 5m, on: [timeout, verify_failed, crash] } # requeue on these classes up to max, then retries_exhausted + profile: backend-engineer # inherit persona + verify/caps/engine-class/scope/review-policy (job fields override) + deps: [other-key] # block until each idempotency-key is shipped/ (or testing/ if deps-mode: soft) + deps-mode: soft # soft = a dep also counts as satisfied while in testing/ # --- reserved (parsed + shown in status, but no-op until a later phase) --- - profile: prefers: budget: deps: deps-mode: review-policy: artifacts: tracker-item: + prefers: budget: review-policy: artifacts: tracker-item: --- +${C_BOLD}PROFILES${C_RESET} profiles/.md presets persona + capabilities + default-verify + engine-class + + prefers-engine + allowed-scope + review-policy. A job's own fields always override. + Catalog: developer, backend-engineer, frontend-engineer, ux-designer, ui-designer, + qa, reviewer, docs-writer, planner(reserved). + ${C_BOLD}RESILIENCE${C_RESET} crash-safe: orphaned building/ jobs (dead worker) are recovered to inbox/ on 'run' startup; git-repo cwd work is checkpointed to branch aq/wip/ on every exit (resumed on retry); 'retry' requeues failures with backoff. See 'insights'.