feat(agent-queue): profiles (persona + presets) and single-host deps/DAG (P1-S2)
Implements roadmap §6 (profiles) and §5 deps on the bash runner, backward-compatible (jobs without profile/deps behave exactly as before). Profiles (§6): - profile_get / profile_persona / fm_eff helpers + PROFILES_DIR (AGENT_QUEUE_PROFILES override). A job's `profile:` inherits verify (<- default-verify), capabilities, engine-class, prefers-engine, allowed-scope, review-policy when the job omits them; job fields always override (precedence job > profile > default). Resolution runs via fm_eff inside the capability gate and resolve_engine, so inherited caps/engine-class take effect before launch. - persona injection: the profile's persona block is prepended to the stripped body fed to the engine (job .md unchanged on disk; nothing secret logged). - allowed-scope guardrail (WARN-ONLY): scope_check logs a non-blocking WARNING + records scope_warning= for changed paths outside the globs; path_in_scope is a pure, unit-testable matcher (`dir/**` = subtree). deps / DAG, single host (§5): - deps reference other jobs by idempotency-key. dep_satisfied: shipped/ (hard) or shipped/+testing/ (deps-mode: soft). deps_unmet drives a block-with-reason skip in inbox selection (never launched/failed); cmd_status surfaces "blocked (waiting on <keys>)". deps_would_cycle rejects cyclic submits on `add`. - _drain_pending: `--once` drains past dep-blocked jobs (idle can't satisfy them) while still waiting on retry/recovery backoff timers. Meta now records effective (inherited) capabilities/engine-class/prefers-engine/ review-policy/allowed-scope so `status` reflects resolved config.
This commit is contained in:
saravanakumardb1
parent
7c4f5bc9b0
commit
3d99f04427
@ -29,6 +29,8 @@ set -uo pipefail
|
||||
# ── Resolve paths ───────────────────────────────────────────────────
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
QUEUE_ROOT="${AGENT_QUEUE_ROOT:-$SCRIPT_DIR/queue}"
|
||||
# Profile catalog dir (persona + capability presets). Override for tests.
|
||||
PROFILES_DIR="${AGENT_QUEUE_PROFILES:-$SCRIPT_DIR/profiles}"
|
||||
INBOX="$QUEUE_ROOT/inbox"
|
||||
BUILDING="$QUEUE_ROOT/building"
|
||||
REVIEW="$QUEUE_ROOT/review"
|
||||
@ -120,6 +122,51 @@ lock_key_for() {
|
||||
# _keyhash <key> -> stable filename-safe token for a lock key
|
||||
_keyhash() { printf '%s' "$1" | cksum | awk '{print $1}'; }
|
||||
|
||||
# ── Profiles (§6): persona + capability/engine/scope presets ─────────
|
||||
#
|
||||
# profile_get <profile-name> <profile-key> [default] -> a single-line value from
|
||||
# profiles/<name>.md, else the default.
|
||||
profile_get() {
|
||||
local pf="$PROFILES_DIR/$1.md"
|
||||
[[ -f "$pf" ]] || { printf '%s' "${3:-}"; return; }
|
||||
fm_get "$pf" "$2" "${3:-}"
|
||||
}
|
||||
|
||||
# profile_persona <profile-name> -> the multi-line `persona: |` block (2-space
|
||||
# indent stripped), or empty. Used to prepend a persona overlay to the job body.
|
||||
profile_persona() {
|
||||
local pf="$PROFILES_DIR/$1.md"
|
||||
[[ -f "$pf" ]] || return 0
|
||||
awk '
|
||||
NR==1 && $0!="---" { exit }
|
||||
NR==1 { infm=1; next }
|
||||
infm && $0=="---" { exit }
|
||||
!infm { next }
|
||||
inpersona {
|
||||
if ($0 ~ /^[A-Za-z0-9_-]+[ \t]*:/) { inpersona=0 }
|
||||
else { line=$0; sub(/^ /,"",line); print line; next }
|
||||
}
|
||||
$0 ~ /^persona[ \t]*:[ \t]*\|[ \t]*$/ { inpersona=1; next }
|
||||
' "$pf"
|
||||
}
|
||||
|
||||
# fm_eff <file> <job-key> [default] [profile-key] -> effective value with
|
||||
# precedence job > profile > built-in default (§6 resolution). The profile is the
|
||||
# job's `profile:` frontmatter; `profile-key` defaults to `job-key` (e.g. `verify`
|
||||
# inherits the profile's `default-verify`). Inheritable: verify, capabilities,
|
||||
# engine-class, prefers-engine, allowed-scope, review-policy.
|
||||
fm_eff() {
|
||||
local file=$1 jkey=$2 def=${3:-} pkey=${4:-$2} v prof pv
|
||||
v=$(fm_get "$file" "$jkey" "")
|
||||
if [[ -n "$v" ]]; then printf '%s' "$v"; return; fi
|
||||
prof=$(fm_get "$file" profile "")
|
||||
if [[ -n "$prof" ]]; then
|
||||
pv=$(profile_get "$prof" "$pkey" "")
|
||||
[[ -n "$pv" ]] && { printf '%s' "$pv"; return; }
|
||||
fi
|
||||
printf '%s' "$def"
|
||||
}
|
||||
|
||||
# _mtime <file> -> file modification time in epoch seconds (BSD or GNU stat); empty if missing
|
||||
_mtime() {
|
||||
[[ -e "$1" ]] || { echo ""; return; }
|
||||
@ -356,10 +403,10 @@ resolve_engine() {
|
||||
local f=$1 eng cls prefers
|
||||
eng=$(fm_get "$f" engine "")
|
||||
if [[ -n "$eng" ]]; then printf '%s' "$eng"; return 0; fi
|
||||
cls=$(fm_get "$f" engine-class "")
|
||||
cls=$(fm_eff "$f" engine-class "") # inherit engine-class from the job's profile
|
||||
if [[ -z "$cls" ]]; then printf '%s' "$DEFAULT_ENGINE"; return 0; fi
|
||||
local class_engines; class_engines=$(engine_class_engines "$cls")
|
||||
prefers=$(fm_get "$f" prefers-engine "")
|
||||
prefers=$(fm_eff "$f" prefers-engine "")
|
||||
local ordered=() seen=" " p c
|
||||
if [[ -n "$prefers" ]]; then
|
||||
while IFS= read -r p; do
|
||||
@ -377,6 +424,134 @@ resolve_engine() {
|
||||
printf '%s' ""
|
||||
}
|
||||
|
||||
# ── deps / DAG, single host (§5) ─────────────────────────────────────
|
||||
# deps reference other jobs by their (author-controlled) `idempotency-key`.
|
||||
#
|
||||
# _key_in_dir <key> <dir> -> 0 if some .md in <dir> has idempotency-key == key.
|
||||
_key_in_dir() {
|
||||
local key=$1 d=$2 ef
|
||||
for ef in "$d"/*.md; do
|
||||
[[ -e "$ef" ]] || continue
|
||||
[[ "$(fm_get "$ef" idempotency-key "")" == "$key" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# dep_satisfied <key> <mode> -> 0 when the dep is met: a job with <key> is in
|
||||
# shipped/ (default), or shipped/ OR testing/ when mode is `soft`.
|
||||
dep_satisfied() {
|
||||
local key=$1 mode=$2
|
||||
_key_in_dir "$key" "$SHIPPED" && return 0
|
||||
[[ "$mode" == soft ]] && _key_in_dir "$key" "$TESTING" && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
# deps_unmet <file> -> space-separated list of this job's UNMET dep keys (empty if
|
||||
# none / no deps). `deps-mode` (hard|soft) is job-level.
|
||||
deps_unmet() {
|
||||
local f=$1 keys mode k unmet=""
|
||||
keys=$(parse_list "$(fm_get "$f" deps "")" | tr '\n' ' ')
|
||||
[[ -n "${keys// /}" ]] || { printf ''; return 0; }
|
||||
mode=$(fm_get "$f" deps-mode "hard")
|
||||
for k in $keys; do
|
||||
[[ -n "$k" ]] || continue
|
||||
dep_satisfied "$k" "$mode" || unmet+="$k "
|
||||
done
|
||||
printf '%s' "${unmet% }"
|
||||
}
|
||||
|
||||
# _deps_of_key <key> -> dep keys (space-separated) of the job carrying <key>,
|
||||
# scanned across inbox + active stages.
|
||||
_deps_of_key() {
|
||||
local key=$1 d ef
|
||||
for d in "$INBOX" "$BUILDING" "$REVIEW" "$TESTING" "$SHIPPED"; do
|
||||
for ef in "$d"/*.md; do
|
||||
[[ -e "$ef" ]] || continue
|
||||
[[ "$(fm_get "$ef" idempotency-key "")" == "$key" ]] || continue
|
||||
parse_list "$(fm_get "$ef" deps "")" | tr '\n' ' '
|
||||
return 0
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# deps_would_cycle <new-key> <new-deps-space> -> 0 if adding a job with <new-key>
|
||||
# depending on <new-deps> would create a cycle (BFS over existing key->deps edges
|
||||
# back to new-key; also catches self-dependency).
|
||||
deps_would_cycle() {
|
||||
local newkey=$1 newdeps=$2 visited=" " frontier next k d kd
|
||||
[[ -n "$newkey" ]] || return 1
|
||||
_in_list "$newkey" "$newdeps" && return 0
|
||||
frontier="$newdeps"
|
||||
while [[ -n "${frontier// /}" ]]; do
|
||||
next=""
|
||||
for k in $frontier; do
|
||||
[[ -n "$k" ]] || continue
|
||||
[[ "$k" == "$newkey" ]] && return 0
|
||||
case "$visited" in *" $k "*) continue;; esac
|
||||
visited+="$k "
|
||||
kd=$(_deps_of_key "$k")
|
||||
for d in $kd; do next+="$d "; done
|
||||
done
|
||||
frontier="$next"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# _drain_pending -> 0 if some inbox job can still make progress on its own: it is
|
||||
# runnable now, or it is waiting on a retry/recovery backoff (which elapses with
|
||||
# time). A job blocked ONLY by unmet deps is NOT pending while the loop is idle
|
||||
# (no running job can satisfy its deps), so `--once` can drain past it.
|
||||
_drain_pending() {
|
||||
local cand cj ne now; now=$(date +%s)
|
||||
for cand in "$INBOX"/*.md; do
|
||||
[[ -e "$cand" ]] || continue
|
||||
cj=$(basename "$cand"); cj=${cj%.md}
|
||||
ne=$(grep '^next_eligible=' "$STATE/$cj.meta" 2>/dev/null | tail -1 | cut -d= -f2)
|
||||
if [[ "$ne" =~ ^[0-9]+$ ]] && [[ "$ne" -gt "$now" ]]; then return 0; fi
|
||||
[[ -n "$(deps_unmet "$cand")" ]] && continue
|
||||
return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── allowed-scope guardrail (§6/§12) — WARN-ONLY this phase ───────────
|
||||
# path_in_scope <path> <globs-space> -> 0 if <path> matches any allowed-scope glob
|
||||
# (`dir/**` matches the whole subtree; `*` matches across `/`). Pure + testable.
|
||||
path_in_scope() {
|
||||
local path=$1 globs=$2 g pat
|
||||
for g in $globs; do
|
||||
[[ -n "$g" ]] || continue
|
||||
pat=${g//\*\*/\*}
|
||||
# shellcheck disable=SC2053
|
||||
[[ "$path" == $pat ]] && return 0
|
||||
[[ "$path" == "$g"/* ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# scope_check <cwd> <base> <scope> <logf> <metaf> -> log a WARNING (non-blocking)
|
||||
# for changed paths outside allowed-scope. Records scope_warning= in the meta.
|
||||
scope_check() {
|
||||
local cwd=$1 base=$2 scope=$3 logf=$4 metaf=$5 changed globs p out=""
|
||||
_is_git_repo "$cwd" || return 0
|
||||
if [[ -n "$base" ]]; then
|
||||
changed=$(git -C "$cwd" diff --name-only "$base" HEAD 2>/dev/null)
|
||||
else
|
||||
changed=$(git -C "$cwd" diff --name-only HEAD 2>/dev/null)
|
||||
fi
|
||||
[[ -n "$changed" ]] || return 0
|
||||
globs=$(parse_list "$scope" | tr '\n' ' ')
|
||||
[[ -n "${globs// /}" ]] || return 0
|
||||
while IFS= read -r p; do
|
||||
[[ -n "$p" ]] || continue
|
||||
path_in_scope "$p" "$globs" || out+="$p "
|
||||
done <<< "$changed"
|
||||
if [[ -n "$out" ]]; then
|
||||
echo "WARNING: allowed-scope violation (warn-only) — changed outside [$globs]: ${out% }" >> "$logf"
|
||||
echo "scope_warning=${out% }" >> "$metaf"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Engine driver: builds argv into AGENT_CMD[]; sets AGENT_STDIN if the ──
|
||||
# prompt should be fed on stdin (claude/codex) rather than a flag. $pf is the
|
||||
# frontmatter-STRIPPED body file, so a body starting with '--' is never
|
||||
@ -424,8 +599,9 @@ run_worker() {
|
||||
# The worker only ever APPENDS (ended/exit/result) to avoid a truncation race.
|
||||
|
||||
# ── Capability gate (§5/§8 single-host): if the job declares `capabilities`
|
||||
# this host does not satisfy, route to failed/ WITHOUT launching the agent. ──
|
||||
local req_caps; req_caps=$(parse_list "$(fm_get "$doing_file" capabilities "")" | tr '\n' ' ')
|
||||
# (own or inherited from its profile) this host does not satisfy, route to
|
||||
# failed/ WITHOUT launching the agent. ──
|
||||
local req_caps; req_caps=$(parse_list "$(fm_eff "$doing_file" capabilities "")" | tr '\n' ' ')
|
||||
if [[ -n "${req_caps// /}" ]]; then
|
||||
local avail; avail=$(detect_capabilities)
|
||||
if ! caps_match "$req_caps" "$avail"; then
|
||||
@ -474,6 +650,16 @@ run_worker() {
|
||||
# Strip our frontmatter so the agent only sees the task body.
|
||||
local bodyf="$STATE/$job.body.md"
|
||||
strip_frontmatter "$doing_file" > "$bodyf"
|
||||
# ── Persona injection (§6): prepend the profile's persona to the body fed to
|
||||
# the engine (job body unchanged on disk). Secrets are never logged. ──
|
||||
local prof; prof=$(fm_get "$doing_file" profile "")
|
||||
if [[ -n "$prof" ]]; then
|
||||
local persona; persona=$(profile_persona "$prof")
|
||||
if [[ -n "$persona" ]]; then
|
||||
{ printf '%s\n\n' "$persona"; cat "$bodyf"; } > "$bodyf.tmp" && mv "$bodyf.tmp" "$bodyf"
|
||||
echo "profile: injected persona overlay from '$prof'" >> "$logf"
|
||||
fi
|
||||
fi
|
||||
build_agent_cmd "$engine" "$bodyf" "$yolo"
|
||||
|
||||
# ── WIP checkpoint setup (§25.2): on a git cwd, create/checkout aq/wip/<job>
|
||||
@ -547,6 +733,11 @@ run_worker() {
|
||||
_numstat_into_meta "$cwd" "$WIP_BASE" "$metaf"
|
||||
parse_usage "$engine" "$logf" >> "$metaf"
|
||||
|
||||
# ── allowed-scope guardrail (§6) — WARN-ONLY: flag out-of-scope changes but
|
||||
# never block the job this phase. Scope may be inherited from the profile. ──
|
||||
local scope; scope=$(fm_eff "$doing_file" allowed-scope "" allowed-scope)
|
||||
[[ -n "$scope" ]] && scope_check "$cwd" "$WIP_BASE" "$scope" "$logf" "$metaf"
|
||||
|
||||
if $timed_out; then
|
||||
echo "TIMED OUT after ${tmo}s (rc=$rc): $(date)" >> "$logf"
|
||||
_finish_failure "$job" "$doing_file" "$metaf" "$logf" "timeout" "$rc" "$started"
|
||||
@ -558,7 +749,8 @@ run_worker() {
|
||||
local review_file="$REVIEW/$job.md"
|
||||
echo "exit=$rc" >> "$metaf"
|
||||
echo "completed OK (rc=0): landed in review — $(date)" >> "$logf"
|
||||
local verify; verify=$(fm_get "$review_file" verify "$DEFAULT_VERIFY")
|
||||
# verify is job-level, else inherited from the profile's default-verify.
|
||||
local verify; verify=$(fm_eff "$review_file" verify "$DEFAULT_VERIFY" default-verify)
|
||||
if [[ -z "$verify" ]]; then
|
||||
_meta_end "$metaf" "review" "$started"
|
||||
echo "no verify command — parked in review for manual promote: $(date)" >> "$logf"
|
||||
@ -896,6 +1088,13 @@ cmd_add() {
|
||||
done
|
||||
fi
|
||||
|
||||
# ── dep cycle detection (§5): reject a submit that would create a cycle in the
|
||||
# idempotency-key dependency graph (inbox + active stages). ──
|
||||
local newdeps; newdeps=$(parse_list "$(fm_get "$file" deps "")" | tr '\n' ' ')
|
||||
if [[ -n "${newdeps// /}" ]] && deps_would_cycle "$idem" "$newdeps"; then
|
||||
die "dependency cycle detected: job (key '${idem:-<none>}') with deps [${newdeps% }] would create a cycle — refusing."
|
||||
fi
|
||||
|
||||
local base; base=$(basename "$file")
|
||||
local stamp; stamp=$(date +%Y%m%d-%H%M%S)
|
||||
local dest="$INBOX/${stamp}__${base}"
|
||||
@ -963,6 +1162,8 @@ cmd_run() {
|
||||
cand_job=$(basename "$cand"); cand_job=${cand_job%.md}
|
||||
cand_ne=$(grep '^next_eligible=' "$STATE/$cand_job.meta" 2>/dev/null | tail -1 | cut -d= -f2)
|
||||
if [[ "$cand_ne" =~ ^[0-9]+$ ]] && [[ "$cand_ne" -gt "$now_s" ]]; then continue; fi
|
||||
# skip jobs whose deps (§5 DAG) are unmet — blocked, re-evaluated next loop
|
||||
if [[ -n "$(deps_unmet "$cand")" ]]; then continue; fi
|
||||
next="$cand"; break
|
||||
done < <(inbox_sorted)
|
||||
[[ -z "$next" ]] && break
|
||||
@ -997,16 +1198,17 @@ cmd_run() {
|
||||
echo "attempts=$w_attempts"
|
||||
echo "priority=$(fm_get "$doing_file" priority medium)"
|
||||
echo "profile=$(fm_get "$doing_file" profile "")"
|
||||
echo "engine_class=$(fm_get "$doing_file" engine-class "")"
|
||||
echo "capabilities=$(fm_get "$doing_file" capabilities "")"
|
||||
echo "engine_class=$(fm_eff "$doing_file" engine-class "")"
|
||||
echo "capabilities=$(fm_eff "$doing_file" capabilities "")"
|
||||
echo "prefers=$(fm_get "$doing_file" prefers "")"
|
||||
echo "prefers_engine=$(fm_get "$doing_file" prefers-engine "")"
|
||||
echo "prefers_engine=$(fm_eff "$doing_file" prefers-engine "")"
|
||||
echo "allowed_scope=$(fm_eff "$doing_file" allowed-scope "" allowed-scope)"
|
||||
echo "budget=$(fm_get "$doing_file" budget "")"
|
||||
echo "deps=$(fm_get "$doing_file" deps "")"
|
||||
echo "deps_mode=$(fm_get "$doing_file" deps-mode "")"
|
||||
echo "idempotency_key=$(fm_get "$doing_file" idempotency-key "")"
|
||||
echo "retry=$(fm_get "$doing_file" retry "")"
|
||||
echo "review_policy=$(fm_get "$doing_file" review-policy "")"
|
||||
echo "review_policy=$(fm_eff "$doing_file" review-policy "" review-policy)"
|
||||
echo "artifacts=$(fm_get "$doing_file" artifacts "")"
|
||||
echo "tracker_item=$(fm_get "$doing_file" tracker-item "")"
|
||||
} > "$STATE/$job.meta"
|
||||
@ -1018,8 +1220,11 @@ cmd_run() {
|
||||
done
|
||||
|
||||
if $once; then
|
||||
[[ "$(active_workers)" -eq 0 && -z "$(ls -1 "$INBOX"/*.md 2>/dev/null)" ]] && {
|
||||
log "drain complete — inbox empty, no workers running"; rm -f "$STATE/daemon.pid"; exit 0; }
|
||||
# drain when no worker is running and nothing in inbox can still progress on
|
||||
# its own (backoff jobs still count as pending; dep-blocked jobs do not).
|
||||
if [[ "$(active_workers)" -eq 0 ]] && ! _drain_pending; then
|
||||
log "drain complete — no runnable work, no workers running"; rm -f "$STATE/daemon.pid"; exit 0
|
||||
fi
|
||||
fi
|
||||
sleep "$POLL_SECONDS"
|
||||
done
|
||||
@ -1072,6 +1277,18 @@ cmd_status() {
|
||||
printf ' %s%s%s\n' "$C_DIM" "$(_insights_line "$f")" "$C_RESET"
|
||||
done
|
||||
$printed || printf ' %sno workers running%s\n' "$C_DIM" "$C_RESET"
|
||||
|
||||
# blocked jobs (unmet deps, §5) — waiting in inbox/, re-evaluated each loop
|
||||
local bf bj unmet bprinted=false
|
||||
for bf in "$INBOX"/*.md; do
|
||||
[[ -e "$bf" ]] || continue
|
||||
unmet=$(deps_unmet "$bf")
|
||||
[[ -n "$unmet" ]] || continue
|
||||
if ! $bprinted; then echo; printf ' %sBLOCKED%s\n' "$C_BOLD" "$C_RESET"; bprinted=true; fi
|
||||
bj=$(basename "$bf"); bj=${bj%.md}
|
||||
printf ' %s%-26s%s %sblocked (waiting on: %s)%s\n' \
|
||||
"$C_BOLD" "$bj" "$C_RESET" "$C_YEL" "$unmet" "$C_RESET"
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
@ -1328,10 +1545,18 @@ ${C_BOLD}TASK FRONTMATTER${C_RESET} (top of each .md)
|
||||
capabilities: [os:any, node>=20, has:git] # hard host requirements; unmet -> failed (capability_mismatch)
|
||||
idempotency-key: my-task-1 # re-adding same key+body = no-op; same key+different body = reject/supersede
|
||||
retry: { max: 2, backoff: 5m, on: [timeout, verify_failed, crash] } # requeue on these classes up to max, then retries_exhausted
|
||||
profile: backend-engineer # inherit persona + verify/caps/engine-class/scope/review-policy (job fields override)
|
||||
deps: [other-key] # block until each idempotency-key is shipped/ (or testing/ if deps-mode: soft)
|
||||
deps-mode: soft # soft = a dep also counts as satisfied while in testing/
|
||||
# --- reserved (parsed + shown in status, but no-op until a later phase) ---
|
||||
profile: prefers: budget: deps: deps-mode: review-policy: artifacts: tracker-item:
|
||||
prefers: budget: review-policy: artifacts: tracker-item:
|
||||
---
|
||||
|
||||
${C_BOLD}PROFILES${C_RESET} profiles/<name>.md presets persona + capabilities + default-verify + engine-class +
|
||||
prefers-engine + allowed-scope + review-policy. A job's own fields always override.
|
||||
Catalog: developer, backend-engineer, frontend-engineer, ux-designer, ui-designer,
|
||||
qa, reviewer, docs-writer, planner(reserved).
|
||||
|
||||
${C_BOLD}RESILIENCE${C_RESET} crash-safe: orphaned building/ jobs (dead worker) are recovered to inbox/ on
|
||||
'run' startup; git-repo cwd work is checkpointed to branch aq/wip/<job> on every
|
||||
exit (resumed on retry); 'retry' requeues failures with backoff. See 'insights'.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user