test(agent-queue): profiles + deps/DAG selftest cases (P1-S2)
Adds (never weakens) temp-catalog + temp-git cases: profile verify inheritance + job-override precedence, persona-injection golden, profile capability inheritance, allowed-scope warn-only + path_in_scope unit, deps block->run, deps-mode soft (testing/), and submit-time cycle rejection. Full suite green (46 checks).
This commit is contained in:
saravanakumardb1
parent
f2dabdeb81
commit
71d8a7cd4e
@ -420,4 +420,169 @@ else
|
|||||||
printf '%s\n' "$out" >&2; fail "insights aggregate rollup missing/incorrect"
|
printf '%s\n' "$out" >&2; fail "insights aggregate rollup missing/incorrect"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────
|
||||||
|
# Phase 1 — Slice 2 cases (profiles + deps/DAG, single host).
|
||||||
|
# Uses a temp profile catalog (AGENT_QUEUE_PROFILES) + temp git repos.
|
||||||
|
# ─────────────────────────────────────────────────────────────────────
|
||||||
|
profdir="$tmp/profiles"; mkdir -p "$profdir"
|
||||||
|
printf '%s\n' '---' 'name: vfail' 'persona: |' ' PERSONA-VFAIL' 'default-verify: false' '---' > "$profdir/vfail.md"
|
||||||
|
printf '%s\n' '---' 'name: vpass' 'default-verify: true' '---' > "$profdir/vpass.md"
|
||||||
|
printf '%s\n' '---' 'name: capreq' 'capabilities: [has:definitely-not-installed]' '---' > "$profdir/capreq.md"
|
||||||
|
printf '%s\n' '---' 'name: personap' 'persona: |' ' PERSONA-MARKER-XYZ' ' second persona line' 'default-verify: true' '---' > "$profdir/personap.md"
|
||||||
|
printf '%s\n' '---' 'name: scoped' 'allowed-scope: [backend/**]' '---' > "$profdir/scoped.md"
|
||||||
|
export AGENT_QUEUE_PROFILES="$profdir"
|
||||||
|
funcs="$tmp/aq-funcs.sh"; sed '/^main "\$@"/d' "$AQ" > "$funcs"
|
||||||
|
|
||||||
|
# 19. profile inherits default-verify: vfail (verify=false) → failed/verify_failed;
|
||||||
|
# vpass (verify=true) → testing/.
|
||||||
|
export AGENT_QUEUE_ROOT="$tmp/queue-pverify"
|
||||||
|
"$AQ" init >/dev/null
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'profile: vfail' '---' '' '# pv-fail' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/pvfail.md"
|
||||||
|
DEVIN_BIN="$stub" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
if ls "$AGENT_QUEUE_ROOT"/failed/pvfail.md >/dev/null 2>&1 \
|
||||||
|
&& [ "$(metaval "$AGENT_QUEUE_ROOT/.state/pvfail.meta" result)" = "verify_failed" ]; then
|
||||||
|
pass "profile inherit: default-verify=false → failed/ (verify_failed)"
|
||||||
|
else
|
||||||
|
fail "profile verify=false did not route to failed (result=$(metaval "$AGENT_QUEUE_ROOT/.state/pvfail.meta" result))"
|
||||||
|
fi
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'profile: vpass' '---' '' '# pv-pass' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/pvpass.md"
|
||||||
|
DEVIN_BIN="$stub" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
if ls "$AGENT_QUEUE_ROOT"/testing/pvpass.md >/dev/null 2>&1; then
|
||||||
|
pass "profile inherit: default-verify=true → testing/"
|
||||||
|
else
|
||||||
|
fail "profile verify=true did not reach testing/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 19b. job-level verify overrides the profile (precedence job > profile).
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'profile: vfail' 'verify: true' '---' '' '# override' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/pvoverride.md"
|
||||||
|
DEVIN_BIN="$stub" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
ls "$AGENT_QUEUE_ROOT"/testing/pvoverride.md >/dev/null 2>&1 \
|
||||||
|
&& pass "profile precedence: job verify overrides profile default-verify" \
|
||||||
|
|| fail "job-level verify did not override profile"
|
||||||
|
|
||||||
|
# 20. persona injection (golden): the body fed to the engine begins with the
|
||||||
|
# profile persona. A stub copies its --prompt-file to a sentinel.
|
||||||
|
export AGENT_QUEUE_ROOT="$tmp/queue-persona"
|
||||||
|
sentinel="$tmp/persona-body.txt"; rm -f "$sentinel"
|
||||||
|
copystub="$tmp/copy-engine"
|
||||||
|
cat > "$copystub" <<STUBEOF
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
pf=""
|
||||||
|
while [ \$# -gt 0 ]; do case "\$1" in --prompt-file) pf="\$2"; shift 2;; *) shift;; esac; done
|
||||||
|
[ -n "\$pf" ] && cp "\$pf" "$sentinel"
|
||||||
|
exit 0
|
||||||
|
STUBEOF
|
||||||
|
chmod +x "$copystub"
|
||||||
|
"$AQ" init >/dev/null
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'profile: personap' '---' '' 'TASK-BODY-LINE' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/personajob.md"
|
||||||
|
DEVIN_BIN="$copystub" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
if [ "$(head -1 "$sentinel" 2>/dev/null)" = "PERSONA-MARKER-XYZ" ] \
|
||||||
|
&& grep -q 'TASK-BODY-LINE' "$sentinel" 2>/dev/null; then
|
||||||
|
pass "persona injection: engine body begins with profile persona, task preserved"
|
||||||
|
else
|
||||||
|
echo "body head: $(head -3 "$sentinel" 2>/dev/null)" >&2
|
||||||
|
fail "persona was not prepended to the engine body"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 21. profile capability inheritance: a job omitting capabilities inherits the
|
||||||
|
# profile's → unmet → failed/ capability_mismatch, agent never launched.
|
||||||
|
export AGENT_QUEUE_ROOT="$tmp/queue-pcaps"
|
||||||
|
launchflag="$tmp/pcaps-launched"; rm -f "$launchflag"
|
||||||
|
launchstub3="$tmp/cap-launch3"
|
||||||
|
printf '#!/usr/bin/env bash\ntouch %q\nexit 0\n' "$launchflag" > "$launchstub3"; chmod +x "$launchstub3"
|
||||||
|
"$AQ" init >/dev/null
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'profile: capreq' '---' '' '# pcaps' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/pcapsjob.md"
|
||||||
|
DEVIN_BIN="$launchstub3" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
if ls "$AGENT_QUEUE_ROOT"/failed/pcapsjob.md >/dev/null 2>&1 \
|
||||||
|
&& [ "$(metaval "$AGENT_QUEUE_ROOT/.state/pcapsjob.meta" result)" = "capability_mismatch" ] \
|
||||||
|
&& [ ! -e "$launchflag" ]; then
|
||||||
|
pass "profile caps inheritance: unmet inherited capability → capability_mismatch (no launch)"
|
||||||
|
else
|
||||||
|
fail "profile caps inheritance failed (result=$(metaval "$AGENT_QUEUE_ROOT/.state/pcapsjob.meta" result) launched=$([ -e "$launchflag" ] && echo yes || echo no))"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 22. allowed-scope warn-only: an out-of-scope change logs a WARNING and the job
|
||||||
|
# still succeeds; plus a direct path_in_scope unit check.
|
||||||
|
export AGENT_QUEUE_ROOT="$tmp/queue-scope"
|
||||||
|
repos="$tmp/repo-scope"; mkrepo "$repos"
|
||||||
|
scopestub="$tmp/scope-engine"
|
||||||
|
printf '#!/usr/bin/env bash\nmkdir -p frontend && echo changed > frontend/out.txt\nexit 0\n' > "$scopestub"
|
||||||
|
chmod +x "$scopestub"
|
||||||
|
"$AQ" init >/dev/null
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $repos" 'yolo: true' 'profile: scoped' '---' '' '# scope task' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/scopejob.md"
|
||||||
|
DEVIN_BIN="$scopestub" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
if grep -q 'allowed-scope violation' "$AGENT_QUEUE_ROOT/logs/scopejob.log" 2>/dev/null \
|
||||||
|
&& ls "$AGENT_QUEUE_ROOT"/review/scopejob.md >/dev/null 2>&1; then
|
||||||
|
pass "allowed-scope: out-of-scope change WARNS (warn-only) and job still succeeds"
|
||||||
|
else
|
||||||
|
fail "allowed-scope warn-only did not warn / job did not succeed"
|
||||||
|
fi
|
||||||
|
if bash -c 'set -uo pipefail; source "'"$funcs"'"; path_in_scope "backend/a/b.ts" "backend/**" && ! path_in_scope "frontend/x.ts" "backend/**"'; then
|
||||||
|
pass "allowed-scope: path_in_scope matches subtree, rejects outside (unit)"
|
||||||
|
else
|
||||||
|
fail "path_in_scope unit logic wrong"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 23. deps block→run: B deps:[keyA] stays blocked until A is shipped/, then runs.
|
||||||
|
export AGENT_QUEUE_ROOT="$tmp/queue-deps"
|
||||||
|
"$AQ" init >/dev/null
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'idempotency-key: keyA' '---' '' '# A' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/jobA.md"
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'idempotency-key: keyB' 'deps: [keyA]' '---' '' '# B' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/jobB.md"
|
||||||
|
DEVIN_BIN="$stub" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
if ls "$AGENT_QUEUE_ROOT"/inbox/jobB.md >/dev/null 2>&1 && ls "$AGENT_QUEUE_ROOT"/review/jobA.md >/dev/null 2>&1; then
|
||||||
|
pass "deps: B stays blocked in inbox while A is unshipped"
|
||||||
|
else
|
||||||
|
fail "deps: B should be blocked while A unshipped (A=$(ls "$AGENT_QUEUE_ROOT"/review 2>/dev/null) B-in-inbox=$(ls "$AGENT_QUEUE_ROOT"/inbox 2>/dev/null))"
|
||||||
|
fi
|
||||||
|
# status surfaces the blocked job
|
||||||
|
"$AQ" status 2>/dev/null | grep -q 'blocked (waiting on: keyA)' \
|
||||||
|
&& pass "deps: status surfaces 'blocked (waiting on: keyA)'" \
|
||||||
|
|| fail "deps: status did not surface blocked job"
|
||||||
|
# ship A (review -> testing -> shipped), then B becomes runnable
|
||||||
|
"$AQ" promote jobA >/dev/null 2>&1 # review -> testing
|
||||||
|
"$AQ" promote jobA >/dev/null 2>&1 # testing -> shipped
|
||||||
|
DEVIN_BIN="$stub" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
if ls "$AGENT_QUEUE_ROOT"/review/jobB.md >/dev/null 2>&1; then
|
||||||
|
pass "deps: once A is shipped, B unblocks and completes"
|
||||||
|
else
|
||||||
|
fail "deps: B did not run after A shipped"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 24. deps-mode soft: dep satisfied when the dependency is in testing/.
|
||||||
|
export AGENT_QUEUE_ROOT="$tmp/queue-depsoft"
|
||||||
|
"$AQ" init >/dev/null
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'idempotency-key: keyA' 'verify: true' '---' '' '# A-soft' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/sjobA.md"
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'idempotency-key: keyC' 'deps: [keyA]' 'deps-mode: soft' '---' '' '# C-soft' \
|
||||||
|
> "$AGENT_QUEUE_ROOT/inbox/sjobC.md"
|
||||||
|
DEVIN_BIN="$stub" "$AQ" run --once >/dev/null 2>&1
|
||||||
|
if ls "$AGENT_QUEUE_ROOT"/testing/sjobA.md >/dev/null 2>&1 && ls "$AGENT_QUEUE_ROOT"/review/sjobC.md >/dev/null 2>&1; then
|
||||||
|
pass "deps-mode soft: dep satisfied while dependency is in testing/"
|
||||||
|
else
|
||||||
|
fail "deps-mode soft did not unblock from testing/ (A=$(ls "$AGENT_QUEUE_ROOT"/testing 2>/dev/null) C=$(ls "$AGENT_QUEUE_ROOT"/review 2>/dev/null))"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 25. cycle detection: adding A deps:[keyB] while B deps:[keyA] exists is rejected.
|
||||||
|
export AGENT_QUEUE_ROOT="$tmp/queue-cycle"
|
||||||
|
"$AQ" init >/dev/null
|
||||||
|
cycB="$tmp/cyc-b.md"
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'idempotency-key: keyB' 'deps: [keyA]' '---' '' '# cyc B' > "$cycB"
|
||||||
|
"$AQ" add "$cycB" >/dev/null 2>&1 # B added (blocked on keyA — allowed)
|
||||||
|
cycA="$tmp/cyc-a.md"
|
||||||
|
printf '%s\n' '---' 'engine: devin' "cwd: $work" 'yolo: true' 'idempotency-key: keyA' 'deps: [keyB]' '---' '' '# cyc A' > "$cycA"
|
||||||
|
if DEVIN_BIN="$stub" "$AQ" add "$cycA" >/dev/null 2>&1; then
|
||||||
|
fail "cycle detection: adding A deps:[keyB] while B deps:[keyA] should be rejected"
|
||||||
|
else
|
||||||
|
pass "cycle detection: dependency cycle on add is rejected"
|
||||||
|
fi
|
||||||
|
unset AGENT_QUEUE_PROFILES
|
||||||
|
|
||||||
echo "self-test PASS"
|
echo "self-test PASS"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user