diff --git a/agent-queue/.gitignore b/agent-queue/.gitignore new file mode 100644 index 0000000..fd0b7d6 --- /dev/null +++ b/agent-queue/.gitignore @@ -0,0 +1,2 @@ +# Runtime queue state — never commit prompts, logs, or heartbeats +queue/ diff --git a/agent-queue/README.md b/agent-queue/README.md new file mode 100644 index 0000000..cac0acc --- /dev/null +++ b/agent-queue/README.md @@ -0,0 +1,127 @@ +# agent-queue + +A zero-dependency **folder "kanban" runner** for headless coding-agent CLIs — +**Devin**, **Claude Code**, and **OpenAI Codex**. Drop prompt `.md` files into a folder, +and they get executed (in auto-approve mode) one slot at a time, moving through +`inbox → doing → done/failed` with live status. + +> **Why this exists:** the agent CLIs ship a minimal local interface (no built-in +> batch/queue/dashboard — that lives in their *cloud* products). This is the ~250-line +> glue that turns "run one prompt interactively" into "queue many and walk away." + +--- + +## Quick start + +```bash +cd learning_ai_devops_tools/agent-queue +chmod +x agent-queue.sh +./agent-queue.sh init + +# queue a roadmap for Devin, running in the tracker-web repo, auto-approving everything +./agent-queue.sh add ~/roadmaps/UX-2.md \ + --engine devin \ + --cwd /Users/sd9235/code/mygh/learning_ai_common_plat/dashboards/tracker-web \ + --yolo + +# start processing (foreground; Ctrl-C to stop). Run up to 2 agents at once. +./agent-queue.sh run --max 2 +``` + +In a **second terminal**, watch progress: + +```bash +./agent-queue.sh watch +``` + +``` + AGENT QUEUE /…/agent-queue/queue + inbox 3 doing 2 done 5 failed 0 running 2/2 + + RUNNING + 20260528-2130__UX-2 devin 4m12s pid 51234 ⏺ Edited src/app/dashboard/items/page.tsx + 20260528-2131__UX-3 claude 1m02s pid 51290 Running: pnpm typecheck +``` + +--- + +## How a task is configured + +Each `.md` carries optional **frontmatter** telling the runner which engine to use, +which directory to run in, and whether to auto-approve: + +```md +--- +engine: devin # devin | claude | codex (default: $AGENT_QUEUE_ENGINE) +cwd: /abs/path/to/repo # where the agent executes (default: cwd when added) +yolo: true # auto-approve ALL tools (default: true) +--- + +# Your task / roadmap goes here +... +``` + +`add --engine/--cwd/--yolo` will inject this frontmatter for you if the file doesn't +already have a `---` block. + +## Engine mapping + +| `engine:` | Command run | Auto-approve flag (`yolo: true`) | +| --------- | ----------- | -------------------------------- | +| `devin` | `devin -p --prompt-file ` | `--permission-mode dangerous` | +| `claude` | `claude -p ""` | `--dangerously-skip-permissions` | +| `codex` | `codex exec ""` | `--dangerously-bypass-approvals-and-sandbox` | + +> Flags drift between CLI versions — if one changes, edit `build_agent_cmd()` in +> `agent-queue.sh` (it's the single place each engine is mapped). + +## Commands + +| Command | What it does | +| ------- | ------------ | +| `init` | create the `queue/` folders | +| `add [--engine E] [--cwd P] [--yolo\|--no-yolo]` | queue a prompt into `inbox/` | +| `run [--max N] [--engine E] [--once]` | process the inbox (foreground loop) | +| `status` | kanban counts + running-worker table | +| `watch [interval]` | live `status`, redrawn every N seconds (default 2) | +| `stop` | kill running workers + the run loop | +| `logs [-f]` | print / follow a job's log | + +## Folder layout + +``` +queue/ + inbox/ # drop / queued .md files (oldest picked first) + doing/ # currently executing + done/ # exited 0 + failed/ # non-zero exit (or bad cwd) + logs/ # .log — full agent output + .state/ # .meta heartbeats + daemon.pid (runtime only) +``` + +## Config (env overrides) + +| Var | Default | Meaning | +| --- | ------- | ------- | +| `AGENT_QUEUE_ROOT` | `./queue` | where the kanban folders live | +| `AGENT_QUEUE_MAX` | `2` | max concurrent agents | +| `AGENT_QUEUE_ENGINE` | `devin` | default engine when none in frontmatter | +| `AGENT_QUEUE_POLL` | `3` | inbox poll interval (seconds) | +| `DEVIN_BIN` / `CLAUDE_BIN` / `CODEX_BIN` | autodetected | override CLI binary paths | + +## ⚠️ Safety + +Running agents with `yolo: true` means **no approval prompts** — they will edit files, +run shell commands, and commit unattended. Mitigate: + +- Prefer **scope-locked** prompt files (e.g. "edit only under `dashboards/tracker-web/`"). +- Tell prompts **not to `git push`** — review commits before they leave your machine. +- Avoid queueing two tasks that touch the **same repo** concurrently (git contention). + Use `--max 1` if all tasks share a repo. +- Watch cost: each job is a full agent session. + +## Roadmap / nice-to-haves + +- `--push` opt-in + per-repo lock to serialize same-repo jobs automatically. +- Node/TS rewrite with a richer live TUI dashboard. +- `done`-folder retention / archive by date. diff --git a/agent-queue/agent-queue.sh b/agent-queue/agent-queue.sh new file mode 100755 index 0000000..16f2f22 --- /dev/null +++ b/agent-queue/agent-queue.sh @@ -0,0 +1,380 @@ +#!/usr/bin/env bash +# +# agent-queue — a folder-based "kanban" runner for headless coding-agent CLIs. +# +# Drop a prompt .md file into queue/inbox/, and `agent-queue run` will: +# 1. pick the oldest file (respecting --max concurrency), +# 2. move it inbox/ -> doing/, +# 3. launch the chosen agent CLI (devin | claude | codex) in --yolo mode, +# 4. on success move doing/ -> done/, on failure -> failed/, +# 5. write a per-job log + live state so `status`/`watch` can show progress. +# +# Per-task config travels in YAML-ish frontmatter at the top of the .md: +# --- +# engine: devin # devin | claude | codex (default: $DEFAULT_ENGINE) +# cwd: /abs/path/repo # where the agent runs (default: $PWD when added) +# yolo: true # auto-approve all tools (default: true) +# --- +# +# Subcommands: init | add | run | status | watch | stop | logs | help +# +set -uo pipefail + +# ── Resolve paths ─────────────────────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +QUEUE_ROOT="${AGENT_QUEUE_ROOT:-$SCRIPT_DIR/queue}" +INBOX="$QUEUE_ROOT/inbox" +DOING="$QUEUE_ROOT/doing" +DONE="$QUEUE_ROOT/done" +FAILED="$QUEUE_ROOT/failed" +LOGS="$QUEUE_ROOT/logs" +STATE="$QUEUE_ROOT/.state" + +# ── Config (env-overridable) ──────────────────────────────────────── +MAX_CONCURRENCY="${AGENT_QUEUE_MAX:-2}" +DEFAULT_ENGINE="${AGENT_QUEUE_ENGINE:-devin}" +POLL_SECONDS="${AGENT_QUEUE_POLL:-3}" + +DEVIN_BIN="${DEVIN_BIN:-$(command -v devin || echo "$HOME/.local/bin/devin")}" +CLAUDE_BIN="${CLAUDE_BIN:-$(command -v claude || echo claude)}" +CODEX_BIN="${CODEX_BIN:-$(command -v codex || echo codex)}" + +# ── Colors ────────────────────────────────────────────────────────── +if [[ -t 1 ]]; then + C_RESET=$'\033[0m'; C_DIM=$'\033[2m'; C_BOLD=$'\033[1m' + C_BLUE=$'\033[34m'; C_GREEN=$'\033[32m'; C_RED=$'\033[31m'; C_YEL=$'\033[33m'; C_CYAN=$'\033[36m' +else + C_RESET=""; C_DIM=""; C_BOLD=""; C_BLUE=""; C_GREEN=""; C_RED=""; C_YEL=""; C_CYAN="" +fi + +log() { printf '%s[agent-queue]%s %s\n' "$C_CYAN" "$C_RESET" "$*"; } +err() { printf '%s[agent-queue]%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; } +die() { err "$*"; exit 1; } + +# ── Init ──────────────────────────────────────────────────────────── +ensure_dirs() { mkdir -p "$INBOX" "$DOING" "$DONE" "$FAILED" "$LOGS" "$STATE"; } + +# ── Frontmatter parsing ───────────────────────────────────────────── +# fm_get +fm_get() { + local file=$1 key=$2 def=${3:-} + local val + # only scan a leading --- ... --- block + val=$(awk -v k="$key" ' + NR==1 && $0!="---" { exit } + NR==1 { infm=1; next } + infm && $0=="---" { exit } + infm { + line=$0 + sub(/^[ \t]*/,"",line) + if (line ~ "^" k "[ \t]*:") { + sub("^" k "[ \t]*:[ \t]*","",line) + gsub(/^["'\''[:space:]]+|["'\''[:space:]]+$/,"",line) + print line; exit + } + }' "$file" 2>/dev/null) + [[ -n "$val" ]] && printf '%s' "$val" || printf '%s' "$def" +} + +# strip_frontmatter -> prints the body (everything after a leading ---..--- block) +strip_frontmatter() { + awk 'NR==1 && $0=="---" { infm=1; next } + infm && $0=="---" { infm=0; next } + { if (!infm) print }' "$1" +} + +# ── Engine driver: builds argv into AGENT_CMD[]; sets AGENT_STDIN if the ── +# prompt should be fed on stdin (claude/codex) rather than a flag. $pf is the +# frontmatter-STRIPPED body file, so a body starting with '--' is never +# misparsed as a CLI option. +build_agent_cmd() { + local engine=$1 pf=$2 yolo=$3 + AGENT_CMD=(); AGENT_STDIN="" + case "$engine" in + devin) + AGENT_CMD=( "$DEVIN_BIN" -p --prompt-file "$pf" ) + [[ "$yolo" == "true" ]] && AGENT_CMD+=( --permission-mode dangerous ) + ;; + claude) + AGENT_CMD=( "$CLAUDE_BIN" -p ) + [[ "$yolo" == "true" ]] && AGENT_CMD+=( --dangerously-skip-permissions ) + AGENT_STDIN="$pf" + ;; + codex) + AGENT_CMD=( "$CODEX_BIN" exec ) + [[ "$yolo" == "true" ]] && AGENT_CMD+=( --dangerously-bypass-approvals-and-sandbox ) + AGENT_STDIN="$pf" + ;; + *) die "unknown engine '$engine' (use: devin | claude | codex)";; + esac +} + +# ── Worker: runs one job to completion (invoked in background) ─────── +run_worker() { + local doing_file=$1 + local job; job=$(basename "$doing_file") + job=${job%.md} + local engine cwd yolo logf metaf + engine=$(fm_get "$doing_file" engine "$DEFAULT_ENGINE") + cwd=$(fm_get "$doing_file" cwd "$PWD") + yolo=$(fm_get "$doing_file" yolo "true") + logf="$LOGS/$job.log" + metaf="$STATE/$job.meta" + # NOTE: the parent (cmd_run) creates $metaf with job/engine/cwd/started/pid. + # The worker only ever APPENDS (ended/exit/result) to avoid a truncation race. + + { + echo "===== agent-queue job: $job =====" + echo "engine=$engine cwd=$cwd yolo=$yolo" + echo "started: $(date)" + echo "=================================" + } >> "$logf" + + if [[ ! -d "$cwd" ]]; then + echo "FATAL: cwd does not exist: $cwd" >> "$logf" + mv "$doing_file" "$FAILED/" 2>/dev/null + echo "result=failed" >> "$metaf"; echo "ended=$(date +%s)" >> "$metaf" + return 1 + fi + + # Strip our frontmatter so the agent only sees the task body. + local bodyf="$STATE/$job.body.md" + strip_frontmatter "$doing_file" > "$bodyf" + build_agent_cmd "$engine" "$bodyf" "$yolo" + local rc + if [[ -n "$AGENT_STDIN" ]]; then + ( cd "$cwd" && "${AGENT_CMD[@]}" < "$AGENT_STDIN" ) >> "$logf" 2>&1 + else + ( cd "$cwd" && "${AGENT_CMD[@]}" ) >> "$logf" 2>&1 + fi + rc=$? + + echo "ended=$(date +%s)" >> "$metaf" + echo "exit=$rc" >> "$metaf" + if [[ $rc -eq 0 ]]; then + mv "$doing_file" "$DONE/" 2>/dev/null + echo "result=done" >> "$metaf" + echo "completed OK (rc=0): $(date)" >> "$logf" + else + mv "$doing_file" "$FAILED/" 2>/dev/null + echo "result=failed" >> "$metaf" + echo "FAILED (rc=$rc): $(date)" >> "$logf" + fi +} + +# count live workers by checking recorded pids +live_workers() { + local n=0 f pid + for f in "$STATE"/*.meta; do + [[ -e "$f" ]] || continue + grep -q '^ended=' "$f" && continue + pid=$(grep '^pid=' "$f" | head -1 | cut -d= -f2) + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && n=$((n+1)) + done + echo "$n" +} + +# ── Commands ──────────────────────────────────────────────────────── +cmd_init() { ensure_dirs; log "queue initialized at $C_BOLD$QUEUE_ROOT$C_RESET"; } + +cmd_add() { + ensure_dirs + local file="" engine="" cwd="" yolo="" + while [[ $# -gt 0 ]]; do + case "$1" in + --engine) engine=$2; shift 2;; + --cwd) cwd=$2; shift 2;; + --yolo) yolo=true; shift;; + --no-yolo) yolo=false; shift;; + *) file=$1; shift;; + esac + done + [[ -n "$file" && -f "$file" ]] || die "usage: add [--engine devin|claude|codex] [--cwd PATH] [--yolo|--no-yolo]" + local base; base=$(basename "$file") + local stamp; stamp=$(date +%Y%m%d-%H%M%S) + local dest="$INBOX/${stamp}__${base}" + + # If user passed flags AND the file has no frontmatter, inject one. + if [[ -n "$engine$cwd$yolo" ]] && [[ "$(head -1 "$file")" != "---" ]]; then + { + echo "---" + echo "engine: ${engine:-$DEFAULT_ENGINE}" + echo "cwd: ${cwd:-$PWD}" + echo "yolo: ${yolo:-true}" + echo "---" + echo + cat "$file" + } > "$dest" + else + cp "$file" "$dest" + fi + log "queued $C_BOLD$(basename "$dest")$C_RESET (engine=$(fm_get "$dest" engine "$DEFAULT_ENGINE"), cwd=$(fm_get "$dest" cwd "$PWD"))" +} + +cmd_run() { + ensure_dirs + local once=false + while [[ $# -gt 0 ]]; do + case "$1" in + --max) MAX_CONCURRENCY=$2; shift 2;; + --engine) DEFAULT_ENGINE=$2; shift 2;; + --once|--drain) once=true; shift;; + *) die "run: unknown arg '$1'";; + esac + done + echo "$$" > "$STATE/daemon.pid" + trap 'rm -f "$STATE/daemon.pid"; log "run loop stopped"; exit 0' INT TERM + log "run loop started (max=$MAX_CONCURRENCY, default engine=$DEFAULT_ENGINE). Ctrl-C to stop." + + while true; do + local running; running=$(live_workers) + # launch jobs while we have capacity and inbox files + while [[ "$running" -lt "$MAX_CONCURRENCY" ]]; do + local next; next=$(ls -1 "$INBOX"/*.md 2>/dev/null | sort | head -1) + [[ -z "$next" ]] && break + local job; job=$(basename "$next"); job=${job%.md} + local doing_file="$DOING/$(basename "$next")" + mv "$next" "$doing_file" + local w_eng w_cwd w_yolo + w_eng=$(fm_get "$doing_file" engine "$DEFAULT_ENGINE") + w_cwd=$(fm_get "$doing_file" cwd "$PWD") + w_yolo=$(fm_get "$doing_file" yolo "true") + # write meta BEFORE launch (no pid yet), then append the worker pid from $! + { + echo "job=$job" + echo "engine=$w_eng" + echo "cwd=$w_cwd" + echo "yolo=$w_yolo" + echo "started=$(date +%s)" + } > "$STATE/$job.meta" + run_worker "$doing_file" & + echo "pid=$!" >> "$STATE/$job.meta" + log "▶ launching $C_BOLD$job$C_RESET (engine=$w_eng)" + sleep 1 + running=$(live_workers) + done + + if $once; then + [[ "$(live_workers)" -eq 0 && -z "$(ls -1 "$INBOX"/*.md 2>/dev/null)" ]] && { + log "drain complete — inbox empty, no workers running"; rm -f "$STATE/daemon.pid"; exit 0; } + fi + sleep "$POLL_SECONDS" + done +} + +_count() { ls -1 "$1"/*.md 2>/dev/null | wc -l | tr -d ' '; } + +cmd_status() { + ensure_dirs + local ib dg dn fl + ib=$(_count "$INBOX"); dg=$(_count "$DOING"); dn=$(_count "$DONE"); fl=$(_count "$FAILED") + local running; running=$(live_workers) + echo + printf '%s AGENT QUEUE %s %s\n' "$C_BOLD" "$C_DIM$QUEUE_ROOT$C_RESET" "" + printf ' %sinbox%s %-3s %sdoing%s %-3s %sdone%s %-3s %sfailed%s %-3s %srunning%s %s/%s\n\n' \ + "$C_BLUE" "$C_RESET" "$ib" "$C_YEL" "$C_RESET" "$dg" \ + "$C_GREEN" "$C_RESET" "$dn" "$C_RED" "$C_RESET" "$fl" \ + "$C_BOLD" "$C_RESET" "$running" "$MAX_CONCURRENCY" + + # running table + local f + local printed=false + for f in "$STATE"/*.meta; do + [[ -e "$f" ]] || continue + grep -q '^ended=' "$f" && continue + local pid; pid=$(grep '^pid=' "$f" | cut -d= -f2) + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null || continue + if ! $printed; then printf ' %sRUNNING%s\n' "$C_BOLD" "$C_RESET"; printed=true; fi + local job eng start now el last + job=$(grep '^job=' "$f" | cut -d= -f2) + eng=$(grep '^engine=' "$f" | cut -d= -f2) + start=$(grep '^started=' "$f" | cut -d= -f2) + now=$(date +%s); el=$(( now - ${start:-$now} )) + last=$(tail -n 1 "$LOGS/$job.log" 2>/dev/null | cut -c1-60) + printf ' %s%-26s%s %-7s %3dm%02ds pid %-6s %s%s%s\n' \ + "$C_BOLD" "$job" "$C_RESET" "$eng" $((el/60)) $((el%60)) "$pid" "$C_DIM" "$last" "$C_RESET" + done + $printed || printf ' %sno workers running%s\n' "$C_DIM" "$C_RESET" + echo +} + +cmd_watch() { + local interval="${1:-2}" + while true; do clear; cmd_status; sleep "$interval"; done +} + +cmd_stop() { + ensure_dirs + local killed=0 f pid + for f in "$STATE"/*.meta; do + [[ -e "$f" ]] || continue + grep -q '^ended=' "$f" && continue + pid=$(grep '^pid=' "$f" | cut -d= -f2) + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && { kill "$pid" 2>/dev/null && killed=$((killed+1)); } + done + [[ -f "$STATE/daemon.pid" ]] && kill "$(cat "$STATE/daemon.pid")" 2>/dev/null + rm -f "$STATE/daemon.pid" + log "stopped $killed running worker(s) + run loop" +} + +cmd_logs() { + local job="${1:-}" follow="" + [[ "${2:-}" == "-f" || "$job" == "-f" ]] && follow="-f" + [[ "$job" == "-f" ]] && job="${2:-}" + [[ -n "$job" ]] || die "usage: logs [-f]" + local lf="$LOGS/$job.log" + [[ -f "$lf" ]] || lf=$(ls -1t "$LOGS"/*"$job"*.log 2>/dev/null | head -1) + [[ -f "$lf" ]] || die "no log found for '$job'" + if [[ -n "$follow" ]]; then tail -f "$lf"; else cat "$lf"; fi +} + +usage() { + cat < [args] + +${C_BOLD}COMMANDS${C_RESET} + init create the queue/ folders + add [opts] queue a prompt file into inbox/ + --engine devin|claude|codex --cwd PATH --yolo | --no-yolo + run [--max N] [--engine E] [--once] + process inbox/ (foreground loop; Ctrl-C to stop) + status show kanban counts + running workers + watch [interval] live status (default 2s) + stop kill running workers + the run loop + logs [-f] print (or follow) a job's log + help this message + +${C_BOLD}KANBAN${C_RESET} inbox → doing → done / failed (logs/ + .state/ alongside) + +${C_BOLD}TASK FRONTMATTER${C_RESET} (top of each .md) + --- + engine: devin + cwd: /Users/you/code/repo + yolo: true + --- + +${C_BOLD}ENV${C_RESET} + AGENT_QUEUE_ROOT (=$QUEUE_ROOT) AGENT_QUEUE_MAX (=$MAX_CONCURRENCY) + AGENT_QUEUE_ENGINE (=$DEFAULT_ENGINE) DEVIN_BIN / CLAUDE_BIN / CODEX_BIN +EOF +} + +main() { + local cmd="${1:-help}"; shift || true + case "$cmd" in + init) cmd_init "$@";; + add) cmd_add "$@";; + run) cmd_run "$@";; + status) cmd_status "$@";; + watch) cmd_watch "$@";; + stop) cmd_stop "$@";; + logs) cmd_logs "$@";; + help|-h|--help) usage;; + *) err "unknown command: $cmd"; echo; usage; exit 1;; + esac +} + +main "$@"