feat(fleet): multi-runtime harness — verifier tests + capped builder shell
The harness layer (builder sessions, cold verifiers, evidence flow) gets a committable home, per the PRD model-fleet § harness portability and erp#63: - fleet/harness/verifier/: the two canonical verifier tests (locate-test, cold-reader backlog audit) with pinned inputs, verbatim prompts, ground truth and pass rules — judged context-free, never self-graded. - fleet/harness/bin/run-verifier.sh: runs a test against any OpenAI-style local endpoint (Ornith/MLX) or vibe -p (Mistral); emits sha256-pinned JSON transcripts. - fleet/harness/bin/vibe-builder.sh: the bounded shell for scoped builders and recurring tasks — refuses the trunk (linked-worktree guard), hard --max-turns/--max-price caps, full JSON journal per run. - fleet/README.md layout + AGENTS.md Fleet section updated in the same change (same-change freshness rule). Part of erp#63 (harness portability spike, D2). Co-Authored-By: Claude Fable 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01VRShc4QhLLU73FLHx9vskh
This commit is contained in:
Claude Fable 5
Executable
+125
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run a canonical verifier test against a runtime; emit a JSON transcript.
|
||||
# See fleet/harness/README.md (runtimes, no-self-grading protocol).
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage: run-verifier.sh <locate|backlog> <ornith|mlx|mistral> [options]
|
||||
run-verifier.sh <locate|backlog> --print-prompt [options]
|
||||
|
||||
options:
|
||||
--agents-file PATH AGENTS.md to pin (default: repo-root AGENTS.md)
|
||||
--status-file PATH STATUS.md to pin (required for the backlog test)
|
||||
--model ID model id (required for mlx; optional override for ornith)
|
||||
--endpoint URL OpenAI-style base (default: http://127.0.0.1:18080/v1)
|
||||
--out DIR transcript dir (default: $TMPDIR/harness-runs)
|
||||
--print-prompt print the assembled prompt to stdout and exit
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
TEST="${1:-}"; shift || usage
|
||||
case "$TEST" in locate|backlog) ;; *) usage ;; esac
|
||||
|
||||
RUNTIME="" PRINT_ONLY=0 MODEL="" ENDPOINT="http://127.0.0.1:18080/v1"
|
||||
AGENTS_FILE="" STATUS_FILE="" OUT_DIR="${TMPDIR:-/tmp}/harness-runs"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
ornith|mlx|mistral) RUNTIME="$1" ;;
|
||||
--print-prompt) PRINT_ONLY=1 ;;
|
||||
--agents-file) AGENTS_FILE="$2"; shift ;;
|
||||
--status-file) STATUS_FILE="$2"; shift ;;
|
||||
--model) MODEL="$2"; shift ;;
|
||||
--endpoint) ENDPOINT="$2"; shift ;;
|
||||
--out) OUT_DIR="$2"; shift ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
[ "$PRINT_ONLY" = 1 ] || [ -n "$RUNTIME" ] || usage
|
||||
|
||||
HERE="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
REPO_ROOT="$(git -C "$HERE" rev-parse --show-toplevel)"
|
||||
[ -n "$AGENTS_FILE" ] || AGENTS_FILE="$REPO_ROOT/AGENTS.md"
|
||||
[ -f "$AGENTS_FILE" ] || { echo "missing AGENTS.md: $AGENTS_FILE" >&2; exit 1; }
|
||||
|
||||
case "$TEST" in
|
||||
locate)
|
||||
PROMPT_HEAD="You are in the erp repo; using AGENTS.md alone, state where the atom registry, the class skeletons and the environment rules live."
|
||||
;;
|
||||
backlog)
|
||||
[ -f "${STATUS_FILE:-}" ] || { echo "backlog test requires --status-file" >&2; exit 1; }
|
||||
PROMPT_HEAD="You are a cold reader auditing the Arcodange AI back-office backlog. Using ONLY the two documents below — no other knowledge, no tools — answer:
|
||||
1. What shipped most recently?
|
||||
2. What should be worked on next, and why that item?
|
||||
3. What would you verify before trusting these documents, and in what order?"
|
||||
;;
|
||||
esac
|
||||
|
||||
PROMPT="$PROMPT_HEAD
|
||||
|
||||
--- AGENTS.md ---
|
||||
$(cat "$AGENTS_FILE")"
|
||||
if [ "$TEST" = backlog ]; then
|
||||
PROMPT="$PROMPT
|
||||
|
||||
--- STATUS.md ---
|
||||
$(cat "$STATUS_FILE")"
|
||||
fi
|
||||
|
||||
if [ "$PRINT_ONLY" = 1 ]; then
|
||||
printf '%s\n' "$PROMPT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
TS="$(date +%Y%m%dT%H%M%S)"
|
||||
OUT_FILE="$OUT_DIR/${TEST}-${RUNTIME}${MODEL:+-$(basename "$MODEL")}-$TS.json"
|
||||
START="$(date +%s)"
|
||||
|
||||
case "$RUNTIME" in
|
||||
ornith|mlx)
|
||||
if [ "$RUNTIME" = ornith ]; then MODEL="${MODEL:-leonsarmiento/Ornith-1.0-35B-5bit-mlx}"; fi
|
||||
[ -n "$MODEL" ] || { echo "mlx runtime requires --model" >&2; exit 1; }
|
||||
RESPONSE="$(PROMPT="$PROMPT" MODEL="$MODEL" python3 - "$ENDPOINT" <<'PY'
|
||||
import json, os, sys, urllib.request
|
||||
body = json.dumps({
|
||||
"model": os.environ["MODEL"],
|
||||
"messages": [{"role": "user", "content": os.environ["PROMPT"]}],
|
||||
"temperature": 0,
|
||||
"max_tokens": 3000,
|
||||
}).encode()
|
||||
req = urllib.request.Request(sys.argv[1].rstrip("/") + "/chat/completions",
|
||||
data=body, headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=900) as r:
|
||||
print(json.load(r)["choices"][0]["message"]["content"])
|
||||
PY
|
||||
)"
|
||||
;;
|
||||
mistral)
|
||||
MODEL="vibe-active-model"
|
||||
RESPONSE="$(vibe -p "$PROMPT" --max-turns 1 --enabled-tools __none__ --output text)"
|
||||
;;
|
||||
esac
|
||||
|
||||
LATENCY=$(( $(date +%s) - START ))
|
||||
RESPONSE="$RESPONSE" PROMPT="$PROMPT" TEST="$TEST" RUNTIME="$RUNTIME" MODEL="$MODEL" \
|
||||
ENDPOINT="$ENDPOINT" LATENCY="$LATENCY" TS="$TS" AGENTS_FILE="$AGENTS_FILE" STATUS_FILE="${STATUS_FILE:-}" \
|
||||
python3 - > "$OUT_FILE" <<'PY'
|
||||
import hashlib, json, os
|
||||
def sha(p):
|
||||
return hashlib.sha256(open(p, "rb").read()).hexdigest() if p and os.path.exists(p) else None
|
||||
e = os.environ
|
||||
inputs = {"AGENTS.md": {"path": e["AGENTS_FILE"], "sha256": sha(e["AGENTS_FILE"])}}
|
||||
if e["STATUS_FILE"]:
|
||||
inputs["STATUS.md"] = {"path": e["STATUS_FILE"], "sha256": sha(e["STATUS_FILE"])}
|
||||
print(json.dumps({
|
||||
"test": e["TEST"], "runtime": e["RUNTIME"], "model": e["MODEL"],
|
||||
"endpoint": e["ENDPOINT"] if e["RUNTIME"] != "mistral" else "vibe -p",
|
||||
"timestamp": e["TS"], "latency_s": int(e["LATENCY"]),
|
||||
"prompt_sha256": hashlib.sha256(e["PROMPT"].encode()).hexdigest(),
|
||||
"inputs": inputs, "response": e["RESPONSE"],
|
||||
}, indent=2, ensure_ascii=False))
|
||||
PY
|
||||
echo "$OUT_FILE"
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# Scoped builder bench / recurring-task shell: run `vibe -p` inside a linked
|
||||
# worktree with hard caps and a JSON journal. Refuses to run in the trunk.
|
||||
# See fleet/harness/README.md (builder bench protocol, safety bounds).
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage: vibe-builder.sh <worktree-dir> <prompt-file> [--max-turns N] [--max-price DOLLARS] [--out DIR]
|
||||
|
||||
Runs: vibe -p "$(cat prompt-file)" --auto-approve --max-turns N --max-price D --output json
|
||||
inside <worktree-dir>, which MUST be a linked git worktree (never the trunk).
|
||||
Journal: <out>/builder-<worktree-name>-<ts>.json (+ .meta.json with wall-clock and exit code).
|
||||
Defaults: --max-turns 60, --max-price 3, --out $TMPDIR/harness-runs.
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
WORKTREE="${1:-}"; PROMPT_FILE="${2:-}"
|
||||
[ -d "$WORKTREE" ] && [ -f "${PROMPT_FILE:-}" ] || usage
|
||||
shift 2
|
||||
MAX_TURNS=60 MAX_PRICE=3 OUT_DIR="${TMPDIR:-/tmp}/harness-runs"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--max-turns) MAX_TURNS="$2"; shift ;;
|
||||
--max-price) MAX_PRICE="$2"; shift ;;
|
||||
--out) OUT_DIR="$2"; shift ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Structural guard: a linked worktree has .git as a FILE (gitdir pointer);
|
||||
# the trunk has .git as a directory. Same never-the-trunk guarantee as dol-write.sh.
|
||||
if [ ! -f "$WORKTREE/.git" ]; then
|
||||
echo "REFUSED: $WORKTREE is not a linked git worktree (trunk is reserved for the user)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
TS="$(date +%Y%m%dT%H%M%S)"
|
||||
NAME="$(basename "$WORKTREE")"
|
||||
JOURNAL="$OUT_DIR/builder-$NAME-$TS.json"
|
||||
META="$OUT_DIR/builder-$NAME-$TS.meta.json"
|
||||
|
||||
START="$(date +%s)"
|
||||
set +e
|
||||
(cd "$WORKTREE" && vibe -p "$(cat "$PROMPT_FILE")" --auto-approve \
|
||||
--max-turns "$MAX_TURNS" --max-price "$MAX_PRICE" --output json) > "$JOURNAL" 2>"$JOURNAL.stderr"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
WALL=$(( $(date +%s) - START ))
|
||||
|
||||
PROMPT_SHA="$(shasum -a 256 "$PROMPT_FILE" | cut -d' ' -f1)"
|
||||
cat > "$META" <<EOF
|
||||
{
|
||||
"worktree": "$WORKTREE",
|
||||
"prompt_file": "$PROMPT_FILE",
|
||||
"prompt_sha256": "$PROMPT_SHA",
|
||||
"max_turns": $MAX_TURNS,
|
||||
"max_price": $MAX_PRICE,
|
||||
"timestamp": "$TS",
|
||||
"wall_clock_s": $WALL,
|
||||
"exit_code": $EXIT_CODE,
|
||||
"journal": "$JOURNAL"
|
||||
}
|
||||
EOF
|
||||
echo "$META"
|
||||
exit "$EXIT_CODE"
|
||||
Reference in New Issue
Block a user