#!/usr/bin/env bash
# Checks every task is a real test: verify.sh must fail on the untouched repo
# and pass once solution.sh has been applied. A task whose verifier passes
# before the agent does anything measures nothing.
set -uo pipefail
cd "$(dirname "$0")/.."
tasks_dir=evals/tasks
fails=0

for dir in "$tasks_dir"/*/; do
  name=$(basename "$dir")
  task_abs=$(cd "$dir" && pwd)
  work=$(mktemp -d "/tmp/selftest-$name-XXXX")
  export WORK_DIR="$work" TASK_DIR="$task_abs" AGENT_ANSWER_FILE="$work/.agent-answer.txt"

  if ! out=$(cd "$work" && bash "$task_abs/setup.sh" 2>&1); then
    printf '%-26s SETUP FAILED\n%s\n' "$name" "$out"; fails=$((fails+1)); rm -rf "$work"; continue
  fi

  before_out=$(cd "$work" && bash "$task_abs/verify.sh" 2>&1); before=$?
  (cd "$work" && bash "$task_abs/solution.sh" >/dev/null 2>&1)
  after_out=$(cd "$work" && bash "$task_abs/verify.sh" 2>&1); after=$?

  case "$before:$after" in
    0:*) printf '%-26s BAD: verifier passes before any work\n' "$name"; fails=$((fails+1));;
    *:0) printf '%-26s ok   (fails unsolved, passes solved)\n' "$name";;
    *:*) printf '%-26s BAD: verifier still fails after the reference solution\n%s\n' "$name" "$after_out"; fails=$((fails+1));;
  esac
  rm -rf "$work"
done

if [ "$fails" -ne 0 ]; then echo; echo "$fails task(s) are not measuring anything"; exit 1; fi
echo; echo "all tasks verified"