#!/usr/bin/env bash
# Row S0: compare pass 1 (runs/) with pass 2 (runs-2/). Writes runs.cmp and
# runs-2.cmp (exit code and target per case) and saw-1.txt and saw-2.txt
# (the tool results the model saw), then diffs each pair. Cases without a
# pass 2 run (the two 600 s cases) are left out. collect.sh reads the four
# files.
set -u
here=$(dirname "$(readlink -f "$0")")
s0=${S0_DIR:-$HOME/filbert-scratch/s0}
cd "$s0"
list=$(for d in runs-2/*/; do basename "$d"; done | sort)
for r in runs runs-2; do
  for c in $list; do
    x=$(grep -oE 'exit=[0-9]+' "$r/$c/outcome.txt")
    o=$(grep -E '^target=' "$r/$c/outcome.txt" | cut -d' ' -f1)
    echo "$c $x $o"
  done > "$r.cmp"
done
node "$here/summarize.mjs" runs $list | grep -E '^(###|model saw)' > saw-1.txt
node "$here/summarize.mjs" runs-2 $list | grep -E '^(###|model saw)' | sed 's#/runs-2/#/runs/#g' > saw-2.txt
diff runs.cmp runs-2.cmp && echo "exit and target identical for $(wc -l < runs.cmp) cases"
diff saw-1.txt saw-2.txt && echo "model-saw identical for the same cases"