#!/usr/bin/env bash
# Row 233 (I3, #1735): root's upgrade and rollback for a stack whose units are root's system units
# (stack-release.sh --units external, row 232). r6's activate A1-A5, A2f, R1-R4 and rollback
# (agents/darkwing/work/queue-195/DESIGN.md), as the I3 brief changes them
# (docs/plans/2026-10-20_installer-rows.md). Deviations and decisions:
# agents/darkwing/work/queue-233/BUILD.md.
#
#   stack-upgrade.sh upgrade --instance dev --bundle FILE --sha SHA [--migrate] [--no-rehearsal]
#                    [--rehearse-port N] [--next-args] [--timeout S]
#   stack-upgrade.sh rollback --instance dev --to ID [--restore ACT [--events N]] [--next-args]
#                    [--timeout S]
#
# Root runs it from a root-owned checkout of the reviewed commit, never from a copy, because the
# stack's user can write the copies. The unit values come from /etc/mosaic-stack/<instance>.args,
# which I2's install writes: one case2-unit.sh argument per line, the serve flags after `--`. Only
# --copy changes, and the file changes only after confirm passes. --org is the same option as
# --business in each of the file's two scopes, case2-unit.sh's own flags before `--` and the serve
# flags after it: each scope takes each spelling once, a second spelling only with the same value,
# and both scopes must name the business, the same one (rename design r2). The stack's user runs
# its steps from its own clone, <home>/src/mosaic-stack, through sudo. Root reads none of that user's
# output but exit codes and the rehearsal's BACKUP_STORE line (sage-1653), which it copies into its
# step log after a match against a fixed pattern and decides nothing on.
#
# Before anything stops, case2-unit.sh --check renders the installed copy from the args file and
# compares the installed units with it, then checks the target's render the same way; either
# refusal is render-refused, with the units left running (#1873 M3).
#
# --next-args changes the serve flags, which a re-render otherwise refuses (row 195's installed-unit
# match): /etc/mosaic-stack/<instance>.args.next, root's with no group or other write, must equal
# the args file before `--` but for --copy's value, and differ from it after `--`. The target
# renders from it through case2-unit.sh --check-new before anything stops. After A3, root moves the
# installed unit files into a new <instance>.replaced.XXXXXX directory beside the args file, and A4
# renders from the next args as a first install. A failure from there moves the replaced files back
# unchanged, restarts them on the old copy and gates it, with no render. After confirm, the next
# args become the args file, and the replaced directory keeps the old unit files and args file for
# a later rollback with --next-args (docs/guides/host-run-sheet.md section 7).
#
# --no-rehearsal is for a stack whose log records no backup. The rehearsal runs either way: exit 3
# from it (no-backup) goes on only with the flag, and exit 0 with the flag refuses
# rehearsal-required (sage-1650, sage-1652).
#
# The lock /run/mosaic-stack/upgrade-<instance>.lock is held from before A1 to the end of the run,
# rollback included; every child runs with the lock's descriptor closed. The step log
# /var/log/mosaic-stack/upgrade-<instance>.log gets one line per step, `<time> <step> rc=N argv:
# <argv>`, and a last `<time> result exit=N <code> <text>` line.
#
# Exit 0 ok, 1 failed with the old stack running, 2 rollback-failed with the units stopped,
# 3 refused with nothing stopped, 4 usage, 5 args-failed: the new stack is live and confirmed, but
# the args file still names the old copy.
#
# Test seams, for packages/cli/tests/stack-upgrade.test.mjs only. Run as root, it refuses any:
#   STACK_UPGRADE_ETC        default /etc/mosaic-stack
#   STACK_UPGRADE_RUN        default /run/mosaic-stack
#   STACK_UPGRADE_LOG_DIR    default /var/log/mosaic-stack
#   STACK_UPGRADE_UNIT_DIR   default /etc/systemd/system
#   STACK_UPGRADE_HOSTCHECK  default /usr/local/lib/mosaic-stack/hostcheck.sh (row 240)
#   STACK_UPGRADE_RENDER     default scripts/case2-unit.sh in this checkout
#   STACK_UPGRADE_SUDO       default sudo
#   STACK_UPGRADE_SYSTEMCTL  default systemctl
set -uo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

usage() {
    cat >&2 <<'USAGE'
usage: stack-upgrade.sh upgrade --instance dev --bundle FILE --sha SHA [--migrate] [--no-rehearsal] [--rehearse-port N] [--next-args] [--timeout S]
       stack-upgrade.sh rollback --instance dev --to ID [--restore ACT [--events N]] [--next-args] [--timeout S]
USAGE
    exit 4
}

if [ "$(id -u)" = 0 ]; then
    for v in ${!STACK_UPGRADE_@}; do echo "stack-upgrade: refusing: $v is a test seam, and root takes none" >&2; exit 3; done
fi
ETC="${STACK_UPGRADE_ETC:-/etc/mosaic-stack}"
RUN_DIR="${STACK_UPGRADE_RUN:-/run/mosaic-stack}"
LOG_DIR="${STACK_UPGRADE_LOG_DIR:-/var/log/mosaic-stack}"
UNIT_DIR="${STACK_UPGRADE_UNIT_DIR:-/etc/systemd/system}"
HOSTCHECK="${STACK_UPGRADE_HOSTCHECK:-/usr/local/lib/mosaic-stack/hostcheck.sh}"
RENDER="${STACK_UPGRADE_RENDER:-$REPO/scripts/case2-unit.sh}"
SUDO="${STACK_UPGRADE_SUDO:-sudo}"
SYSTEMCTL="${STACK_UPGRADE_SYSTEMCTL:-systemctl}"
ID_RE='^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}(-[1-9])?$'

ARGV=("$@")
[ $# -gt 0 ] || usage
OP="$1"; shift
case "$OP" in upgrade|rollback) ;; *) usage ;; esac
INSTANCE="" BUNDLE="" SHA="" MIGRATE=0 NO_REHEARSAL=0 R_PORT=7339 TIMEOUT=60 TO="" RESTORE="" EVENTS="" NEXT=0 ASIDE="" MOVED=() REPLACED_NOTE=""
while [ $# -gt 0 ]; do
    case "$1" in
        --instance) [ $# -ge 2 ] || usage; INSTANCE="$2"; shift 2 ;;
        --bundle) [ "$OP" = upgrade ] && [ $# -ge 2 ] && [[ "$2" =~ ^/[A-Za-z0-9/._+-]+$ ]] || usage; BUNDLE="$2"; shift 2 ;;
        --sha) [ "$OP" = upgrade ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9a-f]{40}$ ]] || usage; SHA="$2"; shift 2 ;;
        --migrate) [ "$OP" = upgrade ] || usage; MIGRATE=1; shift ;;
        --no-rehearsal) [ "$OP" = upgrade ] || usage; NO_REHEARSAL=1; shift ;;
        --rehearse-port) [ "$OP" = upgrade ] && [ $# -ge 2 ] && [[ "$2" =~ ^[1-9][0-9]{3,4}$ ]] && [ "$2" -le 65535 ] || usage; R_PORT="$2"; shift 2 ;;
        --next-args) NEXT=1; shift ;;
        --timeout) [ $# -ge 2 ] && [[ "$2" =~ ^[1-9][0-9]{0,4}$ ]] || usage; TIMEOUT="$2"; shift 2 ;;
        --to) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9a-f]{40}$ ]] || usage; TO="$2"; shift 2 ;;
        --restore) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ $ID_RE ]] || usage; RESTORE="$2"; shift 2 ;;
        --events) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ ^(0|[1-9][0-9]*)$ ]] || usage; EVENTS="$2"; shift 2 ;;
        *) usage ;;
    esac
done
# The default instance only, whose name is dev (row 190); a named instance's units are a later row.
[ "$INSTANCE" = dev ] || usage
if [ "$OP" = upgrade ]; then [ -n "$BUNDLE" ] && [ -n "$SHA" ] || usage; else [ -n "$TO" ] || usage; fi
[ -z "$EVENTS" ] || [ -n "$RESTORE" ] || usage

# --- The step log and the lock ---

umask 027
LOG="$LOG_DIR/upgrade-$INSTANCE.log"
LOCK="$RUN_DIR/upgrade-$INSTANCE.lock"
{ mkdir -p "$LOG_DIR" "$RUN_DIR" && touch "$LOG" && chmod 0640 "$LOG"; } 2>/dev/null ||
    { echo "stack-upgrade: refusing log-failed: cannot write $LOG or make $RUN_DIR; nothing was changed" >&2; exit 3; }
logl() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >> "$LOG" || true; }
# The run's last line, in the log and on the terminal, then the exit.
finish() {
    local rc="$1" code="$2"; shift 2
    logl "result exit=$rc $code $*"
    if [ "$rc" = 0 ]; then echo "stack-upgrade: $*"; else echo "stack-upgrade: $code: $*" >&2; fi
    exit "$rc"
}
refuse() { local code="$1"; shift; finish 3 "$code" "$* (nothing was stopped)"; }
# One step: $1 its name, the rest the command, run with the lock's descriptor closed. One log line
# with its exit and argv; the command's output goes to the terminal only.
step() {
    local name="$1" rc=0; shift
    "$@" 9>&- || rc=$?
    logl "$name rc=$rc argv:$(printf ' %q' "$@")"
    return "$rc"
}

exec 9>> "$LOCK" || refuse lock-failed "cannot open $LOCK"
flock -n 9 || refuse upgrade-running "another stack-upgrade.sh run holds $LOCK"
logl "start rc=0 argv:$(printf ' %q' "$0" "${ARGV[@]}")"

# --- The args file and the stack's user ---

# The selector scopes: selector.sh defines functions only, from this root-owned checkout, as RENDER is.
. "$REPO/scripts/selector.sh" || refuse checkout-unreadable "cannot read $REPO/scripts/selector.sh"
args_business_ok() { [[ "$1" =~ ^[a-z0-9][a-z0-9-]*$ ]]; }
args_outer_refused() { refuse args-unreadable "$ARGS_FILE before --: $2"; }
args_serve_refused() { refuse args-unreadable "$ARGS_FILE after --: $2"; }

# I2's file: one case2-unit.sh argument per line, owned by root with no group or other write. Sets
# ARGS (the lines), S_USER, S_HOME, BUSINESS, NODE, ARGS_COPY, ARGS_PROFILE and ARGS_PROXY, and the paths derived
# from them.
# #385 R2-F1: base case2-unit.sh required --send for the library profile (its default, and the last
# --profile wins, as there), and the current one refuses it. A2 would stop both units, and A4 and
# render-old would both fail on the same args, so a library file holding --send after -- refuses
# here, before anything stops. The file is never rewritten; the operator removes the line.
# Rename r2: case2-unit.sh also refuses a file whose serve flags name no selector or another
# business, so that refuses here too. The serve flags are read with case2-unit.sh's grammar, which
# says which of them take a value, so a value is never taken for a selector; a token that grammar
# lacks refuses, as the render would.
read_args() {
    ARGS_FILE="${1:-$ETC/$INSTANCE.args}"
    local f="$ARGS_FILE" st i sep copies=0 v profile=library served
    [ -f "$f" ] && [ ! -L "$f" ] || refuse args-unreadable "no regular file $f"
    st="$(stat -c '%u %a' "$f")" || refuse args-unreadable "cannot stat $f"
    [ "${st% *}" = "$(id -u)" ] && (( (8#${st#* } & 8#022) == 0 )) || refuse args-unreadable "$f must be $(id -un)'s with no group or other write (it is uid ${st% *}, mode ${st#* })"
    mapfile -t ARGS < "$f" || refuse args-unreadable "cannot read $f"
    S_USER="" S_HOME="" BUSINESS="" NODE="" ARGS_COPY="" ARGS_PROXY=0
    for ((i = 0; i < ${#ARGS[@]}; i++)); do
        case "${ARGS[i]}" in
            --) break ;;
            --pm) ;;
            --proxy) ARGS_PROXY=1 ;;
            --business|--org)
                selector_take outer id args_outer_refused args_business_ok "${ARGS[i]}" "${ARGS[i + 1]:-}"
                BUSINESS="${ARGS[i + 1]}"; i=$((i + 1)) ;;
            --user|--home|--copy|--node|--profile|--claude)
                v="${ARGS[i + 1]:-}"
                case "${ARGS[i]}" in
                    --user) S_USER="$v" ;; --home) S_HOME="$v" ;; --node) NODE="$v" ;;
                    --copy) ARGS_COPY="$v"; copies=$((copies + 1)) ;;
                    --profile) profile="$v" ;;
                esac
                i=$((i + 1)) ;;
            *) refuse args-unreadable "$f has ${ARGS[i]} before --, which case2-unit.sh does not take" ;;
        esac
    done
    sep=$i
    [ "$copies" = 1 ] || refuse args-unreadable "$f needs exactly one --copy before --, not $copies"
    ARGS_PROFILE="$profile"
    if [ "$profile" = library ]; then
        for ((i = i + 1; i < ${#ARGS[@]}; i++)); do
            [ "${ARGS[i]}" != --send ] ||
                refuse args-unreadable "$f has --send after --, which case2-unit.sh refuses for the library profile: remove the --send line, then rerun; the library Console is read-only until #412 (#385)"
        done
    fi
    [[ "$S_USER" =~ ^[a-z_][a-z0-9_-]*$ ]] && [[ "$S_HOME" =~ ^/[A-Za-z0-9/._+-]+$ ]] && [[ "$BUSINESS" =~ ^[a-z0-9][a-z0-9-]*$ ]] &&
        [[ "$NODE" =~ ^/[A-Za-z0-9/._+-]+$ ]] && [[ "$ARGS_COPY" =~ ^[0-9a-f]{40}$ ]] ||
        refuse args-unreadable "$f lacks a plain --user, --home, --business, --node or --copy"
    for ((i = sep + 1; i < ${#ARGS[@]}; i++)); do
        case "${ARGS[i]}" in
            --business|--org)
                selector_take serve id args_serve_refused args_business_ok "${ARGS[i]}" "${ARGS[i + 1]:-}"; i=$((i + 1)) ;;
            --port|--public-host|--auth-header|--allow-user|--proof-header|--proof-file) i=$((i + 1)) ;;
            --tester-socket|--sso|--send) ;;
            *) refuse args-unreadable "$f has ${ARGS[i]} after --, which case2-unit.sh does not take" ;;
        esac
    done
    served="$(selector_value serve)"
    [ -n "$served" ] || refuse args-unreadable "$f needs --business or --org after --, as case2-unit.sh does"
    [ "$served" = "$BUSINESS" ] || refuse args-unreadable "$f names business $BUSINESS before -- and $served after it"
    COPIES_DIR="$S_HOME/.local/share/mosaic-stack/copies"
    CLONE="$S_HOME/src/mosaic-stack"
    SR="$CLONE/scripts/stack-release.sh"
    BUS_UNIT="mosaic-bus@$BUSINESS.service" CON_UNIT=mosaic-webui.service PROXY_UNIT=mosaic-proxy.service
    STOP_UNITS=("$CON_UNIT" "$BUS_UNIT")
    STOPPED_UNITS="both units"
    if [ "$ARGS_PROXY" = 1 ]; then STOP_UNITS+=("$PROXY_UNIT"); STOPPED_UNITS="all three units"; fi
    # The stack's user's steps: its own HOME and default config, the pinned node first on PATH.
    AS_USER=("$SUDO" -u "$S_USER" -H env "PATH=$(dirname "$NODE"):/usr/local/bin:/usr/bin:/bin")
}

# #385 R3-F1: base case2-unit.sh also put --send in the library Console unit's ExecStart. The current
# one checks an installed unit's ExecStart against its own read-only command, so A4 and render-old
# would both refuse that unit after A2 stopped both. No reviewed migration exists, so a library host
# whose installed Console unit has a --send token after serve refuses here, before anything runs. A
# missing unit is read_old's to refuse.
legacy_console() {
    local f="$UNIT_DIR/$CON_UNIT" line t past
    local -a lines words
    [ "$ARGS_PROFILE" = library ] && [ -f "$f" ] && [ ! -L "$f" ] || return 0
    mapfile -t lines < "$f" || refuse units-unreadable "cannot read $f"
    for line in "${lines[@]}"; do
        [[ "$line" =~ ^[[:space:]]*ExecStart[[:space:]]*=(.*)$ ]] || continue
        read -r -a words <<< "${BASH_REMATCH[1]}"
        past=""
        for t in "${words[@]}"; do
            if [ -n "$past" ]; then
                [ "$t" != --send ] ||
                    refuse console-unit-legacy "$f runs serve with --send, so it was rendered before #385, and stack-upgrade.sh cannot migrate it to the read-only library Console: the units are left running; stop and report (docs/guides/host-run-sheet.md section 7)"
            elif [ "$t" = serve ]; then past=1
            fi
        done
    done
}

# RENDER_ARGS: the lines of array $2 (default ARGS) with copy $1 in place of --copy's value.
args_with() {
    local i past=""
    local -n lines_="${2:-ARGS}"
    RENDER_ARGS=()
    for ((i = 0; i < ${#lines_[@]}; i++)); do
        RENDER_ARGS+=("${lines_[i]}")
        [ "${lines_[i]}" != -- ] || past=1
        if [ -z "$past" ] && [ "${lines_[i]}" = --copy ]; then RENDER_ARGS+=("$1"); i=$((i + 1)); fi
    done
}

# OLD_ARGS: the args file's lines, which the installed units were rendered from. Under --next-args,
# ARGS becomes the next file's lines, which must equal OLD_ARGS before -- but for --copy's value and
# differ after it.
read_next() {
    OLD_ARGS=("${ARGS[@]}")
    [ "$NEXT" = 1 ] || return 0
    local f="$ETC/$INSTANCE.args.next" a b i copy="$ARGS_COPY"
    unset SELECTOR_SEEN_outer SELECTOR_VALUE_outer SELECTOR_SEEN_serve SELECTOR_VALUE_serve
    read_args "$f"
    # read_old checks the installed units against the args file's copy, not the next file's.
    ARGS_COPY="$copy"
    args_with COPY OLD_ARGS; a=("${RENDER_ARGS[@]}")
    args_with COPY; b=("${RENDER_ARGS[@]}")
    for ((i = 0; i < ${#a[@]} && i < ${#b[@]}; i++)); do
        [ "${a[i]}" = "${b[i]}" ] || refuse next-args-differ "$f must equal $ETC/$INSTANCE.args before -- but for --copy's value; line $((i + 1)) differs"
        [ "${a[i]}" != -- ] || break
    done
    [ "${a[*]}" != "${b[*]}" ] || [ "${#a[@]}" != "${#b[@]}" ] ||
        refuse next-args-same "$f has the same serve flags as $ETC/$INSTANCE.args; run without --next-args"
}

# Before anything stops: the installed units against OLD's render from the args file, so a rollback
# render-old works, then copy $1's render, against the installed units or, under --next-args, as a
# first install from the next args.
render_checks() {
    args_with "$OLD" OLD_ARGS
    step render-old-check "$RENDER" --check "${RENDER_ARGS[@]}" ||
        refuse render-refused "case2-unit.sh --check refuses $OLD's units from $ETC/$INSTANCE.args, so a rollback could not render them; its message is above"
    args_with "$1"
    if [ "$NEXT" = 1 ]; then
        step render-check "$RENDER" --check-new "${RENDER_ARGS[@]}" ||
            refuse render-refused "case2-unit.sh --check-new refuses $1's units from $ETC/$INSTANCE.args.next; its message is above"
    else
        step render-check "$RENDER" --check "${RENDER_ARGS[@]}" ||
            refuse render-refused "case2-unit.sh --check refuses $1's units from $ETC/$INSTANCE.args; its message is above. A re-pin that changes the serve flags takes --next-args (docs/guides/host-run-sheet.md section 7)"
    fi
}

# --next-args, after A3: the installed unit files into a new directory beside the args file, so A4
# renders as a first install. MOVED lists the files moved, for units_back.
units_aside() {
    local u rc=0
    local -a files=(mosaic-bus@.service "$CON_UNIT")
    [ "$ARGS_PROXY" = 0 ] || files+=("$PROXY_UNIT")
    ASIDE="$(mktemp -d "$ETC/$INSTANCE.replaced.XXXXXX")" && chmod 0700 "$ASIDE" || rc=1
    if [ "$rc" = 0 ]; then
        for u in "${files[@]}"; do
            mv -T -- "$UNIT_DIR/$u" "$ASIDE/$u" || { rc=1; break; }
            MOVED+=("$u")
        done
    fi
    logl "units-aside rc=$rc dir: ${ASIDE:-none} moved: ${MOVED[*]}"
    return "$rc"
}

# The moved files back, unchanged, over whatever A4 wrote, then the replaced directory removed.
units_back() {
    local u rc=0
    for u in "${MOVED[@]}"; do mv -fT -- "$ASIDE/$u" "$UNIT_DIR/$u" || rc=1; done
    [ "$rc" != 0 ] || rmdir -- "$ASIDE" || rc=1
    logl "units-back rc=$rc dir: $ASIDE moved: ${MOVED[*]}"
    return "$rc"
}

# The new args file, written beside the old one before anything stops, moved into place only after
# confirm passes. $1 is the copy.
prepare_args() {
    args_with "$1"
    NEW_ARGS="$(mktemp "$ETC/.$INSTANCE.args.XXXXXX")" &&
        printf '%s\n' "${RENDER_ARGS[@]}" > "$NEW_ARGS" && chmod --reference="$ETC/$INSTANCE.args" "$NEW_ARGS" ||
        { rm -f "${NEW_ARGS:-}"; refuse args-failed "cannot write the new args file in $ETC"; }
}
drop_args() { rm -f "${NEW_ARGS:-}"; }

# --- A1 ---

# OLD: the copy all selected installed units run, from their WorkingDirectory under the stack's copies.
read_old() {
    local f ids=() id n
    local files=("$UNIT_DIR/mosaic-bus@.service" "$UNIT_DIR/$CON_UNIT")
    if [ "$ARGS_PROXY" = 1 ]; then files+=("$UNIT_DIR/$PROXY_UNIT")
    else
        [ ! -e "$UNIT_DIR/$PROXY_UNIT" ] && [ ! -L "$UNIT_DIR/$PROXY_UNIT" ] ||
            refuse units-unreadable "installed $UNIT_DIR/$PROXY_UNIT requires --proxy"
    fi
    for f in "${files[@]}"; do
        [ -f "$f" ] && [ ! -L "$f" ] || refuse units-unreadable "no regular file $f"
        n="$(grep -c '^WorkingDirectory=' "$f")"
        id="$(awk -v c="WorkingDirectory=$COPIES_DIR/" 'index($0, c) == 1 { print substr($0, length(c) + 1) }' "$f")"
        [ "$n" = 1 ] && [[ "$id" =~ ^[0-9a-f]{40}$ ]] || refuse units-unreadable "$f names no copy under $COPIES_DIR"
        ids+=("$id")
    done
    for id in "${ids[@]}"; do
        [ "${ids[0]}" = "$id" ] || refuse units-disagree "$UNIT_DIR's units name ${ids[0]} and $id"
    done
    OLD="${ids[0]}"
    logl "units rc=0 old: $OLD"
    [ "$ARGS_COPY" = "$OLD" ] || refuse args-differ "$ETC/$INSTANCE.args names $ARGS_COPY and the units name $OLD"
}

# r6 A1 and row 240: the host's profile still matches, on every host.
hostcheck() {
    [ -f "$HOSTCHECK" ] && [ -x "$HOSTCHECK" ] || refuse profile-drift "no executable $HOSTCHECK (row 240)"
    step hostcheck "$HOSTCHECK" --check || refuse profile-drift "$HOSTCHECK --check failed"
}

# The stack's user's --plan for the run, $@ its op and arguments; any refusal stops here.
plan() {
    local rc=0
    step plan "${AS_USER[@]}" "$SR" "$@" --units external --copies "$COPIES_DIR" --plan --expect-from "$OLD" || rc=$?
    [ "$rc" = 0 ] || refuse plan-refused "the stack's user's $1 --plan exited $rc; its output above names the refusal"
}

# --- The units ---

stop_wait() {
    local u s deadline=$((SECONDS + TIMEOUT)) rc=0
    step stop-console "$SYSTEMCTL" stop "$CON_UNIT" || rc=1
    step stop-bus "$SYSTEMCTL" stop "$BUS_UNIT" || rc=1
    if [ "$ARGS_PROXY" = 1 ]; then step stop-proxy "$SYSTEMCTL" stop "$PROXY_UNIT" || rc=1; fi
    for u in "${STOP_UNITS[@]}"; do
        while :; do
            s="$("$SYSTEMCTL" is-active "$u" 9>&- 2>/dev/null)"
            case "$s" in inactive|failed) break ;; esac
            [ "$SECONDS" -lt "$deadline" ] || { rc=1; break; }
            sleep 1
        done
        case "$s" in inactive|failed) logl "wait-down rc=0 unit: $u state: $s" ;; *) logl "wait-down rc=1 unit: $u state: ${s:-unknown}" ;; esac
    done
    return "$rc"
}
stop_units() {
    local u
    for u in "${STOP_UNITS[@]}"; do "$SYSTEMCTL" stop "$u" 9>&- || true; done
    logl "stop-units rc=0 units: ${STOP_UNITS[*]}"
}

# A2f: the old units back on the copy they never left, gated on a start at or after this second.
# An A2 that timed out leaves a unit on its old start, which the gate refuses as stale-start.
a2f() {
    local cause="$1" since
    drop_args
    since="$(date -u +%s)"
    if [ "$ARGS_PROXY" = 1 ]; then
        step start-proxy "$SYSTEMCTL" start "$PROXY_UNIT" ||
            { stop_units; finish 2 rollback-failed "$cause failed and the proxy did not start for $OLD; $STOPPED_UNITS are stopped"; }
    fi
    step start-bus "$SYSTEMCTL" start "$BUS_UNIT"
    step start-console "$SYSTEMCTL" start "$CON_UNIT"
    if step gate "${AS_USER[@]}" "$SR" gate "$OLD" "$BUSINESS" --units external --copies "$COPIES_DIR" --since "$since" --timeout "$TIMEOUT"; then
        finish 1 failed "$cause failed; $OLD is running again and passed the gate"
    fi
    stop_units
    finish 2 rollback-failed "$cause failed and $OLD failed its gate after the restart; $STOPPED_UNITS are stopped"
}

# R1-R4: back to OLD after the stack's user staged $1. $2 is the cause: A3, A4 or A5.
rollback_to_old() {
    local id="$1" cause="$2"
    drop_args
    stop_wait || { stop_units; finish 2 rollback-failed "$cause failed and the units did not stop for the rollback; $STOPPED_UNITS are stopped"; }
    step revert "${AS_USER[@]}" "$SR" revert "$id" "$BUSINESS" --units external --copies "$COPIES_DIR" ||
        { stop_units; finish 2 rollback-failed "$cause failed and revert failed; $STOPPED_UNITS are stopped"; }
    if [ -n "$ASIDE" ]; then
        # --next-args: the replaced files back as they were, then the start render-old would do.
        units_back ||
            { stop_units; finish 2 rollback-failed "$cause failed and the replaced units in $ASIDE did not all move back; $STOPPED_UNITS are stopped"; }
        step daemon-reload "$SYSTEMCTL" daemon-reload ||
            { stop_units; finish 2 rollback-failed "$cause failed and daemon-reload failed after the replaced units moved back; $STOPPED_UNITS are stopped"; }
        if [ "$ARGS_PROXY" = 1 ]; then
            step start-proxy "$SYSTEMCTL" start "$PROXY_UNIT" ||
                { stop_units; finish 2 rollback-failed "$cause failed and the proxy did not start for $OLD; $STOPPED_UNITS are stopped"; }
        fi
        step start-bus "$SYSTEMCTL" start "$BUS_UNIT" ||
            { stop_units; finish 2 rollback-failed "$cause failed and the bus did not start for $OLD; $STOPPED_UNITS are stopped"; }
        step start-console "$SYSTEMCTL" start "$CON_UNIT" ||
            { stop_units; finish 2 rollback-failed "$cause failed and the Console did not start for $OLD; $STOPPED_UNITS are stopped"; }
    else
        args_with "$OLD" OLD_ARGS
        step render-old "$RENDER" "${RENDER_ARGS[@]}" ||
            { stop_units; finish 2 rollback-failed "$cause failed and the render of $OLD failed; $STOPPED_UNITS are stopped"; }
    fi
    # confirm --rollback exits 1 when OLD passed its gate and 2 when it failed; anything else is a
    # refusal, with OLD's gate never run.
    local rc=0
    step confirm-rollback "${AS_USER[@]}" "$SR" confirm "$id" "$BUSINESS" --units external --copies "$COPIES_DIR" --rollback "$OLD" --cause "$cause" --timeout "$TIMEOUT" || rc=$?
    [ "$rc" != 1 ] || finish 1 failed "$cause failed; rolled back to $OLD, which passed the gate"
    stop_units
    finish 2 rollback-failed "$cause failed and the rollback to $OLD did not pass (confirm --rollback exited $rc); $STOPPED_UNITS are stopped"
}

# A2 to A5 for copy $1, once A1 and any rehearsal passed. $@ after it is the stack's user's op.
run_steps() {
    local id="$1" rc=0; shift
    prepare_args "$id"
    # A2
    stop_wait || a2f A2
    # A3
    step stage "${AS_USER[@]}" "$SR" "$@" --units external --copies "$COPIES_DIR" || rc=$?
    case "$rc" in
        0) ;;
        3) a2f A3 ;;
        *) rollback_to_old "$id" A3 ;;
    esac
    # A4
    if [ "$NEXT" = 1 ]; then units_aside || rollback_to_old "$id" A4; fi
    args_with "$id"
    step render "$RENDER" "${RENDER_ARGS[@]}" || rollback_to_old "$id" A4
    # A5
    step confirm "${AS_USER[@]}" "$SR" confirm "$id" "$BUSINESS" --units external --copies "$COPIES_DIR" --timeout "$TIMEOUT" ||
        rollback_to_old "$id" A5
    if [ "$NEXT" = 1 ]; then
        # The old args file joins the replaced units before the next one takes its place.
        cp -p -- "$ETC/$INSTANCE.args" "$ASIDE/$INSTANCE.args" && rm -f -- "$ETC/$INSTANCE.args.next" &&
            logl "next-args rc=0 replaced: $ASIDE" ||
            { logl "next-args rc=1 replaced: $ASIDE"; finish 5 args-failed "$id is live and confirmed, but the old args file did not reach $ASIDE; $ETC/$INSTANCE.args still names $OLD; copy it there, then put $NEW_ARGS in its place and remove $ETC/$INSTANCE.args.next"; }
    fi
    if mv -f "$NEW_ARGS" "$ETC/$INSTANCE.args"; then logl "args rc=0 copy: $id"
    else logl "args rc=1 copy: $id"; finish 5 args-failed "$id is live and confirmed, but $ETC/$INSTANCE.args still names $OLD; put $NEW_ARGS in its place"
    fi
    REPLACED_NOTE="${ASIDE:+; the replaced units and args file are in $ASIDE}"
}

do_upgrade() {
    read_args
    read_next
    legacy_console
    # Before A1: the bundle into the stack's user's clone, then the copy from it (I1).
    step fetch "${AS_USER[@]}" git -C "$CLONE" fetch -q "$BUNDLE" refs/heads/refactor:refs/remotes/origin/refactor ||
        refuse fetch-failed "the stack's user could not fetch $BUNDLE into $CLONE"
    step ff "${AS_USER[@]}" git -C "$CLONE" merge -q --ff-only "$SHA" ||
        refuse fetch-failed "the stack's user could not fast-forward $CLONE's refactor to $SHA"
    step copy "${AS_USER[@]}" "$CLONE/scripts/stack-copy.sh" copy --bundle "$BUNDLE" "$SHA" --copies "$COPIES_DIR" ||
        refuse copy-failed "the stack's user could not make copy $SHA from $BUNDLE"
    # A1
    local head
    head="$(git -C "$REPO" rev-parse HEAD 2>/dev/null)"
    [ "$head" = "$SHA" ] || refuse head-mismatch "this checkout, $REPO, is at ${head:-unknown}, not $SHA"
    [ -x "$COPIES_DIR/$SHA/scripts/mosaic" ] || refuse copy-missing "no $COPIES_DIR/$SHA/scripts/mosaic"
    hostcheck
    read_old
    local -a migrate=()
    [ "$MIGRATE" = 0 ] || migrate=(--migrate)
    plan activate "$SHA" "$BUSINESS" "${migrate[@]}"
    render_checks "$SHA"
    rehearse
    run_steps "$SHA" activate "$SHA" "$BUSINESS" "${migrate[@]}"
    finish 0 ok "upgrade ok: $SHA is live for $BUSINESS$REPLACED_NOTE"
}

# The rehearsal against the log's newest backup, before anything stops. Its output goes to the
# terminal; root keeps only the BACKUP_STORE line, so the go and the log show which store it used.
rehearse() {
    local cap rc bs
    cap="$(mktemp "$RUN_DIR/rehearse-$INSTANCE.XXXXXX")" || refuse rehearsal-failed "cannot make a capture file in $RUN_DIR"
    local -a cmd=("${AS_USER[@]}" "$SR" rehearse "$SHA" "$BUSINESS" --copies "$COPIES_DIR" --backup last --port "$R_PORT" --timeout "$TIMEOUT")
    "${cmd[@]}" 9>&- 2>&1 | tee "$cap" 9>&-
    rc="${PIPESTATUS[0]}"
    logl "rehearse rc=$rc argv:$(printf ' %q' "${cmd[@]}")"
    bs="$(grep -E '^BACKUP_STORE [0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}(-[1-9])? schema [0-9a-z]{1,16}$' "$cap" | tail -n 1)"
    rm -f "$cap"
    logl "rehearse-store rc=0 ${bs:-BACKUP_STORE none}"
    case "$rc" in
        0) [ "$NO_REHEARSAL" = 0 ] || refuse rehearsal-required "the log records a backup and the rehearsal ran; --no-rehearsal is only for a stack with none" ;;
        3) [ "$NO_REHEARSAL" = 1 ] || refuse no-backup "the log records no backup to rehearse against; only a go that names --no-rehearsal goes on"
           logl "rehearsal-skipped rc=0 the log records no backup and the go names --no-rehearsal" ;;
        *) refuse rehearsal-failed "the rehearsal exited $rc" ;;
    esac
}

# r6 rollback: to the log's previous copy, which the stack's user checks against --to. No HEAD
# check: the target is an older copy than this checkout, and no rehearsal.
do_rollback() {
    read_args
    read_next
    legacy_console
    [ -x "$COPIES_DIR/$TO/scripts/mosaic" ] || refuse copy-missing "no $COPIES_DIR/$TO/scripts/mosaic"
    hostcheck
    read_old
    local -a restore=()
    [ -z "$RESTORE" ] || restore=(--restore "$RESTORE")
    [ -z "$EVENTS" ] || restore+=(--events "$EVENTS")
    plan rollback "$BUSINESS" --to "$TO" "${restore[@]}"
    render_checks "$TO"
    run_steps "$TO" rollback "$BUSINESS" --to "$TO" "${restore[@]}"
    finish 0 ok "rollback ok: $TO is live for $BUSINESS$REPLACED_NOTE"
}

case "$OP" in
    upgrade) do_upgrade ;;
    rollback) do_rollback ;;
esac