stack/scripts/stack-upgrade.sh

#!/usr/bin/env bash
# Row 233 (I3, #1735): root's upgrade and rollback for a stack whose units are root's system units
# (stack-release.sh --units external, row 232). r6's activate A1-A5, A2f, R1-R4 and rollback
# (agents/darkwing/work/queue-195/DESIGN.md), as the I3 brief changes them
# (docs/plans/2026-10-20_installer-rows.md). Deviations and decisions:
# agents/darkwing/work/queue-233/BUILD.md.
#
#   stack-upgrade.sh upgrade --instance dev --bundle FILE --sha SHA [--migrate] [--no-rehearsal]
#                    [--rehearse-port N] [--timeout S]
#   stack-upgrade.sh rollback --instance dev --to ID [--restore ACT [--events N]] [--timeout S]
#
# Root runs it from a root-owned checkout of the reviewed commit, never from a copy, because the
# stack's user can write the copies. The unit values come from /etc/mosaic-stack/<instance>.args,
# which I2's install writes: one case2-unit.sh argument per line, the serve flags after `--`. Only
# --copy changes, and the file changes only after confirm passes. The stack's user runs its steps
# from its own clone, <home>/src/mosaic-stack, through sudo. Root reads none of that user's output
# but exit codes and the rehearsal's BACKUP_STORE line (sage-1653), which it copies into its step
# log after a match against a fixed pattern and decides nothing on.
#
# --no-rehearsal is for a stack whose log records no backup. The rehearsal runs either way: exit 3
# from it (no-backup) goes on only with the flag, and exit 0 with the flag refuses
# rehearsal-required (sage-1650, sage-1652).
#
# The lock /run/mosaic-stack/upgrade-<instance>.lock is held from before A1 to the end of the run,
# rollback included; every child runs with the lock's descriptor closed. The step log
# /var/log/mosaic-stack/upgrade-<instance>.log gets one line per step, `<time> <step> rc=N argv:
# <argv>`, and a last `<time> result exit=N <code> <text>` line.
#
# Exit 0 ok, 1 failed with the old stack running, 2 rollback-failed with the units stopped,
# 3 refused with nothing stopped, 4 usage.
#
# Test seams, for packages/cli/tests/stack-upgrade.test.mjs only. Run as root, it refuses any:
#   STACK_UPGRADE_ETC        default /etc/mosaic-stack
#   STACK_UPGRADE_RUN        default /run/mosaic-stack
#   STACK_UPGRADE_LOG_DIR    default /var/log/mosaic-stack
#   STACK_UPGRADE_UNIT_DIR   default /etc/systemd/system
#   STACK_UPGRADE_HOSTCHECK  default /usr/local/lib/mosaic-stack/hostcheck.sh (row 240)
#   STACK_UPGRADE_RENDER     default scripts/case2-unit.sh in this checkout
#   STACK_UPGRADE_SUDO       default sudo
#   STACK_UPGRADE_SYSTEMCTL  default systemctl
set -uo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

usage() {
    cat >&2 <<'USAGE'
usage: stack-upgrade.sh upgrade --instance dev --bundle FILE --sha SHA [--migrate] [--no-rehearsal] [--rehearse-port N] [--timeout S]
       stack-upgrade.sh rollback --instance dev --to ID [--restore ACT [--events N]] [--timeout S]
USAGE
    exit 4
}

if [ "$(id -u)" = 0 ]; then
    for v in ${!STACK_UPGRADE_@}; do echo "stack-upgrade: refusing: $v is a test seam, and root takes none" >&2; exit 3; done
fi
ETC="${STACK_UPGRADE_ETC:-/etc/mosaic-stack}"
RUN_DIR="${STACK_UPGRADE_RUN:-/run/mosaic-stack}"
LOG_DIR="${STACK_UPGRADE_LOG_DIR:-/var/log/mosaic-stack}"
UNIT_DIR="${STACK_UPGRADE_UNIT_DIR:-/etc/systemd/system}"
HOSTCHECK="${STACK_UPGRADE_HOSTCHECK:-/usr/local/lib/mosaic-stack/hostcheck.sh}"
RENDER="${STACK_UPGRADE_RENDER:-$REPO/scripts/case2-unit.sh}"
SUDO="${STACK_UPGRADE_SUDO:-sudo}"
SYSTEMCTL="${STACK_UPGRADE_SYSTEMCTL:-systemctl}"
ID_RE='^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}(-[1-9])?$'

ARGV=("$@")
[ $# -gt 0 ] || usage
OP="$1"; shift
case "$OP" in upgrade|rollback) ;; *) usage ;; esac
INSTANCE="" BUNDLE="" SHA="" MIGRATE=0 NO_REHEARSAL=0 R_PORT=7339 TIMEOUT=60 TO="" RESTORE="" EVENTS=""
while [ $# -gt 0 ]; do
    case "$1" in
        --instance) [ $# -ge 2 ] || usage; INSTANCE="$2"; shift 2 ;;
        --bundle) [ "$OP" = upgrade ] && [ $# -ge 2 ] && [[ "$2" =~ ^/[A-Za-z0-9/._+-]+$ ]] || usage; BUNDLE="$2"; shift 2 ;;
        --sha) [ "$OP" = upgrade ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9a-f]{40}$ ]] || usage; SHA="$2"; shift 2 ;;
        --migrate) [ "$OP" = upgrade ] || usage; MIGRATE=1; shift ;;
        --no-rehearsal) [ "$OP" = upgrade ] || usage; NO_REHEARSAL=1; shift ;;
        --rehearse-port) [ "$OP" = upgrade ] && [ $# -ge 2 ] && [[ "$2" =~ ^[1-9][0-9]{3,4}$ ]] && [ "$2" -le 65535 ] || usage; R_PORT="$2"; shift 2 ;;
        --timeout) [ $# -ge 2 ] && [[ "$2" =~ ^[1-9][0-9]{0,4}$ ]] || usage; TIMEOUT="$2"; shift 2 ;;
        --to) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9a-f]{40}$ ]] || usage; TO="$2"; shift 2 ;;
        --restore) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ $ID_RE ]] || usage; RESTORE="$2"; shift 2 ;;
        --events) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ ^(0|[1-9][0-9]*)$ ]] || usage; EVENTS="$2"; shift 2 ;;
        *) usage ;;
    esac
done
# The default instance only, whose name is dev (row 190); a named instance's units are a later row.
[ "$INSTANCE" = dev ] || usage
if [ "$OP" = upgrade ]; then [ -n "$BUNDLE" ] && [ -n "$SHA" ] || usage; else [ -n "$TO" ] || usage; fi
[ -z "$EVENTS" ] || [ -n "$RESTORE" ] || usage

# --- The step log and the lock ---

umask 027
LOG="$LOG_DIR/upgrade-$INSTANCE.log"
LOCK="$RUN_DIR/upgrade-$INSTANCE.lock"
{ mkdir -p "$LOG_DIR" "$RUN_DIR" && touch "$LOG" && chmod 0640 "$LOG"; } 2>/dev/null ||
    { echo "stack-upgrade: refusing log-failed: cannot write $LOG or make $RUN_DIR; nothing was changed" >&2; exit 3; }
logl() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >> "$LOG" || true; }
# The run's last line, in the log and on the terminal, then the exit.
finish() {
    local rc="$1" code="$2"; shift 2
    logl "result exit=$rc $code $*"
    if [ "$rc" = 0 ]; then echo "stack-upgrade: $*"; else echo "stack-upgrade: $code: $*" >&2; fi
    exit "$rc"
}
refuse() { local code="$1"; shift; finish 3 "$code" "$* (nothing was stopped)"; }
# One step: $1 its name, the rest the command, run with the lock's descriptor closed. One log line
# with its exit and argv; the command's output goes to the terminal only.
step() {
    local name="$1" rc=0; shift
    "$@" 9>&- || rc=$?
    logl "$name rc=$rc argv:$(printf ' %q' "$@")"
    return "$rc"
}

exec 9>> "$LOCK" || refuse lock-failed "cannot open $LOCK"
flock -n 9 || refuse upgrade-running "another stack-upgrade.sh run holds $LOCK"
logl "start rc=0 argv:$(printf ' %q' "$0" "${ARGV[@]}")"

# --- The args file and the stack's user ---

# I2's file: one case2-unit.sh argument per line, owned by root with no group or other write. Sets
# ARGS (the lines), S_USER, S_HOME, BUSINESS, NODE and ARGS_COPY, and the paths derived from them.
read_args() {
    local f="$ETC/$INSTANCE.args" st i copies=0 v
    [ -f "$f" ] && [ ! -L "$f" ] || refuse args-unreadable "no regular file $f"
    st="$(stat -c '%u %a' "$f")" || refuse args-unreadable "cannot stat $f"
    [ "${st% *}" = "$(id -u)" ] && (( (8#${st#* } & 8#022) == 0 )) || refuse args-unreadable "$f must be $(id -un)'s with no group or other write (it is uid ${st% *}, mode ${st#* })"
    mapfile -t ARGS < "$f" || refuse args-unreadable "cannot read $f"
    S_USER="" S_HOME="" BUSINESS="" NODE="" ARGS_COPY=""
    for ((i = 0; i < ${#ARGS[@]}; i++)); do
        case "${ARGS[i]}" in
            --) break ;;
            --pm) ;;
            --user|--home|--business|--copy|--node|--profile|--claude)
                v="${ARGS[i + 1]:-}"
                case "${ARGS[i]}" in
                    --user) S_USER="$v" ;; --home) S_HOME="$v" ;; --business) BUSINESS="$v" ;; --node) NODE="$v" ;;
                    --copy) ARGS_COPY="$v"; copies=$((copies + 1)) ;;
                esac
                i=$((i + 1)) ;;
            *) refuse args-unreadable "$f has ${ARGS[i]} before --, which case2-unit.sh does not take" ;;
        esac
    done
    [ "$copies" = 1 ] || refuse args-unreadable "$f needs exactly one --copy before --, not $copies"
    [[ "$S_USER" =~ ^[a-z_][a-z0-9_-]*$ ]] && [[ "$S_HOME" =~ ^/[A-Za-z0-9/._+-]+$ ]] && [[ "$BUSINESS" =~ ^[a-z0-9][a-z0-9-]*$ ]] &&
        [[ "$NODE" =~ ^/[A-Za-z0-9/._+-]+$ ]] && [[ "$ARGS_COPY" =~ ^[0-9a-f]{40}$ ]] ||
        refuse args-unreadable "$f lacks a plain --user, --home, --business, --node or --copy"
    COPIES_DIR="$S_HOME/.local/share/mosaic-stack/copies"
    CLONE="$S_HOME/src/mosaic-stack"
    SR="$CLONE/scripts/stack-release.sh"
    BUS_UNIT="mosaic-bus@$BUSINESS.service" CON_UNIT=mosaic-webui.service
    # The stack's user's steps: its own HOME and default config, the pinned node first on PATH.
    AS_USER=("$SUDO" -u "$S_USER" -H env "PATH=$(dirname "$NODE"):/usr/local/bin:/usr/bin:/bin")
}

# RENDER_ARGS: the args file's lines with copy $1 in place of --copy's value.
args_with() {
    local i past=""
    RENDER_ARGS=()
    for ((i = 0; i < ${#ARGS[@]}; i++)); do
        RENDER_ARGS+=("${ARGS[i]}")
        [ "${ARGS[i]}" != -- ] || past=1
        if [ -z "$past" ] && [ "${ARGS[i]}" = --copy ]; then RENDER_ARGS+=("$1"); i=$((i + 1)); fi
    done
}

# The new args file, written beside the old one before anything stops, moved into place only after
# confirm passes. $1 is the copy.
prepare_args() {
    args_with "$1"
    NEW_ARGS="$(mktemp "$ETC/.$INSTANCE.args.XXXXXX")" &&
        printf '%s\n' "${RENDER_ARGS[@]}" > "$NEW_ARGS" && chmod --reference="$ETC/$INSTANCE.args" "$NEW_ARGS" ||
        { rm -f "${NEW_ARGS:-}"; refuse args-failed "cannot write the new args file in $ETC"; }
}
drop_args() { rm -f "${NEW_ARGS:-}"; }

# --- A1 ---

# OLD: the copy both installed units run, from their WorkingDirectory under the stack's copies.
read_old() {
    local f ids=() id n
    for f in "$UNIT_DIR/[email protected]" "$UNIT_DIR/$CON_UNIT"; do
        [ -f "$f" ] && [ ! -L "$f" ] || refuse units-unreadable "no regular file $f"
        n="$(grep -c '^WorkingDirectory=' "$f")"
        id="$(awk -v c="WorkingDirectory=$COPIES_DIR/" 'index($0, c) == 1 { print substr($0, length(c) + 1) }' "$f")"
        [ "$n" = 1 ] && [[ "$id" =~ ^[0-9a-f]{40}$ ]] || refuse units-unreadable "$f names no copy under $COPIES_DIR"
        ids+=("$id")
    done
    [ "${ids[0]}" = "${ids[1]}" ] || refuse units-disagree "$UNIT_DIR's units name ${ids[0]} and ${ids[1]}"
    OLD="${ids[0]}"
    logl "units rc=0 old: $OLD"
    [ "$ARGS_COPY" = "$OLD" ] || refuse args-differ "$ETC/$INSTANCE.args names $ARGS_COPY and the units name $OLD"
}

# r6 A1 and row 240: the host's profile still matches, on every host.
hostcheck() {
    [ -f "$HOSTCHECK" ] && [ -x "$HOSTCHECK" ] || refuse profile-drift "no executable $HOSTCHECK (row 240)"
    step hostcheck "$HOSTCHECK" --check || refuse profile-drift "$HOSTCHECK --check failed"
}

# The stack's user's --plan for the run, $@ its op and arguments; any refusal stops here.
plan() {
    local rc=0
    step plan "${AS_USER[@]}" "$SR" "$@" --units external --copies "$COPIES_DIR" --plan --expect-from "$OLD" || rc=$?
    [ "$rc" = 0 ] || refuse plan-refused "the stack's user's $1 --plan exited $rc; its output above names the refusal"
}

# --- The units ---

stop_wait() {
    local u s deadline=$((SECONDS + TIMEOUT)) rc=0
    step stop-console "$SYSTEMCTL" stop "$CON_UNIT" || rc=1
    step stop-bus "$SYSTEMCTL" stop "$BUS_UNIT" || rc=1
    for u in "$CON_UNIT" "$BUS_UNIT"; do
        while :; do
            s="$("$SYSTEMCTL" is-active "$u" 9>&- 2>/dev/null)"
            case "$s" in inactive|failed) break ;; esac
            [ "$SECONDS" -lt "$deadline" ] || { rc=1; break; }
            sleep 1
        done
        case "$s" in inactive|failed) logl "wait-down rc=0 unit: $u state: $s" ;; *) logl "wait-down rc=1 unit: $u state: ${s:-unknown}" ;; esac
    done
    return "$rc"
}
stop_units() { "$SYSTEMCTL" stop "$CON_UNIT" 9>&- || true; "$SYSTEMCTL" stop "$BUS_UNIT" 9>&- || true; logl "stop-units rc=0 units: $CON_UNIT $BUS_UNIT"; }

# A2f: the old units back on the copy they never left, gated on a start at or after this second.
# An A2 that timed out leaves a unit on its old start, which the gate refuses as stale-start.
a2f() {
    local cause="$1" since
    drop_args
    since="$(date -u +%s)"
    step start-bus "$SYSTEMCTL" start "$BUS_UNIT"
    step start-console "$SYSTEMCTL" start "$CON_UNIT"
    if step gate "${AS_USER[@]}" "$SR" gate "$OLD" "$BUSINESS" --units external --copies "$COPIES_DIR" --since "$since" --timeout "$TIMEOUT"; then
        finish 1 failed "$cause failed; $OLD is running again and passed the gate"
    fi
    stop_units
    finish 2 rollback-failed "$cause failed and $OLD failed its gate after the restart; both units are stopped"
}

# R1-R4: back to OLD after the stack's user staged $1. $2 is the cause: A3, A4 or A5.
rollback_to_old() {
    local id="$1" cause="$2"
    drop_args
    stop_wait || { stop_units; finish 2 rollback-failed "$cause failed and the units did not stop for the rollback; both units are stopped"; }
    step revert "${AS_USER[@]}" "$SR" revert "$id" "$BUSINESS" --units external --copies "$COPIES_DIR" ||
        { stop_units; finish 2 rollback-failed "$cause failed and revert failed; both units are stopped"; }
    args_with "$OLD"
    step render-old "$RENDER" "${RENDER_ARGS[@]}" ||
        { stop_units; finish 2 rollback-failed "$cause failed and the render of $OLD failed; both units are stopped"; }
    # confirm --rollback exits 1 when OLD passed its gate and 2 when it failed; anything else is a
    # refusal, with OLD's gate never run.
    local rc=0
    step confirm-rollback "${AS_USER[@]}" "$SR" confirm "$id" "$BUSINESS" --units external --copies "$COPIES_DIR" --rollback "$OLD" --cause "$cause" --timeout "$TIMEOUT" || rc=$?
    [ "$rc" != 1 ] || finish 1 failed "$cause failed; rolled back to $OLD, which passed the gate"
    stop_units
    finish 2 rollback-failed "$cause failed and the rollback to $OLD did not pass (confirm --rollback exited $rc); both units are stopped"
}

# A2 to A5 for copy $1, once A1 and any rehearsal passed. $@ after it is the stack's user's op.
run_steps() {
    local id="$1" rc=0; shift
    prepare_args "$id"
    # A2
    stop_wait || a2f A2
    # A3
    step stage "${AS_USER[@]}" "$SR" "$@" --units external --copies "$COPIES_DIR" || rc=$?
    case "$rc" in
        0) ;;
        3) a2f A3 ;;
        *) rollback_to_old "$id" A3 ;;
    esac
    # A4
    args_with "$id"
    step render "$RENDER" "${RENDER_ARGS[@]}" || rollback_to_old "$id" A4
    # A5
    step confirm "${AS_USER[@]}" "$SR" confirm "$id" "$BUSINESS" --units external --copies "$COPIES_DIR" --timeout "$TIMEOUT" ||
        rollback_to_old "$id" A5
    if mv -f "$NEW_ARGS" "$ETC/$INSTANCE.args"; then logl "args rc=0 copy: $id"
    else logl "args rc=1 copy: $id"; finish 1 args-failed "$id is live and confirmed, but $ETC/$INSTANCE.args still names $OLD; put $NEW_ARGS in its place"
    fi
}

do_upgrade() {
    read_args
    # Before A1: the bundle into the stack's user's clone, then the copy from it (I1).
    step fetch "${AS_USER[@]}" git -C "$CLONE" fetch -q "$BUNDLE" refs/heads/refactor:refs/remotes/origin/refactor ||
        refuse fetch-failed "the stack's user could not fetch $BUNDLE into $CLONE"
    step ff "${AS_USER[@]}" git -C "$CLONE" merge -q --ff-only "$SHA" ||
        refuse fetch-failed "the stack's user could not fast-forward $CLONE's refactor to $SHA"
    step copy "${AS_USER[@]}" "$CLONE/scripts/stack-copy.sh" copy --bundle "$BUNDLE" "$SHA" --copies "$COPIES_DIR" ||
        refuse copy-failed "the stack's user could not make copy $SHA from $BUNDLE"
    # A1
    local head
    head="$(git -C "$REPO" rev-parse HEAD 2>/dev/null)"
    [ "$head" = "$SHA" ] || refuse head-mismatch "this checkout, $REPO, is at ${head:-unknown}, not $SHA"
    [ -x "$COPIES_DIR/$SHA/scripts/mosaic" ] || refuse copy-missing "no $COPIES_DIR/$SHA/scripts/mosaic"
    hostcheck
    read_old
    local -a migrate=()
    [ "$MIGRATE" = 0 ] || migrate=(--migrate)
    plan activate "$SHA" "$BUSINESS" "${migrate[@]}"
    rehearse
    run_steps "$SHA" activate "$SHA" "$BUSINESS" "${migrate[@]}"
    finish 0 ok "upgrade ok: $SHA is live for $BUSINESS"
}

# The rehearsal against the log's newest backup, before anything stops. Its output goes to the
# terminal; root keeps only the BACKUP_STORE line, so the go and the log show which store it used.
rehearse() {
    local cap rc bs
    cap="$(mktemp "$RUN_DIR/rehearse-$INSTANCE.XXXXXX")" || refuse rehearsal-failed "cannot make a capture file in $RUN_DIR"
    local -a cmd=("${AS_USER[@]}" "$SR" rehearse "$SHA" "$BUSINESS" --copies "$COPIES_DIR" --backup last --port "$R_PORT" --timeout "$TIMEOUT")
    "${cmd[@]}" 9>&- 2>&1 | tee "$cap" 9>&-
    rc="${PIPESTATUS[0]}"
    logl "rehearse rc=$rc argv:$(printf ' %q' "${cmd[@]}")"
    bs="$(grep -E '^BACKUP_STORE [0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}(-[1-9])? schema [0-9a-z]{1,16}$' "$cap" | tail -n 1)"
    rm -f "$cap"
    logl "rehearse-store rc=0 ${bs:-BACKUP_STORE none}"
    case "$rc" in
        0) [ "$NO_REHEARSAL" = 0 ] || refuse rehearsal-required "the log records a backup and the rehearsal ran; --no-rehearsal is only for a stack with none" ;;
        3) [ "$NO_REHEARSAL" = 1 ] || refuse no-backup "the log records no backup to rehearse against; only a go that names --no-rehearsal goes on"
           logl "rehearsal-skipped rc=0 the log records no backup and the go names --no-rehearsal" ;;
        *) refuse rehearsal-failed "the rehearsal exited $rc" ;;
    esac
}

# r6 rollback: to the log's previous copy, which the stack's user checks against --to. No HEAD
# check: the target is an older copy than this checkout, and no rehearsal.
do_rollback() {
    read_args
    [ -x "$COPIES_DIR/$TO/scripts/mosaic" ] || refuse copy-missing "no $COPIES_DIR/$TO/scripts/mosaic"
    hostcheck
    read_old
    local -a restore=()
    [ -z "$RESTORE" ] || restore=(--restore "$RESTORE")
    [ -z "$EVENTS" ] || restore+=(--events "$EVENTS")
    plan rollback "$BUSINESS" --to "$TO" "${restore[@]}"
    run_steps "$TO" rollback "$BUSINESS" --to "$TO" "${restore[@]}"
    finish 0 ok "rollback ok: $TO is live for $BUSINESS"
}

case "$OP" in
    upgrade) do_upgrade ;;
    rollback) do_rollback ;;
esac