jason.woltje
57f59e0c39
Lands Darkwing's round 1 candidate, manifest 464ce9eb177a (1a1dc80caon darkwing's branch), approved by Filbert (29922, rev 2659). Seven paths: scripts/stack-upgrade.sh (new), scripts/stack-release.sh, docs/guides/host-run-sheet.md (new), docs/TOOLS.md, packages/cli/tests/stack-upgrade.test.mjs (new), packages/cli/tests/stack-release.test.mjs and agents/darkwing/work/queue-233/BUILD.md. Evidence: Darkwing's gate ond99166ade(basee65a3ab4d) 1201/0, mutants m1-m8 8 of 8 killed. The N1 fix08db6ac3f(flock sleep leak in stack-upgrade.test.mjs) reran c0 20/0 with m1 killed (sage-1708); BUILD.md doesn't record that rerun. Sincee65a3ab4dno commit touches the seven paths. Rows 241 and 242 changed bus and cli files. Only cli tests read the seven paths, so Sage ran the cli suite on5465fd52dplus the seven paths: 617 pass, 0 fail (out233/pkg-cli.log, 05:26:33Z to 05:35:00Z). A combined gate with row 234 failed bus schema.test.mjs:522 on 234's new tables, and Gizmoduck found a second 234 failure (stack-release.test.mjs T46), so Sage stopped it. Row 234 round 3 fixes both and rebuilds on this landing, since its T46 lines go in a file this commit changes. Filbert's N1, N4 and N5 go to row 240 (briefb00d7d94d). N2 and N3 are in DEFERRED.md.
2175 lines
119 KiB
Bash
Executable File
2175 lines
119 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Stack activate and rollback, health-gated (D1, #1659; docs/plans/2026-10-18_stack-release.md).
|
|
# Moves the bus, the Console and the human CLI from the pointer's copy to another copy together,
|
|
# behind a health gate, and puts the old stack back inside the same run when the gate fails.
|
|
# Console tester listener flags follow target support (row 176, #1674); so does the row 178 proof
|
|
# pair, which activate sets or drops (row 187, #1685). activate --no-send drops --send (row 189, #1687).
|
|
# activate --sso swaps the preview set and the pair for the Console's own sign-in (row 200, #1698).
|
|
# Design and tests: agents/darkwing/work/queue-162/BUILD.md.
|
|
#
|
|
# Who runs it. Decision 96 (Mos, mos-0716) lets this script stop and start the live units under
|
|
# four conditions:
|
|
# 1. every adopt, activate or rollback runs on Mos's go, and Jason or Mos runs it;
|
|
# 2. a run that migrates the store (activate --migrate) happens with Jason present;
|
|
# 3. a rollback --restore with events written after the backup waits for Jason, which is why it
|
|
# needs --events N;
|
|
# 4. this script never prunes backups, rollback sets, post-attempt directories or copies.
|
|
# Decisions 94 and 102 still decide who starts the Console; webui-service.sh keeps having no start
|
|
# or stop verb. status and --plan only read, and every seat may run them.
|
|
# rehearse (D2, row 163) stops and starts no unit and writes only its own rehearsal directory, but
|
|
# it reads the live config and a backup of the live store, so Jason or Mos runs it.
|
|
#
|
|
# --units external (row 232, #1734; r6 section 4 at agents/darkwing/work/queue-195/DESIGN.md) is
|
|
# for a host whose units are root's system units. This script then reads those units and never
|
|
# starts or stops one: root stops them, activate or rollback stages the copy (A3), root installs
|
|
# and starts them, and confirm gates them and writes the pointer (A5). revert and confirm
|
|
# --rollback put the old copy back when a step fails (R2, R4), and gate checks a copy root
|
|
# restarted with no activation open (A2f). Without it, everything runs as before.
|
|
#
|
|
# Most functions below run as `f || handler`, and bash ignores set -e inside such a call, so
|
|
# every step that can fail carries its own `|| return 1`.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
REPO="$(pwd)"
|
|
source scripts/common.sh
|
|
source scripts/instance.sh
|
|
|
|
COPIES="${XDG_DATA_HOME:-$HOME/.local/share}/mosaic-stack/copies"
|
|
BIN="$HOME/.npm-global/bin"
|
|
UNIT_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user"
|
|
TIMEOUT=60 WANT_PM=0 AFTER_HARNESS=0 MIGRATE=0 PLAN=0 RESTORE="" EVENTS="" TESTER_PORT=""
|
|
PROOF_HEADER="" PROOF_FILE="" NO_PROOF=0 NO_SEND=0 SSO=0 R_BACKUP="" R_PORT=""
|
|
UNITS=user TO_FLAG="" EXPECT_FROM="" SINCE="" ROLLBACK_OLD="" CAUSE=""
|
|
|
|
usage() {
|
|
cat >&2 <<'USAGE'
|
|
usage: stack-release.sh adopt BUSINESS
|
|
stack-release.sh activate ID BUSINESS [--pm | --pm-after-harness] [--migrate] [--tester-port N]
|
|
[--proof-header NAME --proof-file PATH | --no-proof | --sso] [--no-send] [--timeout S] [--plan]
|
|
stack-release.sh pm-on BUSINESS [--timeout S]
|
|
stack-release.sh rollback BUSINESS [--restore ACTIVATION [--events N]] [--timeout S] [--plan]
|
|
stack-release.sh status BUSINESS
|
|
stack-release.sh rehearse ID BUSINESS --backup ACTIVATION|PATH --port N [--timeout S]
|
|
stack-release.sh confirm ID BUSINESS --units external [--rollback OLD --cause A3|A4|A5] [--timeout S]
|
|
stack-release.sh revert ID BUSINESS --units external
|
|
stack-release.sh gate ID BUSINESS --units external --since EPOCH [--timeout S]
|
|
common options: --copies DIR, --bin DIR, --instance NAME, --units user|external
|
|
--instance NAME (row 190) runs on stack instance NAME: its config, data root, units and the wrapper
|
|
mosaic-NAME. Without it, the default instance, as before.
|
|
--events N is the total of the per-table counts the post-backup-events refusal prints: events and
|
|
every table in COUNTED.
|
|
--units external (row 232) reads root's system units and starts or stops none. adopt, activate,
|
|
rollback, status, confirm, revert and gate take it; rollback then needs --to ID, the previous copy.
|
|
activate and rollback --plan take --expect-from OLD. --pm, --pm-after-harness, --tester-port, the
|
|
proof flags, --no-proof, --no-send, --sso and pm-on are not for it.
|
|
USAGE
|
|
exit "${1:-4}"
|
|
}
|
|
say() { echo "stack-release: $*"; [ -z "${STEPS:-}" ] || echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >> "$STEPS" || true; }
|
|
now() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
now_ms() { date -u +%Y-%m-%dT%H:%M:%S.%3NZ; }
|
|
|
|
# One JSON object from KEY VALUE pairs; the value @null is JSON null.
|
|
jline() {
|
|
node -e 'const a=process.argv.slice(1),o={};for(let i=0;i<a.length;i+=2)o[a[i]]=a[i+1]==="@null"?null:a[i+1];process.stdout.write(JSON.stringify(o)+"\n")' -- "$@"
|
|
}
|
|
log_line() {
|
|
[ "$UNITS" = user ] || set -- "$@" units external
|
|
jline at "$(now)" op "$OP" from "${FROM:-@null}" to "${TO:-@null}" backup "${BACKUP:-@null}" activation "${ACT:-@null}" "$@" >> "$LOG"
|
|
}
|
|
|
|
# --- The units backend (row 232) ---
|
|
|
|
# Every systemctl, journalctl and systemd-analyze call goes through these three. user is the
|
|
# systemd user manager, as before. external reads root's system units, which needs no privilege,
|
|
# and refuses any verb that would change one.
|
|
sc() {
|
|
if [ "$UNITS" = user ]; then systemctl --user "$@"; return; fi
|
|
case "$1" in
|
|
show|is-active|list-units) systemctl "$@" ;;
|
|
*) echo "stack-release: --units external never runs systemctl $1; root does" >&2; return 1 ;;
|
|
esac
|
|
}
|
|
# The bus unit's journal is a user journal only: under external it lands in the system journal,
|
|
# which the stack's user can't read (r6 section 4), so the external PM check reads launches list.
|
|
jctl() { [ "$UNITS" = user ] || return 1; journalctl --user "$@"; }
|
|
analyze() { [ "$UNITS" = user ] || return 1; systemd-analyze --user "$@"; }
|
|
|
|
# A refusal before anything stopped: exit 3, and one log line with the code, except for
|
|
# run-locked (the lock holder may be writing the log) and in --plan, which writes nothing.
|
|
refuse() {
|
|
local code="$1"; shift
|
|
echo "stack-release: refusing $code: $*" >&2
|
|
[ "$code" = run-locked ] || [ "$PLAN" = 1 ] || log_line result refused code "$code"
|
|
exit 3
|
|
}
|
|
|
|
# Reject tunnel/main ports and anything other than a whole high port (exit 4 via usage).
|
|
valid_tester_port() {
|
|
[[ "$1" =~ ^[0-9]{1,5}$ ]] && [ "$((10#$1))" -ge 1024 ] && [ "$((10#$1))" -le 65535 ] &&
|
|
[ "$((10#$1))" -ne 7332 ] && [ "$((10#$1))" -ne "$2" ] || return 1
|
|
# Row 190: a named instance's Console never takes 7330, 7332 or 7333.
|
|
local r
|
|
for r in "${INSTANCE_RESERVED[@]}"; do [ "$((10#$1))" -ne "$r" ] || return 1; done
|
|
}
|
|
|
|
parse_opts() {
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--copies) [ $# -ge 2 ] || usage; COPIES="$2"; shift 2 ;;
|
|
--bin) [ $# -ge 2 ] || usage; BIN="$2"; shift 2 ;;
|
|
--timeout) [ "$OP" != status ] && [ "$OP" != adopt ] && [ "$OP" != revert ] && [ $# -ge 2 ] && [[ "$2" =~ ^[1-9][0-9]{0,4}$ ]] || usage; TIMEOUT="$2"; shift 2 ;;
|
|
--plan) [ "$OP" = activate ] || [ "$OP" = rollback ] || usage; PLAN=1; shift ;;
|
|
--pm) [ "$OP" = activate ] || usage; WANT_PM=1; shift ;;
|
|
--pm-after-harness) [ "$OP" = activate ] || usage; AFTER_HARNESS=1; shift ;;
|
|
--tester-port) [ "$OP" = activate ] && [ $# -ge 2 ] && [ -n "$2" ] || usage; TESTER_PORT="$2"; shift 2 ;;
|
|
--proof-header) [ "$OP" = activate ] && [ $# -ge 2 ] && [ -n "$2" ] || usage; PROOF_HEADER="$2"; shift 2 ;;
|
|
--proof-file) [ "$OP" = activate ] && [ $# -ge 2 ] && [ -n "$2" ] || usage; PROOF_FILE="$2"; shift 2 ;;
|
|
--no-proof) [ "$OP" = activate ] || usage; NO_PROOF=1; shift ;;
|
|
--no-send) [ "$OP" = activate ] || usage; NO_SEND=1; shift ;;
|
|
--sso) [ "$OP" = activate ] || usage; SSO=1; shift ;;
|
|
--migrate) [ "$OP" = activate ] || usage; MIGRATE=1; shift ;;
|
|
--restore) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}(-[1-9])?$ ]] || usage; RESTORE="$2"; shift 2 ;;
|
|
--backup) [ "$OP" = rehearse ] && [ $# -ge 2 ] && { [ "$2" = last ] || [[ "$2" =~ ^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}(-[1-9])?$ ]] || [[ "$2" =~ ^/[A-Za-z0-9/._+-]*$ ]]; } || usage; R_BACKUP="$2"; shift 2 ;;
|
|
--port) [ "$OP" = rehearse ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9]{1,5}$ ]] && [ "$((10#$2))" -ge 1024 ] && [ "$((10#$2))" -le 65535 ] || usage; R_PORT="$((10#$2))"; shift 2 ;;
|
|
--events) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ ^(0|[1-9][0-9]*)$ ]] || usage; EVENTS="$2"; shift 2 ;;
|
|
--units) [ "$OP" != rehearse ] && [ $# -ge 2 ] && { [ "$2" = user ] || [ "$2" = external ]; } || usage; UNITS="$2"; shift 2 ;;
|
|
--to) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9a-f]{40}$ ]] || usage; TO_FLAG="$2"; shift 2 ;;
|
|
--expect-from) { [ "$OP" = activate ] || [ "$OP" = rollback ]; } && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9a-f]{40}$ ]] || usage; EXPECT_FROM="$2"; shift 2 ;;
|
|
--since) [ "$OP" = gate ] && [ $# -ge 2 ] && [[ "$2" =~ ^(0|[1-9][0-9]{0,11})$ ]] || usage; SINCE="$2"; shift 2 ;;
|
|
--rollback) [ "$OP" = confirm ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9a-f]{40}$ ]] || usage; ROLLBACK_OLD="$2"; shift 2 ;;
|
|
--cause) [ "$OP" = confirm ] && [ $# -ge 2 ] && [[ "$2" =~ ^A[345]$ ]] || usage; CAUSE="$2"; shift 2 ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
# Row 232: confirm, revert, gate, --to and --expect-from are external only, and an external
|
|
# rollback names its target. gate needs --since, its start floor (r6 N2), and --expect-from
|
|
# goes with --plan. The flags that change a unit's render or the PM are root's under external.
|
|
if [ "$UNITS" = user ]; then
|
|
[ "$OP" != confirm ] && [ "$OP" != revert ] && [ "$OP" != gate ] && [ -z "$TO_FLAG" ] && [ -z "$EXPECT_FROM" ] || usage
|
|
else
|
|
[ "$OP" != rollback ] || [ -n "$TO_FLAG" ] || usage
|
|
[ "$WANT_PM" = 0 ] && [ -z "$TESTER_PORT" ] && [ -z "$PROOF_HEADER" ] && [ -z "$PROOF_FILE" ] && [ "$NO_PROOF" = 0 ] && [ "$NO_SEND" = 0 ] && [ "$SSO" = 0 ] || usage
|
|
fi
|
|
[ "$OP" != gate ] || [ -n "$SINCE" ] || usage
|
|
[ -z "$EXPECT_FROM" ] || [ "$PLAN" = 1 ] || usage
|
|
[ "${ROLLBACK_OLD:+1}" = "${CAUSE:+1}" ] || usage
|
|
[ -z "$EVENTS" ] || [ -n "$RESTORE" ] || usage
|
|
[ "$OP" != rehearse ] || { [ -n "$R_BACKUP" ] && [ -n "$R_PORT" ]; } || usage
|
|
# A bad tester port is a usage error; --plan lists it as tester-port-invalid instead (row 180).
|
|
[ -z "$TESTER_PORT" ] || [ "$PLAN" = 1 ] || valid_tester_port "$TESTER_PORT" 7332 || usage
|
|
# --pm-after-harness replaces --pm (row 169 item 1), so the two never go together.
|
|
[ "$WANT_PM" = 0 ] || [ "$AFTER_HARNESS" = 0 ] || usage
|
|
# The proof pair goes together, and --no-proof goes alone (row 187).
|
|
[ "${PROOF_HEADER:+1}" = "${PROOF_FILE:+1}" ] || usage
|
|
[ "$NO_PROOF" = 0 ] || [ -z "$PROOF_HEADER" ] || usage
|
|
# --sso drops the pair itself, so it takes neither a pair nor --no-proof (row 200).
|
|
[ "$SSO" = 0 ] || { [ -z "$PROOF_HEADER" ] && [ "$NO_PROOF" = 0 ]; } || usage
|
|
local d
|
|
for d in "$COPIES" "$BIN"; do
|
|
case "$d" in /*) ;; *) usage ;; esac
|
|
case "$d" in *[!A-Za-z0-9/._+-]*) usage ;; esac
|
|
done
|
|
}
|
|
|
|
# --- The pointer and the log (item 1) ---
|
|
|
|
# Sets P_COPY P_BUSINESS P_PM P_PENDING P_CLAUDE P_SCHEMA P_ACT and the array P_CONSOLE; returns 1
|
|
# with no pointer. A pointer that is there but malformed stops the script: Jason or Mos settles it.
|
|
# pmPending (row 169) is missing or true, and true only beside pm false.
|
|
read_pointer() {
|
|
[ -f "$POINTER" ] || return 1
|
|
local fields
|
|
fields="$(node -e '
|
|
const p=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));
|
|
const ok=/^[0-9a-f]{40}$/.test(p.copy)&&typeof p.business==="string"&&typeof p.pm==="boolean"&&(p.pmPending===undefined||(p.pmPending===true&&p.pm===false))&&typeof p.claude==="string"&&typeof p.schema==="string"&&typeof p.activation==="string"&&Array.isArray(p.console)&&p.console.every((t)=>typeof t==="string"&&t.length>0&&!/[\n\r]/.test(t));
|
|
if(!ok)throw new Error("pointer fields are missing or malformed");
|
|
process.stdout.write([p.copy,p.business,p.pm?1:0,p.pmPending?1:0,p.claude,p.schema,p.activation,...p.console].join("\n")+"\n");' -- "$POINTER")" || { echo "stack-release: $POINTER is unreadable; Jason or Mos settles it" >&2; exit 1; }
|
|
local -a f
|
|
mapfile -t f <<< "$fields"
|
|
P_COPY="${f[0]}" P_BUSINESS="${f[1]}" P_PM="${f[2]}" P_PENDING="${f[3]}" P_CLAUDE="${f[4]}" P_SCHEMA="${f[5]}" P_ACT="${f[6]}"
|
|
P_CONSOLE=("${f[@]:7}")
|
|
}
|
|
|
|
# Writes the pointer through a temp file in state/ and a rename. Args: copy pm pending claude schema
|
|
# console... A pending of 1 writes pmPending true (row 169); 0 leaves the key out.
|
|
write_pointer() {
|
|
local tmp
|
|
tmp="$(mktemp "$STATE/.stack-active.XXXXXX")"
|
|
node -e '
|
|
const [copy,business,pm,pending,claude,schema,activation,at,...console]=process.argv.slice(1);
|
|
process.stdout.write(JSON.stringify({copy,business,pm:pm==="1",...(pending==="1"?{pmPending:true}:{}),claude,console,schema,activation,at})+"\n");' -- \
|
|
"$1" "$BUSINESS" "$2" "$3" "$4" "$5" "$ACT" "$(now)" "${@:6}" > "$tmp"
|
|
chmod 0644 "$tmp"
|
|
mv -f "$tmp" "$POINTER"
|
|
}
|
|
|
|
# Prints the activation id of each `started` line with no final line after it.
|
|
interrupted() {
|
|
[ -f "$LOG" ] || return 0
|
|
node -e '
|
|
const open=new Map();
|
|
for(const l of require("fs").readFileSync(process.argv[1],"utf8").split("\n")){
|
|
if(!l.trim())continue;
|
|
let o;try{o=JSON.parse(l)}catch{continue}
|
|
if(!o.activation)continue;
|
|
if(o.result==="started")open.set(o.activation,o);else open.delete(o.activation);
|
|
}
|
|
for(const k of open.keys())process.stdout.write(k+"\n");' -- "$LOG"
|
|
}
|
|
|
|
# Reads the log's ok lines, newest first. `previous` prints rollback's target: the first `to`
|
|
# that is not the pointer's copy (release.sh's rule). `backup` prints the backup field of the
|
|
# line that made the pointer's copy live, the backup a rollback would restore. `newest` prints the
|
|
# backup field of the newest line that has one, rehearse's --backup last (row 233). `arrival` prints the
|
|
# activation and `from` of the line that moved the stack onto the pointer's copy from another one,
|
|
# past any same-copy re-pin after it (row 200, sage-1389).
|
|
from_log() {
|
|
[ -f "$LOG" ] || return 1
|
|
node -e '
|
|
const [file,mode,current]=process.argv.slice(1);
|
|
const lines=require("fs").readFileSync(file,"utf8").split("\n").filter((l)=>l.trim());
|
|
for(const l of lines.reverse()){
|
|
let o;try{o=JSON.parse(l)}catch{continue}
|
|
if(o.result!=="ok"||!["adopt","activate","rollback"].includes(o.op)||!o.to)continue;
|
|
if(mode==="previous"&&o.to!==current){process.stdout.write(o.to+"\n");process.exit(0)}
|
|
if(mode==="backup"&&o.to===current){if(o.backup)process.stdout.write(o.backup+"\n");process.exit(o.backup?0:1)}
|
|
if(mode==="newest"&&o.backup){process.stdout.write(o.backup+"\n");process.exit(0)}
|
|
if(mode==="arrival"&&o.to===current&&o.from!==current){process.stdout.write(`${o.activation??""} ${o.from??""}\n`);process.exit(0)}
|
|
}
|
|
process.exit(1);' -- "$LOG" "$1" "$P_COPY"
|
|
}
|
|
|
|
# --- Reading the units ---
|
|
|
|
show() { sc show "$1" -p LoadState -p WorkingDirectory -p ExecStart -p Environment -p MainPID -p DropInPaths 2>/dev/null; }
|
|
argv_of() { sed -n 's/^ExecStart=.*argv\[\]=\([^;]*\) ;.*/\1/p' <<< "$1"; }
|
|
# systemctl show quotes a value that holds a space and escapes with a backslash, and the read -a
|
|
# split below can't undo either, so such a line is unreadable rather than read wrong (row 166 item 4).
|
|
unquoted() { [[ "$1" != *[\"\'\\]* ]]; }
|
|
|
|
# Sets U_PM U_CLAUDE U_CONSOLE (array) from the effective units; returns 1 if a unit can't be read.
|
|
read_units() {
|
|
local bus console tok bus_argv console_argv env
|
|
bus="$(show "$BUS_UNIT")" && console="$(show "$CONSOLE_UNIT")" || return 1
|
|
[ "$(sed -n 's/^LoadState=//p' <<< "$bus")" = loaded ] && [ "$(sed -n 's/^LoadState=//p' <<< "$console")" = loaded ] || return 1
|
|
bus_argv="$(argv_of "$bus")" console_argv="$(argv_of "$console")" env="$(sed -n 's/^Environment=//p' <<< "$bus")"
|
|
unquoted "$bus_argv" && unquoted "$console_argv" || return 1
|
|
unquoted "$env" || return 1
|
|
local -a argv
|
|
read -r -a argv <<< "$bus_argv"
|
|
U_PM=0
|
|
for tok in "${argv[@]:1}"; do [ "$tok" != --pm ] || U_PM=1; done
|
|
U_CLAUDE=""
|
|
read -r -a argv <<< "$env"
|
|
for tok in "${argv[@]}"; do case "$tok" in MOSAIC_CLAUDE_BIN=*) U_CLAUDE="${tok#MOSAIC_CLAUDE_BIN=}" ;; esac; done
|
|
serve_flags "$console_argv" || return 1
|
|
U_CONSOLE=("${SERVE_FLAGS[@]}")
|
|
}
|
|
|
|
# Sets SERVE_FLAGS (array) to the Console argv $1 after `serve`, without --checkout and its value;
|
|
# returns 1 with no `serve`.
|
|
serve_flags() {
|
|
local -a argv
|
|
read -r -a argv <<< "$1"
|
|
SERVE_FLAGS=()
|
|
local i serve=0
|
|
for ((i = 0; i < ${#argv[@]}; i++)); do
|
|
if [ "$serve" = 0 ]; then [ "${argv[i]}" != serve ] || serve=1; continue; fi
|
|
if [ "${argv[i]}" = --checkout ]; then i=$((i + 1)); continue; fi
|
|
SERVE_FLAGS+=("${argv[i]}")
|
|
done
|
|
[ "$serve" = 1 ]
|
|
}
|
|
|
|
console_port() {
|
|
local i
|
|
for ((i = 0; i < ${#P_CONSOLE[@]}; i++)); do
|
|
[ "${P_CONSOLE[i]}" != --port ] || { echo "${P_CONSOLE[i + 1]:-7332}"; return; }
|
|
done
|
|
echo 7332
|
|
}
|
|
|
|
# --- Copies ---
|
|
|
|
# stack-copy.sh's own helpers, in a subshell so its die, parse and globals stay out of this script.
|
|
copy_ok() ( source "$REPO/scripts/stack-copy.sh"; ID="$1"; COPIES="$COPIES_DIR"; copy_root >/dev/null && reachable "$ID" )
|
|
copy_exists() ( source "$REPO/scripts/stack-copy.sh"; ID="$1"; COPIES="$COPIES_DIR"; copy_root >/dev/null )
|
|
|
|
# Prints the copy's SCHEMA_VERSION, then each `from:` of its migration chain, one per line. Text
|
|
# only: the copy's code is never imported or run for this.
|
|
copy_schema() {
|
|
local f="$COPIES/$1/packages/bus/src/store.mjs" v
|
|
[ -f "$f" ] || return 1
|
|
v="$(grep -E "^export const SCHEMA_VERSION = '[A-Za-z0-9]+';\$" "$f")" || return 1
|
|
[ "$(wc -l <<< "$v")" = 1 ] || return 1
|
|
v="${v#*\'}"; echo "${v%%\'*}"
|
|
{ grep -E "^[[:space:]]*from: '[A-Za-z0-9]+',\$" "$f" || true; } | sed "s/.*from: '\\([A-Za-z0-9]*\\)',\$/\\1/"
|
|
}
|
|
|
|
# Sets T_SCHEMA and T_CHAIN (array) for copy $1; returns 1 when either read fails.
|
|
target_schema() {
|
|
local out
|
|
out="$(copy_schema "$1")" || return 1
|
|
local -a l
|
|
mapfile -t l <<< "$out"
|
|
T_SCHEMA="${l[0]}"
|
|
T_CHAIN=("${l[@]:1}")
|
|
}
|
|
|
|
in_chain() { local v; for v in "${T_CHAIN[@]}"; do [ "$v" != "$1" ] || return 0; done; return 1; }
|
|
|
|
# The item 3 checks on the target copy, in order: copy, schema, already-active. An activate that
|
|
# sets a proof pair, drops the pointer's pair with --no-proof, drops the pointer's --send with
|
|
# --no-send, or turns on --sso, may name the pointer's copy: it re-pins the live copy with new
|
|
# Console flags (row 187, Part B's B3 and B-R1; row 189, mos-0972; row 200, the phase 2 window).
|
|
check_target() {
|
|
copy_exists "$TO" 2>/dev/null || refuse copy-missing "no copy $TO in $COPIES"
|
|
copy_ok "$TO" 2>/dev/null || refuse copy-unreachable "$TO is not reachable from refactor and origin/refactor"
|
|
target_schema "$TO" || refuse copy-schema-unknown "cannot read SCHEMA_VERSION and its chain from $COPIES/$TO/packages/bus/src/store.mjs"
|
|
[ "$TO" != "$P_COPY" ] || [ -n "$PROOF_HEADER" ] ||
|
|
{ [ "$NO_PROOF" = 1 ] && [[ " ${P_CONSOLE[*]} " == *" --proof-header "* ]]; } ||
|
|
{ [ "$NO_SEND" = 1 ] && [[ " ${P_CONSOLE[*]} " == *" --send "* ]]; } ||
|
|
{ [ "$SSO" = 1 ] && [[ " ${P_CONSOLE[*]} " != *" --sso "* ]]; } ||
|
|
refuse already-active "$TO is the pointer's copy"
|
|
}
|
|
|
|
# --- Plug-in kinds (row 169) ---
|
|
|
|
# Unlike copy_schema, these import the copy's own code, as a business-check does: its kinds.mjs
|
|
# (no imports) and, for `harness`, its business.mjs loader. Modes, for copy $2:
|
|
# has KIND exits 0 when the copy's KINDS has KIND
|
|
# unknown prints the business file's plugins keys the copy's KINDS lacks, from a plain JSON read
|
|
# harness prints plugins.harness as JSON, or `none`, from the copy's loader
|
|
# launchby prints launch.by, the PM's instance, from a plain JSON read (row 232); it imports
|
|
# only business.mjs, for the business file's path
|
|
# A failure exits 1 and prints its first line.
|
|
PLUGINS_JS='
|
|
const {readFileSync}=require("node:fs");const {pathToFileURL}=require("node:url");
|
|
const [mode,copy,id,kind]=process.argv.slice(1);
|
|
const imp=(p)=>import(pathToFileURL(`${copy}/packages/${p}`).href);
|
|
(async()=>{
|
|
if(mode==="launchby"){
|
|
const {businessFilePath}=await imp("business/src/business.mjs");
|
|
const by=JSON.parse(readFileSync(businessFilePath(id),"utf8")).launch?.by;
|
|
if(typeof by!=="string"||!by)throw new Error("the business file has no launch.by");
|
|
return process.stdout.write(by);
|
|
}
|
|
const {KINDS}=await imp("plugins/src/kinds.mjs");
|
|
if(mode==="has")process.exit(Object.hasOwn(KINDS,kind)?0:1);
|
|
const b=await imp("business/src/business.mjs");
|
|
if(mode==="unknown"){
|
|
const p=JSON.parse(readFileSync(b.businessFilePath(id),"utf8")).plugins;
|
|
if(p===undefined)return;
|
|
if(p===null||typeof p!=="object"||Array.isArray(p))throw new Error("its plugins is not an object");
|
|
process.stdout.write(Object.keys(p).filter((k)=>!Object.hasOwn(KINDS,k)).join(" "));
|
|
}else{
|
|
const h=b.loadBusiness(id,{rolesDir:`${copy}/roles`}).plugins.harness;
|
|
process.stdout.write(h===undefined?"none":JSON.stringify(h));
|
|
}
|
|
})().catch((e)=>{process.stdout.write(String(e?.message??e).split("\n")[0]);process.exit(1)});'
|
|
plugins() { node -e "$PLUGINS_JS" -- "$1" "$COPIES/$2" "$BUSINESS" "${3:-}"; }
|
|
|
|
# A refusal --plan lists and goes on from; outside --plan it is refuse, which exits. A caller
|
|
# returns after it, so the checks that depend on this one are skipped.
|
|
check_refuse() {
|
|
[ "$PLAN" = 1 ] || refuse "$@"
|
|
local code="$1"; shift
|
|
PLAN_REFUSE+=("$code: $*")
|
|
}
|
|
|
|
# Item 1 and item 4, before anything stops, for target $TO: no plug-in kind the target lacks, then
|
|
# for --pm-after-harness the target's harness kind and no harness entry yet.
|
|
plugin_checks() {
|
|
local out fix=""
|
|
out="$(plugins unknown "$TO" 2>&1)" || { check_refuse business-unreadable "cannot read the business file's plugins with $TO's code: $out"; return 0; }
|
|
if [ -n "$out" ]; then
|
|
[ "$OP" != rollback ] || fix="; set each to null with plugins set through the live bus first"
|
|
check_refuse unknown-kind-for-target "the business file names plug-in kind(s) $out, which $TO does not know$fix"
|
|
return 0
|
|
fi
|
|
[ "$AFTER_HARNESS" = 1 ] || return 0
|
|
plugins has "$TO" harness 2>/dev/null || { check_refuse pm-after-harness-target "$TO has no harness plug-in kind, so there is nothing to wait for; use --pm"; return 0; }
|
|
out="$(plugins harness "$TO" 2>&1)" || { check_refuse business-unreadable "$TO's loader refuses the business file: $out"; return 0; }
|
|
[ "$out" = none ] || check_refuse pm-after-harness-set "the business file already sets harness $out; use --pm"
|
|
}
|
|
|
|
# --- The store ---
|
|
|
|
# Row 181: the tables a bus writer fills without an event in the same transaction, in schema
|
|
# order. A restore drops their rows after STAMP as it drops events, so the guard counts them too.
|
|
# BUILD.md for row 181 names each writer; the T46 guard classifies every table in the schema.
|
|
COUNTED="role_claims thread_claims decision_events messages deliveries task_snapshots projects threads thread_opens instance_names usage_records"
|
|
|
|
STORE_JS='
|
|
const {DatabaseSync}=require("node:sqlite");const {pathToFileURL}=require("node:url");
|
|
const [mode,file,stamp,tables=""]=process.argv.slice(1);
|
|
// immutable=1 on a stopped store with no -wal or -shm: it creates no file and leaves the main
|
|
// file mtime and sha256 as they were (the row 162 probe). A plain readOnly open of a stopped
|
|
// WAL store creates -shm and -wal, so it is used only on a store the running bus holds.
|
|
const db=mode.startsWith("held")?new DatabaseSync(file,{readOnly:true}):new DatabaseSync(pathToFileURL(file).href+"?immutable=1",{readOnly:true});
|
|
try{
|
|
if(mode.endsWith("version"))process.stdout.write(db.prepare("SELECT value FROM meta WHERE key=?").get("schema_version").value+"\n");
|
|
else{
|
|
// "TOTAL<tab>TEXT": events, then each listed table with rows after STAMP, in list order. A
|
|
// listed table the store lacks counts 0 (no instance_names before 3g, no usage_records before
|
|
// 3h). One that is there but cannot be read throws, as a failed events count does.
|
|
const ev=db.prepare("SELECT count(*) AS n FROM events WHERE at > ?").get(stamp).n;let total=ev,text=ev+" event(s)";
|
|
for(const t of tables.split(" ").filter(Boolean)){
|
|
if(!db.prepare("SELECT 1 FROM sqlite_master WHERE type=? AND name=?").get("table",t))continue;
|
|
const n=db.prepare("SELECT count(*) AS n FROM "+t+" WHERE at > ?").get(stamp).n;
|
|
if(n){total+=n;text+=", "+n+" "+t+" row(s)"}
|
|
}
|
|
process.stdout.write(total+"\t"+text+"\n");
|
|
}
|
|
}finally{db.close()}'
|
|
|
|
# Sets UNCLEAN and returns 0 when a -wal or -shm sits beside the store. An immutable read would
|
|
# skip that WAL and could give a stale answer, so the caller refuses store-unclean and leaves the
|
|
# store as it is. Called directly, never inside $(...), so UNCLEAN reaches the caller.
|
|
store_unclean() {
|
|
local f
|
|
for f in "$STORE-wal" "$STORE-shm"; do
|
|
if [ -e "$f" ] || [ -L "$f" ]; then UNCLEAN="$f"; return 0; fi
|
|
done
|
|
return 1
|
|
}
|
|
# A read of the stopped store: version, or count STAMP TABLES. Call store_unclean first.
|
|
store_read() { node -e "$STORE_JS" -- "$1" "$STORE" "${2:-}" "${3:-}"; }
|
|
|
|
# No writer.lock, and no process with a file open under <dataRoot>/bus/ (stopped-check.sh's
|
|
# test). One find over the /proc fd links, not a readlink per link: this host has about 16,000.
|
|
# find's status is ignored, since other users' fd dirs are unreadable; so were they in the loop.
|
|
stopped() {
|
|
[ ! -e "$DATA/bus/writer.lock" ] || return 1
|
|
local pat
|
|
pat="$(sed 's/[][*?\\]/\\&/g' <<< "$DATA/bus")/*"
|
|
[ -z "$(find /proc/[0-9]*/fd -mindepth 1 -maxdepth 1 -lname "$pat" -print -quit 2>/dev/null || true)" ]
|
|
}
|
|
|
|
sums() { (cd "$1" && find . -type f ! -name SHA256SUMS ! -name steps.log -print0 | sort -z | xargs -0 -r sha256sum); }
|
|
sum_of() { awk -v f="./$2" '$2 == f { print $1 }' "$1/SHA256SUMS"; }
|
|
|
|
# The backup of the stopped store (backup.sh's steps). Only bus.sqlite: a -wal or -shm was
|
|
# already refused as store-unclean. Every activate with a store takes one (row 185). Sets BACKUP
|
|
# once the backup is whole, for the log line; TAKEN is set by the caller only before a migration.
|
|
backup_store() {
|
|
local o="$DATA/backups/stack/$ACT"
|
|
(
|
|
umask 077
|
|
mkdir -p "$DATA/backups/stack" &&
|
|
mkdir -m 700 "$o" "$o/bus" &&
|
|
(cd "$DATA/bus" && sha256sum bus.sqlite) > "$o/SOURCE.sha256" &&
|
|
cp -p "$STORE" "$o/bus/" &&
|
|
(cd "$o/bus" && sha256sum bus.sqlite) > "$o/SHA256SUMS" &&
|
|
cmp -s "$o/SOURCE.sha256" "$o/SHA256SUMS" &&
|
|
stat -c '%n %a %U %s %y' "$STORE" > "$o/MODES.txt" &&
|
|
ls -1A "$DATA/bus" > "$o/LISTING.txt" &&
|
|
echo "$DATA" > "$o/DATA_ROOT" &&
|
|
now_ms > "$o/STAMP" &&
|
|
echo "$STORE_SCHEMA" > "$o/SCHEMA" &&
|
|
echo "$RS" > "$o/ROLLBACK_SET"
|
|
) || return 1
|
|
stopped || return 1
|
|
BACKUP="$ACT"
|
|
say "backup: $o"
|
|
}
|
|
|
|
# Puts backup $1 back as restore.sh does: the current bus/ moves to $POST, nothing is deleted.
|
|
# Sets RESTORED once the old bus/ has moved, so the caller knows the store was swapped.
|
|
restore_store() {
|
|
local o="$DATA/backups/stack/$1"
|
|
[ "$(cat "$o/DATA_ROOT" 2>/dev/null)" = "$DATA" ] || return 1
|
|
stopped || return 1
|
|
(cd "$o/bus" && sha256sum -c --quiet ../SHA256SUMS) || return 1
|
|
[ ! -e "$POST" ] || return 1
|
|
mv "$DATA/bus" "$POST" || return 1
|
|
RESTORED=1
|
|
(umask 077; mkdir -m 700 "$DATA/bus") || return 1
|
|
cp -p "$o"/bus/* "$DATA/bus/" || return 1
|
|
(cd "$DATA/bus" && sha256sum -c --quiet "$o/SHA256SUMS") || return 1
|
|
[ "$(ls -1A "$DATA/bus")" = "$(ls -1A "$o/bus")" ] || return 1
|
|
say "restored $o; the store from before the restore is at $POST"
|
|
}
|
|
|
|
# --- The run (item 4) ---
|
|
|
|
# Step 1: the rollback set. A failure here has changed nothing but the set itself.
|
|
capture() {
|
|
mkdir -p -m 0700 "$STATE/stack-rollback" 2>/dev/null || return 1
|
|
mkdir -m 0700 "$RS" 2>/dev/null || return 1
|
|
STEPS="$RS/steps.log"
|
|
local u d
|
|
for u in "$BUS_TEMPLATE" "$CONSOLE_UNIT"; do
|
|
[ -f "$UNIT_DIR/$u" ] || { say "capture: no $UNIT_DIR/$u"; return 1; }
|
|
cp -p "$UNIT_DIR/$u" "$RS/$u" || return 1
|
|
done
|
|
# This instance's own directories only; the shared prefix ones are never captured (row 190).
|
|
for d in "$BUS_TEMPLATE.d" "$BUS_UNIT.d" "$CONSOLE_UNIT.d"; do
|
|
[ ! -d "$UNIT_DIR/$d" ] || cp -pr "$UNIT_DIR/$d" "$RS/$d" || return 1
|
|
done
|
|
[ -f "$BIN/$WRAPPER_NAME" ] || { say "capture: no wrapper $BIN/$WRAPPER_NAME"; return 1; }
|
|
cp -p "$BIN/$WRAPPER_NAME" "$RS/wrapper" || return 1
|
|
stat -c %a "$BIN/$WRAPPER_NAME" > "$RS/WRAPPER_MODE" || return 1
|
|
for u in "$BUS_UNIT" "$CONSOLE_UNIT"; do show "$u" > "$RS/show-$u.txt" || return 1; done
|
|
sums "$RS" > "$RS/SHA256SUMS" || return 1
|
|
say "captured the rollback set: $RS"
|
|
}
|
|
|
|
# Stops the Console, then the bus, and waits for both and the stopped check, within --timeout.
|
|
stop_both() {
|
|
sc stop "$CONSOLE_UNIT" || return 1
|
|
sc stop "$BUS_UNIT" || return 1
|
|
local deadline=$((SECONDS + TIMEOUT)) u s down
|
|
while :; do
|
|
down=1
|
|
for u in "$CONSOLE_UNIT" "$BUS_UNIT"; do
|
|
s="$(sc is-active "$u" 2>/dev/null || true)"
|
|
case "$s" in inactive|failed) ;; *) down=0 ;; esac
|
|
done
|
|
if [ "$down" = 1 ] && stopped; then say "stopped both units"; return 0; fi
|
|
[ "$SECONDS" -lt "$deadline" ] || return 1
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
start_both() {
|
|
START_EPOCH="$(date +%s)" && START_MS="$(date +%s%3N)" || return 1
|
|
sc start "$BUS_UNIT" && sc start "$CONSOLE_UNIT"
|
|
}
|
|
|
|
# Step 4: installs copy $1 with pm $2, claude $3 and the Console flags after them. Sets FAIL.
|
|
install_stack() {
|
|
local id="$1" pm="$2" claude="$3"; shift 3
|
|
local -a pmflag=()
|
|
[ "$pm" = 0 ] || pmflag=(--pm)
|
|
scripts/bus-service.sh install "${INSTANCE_FLAGS[@]}" --copy "$id" --copies "$COPIES" "${pmflag[@]}" --claude "$claude" >> "$STEPS" 2>&1 || { FAIL=install-bus; return 1; }
|
|
scripts/webui-service.sh install "${INSTANCE_FLAGS[@]}" --copy "$id" --copies "$COPIES" "$@" >> "$STEPS" 2>&1 || { FAIL=install-console; return 1; }
|
|
scripts/stack-copy.sh wrapper "$id" "${INSTANCE_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" >> "$STEPS" 2>&1 || { FAIL=wrapper; return 1; }
|
|
say "installed $id"
|
|
}
|
|
|
|
# --- The health gate (item 5) ---
|
|
|
|
# One pass of every check against copy $1 with pm $2. Sets GATE to the first failing code and
|
|
# GATE_WHY to what it saw.
|
|
gate_once() {
|
|
local copy="$COPIES/$1" pm="$2" main out
|
|
GATE=units-inactive GATE_WHY=""
|
|
sc is-active --quiet "$BUS_UNIT" && sc is-active --quiet "$CONSOLE_UNIT" || return 1
|
|
if [ "$UNITS" = external ]; then
|
|
# r6 N4: root started the units, so this start's time is the bus unit's, in whole seconds,
|
|
# at or after FLOOR. It is a gate code, so polling goes on: Restart=on-failure can bring a
|
|
# fresh start.
|
|
GATE=stale-start
|
|
out="$(sc show --timestamp=unix -p ActiveEnterTimestamp "$BUS_UNIT" 2>/dev/null | sed -n 's/^ActiveEnterTimestamp=@//p')" || return 1
|
|
[[ "$out" =~ ^[0-9]{1,12}$ ]] || { GATE_WHY="ActiveEnterTimestamp is unreadable"; return 1; }
|
|
[ "$out" -ge "$FLOOR" ] || { GATE_WHY="the bus started at @$out, before @$FLOOR"; return 1; }
|
|
START_MS=$((out * 1000))
|
|
fi
|
|
GATE=bus-status
|
|
main="$(sc show "$BUS_UNIT" -p MainPID | sed -n 's/^MainPID=//p')" || return 1
|
|
out="$("$copy/scripts/mosaic" bus status --json 2>/dev/null)" || { GATE_WHY="bus status failed"; return 1; }
|
|
# Every `bus start` writes boot.at (packages/cli/src/boot-status.mjs), so a value from before
|
|
# this start means the host on the socket is not the one this run started.
|
|
GATE_WHY="$(node -e '
|
|
const [b,main,since]=process.argv.slice(2),s=JSON.parse(process.argv[1]);
|
|
const why=!s.host?"no host":s.host.business!==b?`host for ${s.host.business}`:s.host.live!==true?"host not live":String(s.host.pid)!==main?`host pid ${s.host.pid} is not MainPID ${main}`:!s.boot||s.boot.ok!==true?"boot status not ok":s.boot.business!==b?`boot status for ${s.boot.business}`:!(Date.parse(s.boot.at)>=Number(since))?"boot status is from before this start":"";
|
|
process.stdout.write(why);' -- "$out" "$BUSINESS" "$main" "$START_MS" 2>&1)" || { GATE_WHY="bus status unreadable"; return 1; }
|
|
[ -z "$GATE_WHY" ] || return 1
|
|
GATE=healthz
|
|
node -e '
|
|
fetch(`http://127.0.0.1:${process.argv[1]}/healthz`,{signal:AbortSignal.timeout(5000)}).then((r)=>process.exit(r.status===200?0:1),()=>process.exit(1));' -- "$(console_port)" || return 1
|
|
GATE=copy-check
|
|
out="$(scripts/stack-copy.sh check "${INSTANCE_FLAGS[@]}" "${UNITS_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>/dev/null)" || return 1
|
|
# check passes when the three agree on one copy; this run needs it to be the target (row 166 item 3).
|
|
out="$(sed -n 's/^bus: //p' <<< "$out")"
|
|
[ "$out" = "$copy" ] || { GATE_WHY="the bus, Console and CLI agree on $out, not $copy"; return 1; }
|
|
if [ "$pm" = 1 ]; then
|
|
GATE=pm-launch-unconfirmed
|
|
if [ "$UNITS" = user ]; then pm_launched "$copy" || return 1; else pm_external "$1" || return 1; fi
|
|
fi
|
|
GATE=""
|
|
}
|
|
|
|
# Ruling 2 (sage-0736, sage-0738). The PM's run is the one named on the single
|
|
# `launched <instance>: run <run>, <harness> <model>` line that `bus start --pm` prints
|
|
# (packages/cli/src/cli.mjs) to the unit's journal since this start. The gate depends on that
|
|
# wording: if it changes, this check refuses rather than passes. --since takes @<epoch seconds>,
|
|
# because this host's journalctl reads a bare time as local time.
|
|
pm_launched() {
|
|
local journal run list
|
|
journal="$(jctl -u "$BUS_UNIT" --since "@$START_EPOCH" -o cat 2>/dev/null)" || { GATE_WHY="journal read failed"; return 1; }
|
|
local -a lines=()
|
|
mapfile -t lines < <(grep '^launched ' <<< "$journal" || true)
|
|
[ "${#lines[@]}" -gt 0 ] || { GATE_WHY="no launched line since the start"; return 1; }
|
|
[ "${#lines[@]}" = 1 ] || { GATE_WHY="${#lines[@]} launched lines since the start"; return 1; }
|
|
[[ "${lines[0]}" =~ ^launched\ ([^[:space:]]+):\ run\ ([^[:space:]]+),\ ]] || { GATE_WHY="the launched line does not parse: ${lines[0]}"; return 1; }
|
|
run="${BASH_REMATCH[2]}"
|
|
list="$("$1/scripts/mosaic" launches list --json 2>/dev/null)" || { GATE_WHY="launches list failed"; return 1; }
|
|
GATE_WHY="$(node -e '
|
|
const [list,run,b,since]=process.argv.slice(1),s=JSON.parse(list).find((x)=>x.run===run);
|
|
process.stdout.write(!s?`run ${run} is not in launches list`:s.business!==b?`run ${run} is for ${s.business}`:s.live!==true?`run ${run} is not live`:!(Date.parse(s.startedAt)>=Number(since))?`run ${run} started before this start`:"");' -- "$list" "$run" "$BUSINESS" "$START_MS" 2>&1)" || { GATE_WHY="launches list unreadable"; return 1; }
|
|
[ -z "$GATE_WHY" ]
|
|
}
|
|
|
|
# r6 section 4: the PM check under --units external, where the bus unit's journal is the system
|
|
# journal and unreadable here. It passes when exactly one live launches list entry is for the
|
|
# PM's instance (launch.by, packages/cli/src/launcher.mjs:675), and that entry is for BUSINESS and
|
|
# started at or after this start. A run the PM starts for another role has another instance.
|
|
pm_external() {
|
|
local by list
|
|
by="$(plugins launchby "$1" 2>&1)" || { GATE_WHY="cannot read launch.by: $by"; return 1; }
|
|
list="$("$COPIES/$1/scripts/mosaic" launches list --json 2>/dev/null)" || { GATE_WHY="launches list failed"; return 1; }
|
|
GATE_WHY="$(node -e '
|
|
const [list,by,b,since]=process.argv.slice(1),l=JSON.parse(list).filter((x)=>x.instance===by&&x.live===true),s=l[0];
|
|
process.stdout.write(l.length===0?`no live launch of ${by}`:l.length>1?`${l.length} live launches of ${by}`:s.business!==b?`the launch of ${by} is for ${s.business}`:!(Date.parse(s.startedAt)>=Number(since))?`run ${s.run} started before this start`:"");' -- "$list" "$by" "$BUSINESS" "$START_MS" 2>&1)" || { GATE_WHY="launches list unreadable"; return 1; }
|
|
[ -z "$GATE_WHY" ]
|
|
}
|
|
|
|
# Polls every second until every check passes or --timeout runs out.
|
|
gate() {
|
|
local deadline=$((SECONDS + TIMEOUT))
|
|
while :; do
|
|
if gate_once "$1" "$2"; then say "gate passed on $1"; return 0; fi
|
|
[ "$SECONDS" -lt "$deadline" ] || { say "gate failed on $1: $GATE${GATE_WHY:+ ($GATE_WHY)}"; return 1; }
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
# --- Rollback inside the run (item 6) ---
|
|
|
|
# Puts back the captured unit files, drop-ins and wrapper, checked against SHA256SUMS.
|
|
put_back() {
|
|
local u f rel tmp
|
|
for u in "$BUS_TEMPLATE" "$CONSOLE_UNIT"; do
|
|
tmp="$(mktemp "$UNIT_DIR/.$u.XXXXXX")" && cp -p "$RS/$u" "$tmp" && mv -f "$tmp" "$UNIT_DIR/$u" || return 1
|
|
done
|
|
for u in "$BUS_TEMPLATE.d" "$BUS_UNIT.d" "$CONSOLE_UNIT.d"; do
|
|
[ -d "$RS/$u" ] || continue
|
|
while IFS= read -r -d '' f; do
|
|
rel="${f#"$RS/$u/"}"
|
|
! cmp -s "$f" "$UNIT_DIR/$u/$rel" 2>/dev/null || continue
|
|
mkdir -p "$(dirname "$UNIT_DIR/$u/$rel")" || return 1
|
|
tmp="$(mktemp "$UNIT_DIR/$u/.restore.XXXXXX")" && cp -p "$f" "$tmp" && mv -f "$tmp" "$UNIT_DIR/$u/$rel" || return 1
|
|
done < <(find "$RS/$u" -type f -print0)
|
|
done
|
|
tmp="$(mktemp "$BIN/.$WRAPPER_NAME.XXXXXX")" && cp -p "$RS/wrapper" "$tmp" && chmod "$(cat "$RS/WRAPPER_MODE")" "$tmp" && mv -f "$tmp" "$BIN/$WRAPPER_NAME" || return 1
|
|
for u in "$BUS_TEMPLATE" "$CONSOLE_UNIT"; do
|
|
[ "$(sha256sum < "$UNIT_DIR/$u" | cut -d' ' -f1)" = "$(sum_of "$RS" "$u")" ] || return 1
|
|
done
|
|
[ "$(sha256sum < "$BIN/$WRAPPER_NAME" | cut -d' ' -f1)" = "$(sum_of "$RS" wrapper)" ] || return 1
|
|
[ "$(stat -c %a "$BIN/$WRAPPER_NAME")" = "$(cat "$RS/WRAPPER_MODE")" ] || return 1
|
|
sc daemon-reload || return 1
|
|
say "put back the unit files and the wrapper from $RS"
|
|
}
|
|
|
|
# A failure after step 2: put the pointer's stack back and gate it. $1 is the run's code. Only a
|
|
# backup this run took before a migration (TAKEN) is restored; a same-schema activate's backup
|
|
# and a rollback's --restore backup never are.
|
|
rollback_in_run() {
|
|
trap - INT TERM HUP
|
|
local code="$1" rcode=""
|
|
say "rolling back: $code"
|
|
if ! stop_both; then rcode=stop-timeout
|
|
elif ! put_back; then rcode=put-back
|
|
elif [ -n "$TAKEN" ] && ! restore_store "$TAKEN"; then rcode=restore
|
|
elif ! start_both; then rcode=start
|
|
elif ! gate "$P_COPY" "$P_PM"; then rcode="$GATE"
|
|
fi
|
|
if [ -z "$rcode" ]; then
|
|
log_line result gate-failed code "$code" rollback ok
|
|
echo "stack-release: $code; rolled back to $P_COPY, which passed the gate" >&2
|
|
exit 1
|
|
fi
|
|
rollback_failed "$code" "$rcode"
|
|
}
|
|
|
|
# Ruling 4 (T14b): once rollback --restore has swapped the store, any later failure skips item 6,
|
|
# since putting the newer store back would be a second swap. Before the swap, item 6 runs.
|
|
after_fail() {
|
|
[ -z "$RESTORED" ] || rollback_failed "$1" not-attempted
|
|
rollback_in_run "$1"
|
|
}
|
|
|
|
# Stop both units, print both results, log rollback-failed, exit 2. No retry.
|
|
rollback_failed() {
|
|
trap - INT TERM HUP
|
|
sc stop "$CONSOLE_UNIT" || true
|
|
sc stop "$BUS_UNIT" || true
|
|
say "rollback-failed: run $1, rollback $2"
|
|
log_line result rollback-failed code "$1" rollback failed rollbackCode "$2"
|
|
echo "stack-release: rollback-failed: the run failed with $1 and the rollback with $2; both units are stopped. Rollback set: $RS${BACKUP:+, backup $DATA/backups/stack/$BACKUP}${RESTORED:+, post-attempt store $POST}" >&2
|
|
exit 2
|
|
}
|
|
|
|
# --- Refusals (item 3) ---
|
|
|
|
# confirm and revert act on the open activation by design, so they take the lock alone (r6 section 4).
|
|
lock_only() {
|
|
mkdir -p "$STATE"
|
|
exec 9> "$STATE/stack-release.lock"
|
|
flock -n 9 || refuse run-locked "another stack-release run holds $STATE/stack-release.lock"
|
|
}
|
|
|
|
lock() {
|
|
lock_only
|
|
local open
|
|
open="$(interrupted | paste -sd' ' -)"
|
|
[ -z "$open" ] || refuse interrupted-run "activation $open started and has no final line; Jason or Mos settles it (status shows it)"
|
|
}
|
|
|
|
instance_busy() {
|
|
local out unit load state rest
|
|
out="$(sc list-units --all --no-legend --plain "$BUS_GLOB" 2>/dev/null)" || refuse instance-busy "$SC_NAME list-units failed, so the bus state is unknown"
|
|
while read -r unit load state rest; do
|
|
[[ "$unit" == $BUS_GLOB ]] || continue
|
|
if [ "$unit" = "$BUS_UNIT" ]; then
|
|
case "$state" in activating|deactivating|reloading) refuse instance-busy "$unit is $state" ;; esac
|
|
else
|
|
case "$state" in inactive|failed) ;; *) refuse instance-busy "$unit is $state; the copy and wrapper switch reach it too" ;; esac
|
|
fi
|
|
done <<< "$out"
|
|
}
|
|
|
|
# install_guard's drop-in scan (bus-service.sh), done before anything stops. A path it can't read
|
|
# refuses here, as install_guard would refuse it after the stop (row 166 item 1).
|
|
drop_in_override() {
|
|
local paths directory dropdir file content glob
|
|
local -a dropdirs
|
|
paths="$(analyze unit-paths 2>/dev/null)" && [ -n "$paths" ] || refuse drop-in-override "systemd unit paths are unknown"
|
|
while IFS= read -r directory; do
|
|
[ -n "$directory" ] || continue
|
|
case "$directory" in /*) ;; *) refuse drop-in-override "invalid systemd unit path $directory" ;; esac
|
|
[ -e "$directory" ] || [ -L "$directory" ] || continue
|
|
[ -d "$directory" ] && [ -r "$directory" ] && [ -x "$directory" ] || refuse drop-in-override "cannot inspect unit path $directory"
|
|
dropdirs=()
|
|
for glob in "${BUS_DROPIN_GLOBS[@]}"; do dropdirs+=("$directory"/$glob); done
|
|
for dropdir in "${dropdirs[@]}"; do
|
|
[ -e "$dropdir" ] || [ -L "$dropdir" ] || continue
|
|
[ -d "$dropdir" ] && [ -r "$dropdir" ] && [ -x "$dropdir" ] || refuse drop-in-override "cannot inspect drop-ins $dropdir"
|
|
for file in "$dropdir"/*.conf; do
|
|
[ -e "$file" ] || [ -L "$file" ] || continue
|
|
content="$(cat "$file" 2>/dev/null)" || refuse drop-in-override "cannot read drop-in $file"
|
|
if grep -Eq '^[[:space:]]*(ExecStart|WorkingDirectory)[[:space:]]*=' <<< "$content"; then
|
|
refuse drop-in-override "$file overrides ExecStart or WorkingDirectory; the operator removes it as in TOOLS.md step 4"
|
|
fi
|
|
done
|
|
done
|
|
done <<< "$paths"
|
|
}
|
|
|
|
# unit-flags-drift: the effective units against the pointer, and --pm against the pointer's pm.
|
|
# --pm-after-harness stands in for --pm (row 169 item 1). A pending pointer (pm false, unit
|
|
# without --pm) matches its unit; only pm-on and rollback get this far with one, since activate
|
|
# refuses pm-pending first and adopt refuses already-adopted.
|
|
flags_drift() {
|
|
local d=""
|
|
[ "$U_PM" = "$P_PM" ] || d+=$'\n'" pm: pointer $P_PM, unit $U_PM"
|
|
[ "$U_CLAUDE" = "$P_CLAUDE" ] || d+=$'\n'" claude: pointer $P_CLAUDE, unit $U_CLAUDE"
|
|
[ "${P_CONSOLE[*]}" = "${U_CONSOLE[*]}" ] || d+=$'\n'" console: pointer ${P_CONSOLE[*]}, unit ${U_CONSOLE[*]}"
|
|
if [ "$AFTER_HARNESS" = 1 ]; then
|
|
[ "$P_PM" = 1 ] || d+=$'\n'" pm: pointer $P_PM, --pm-after-harness needs a pointer with pm 1"
|
|
else
|
|
# Under --units external the PM is a flag in root's unit, so there is no --pm to compare.
|
|
[ "$UNITS" != user ] || [ "$OP" != activate ] || [ "$WANT_PM" = "$P_PM" ] || d+=$'\n'" pm: pointer $P_PM, --pm given $WANT_PM"
|
|
fi
|
|
[ -z "$d" ] || refuse unit-flags-drift "the units differ from the pointer; settle it first:$d"
|
|
}
|
|
|
|
# Row 178: --proof-header and --proof-file are a pair in P_CONSOLE, which proof_for_target (row
|
|
# 187) carries unchanged, replaces or drops; never read the proof value.
|
|
# Resolve desired flags independently of the pointer and effective units, so a hand-edited
|
|
# unit still refuses drift and a failed attempt can restore exactly the captured stack.
|
|
# Read the target's own Console, because old copies cannot start with the new flag.
|
|
console_for_target() {
|
|
local supported=0 main=7332 inherited="" i
|
|
local -a without_tester=()
|
|
sso_base
|
|
grep -q -- '--tester-port' "$COPIES/$TO/packages/webui/src/cli.mjs" && supported=1
|
|
for ((i = 0; i < ${#BASE_CONSOLE[@]}; i++)); do
|
|
if [ "${BASE_CONSOLE[i]}" = --tester-port ]; then
|
|
i=$((i + 1)); inherited="${BASE_CONSOLE[i]:-}"
|
|
else
|
|
without_tester+=("${BASE_CONSOLE[i]}")
|
|
if [ "${BASE_CONSOLE[i]}" = --port ]; then main="${BASE_CONSOLE[i+1]:-7332}"; fi
|
|
fi
|
|
done
|
|
RUN_CONSOLE=("${BASE_CONSOLE[@]}")
|
|
DROP_TESTER=""
|
|
if [ -n "$TESTER_PORT" ]; then
|
|
# --plan lists both with its other refusals (row 180); outside it they end the run.
|
|
if ! valid_tester_port "$TESTER_PORT" "$main"; then
|
|
[ "$PLAN" = 1 ] || usage
|
|
local other=""
|
|
[ "$main" = 7332 ] || other=" and the Console's --port $main"
|
|
[ "${#INSTANCE_RESERVED[@]}" = 0 ] || other+=" and instance $MOSAIC_INSTANCE's reserved ${INSTANCE_RESERVED[*]}"
|
|
check_refuse tester-port-invalid "--tester-port $TESTER_PORT is not a whole port from 1024 to 65535 other than 7332$other"
|
|
fi
|
|
[ "$supported" = 1 ] || check_refuse tester-port-target "$TO's Console does not support --tester-port"
|
|
RUN_CONSOLE=("${without_tester[@]}" --tester-port "$TESTER_PORT")
|
|
elif [ -n "$inherited" ] && [ "$supported" = 0 ]; then
|
|
RUN_CONSOLE=("${without_tester[@]}")
|
|
DROP_TESTER="--tester-port $inherited"
|
|
fi
|
|
if [ "$SSO" = 1 ]; then sso_for_target; else proof_for_target; fi
|
|
send_for_target
|
|
sso_checks
|
|
}
|
|
|
|
sso_supported() { grep -q -- '--sso' "$COPIES/$TO/packages/webui/src/cli.mjs"; }
|
|
has_flag() { local f="$1"; shift; [[ " $* " == *" $f "* ]]; }
|
|
|
|
# Row 200 (#1698, R1): a rollback from an --sso pointer to a copy whose Console has no --sso. The
|
|
# pointer no longer holds the preview set that Console ran with, so the flags come from the rollback
|
|
# set of the activation that moved the stack onto the pointer's copy from the rollback target. A
|
|
# same-copy re-pin after it has a set of its own, holding the same copy's earlier flags, so the
|
|
# pointer's own activation is the wrong set (sage-1389). The set's show-$CONSOLE_UNIT.txt is
|
|
# the Console unit as it was before that run, checked against the set's SHA256SUMS. Sets
|
|
# BASE_CONSOLE (the flags console_for_target starts from), SSO_PLAN and SSO_STEP. SSO_NO_TARGET
|
|
# skips sso_checks after a refusal here, so --plan lists that refusal alone.
|
|
sso_base() {
|
|
BASE_CONSOLE=("${P_CONSOLE[@]}") SSO_PLAN="" SSO_STEP="" SSO_NO_TARGET=""
|
|
has_flag --sso "${P_CONSOLE[@]}" || return 0
|
|
[ "$SSO" = 1 ] || SSO_PLAN="sso: carried"
|
|
[ "$OP" = rollback ] && ! sso_supported || return 0
|
|
local f="show-$CONSOLE_UNIT.txt" argv="" why="" act="" from="" set
|
|
read -r act from <<< "$(from_log arrival || true)"
|
|
set="$STATE/stack-rollback/$act"
|
|
if [ "$from" != "$TO" ]; then why="the log names no activation that moved the stack from $TO onto $P_COPY"
|
|
elif ! [[ "$act" =~ ^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}(-[1-9])?$ ]]; then why="the activation $act that moved the stack onto $P_COPY names no rollback set"
|
|
elif [ ! -f "$set/$f" ] || [ ! -f "$set/SHA256SUMS" ]; then why="$set has no $f and SHA256SUMS"
|
|
elif [ "$(sum_of "$set" "$f")" != "$(sha256sum < "$set/$f" | cut -d' ' -f1)" ]; then why="$set/$f does not match the set's SHA256SUMS"
|
|
else
|
|
argv="$(argv_of "$(cat "$set/$f")")"
|
|
if ! unquoted "$argv" || ! serve_flags "$argv"; then why="$set/$f holds no Console argv this script can read"
|
|
elif has_flag --sso "${SERVE_FLAGS[@]}"; then why="the Console in $set/$f already ran with --sso"
|
|
fi
|
|
fi
|
|
if [ -n "$why" ]; then
|
|
SSO_NO_TARGET=1
|
|
check_refuse sso-rollback-flags "$TO's Console does not support --sso, and the Console flags it ran with cannot be read: $why"
|
|
return 0
|
|
fi
|
|
# --send and --tester-port follow the pointer, so a --no-send or a tester port set after that
|
|
# activation stays (row 189's rule that rollback carries --send off).
|
|
local i
|
|
BASE_CONSOLE=()
|
|
for ((i = 0; i < ${#SERVE_FLAGS[@]}; i++)); do
|
|
case "${SERVE_FLAGS[i]}" in
|
|
--send) ;;
|
|
--tester-port) i=$((i + 1)) ;;
|
|
*) BASE_CONSOLE+=("${SERVE_FLAGS[i]}") ;;
|
|
esac
|
|
done
|
|
for ((i = 0; i < ${#P_CONSOLE[@]}; i++)); do
|
|
case "${P_CONSOLE[i]}" in
|
|
--send) BASE_CONSOLE+=(--send) ;;
|
|
--tester-port) i=$((i + 1)); BASE_CONSOLE+=(--tester-port "${P_CONSOLE[i]:-}") ;;
|
|
esac
|
|
done
|
|
SSO_PLAN="sso: dropped; $TO's Console does not support --sso, so the rollback takes the Console flags from $set/$f, with --send and --tester-port as the pointer has them"
|
|
SSO_STEP="drop --sso; $TO's Console does not support it, so the Console flags come from $set/$f"
|
|
}
|
|
|
|
# Row 200 (#1698): activate --sso drops the preview set and the proof pair, which serve refuses beside
|
|
# it, and puts --sso where webui-service.sh renders it: after the other flags, before --send and
|
|
# --tester-port. Sets SSO_DROP (the plan's dropped preview flags), PROOF_PLAN and PROOF_STEP.
|
|
sso_for_target() {
|
|
local header="" file="" i
|
|
local -a kept=()
|
|
SSO_DROP=""
|
|
for ((i = 0; i < ${#RUN_CONSOLE[@]}; i++)); do
|
|
case "${RUN_CONSOLE[i]}" in
|
|
--public-host | --auth-header | --allow-user) SSO_DROP+=" ${RUN_CONSOLE[i]} ${RUN_CONSOLE[i + 1]:-}"; i=$((i + 1)) ;;
|
|
--proof-header) i=$((i + 1)); header="${RUN_CONSOLE[i]:-}" ;;
|
|
--proof-file) i=$((i + 1)); file="${RUN_CONSOLE[i]:-}" ;;
|
|
--sso) ;;
|
|
*) kept+=("${RUN_CONSOLE[i]}") ;;
|
|
esac
|
|
done
|
|
RUN_CONSOLE=()
|
|
for ((i = 0; i < ${#kept[@]}; i++)); do
|
|
case "${kept[i]}" in --send | --tester-port) break ;; esac
|
|
RUN_CONSOLE+=("${kept[i]}")
|
|
done
|
|
RUN_CONSOLE+=(--sso "${kept[@]:i}")
|
|
PROOF_PLAN="proof: off" PROOF_STEP=""
|
|
if [ -n "$header" ]; then
|
|
PROOF_PLAN="proof: dropped; --sso drops header $header"
|
|
PROOF_STEP="drop --proof-header $header --proof-file $file; --sso was given"
|
|
fi
|
|
}
|
|
|
|
# Row 200 item 2: whenever the run's Console flags hold --sso, the checks row 192's --sso start runs,
|
|
# before anything stops: a target Console with --sso, console.sso in config.json as the target's own
|
|
# checkConsole reads it, and each provider's slot file. The slot gets lstat, stat and realpath
|
|
# only; this script never opens it. Also sets the plan's sso: line for activate --sso.
|
|
SSO_JS='
|
|
const {readFileSync}=require("node:fs");const {pathToFileURL}=require("node:url");
|
|
const [copy,file]=process.argv.slice(1);
|
|
(async()=>{
|
|
const {checkConsole}=await import(pathToFileURL(`${copy}/packages/webui/src/sso-config.mjs`).href);
|
|
let doc;
|
|
// A JSON.parse message can quote the text, so it is not passed on.
|
|
try{doc=JSON.parse(readFileSync(file,"utf8"))}catch{throw new Error(`cannot read ${file} as JSON`)}
|
|
if(!doc?.console?.sso)throw new Error("there is no console.sso in the system config");
|
|
checkConsole(doc.console);
|
|
for(const p of doc.console.sso.providers)process.stdout.write(`${p.id} ${p.clientSecret.slot}\n`);
|
|
})().catch((e)=>{process.stdout.write(String(e?.message??e).split("\n")[0]);process.exit(1)});'
|
|
sso_checks() {
|
|
has_flag --sso "${RUN_CONSOLE[@]}" || return 0
|
|
if [ "$SSO" = 1 ]; then
|
|
local t carried="" dropped="${SSO_DROP# }"
|
|
for t in "${RUN_CONSOLE[@]}"; do [ "$t" = --sso ] || carried+="${carried:+ }$t"; done
|
|
SSO_PLAN="sso: on; drops ${dropped:-nothing}; carries ${carried:-nothing}"
|
|
SSO_STEP="turn on --sso${SSO_DROP:+; drop$SSO_DROP}"
|
|
fi
|
|
[ -z "$SSO_NO_TARGET" ] || return 0
|
|
sso_supported || { check_refuse sso-target "$TO's Console does not support --sso"; return 0; }
|
|
local cfg="${MOSAIC_CONFIG:-$HOME/.config/mosaic-dev/config.json}" out id slot path why
|
|
# N4 (webby-0019, sage-1403): a pre-phase-1 loader refuses a console key, so with the key not yet
|
|
# written and the stack on another copy, the first step is the re-pin without --sso.
|
|
local first=""
|
|
if ! out="$(node -e "$SSO_JS" -- "$COPIES/$TO" "$cfg" 2>&1)"; then
|
|
out="${out%%$'\n'*}"
|
|
[ "$out" != "there is no console.sso in the system config" ] || [ "$P_COPY" = "$TO" ] ||
|
|
first="; activate $TO without --sso first, then write console.sso, then run activate $TO --sso"
|
|
check_refuse sso-config "$TO's Console would refuse its --sso start: $out$first"
|
|
return 0
|
|
fi
|
|
while read -r id slot; do
|
|
path="$(dirname "$cfg")/secrets/mosaic-stack/$slot"
|
|
why="$(slot_problem "$path")"
|
|
[ -z "$why" ] || check_refuse sso-slot "console.sso provider $id client secret slot $path $why"
|
|
done <<< "$out"
|
|
}
|
|
|
|
# Why a slot file can't serve, or nothing: readSlot's checks in packages/webui/src/sso.mjs, in its
|
|
# order, from lstat, stat and realpath only, then a size of 0 for its one-non-empty-line rule.
|
|
slot_problem() {
|
|
local st uid mode size type real copies repo
|
|
[ -e "$1" ] || [ -L "$1" ] || { echo "is missing"; return; }
|
|
[ ! -L "$1" ] || { echo "is a symbolic link"; return; }
|
|
st="$(stat -c '%u:%a:%s:%F' -- "$1" 2>/dev/null)" || { echo "cannot be read with stat"; return; }
|
|
uid="${st%%:*}" st="${st#*:}"
|
|
mode="${st%%:*}" st="${st#*:}"
|
|
size="${st%%:*}" type="${st#*:}"
|
|
case "$type" in "regular file" | "regular empty file") ;; *) echo "is not a regular file"; return ;; esac
|
|
[ "$uid" = "$(id -u)" ] || { echo "is owned by uid $uid, not $(id -u), the uid this script runs as"; return; }
|
|
(( (8#$mode & 8#077) == 0 )) || { echo "has mode $mode, which gives group or other permission bits"; return; }
|
|
# REPO is $(pwd), which keeps a symlink in the path the script ran by (webby-0028 N1).
|
|
real="$(realpath -e -- "$1")" && copies="$(realpath -m -- "$COPIES")" && repo="$(realpath -m -- "$REPO")" || { echo "cannot be resolved"; return; }
|
|
case "$real" in "$repo" | "$repo"/* | "$copies" | "$copies"/*) echo "is inside the repository or the copies"; return ;; esac
|
|
[ "$size" != 0 ] || echo "is empty"
|
|
}
|
|
|
|
# Row 200 item 3 and the design's phase 2 rollback order: a config.json with a `console` key must be
|
|
# one the target's loader reads, or its bus, Console and seats fail on their next read. Runs the
|
|
# target's own scripts/mosaic-config.mjs validate; its output, the resolved config, is not kept.
|
|
config_checks() {
|
|
local cfg="${MOSAIC_CONFIG:-$HOME/.config/mosaic-dev/config.json}" out loader="$COPIES/$TO/scripts/mosaic-config.mjs" first
|
|
# 0: a console key; 1: no key, or no file; 2: the file doesn't parse, so whether it holds the key
|
|
# is unknown, and the run refuses rather than pass it (webby-0028 N2). load_config already exits on
|
|
# such a file before anything is read or stopped, so this is reached only if the file changes after it.
|
|
local key=0
|
|
node -e 'const fs=require("fs"),f=process.argv[1];if(!fs.existsSync(f))process.exit(1);let d;try{d=JSON.parse(fs.readFileSync(f,"utf8"))}catch{process.exit(2)}
|
|
process.exit(d!==null&&typeof d==="object"&&Object.hasOwn(d,"console")?0:1)' -- "$cfg" 2>/dev/null || key=$?
|
|
[ "$key" != 1 ] || return 0
|
|
if [ "$key" != 0 ]; then
|
|
check_refuse config-console-key "$cfg cannot be read as JSON, so whether $TO's loader reads it is unknown; fix the config first, then run the $OP again"
|
|
return 0
|
|
fi
|
|
if [ ! -f "$loader" ]; then out="it has no scripts/mosaic-config.mjs"
|
|
elif out="$(MOSAIC_CONFIG="$cfg" node "$loader" validate 2>&1 >/dev/null)"; then return 0
|
|
fi
|
|
first="${out%%$'\n'*}"
|
|
check_refuse config-console-key "$cfg holds a console key, and $TO's loader refuses it ($first); remove the key first (restore the config.json backup from the window's step 1), then run the $OP again"
|
|
}
|
|
|
|
# Row 189 (mos-0972): --no-send drops --send from RUN_CONSOLE, so the unit and the pointer both lose
|
|
# it; without the flag the pointer's --send goes on unchanged. Rollback and pm-on never take it, so
|
|
# they carry the pointer as it is. Sets SEND_PLAN (the plan's send: line) and SEND_STEP.
|
|
send_for_target() {
|
|
local had=0 i
|
|
local -a without_send=()
|
|
for ((i = 0; i < ${#RUN_CONSOLE[@]}; i++)); do
|
|
if [ "${RUN_CONSOLE[i]}" = --send ]; then had=1; else without_send+=("${RUN_CONSOLE[i]}"); fi
|
|
done
|
|
SEND_STEP=""
|
|
if [ "$NO_SEND" = 1 ]; then
|
|
RUN_CONSOLE=("${without_send[@]}")
|
|
if [ "$had" = 1 ]; then
|
|
SEND_PLAN="send: dropped; --no-send was given"
|
|
SEND_STEP="drop --send; --no-send was given"
|
|
else
|
|
SEND_PLAN="send: off; --no-send was given and the pointer holds no --send"
|
|
SEND_STEP="--no-send was given and the pointer holds no --send, so there is nothing to drop"
|
|
fi
|
|
elif [ "$had" = 1 ]; then
|
|
SEND_PLAN="send: carried"
|
|
else
|
|
SEND_PLAN="send: off"
|
|
fi
|
|
}
|
|
|
|
# Row 187: the row 178 proof pair in RUN_CONSOLE. A new pair replaces the pointer's and goes last;
|
|
# --no-proof drops it. With neither, a target Console that takes --proof-header carries the pair
|
|
# unchanged. One that doesn't drops it on rollback, the way back, and refuses proof-target on
|
|
# activate, so only --no-proof turns the proof off. Sets PROOF_PLAN (the plan's proof: line) and
|
|
# PROOF_STEP (a steps log line, or empty).
|
|
proof_for_target() {
|
|
local supported=0 had=0 header="" file="" i
|
|
local -a without_proof=()
|
|
grep -q -- '--proof-header' "$COPIES/$TO/packages/webui/src/cli.mjs" && supported=1
|
|
for ((i = 0; i < ${#RUN_CONSOLE[@]}; i++)); do
|
|
case "${RUN_CONSOLE[i]}" in
|
|
--proof-header) had=1; i=$((i + 1)); header="${RUN_CONSOLE[i]:-}" ;;
|
|
--proof-file) had=1; i=$((i + 1)); file="${RUN_CONSOLE[i]:-}" ;;
|
|
*) without_proof+=("${RUN_CONSOLE[i]}") ;;
|
|
esac
|
|
done
|
|
PROOF_PLAN="proof: off" PROOF_STEP=""
|
|
if [ "$NO_PROOF" = 1 ]; then
|
|
RUN_CONSOLE=("${without_proof[@]}")
|
|
if [ "$had" = 1 ]; then
|
|
PROOF_PLAN="proof: dropped; --no-proof drops header $header"
|
|
PROOF_STEP="drop --proof-header $header --proof-file $file; --no-proof was given"
|
|
else
|
|
PROOF_PLAN="proof: off; --no-proof was given and the pointer holds no pair"
|
|
PROOF_STEP="--no-proof was given and the pointer holds no proof pair, so there is nothing to drop"
|
|
fi
|
|
return 0
|
|
fi
|
|
if [ -n "$PROOF_HEADER" ]; then
|
|
# webui-service.sh renders the pair after the preview flags and before --send and
|
|
# --tester-port, and flags_drift compares the pointer with that argv as it stands.
|
|
header="$PROOF_HEADER" file="$PROOF_FILE" RUN_CONSOLE=()
|
|
for ((i = 0; i < ${#without_proof[@]}; i++)); do
|
|
case "${without_proof[i]}" in --send | --tester-port) break ;; esac
|
|
RUN_CONSOLE+=("${without_proof[i]}")
|
|
done
|
|
RUN_CONSOLE+=(--proof-header "$header" --proof-file "$file" "${without_proof[@]:i}")
|
|
PROOF_PLAN="proof: on, header $header, file $file"
|
|
[ "$supported" = 1 ] || check_refuse proof-target "$TO's Console does not support --proof-header"
|
|
elif [ "$had" = 0 ]; then
|
|
return 0
|
|
elif [ "$supported" = 0 ] && [ "$OP" = rollback ]; then
|
|
RUN_CONSOLE=("${without_proof[@]}")
|
|
PROOF_PLAN="proof: dropped; $TO's Console does not support --proof-header, so the rollback drops header $header"
|
|
PROOF_STEP="drop --proof-header $header --proof-file $file; the target Console does not support it"
|
|
return 0
|
|
else
|
|
PROOF_PLAN="proof: carried, header $header"
|
|
[ "$supported" = 1 ] || check_refuse proof-target "$TO's Console does not support --proof-header, which the pointer's Console flags carry; give --no-proof to drop the pair"
|
|
fi
|
|
proof_checks "$header" "$file"
|
|
}
|
|
|
|
# Row 187: proofOptions' own rules (packages/webui/src/serve.mjs), checked before anything stops so
|
|
# the Console can't fail its start on them. The file gets stat only: this script never opens it,
|
|
# and no line it writes holds the value.
|
|
proof_checks() {
|
|
local header="${1,,}" auth="" seen=" " missing="" f i why
|
|
for ((i = 0; i < ${#RUN_CONSOLE[@]}; i++)); do
|
|
seen+="${RUN_CONSOLE[i]} "
|
|
[ "${RUN_CONSOLE[i]}" != --auth-header ] || auth="${RUN_CONSOLE[i + 1]:-}"
|
|
done
|
|
for f in --public-host --auth-header --allow-user; do
|
|
[[ "$seen" == *" $f "* ]] || missing+=" $f"
|
|
done
|
|
[ -z "$missing" ] || check_refuse proof-needs-preview "the proof pair needs --public-host, --auth-header and --allow-user in the Console flags; the run's flags lack$missing"
|
|
if ! [[ "$header" =~ ^[a-z0-9][a-z0-9-]{0,63}$ ]] || [ "$header" = host ] || [ "$header" = origin ] ||
|
|
[ "$header" = x-mosaic-tester ] || [ "$header" = "${auth,,}" ]; then
|
|
check_refuse proof-header-invalid "--proof-header $1 must be a header name of a-z, 0-9 and - (64 at most, lowercased) other than host, origin, x-mosaic-tester and the --auth-header value"
|
|
fi
|
|
why="$(proof_file_problem "$2")"
|
|
[ -z "$why" ] || check_refuse proof-file-invalid "--proof-file $2 $why"
|
|
}
|
|
|
|
# Why the proof file can't serve, or nothing. lstat, stat and realpath only; the order follows
|
|
# webui-service.sh render, which refuses the same files.
|
|
proof_file_problem() {
|
|
local st uid mode type real copies repo
|
|
case "$1" in /*) ;; *) echo "is not an absolute path"; return ;; esac
|
|
case "$1" in *[!A-Za-z0-9/._+-]*) echo "holds a character outside [A-Za-z0-9/._+-]"; return ;; esac
|
|
[ ! -L "$1" ] || { echo "is a symlink"; return; }
|
|
st="$(stat -c '%u:%a:%F' -- "$1" 2>/dev/null)" || { echo "cannot be found"; return; }
|
|
uid="${st%%:*}" st="${st#*:}"
|
|
mode="${st%%:*}" type="${st#*:}"
|
|
case "$type" in "regular file" | "regular empty file") ;; *) echo "is not a regular file"; return ;; esac
|
|
real="$(realpath -e -- "$1")" && copies="$(realpath -m -- "$COPIES")" && repo="$(realpath -m -- "$REPO")" || { echo "cannot be resolved"; return; }
|
|
case "$real" in "$repo" | "$repo"/* | "$copies" | "$copies"/*) echo "is inside the repository or the copies"; return ;; esac
|
|
[ "$mode" = 600 ] || [ "$mode" = 400 ] || { echo "has mode $mode, not 600 or 400"; return; }
|
|
[ "$uid" = "$(id -u)" ] || echo "is owned by uid $uid, not $(id -u), the uid this script runs as"
|
|
}
|
|
|
|
# Row 190: the target Console's --port (7332 when absent) and --tester-port against every other
|
|
# installed Console unit in UNIT_DIR, the default's and each named instance's.
|
|
ports_unclaimed() {
|
|
local main=7332 i holder
|
|
local -a ports=()
|
|
for ((i = 0; i < ${#RUN_CONSOLE[@]}; i++)); do
|
|
case "${RUN_CONSOLE[i]}" in
|
|
--port) main="${RUN_CONSOLE[i + 1]:-7332}" ;;
|
|
--tester-port) ports+=("${RUN_CONSOLE[i + 1]:-}") ;;
|
|
esac
|
|
done
|
|
for i in "$main" "${ports[@]}"; do
|
|
[[ "$i" =~ ^[0-9]{1,5}$ ]] || continue
|
|
if holder="$(instance_port_holder "$UNIT_DIR" "$i")"; then
|
|
check_refuse port-claimed "port $i is taken by $UNIT_DIR/$holder"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# The item 3 checks after the pointer, shared by activate and rollback, for target $TO.
|
|
precheck() {
|
|
check_target
|
|
console_for_target
|
|
ports_unclaimed
|
|
instance_busy
|
|
drop_in_override
|
|
read_units || refuse unit-flags-drift "cannot read $BUS_UNIT or $CONSOLE_UNIT"
|
|
[ -n "$U_CLAUDE" ] || refuse no-claude-pin "$BUS_UNIT has no MOSAIC_CLAUDE_BIN"
|
|
flags_drift
|
|
}
|
|
|
|
pointer_for_business() {
|
|
read_pointer || refuse no-pointer "no $POINTER; run adopt first"
|
|
FROM="$P_COPY"
|
|
[ "$P_BUSINESS" = "$BUSINESS" ] || refuse business-mismatch "the pointer is for $P_BUSINESS"
|
|
}
|
|
|
|
# The activation id is the second and the target's 12-character prefix. A second run in the same
|
|
# second adds -1 to -9, so ids still sort by time and an id without one is still whole (row 180).
|
|
# Callers hold the lock. Sets ACT and POST from one free suffix: no rollback set, backup or
|
|
# post-attempt store with that name.
|
|
next_activation() {
|
|
local second s
|
|
second="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
for s in "" -1 -2 -3 -4 -5 -6 -7 -8 -9; do
|
|
ACT="$second-${TO:0:12}$s" POST="$DATA/bus.post-attempt-$second$s"
|
|
[ -e "$STATE/stack-rollback/$ACT" ] || [ -e "$DATA/backups/stack/$ACT" ] || [ -e "$POST" ] || return 0
|
|
done
|
|
# Every name is another run's, so the refused line names none and closes no started run.
|
|
ACT="" POST=""
|
|
refuse capture-failed "every activation id for $second-${TO:0:12} is taken; nothing was stopped"
|
|
}
|
|
|
|
new_activation() {
|
|
next_activation
|
|
RS="$STATE/stack-rollback/$ACT"
|
|
}
|
|
|
|
# --- Verbs ---
|
|
|
|
do_adopt() {
|
|
lock
|
|
[ ! -f "$POINTER" ] || { read_pointer; FROM="$P_COPY"; refuse already-adopted "$POINTER exists (copy $P_COPY)"; }
|
|
local out
|
|
out="$(scripts/stack-copy.sh check "${INSTANCE_FLAGS[@]}" "${UNITS_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>&1)" || refuse check-failed "stack-copy.sh check did not pass:"$'\n'"$out"
|
|
TO="$(sed -n 's/^bus: //p' <<< "$out")"
|
|
TO="${TO#"$COPIES/"}"
|
|
read_units || refuse check-failed "cannot read $BUS_UNIT or $CONSOLE_UNIT"
|
|
# Under --units external the claude pin is root's render, so adopt records what the unit has.
|
|
[ "$UNITS" = external ] || [ -n "$U_CLAUDE" ] || refuse no-claude-pin "$BUS_UNIT has no MOSAIC_CLAUDE_BIN"
|
|
target_schema "$TO" || refuse copy-schema-unknown "cannot read SCHEMA_VERSION from $COPIES/$TO/packages/bus/src/store.mjs"
|
|
next_activation
|
|
write_pointer "$TO" "$U_PM" 0 "$U_CLAUDE" "$T_SCHEMA" "${U_CONSOLE[@]}"
|
|
log_line result ok
|
|
echo "adopted $TO for $BUSINESS (pm $U_PM, schema $T_SCHEMA)"
|
|
}
|
|
|
|
# The store's version for --plan, from the store itself: readOnly while the bus holds it (adds no
|
|
# file), immutable when it is stopped and clean. Sets S_SCHEMA and S_FROM. Falls back to the
|
|
# pointer's schema, and says so, when there is no store file or it can't be read.
|
|
plan_store() {
|
|
local src="" why=""
|
|
S_SCHEMA="" S_FROM="" S_CHANGED=""
|
|
if [ ! -e "$STORE" ]; then why="there is no store file"
|
|
elif sc is-active --quiet "$BUS_UNIT"; then
|
|
src="the store the running bus holds"
|
|
S_SCHEMA="$(node -e "$STORE_JS" -- heldversion "$STORE" 2>/dev/null)" || S_SCHEMA=""
|
|
# A bus that stops during the readOnly open can leave -wal or -shm beside the stopped
|
|
# store, so the version read can't be planned on (row 166 item 2).
|
|
sc is-active --quiet "$BUS_UNIT" || S_CHANGED=1
|
|
elif store_unclean; then why="the store was not read: $UNCLEAN is present"
|
|
else
|
|
src="the stopped store"
|
|
S_SCHEMA="$(store_read version 2>/dev/null)" || S_SCHEMA=""
|
|
fi
|
|
if [ -n "$S_CHANGED" ]; then S_SCHEMA=unknown S_FROM="the bus stopped during the read, so the store state changed; run --plan again"
|
|
elif [ -n "$S_SCHEMA" ]; then S_FROM="read from $src"
|
|
else S_SCHEMA="$P_SCHEMA" S_FROM="from the pointer; ${why:-$src could not be read}"
|
|
fi
|
|
}
|
|
|
|
# sage-0774 and sage-0778: the commits the run adds (activate) or takes away (rollback), so the
|
|
# rows they carry can be named. Copies where neither is an ancestor of the other refuse the plan
|
|
# before it prints anything. Sets RANGE and VERB.
|
|
plan_range() {
|
|
if git -C "$REPO" merge-base --is-ancestor "$P_COPY" "$TO" 2>/dev/null; then RANGE="$P_COPY..$TO" VERB=adds
|
|
elif git -C "$REPO" merge-base --is-ancestor "$TO" "$P_COPY" 2>/dev/null; then RANGE="$TO..$P_COPY" VERB="takes away"
|
|
else refuse copies-unrelated "neither $P_COPY nor $TO is an ancestor of the other, so the plan cannot list the commits"
|
|
fi
|
|
}
|
|
plan_commits() {
|
|
echo "commits: the run $VERB $(git -C "$REPO" rev-list --count --no-merges "$RANGE") (newest first, at most 200):"
|
|
git -C "$REPO" log --oneline --no-merges --max-count=200 "$RANGE" | sed 's/^/ /'
|
|
}
|
|
|
|
# Each step from $S_SCHEMA to $T_SCHEMA, from the target store.mjs's `from:` and `to:` pairs. Text
|
|
# only, like copy_schema. Prints "cannot be read from FILE" when the lines don't pair up or the
|
|
# steps don't reach $T_SCHEMA.
|
|
plan_steps() {
|
|
local f="$COPIES/$TO/packages/bus/src/store.mjs" v="$S_SCHEMA" out="" i
|
|
local -a l
|
|
local -A next=()
|
|
mapfile -t l < <(grep -E "^[[:space:]]*(from|to): '[A-Za-z0-9]+',\$" "$f" | sed -E "s/^[[:space:]]*(from|to): '([A-Za-z0-9]*)',\$/\\1 \\2/")
|
|
(( ${#l[@]} % 2 == 0 )) || { echo "cannot be read from $f"; return; }
|
|
for ((i = 0; i < ${#l[@]}; i += 2)); do
|
|
[[ "${l[i]}" == "from "* && "${l[i+1]}" == "to "* ]] || { echo "cannot be read from $f"; return; }
|
|
next["${l[i]#from }"]="${l[i+1]#to }"
|
|
done
|
|
for ((i = 0; i < ${#next[@]}; i++)); do
|
|
[ "$v" != "$T_SCHEMA" ] && [ -n "${next[$v]:-}" ] || break
|
|
out+="${out:+, }$v to ${next[$v]}"; v="${next[$v]}"
|
|
done
|
|
if [ "$v" = "$T_SCHEMA" ] && [ -n "$out" ]; then echo "$out"; else echo "cannot be read from $f"; fi
|
|
}
|
|
|
|
# Item 8 and sage-0745: what the run would do, with any migration named plainly. Reads the
|
|
# pointer, the log, the target's store.mjs and the store's version. Writes nothing.
|
|
plan() {
|
|
plan_range
|
|
plan_store
|
|
echo "plan: $OP for $BUSINESS"
|
|
echo "from: $P_COPY"
|
|
echo "to: $TO"
|
|
[ "$TO" != "$P_COPY" ] || echo "note: $TO is the pointer's copy; the run re-pins it and only the Console flags change"
|
|
echo "store schema: $S_SCHEMA ($S_FROM)"
|
|
echo "target schema: $T_SCHEMA"
|
|
[ "$S_SCHEMA" = "$P_SCHEMA" ] || [ -n "$S_CHANGED" ] || echo "note: the pointer says $P_SCHEMA; the run reads the stopped store and goes by that"
|
|
# Row 186: set when run_steps would roll back with store-newer or migration-needs-flag, which
|
|
# come before backup_store. A read the bus stopped under judges nothing, so it keeps the backup line.
|
|
local early=""
|
|
if [ -n "$S_CHANGED" ]; then
|
|
: # no migration or restore line from a read the bus stopped under
|
|
elif [ "$S_SCHEMA" = "$T_SCHEMA" ]; then
|
|
echo "migration: none; the store is at the target's version"
|
|
elif [ "$OP" = activate ] && in_chain "$S_SCHEMA"; then
|
|
echo "migration: $S_SCHEMA to $T_SCHEMA, after a backup of the stopped store"
|
|
echo "migration steps: $(plan_steps)"
|
|
echo "needs: --migrate$([ "$MIGRATE" = 1 ] && echo ' (given)' || echo ' (not given; the run would roll back with migration-needs-flag)')"
|
|
echo "needs: Jason present for this run (decision 96 condition 2)"
|
|
[ "$MIGRATE" = 1 ] || early=1
|
|
# Row 233: root reads only the plan's exit, so under external a missing --migrate refuses here.
|
|
[ "$UNITS" = user ] || [ "$MIGRATE" = 1 ] || PLAN_REFUSE+=("migrate-required: the store is $S_SCHEMA and $TO migrates it to $T_SCHEMA; the go names --migrate")
|
|
elif [ "$OP" = activate ]; then
|
|
echo "migration: none possible; $S_SCHEMA is not in $T_SCHEMA's migration chain, so the run would roll back with store-newer"
|
|
early=1
|
|
else
|
|
echo "restore: the store must go back to $T_SCHEMA; needs --restore $(from_log backup || echo ACTIVATION)"
|
|
fi
|
|
if [ "$OP" != activate ]; then
|
|
: # rollback takes no backup of its own
|
|
elif [ ! -e "$STORE" ]; then
|
|
echo "backup: none; there is no store file"
|
|
elif [ -n "$early" ]; then
|
|
echo "backup: none; the run rolls back before the backup"
|
|
else
|
|
echo "backup: the run backs up the stopped store to $DATA/backups/stack/<activation>/ before any migration"
|
|
fi
|
|
plan_commits
|
|
local flags="${INSTANCE_FLAGS[*]:+${INSTANCE_FLAGS[*]} }"
|
|
if [ "$UNITS" = external ]; then
|
|
echo "units: external; root installs the units for $TO and starts them, then confirm gates them"
|
|
else
|
|
[ "$P_PENDING" = 0 ] || echo "pm: the pointer is pending (activate --pm-after-harness); the bus runs without its PM"
|
|
[ "$RUN_PENDING" = 0 ] || echo "pm: the bus starts without --pm and the PM stays pending; next: Jason's plugins set of harness, then pm-on"
|
|
echo "bus: scripts/bus-service.sh install $flags--copy $TO --copies $COPIES$([ "$RUN_PM" = 0 ] || echo ' --pm') --claude $P_CLAUDE"
|
|
[ -z "$DROP_TESTER" ] || echo "drop $DROP_TESTER; the target Console does not support it"
|
|
echo "$PROOF_PLAN"
|
|
echo "$SEND_PLAN"
|
|
[ -z "$SSO_PLAN" ] || echo "$SSO_PLAN"
|
|
echo "Console: scripts/webui-service.sh install $flags--copy $TO --copies $COPIES${RUN_CONSOLE[*]:+ ${RUN_CONSOLE[*]}}"
|
|
fi
|
|
echo "CLI: scripts/stack-copy.sh wrapper $TO $flags--copies $COPIES --bin $BIN"
|
|
local r
|
|
for r in "${PLAN_REFUSE[@]}"; do echo "refuses $r"; done
|
|
[ "${#PLAN_REFUSE[@]}" = 0 ] || exit 3
|
|
}
|
|
|
|
# Steps 4.1 to 4.5 and the gate for target $TO. $1 is the store step: migrate or restore.
|
|
run_steps() {
|
|
log_line result started
|
|
capture || refuse capture-failed "could not write the rollback set $RS; nothing was stopped"
|
|
[ -z "$DROP_TESTER" ] || say "drop $DROP_TESTER; the target Console does not support it"
|
|
[ -z "$PROOF_STEP" ] || say "$PROOF_STEP"
|
|
[ -z "$SEND_STEP" ] || say "$SEND_STEP"
|
|
[ -z "$SSO_STEP" ] || say "$SSO_STEP"
|
|
trap 'after_fail signal' INT TERM HUP
|
|
stop_both || rollback_in_run stop-timeout
|
|
if [ -e "$STORE" ]; then
|
|
if store_unclean; then say "$UNCLEAN is present; the store is left as it is for Jason or Mos"; rollback_in_run store-unclean; fi
|
|
STORE_SCHEMA="$(store_read version)" && [ -n "$STORE_SCHEMA" ] || rollback_in_run schema-unreadable
|
|
say "stopped store schema: $STORE_SCHEMA"
|
|
if [ "$1" = migrate ]; then
|
|
if [ "$STORE_SCHEMA" != "$T_SCHEMA" ]; then
|
|
in_chain "$STORE_SCHEMA" || rollback_in_run store-newer
|
|
[ "$MIGRATE" = 1 ] || rollback_in_run migration-needs-flag
|
|
fi
|
|
backup_store || rollback_in_run backup
|
|
if [ "$STORE_SCHEMA" != "$T_SCHEMA" ]; then
|
|
TAKEN="$ACT"
|
|
node "$COPIES/$TO/packages/bus/src/migrate-schema.mjs" "$DATA" >> "$STEPS" 2>&1 || rollback_in_run migrate-failed
|
|
say "migrated the store to $T_SCHEMA"
|
|
fi
|
|
else
|
|
[ "$STORE_SCHEMA" = "$P_SCHEMA" ] || rollback_in_run store-differs
|
|
if [ -n "$RESTORE" ]; then
|
|
local n
|
|
n="$(store_read count "$(cat "$DATA/backups/stack/$RESTORE/STAMP")" "$COUNTED")" || rollback_in_run schema-unreadable
|
|
[ "${n%%$'\t'*}" = "$COUNT" ] || rollback_in_run post-backup-events
|
|
restore_store "$RESTORE" || after_fail restore
|
|
fi
|
|
fi
|
|
elif [ -n "$RESTORE" ]; then
|
|
rollback_in_run store-differs
|
|
fi
|
|
install_stack "$TO" "$RUN_PM" "$P_CLAUDE" "${RUN_CONSOLE[@]}" || after_fail "$FAIL"
|
|
start_both || after_fail start
|
|
gate "$TO" "$RUN_PM" || after_fail "$GATE"
|
|
trap - INT TERM HUP
|
|
write_pointer "$TO" "$RUN_PM" "$RUN_PENDING" "$P_CLAUDE" "$T_SCHEMA" "${RUN_CONSOLE[@]}"
|
|
if [ "$RUN_PENDING" = 1 ]; then
|
|
log_line result ok pm pending
|
|
echo "stack-release: $OP ok: $TO is live for $BUSINESS without its PM; the PM is pending: Jason's plugins set of harness, then pm-on"
|
|
else
|
|
log_line result ok
|
|
echo "stack-release: $OP ok: $TO is live for $BUSINESS"
|
|
fi
|
|
}
|
|
|
|
# The pm the run installs and gates, and whether the pointer it writes is pending (row 169).
|
|
# --pm-after-harness installs without --pm and leaves the PM pending; a rollback from a pending
|
|
# pointer installs the --pm the activate replaced.
|
|
run_pm() {
|
|
RUN_PM="$P_PM" RUN_PENDING=0
|
|
if [ "$AFTER_HARNESS" = 1 ]; then RUN_PM=0 RUN_PENDING=1
|
|
elif [ "$P_PENDING" = 1 ]; then RUN_PM=1
|
|
fi
|
|
}
|
|
|
|
do_activate() {
|
|
TO="$ID"
|
|
[ "$PLAN" = 1 ] || lock
|
|
pointer_for_business
|
|
run_pm
|
|
[ "$P_PENDING" = 0 ] || check_refuse pm-pending "the pointer is pending (activate --pm-after-harness); run pm-on or rollback first"
|
|
if [ "$UNITS" = external ]; then
|
|
ext_pointer_checks
|
|
if [ "$PLAN" = 1 ]; then check_target; plugin_checks; config_checks; plan; exit 0; fi
|
|
ext_precheck
|
|
plugin_checks
|
|
config_checks
|
|
new_activation
|
|
run_external migrate
|
|
return
|
|
fi
|
|
if [ "$PLAN" = 1 ]; then check_target; console_for_target; ports_unclaimed; plugin_checks; config_checks; plan; exit 0; fi
|
|
precheck
|
|
plugin_checks
|
|
config_checks
|
|
new_activation
|
|
run_steps migrate
|
|
}
|
|
|
|
do_rollback() {
|
|
[ "$PLAN" = 1 ] || lock
|
|
pointer_for_business
|
|
TO="$(from_log previous)" || refuse no-previous "the log names no earlier copy than $P_COPY"
|
|
run_pm
|
|
if [ "$UNITS" = external ]; then
|
|
# r6: root names the target from the go, and it must be the one rollback reads from the log.
|
|
[ "$TO_FLAG" = "$TO" ] || refuse to-not-previous "--to $TO_FLAG is not the previous copy; the log names $TO"
|
|
ext_pointer_checks
|
|
if [ "$PLAN" = 1 ]; then
|
|
check_target; plugin_checks; config_checks
|
|
[ -z "$RESTORE" ] || { next_activation; restore_checks; }
|
|
plan; exit 0
|
|
fi
|
|
ext_precheck
|
|
plugin_checks
|
|
config_checks
|
|
[ "$P_SCHEMA" = "$T_SCHEMA" ] || [ -n "$RESTORE" ] || refuse schema-differs "the store is at $P_SCHEMA and $TO expects $T_SCHEMA; pass --restore ACTIVATION"
|
|
new_activation
|
|
[ -z "$RESTORE" ] || restore_checks
|
|
run_external restore
|
|
return
|
|
fi
|
|
if [ "$PLAN" = 1 ]; then check_target; console_for_target; ports_unclaimed; plugin_checks; config_checks; plan; exit 0; fi
|
|
precheck
|
|
plugin_checks
|
|
config_checks
|
|
[ "$P_SCHEMA" = "$T_SCHEMA" ] || [ -n "$RESTORE" ] || refuse schema-differs "the store is at $P_SCHEMA and $TO expects $T_SCHEMA; pass --restore ACTIVATION"
|
|
new_activation
|
|
[ -z "$RESTORE" ] || restore_checks
|
|
run_steps restore
|
|
}
|
|
|
|
# rollback --restore's checks on the backup and the rows written after it, before anything stops.
|
|
# Sets COUNT and BACKUP; POST comes from new_activation.
|
|
restore_checks() {
|
|
local o="$DATA/backups/stack/$RESTORE" rc=0 c="" text
|
|
[ -f "$o/SHA256SUMS" ] && [ "$(cat "$o/DATA_ROOT" 2>/dev/null)" = "$DATA" ] && [ "$(cat "$o/SCHEMA" 2>/dev/null)" = "$T_SCHEMA" ] && [ -f "$o/STAMP" ] \
|
|
|| refuse restore-mismatch "$o is not a backup of this root at $T_SCHEMA"
|
|
# Before anything stops. A store the running bus holds is read with readOnly, which adds
|
|
# no file (the probe); a stopped one falls under the store_unclean rule.
|
|
if sc is-active --quiet "$BUS_UNIT"; then
|
|
c="$(node -e "$STORE_JS" -- heldcount "$STORE" "$(cat "$o/STAMP")" "$COUNTED")" || rc=1
|
|
# plan_store's recheck (row 166 item 2), before anything stops.
|
|
sc is-active --quiet "$BUS_UNIT" || refuse restore-mismatch "the bus stopped while the events after $(cat "$o/STAMP") were counted, so the store state changed; nothing was stopped"
|
|
else
|
|
! store_unclean || refuse store-unclean "$UNCLEAN is present; the bus did not close the store cleanly"
|
|
c="$(store_read count "$(cat "$o/STAMP")" "$COUNTED")" || rc=1
|
|
fi
|
|
[ "$rc" = 0 ] || refuse restore-mismatch "cannot count the rows after $(cat "$o/STAMP")"
|
|
COUNT="${c%%$'\t'*}" text="${c#*$'\t'}"
|
|
echo "restore: $o; the store now in place moves to $POST and is kept; $text were written after the backup"
|
|
[ "$COUNT" = 0 ] || [ "$EVENTS" = "$COUNT" ] || refuse post-backup-events "$text were written after the backup; Jason decides, then --events $COUNT"
|
|
BACKUP="$RESTORE"
|
|
}
|
|
|
|
# --- --units external (row 232, r6 section 4) ---
|
|
|
|
# After the pointer, for activate and rollback, in --plan and in the run. OLD is the copy root read
|
|
# from the unit files it installed (r6 A1, N5), so a pointer that names another copy stops the
|
|
# re-pin before root stops anything. The PM is a flag in root's unit, so --pm-after-harness has
|
|
# nothing to wait on. The run's own lock refuses an open activation; --plan takes no lock, so it
|
|
# lists one here.
|
|
ext_pointer_checks() {
|
|
[ -z "$EXPECT_FROM" ] || [ "$EXPECT_FROM" = "$P_COPY" ] || check_refuse pointer-differs "the pointer names $P_COPY and root's units name $EXPECT_FROM; Jason or Mos settles it"
|
|
[ "$AFTER_HARNESS" = 0 ] || check_refuse external-pm "under --units external the PM is a flag in root's bus unit; changing it is a re-install with its own go"
|
|
[ "$PLAN" = 1 ] || return 0
|
|
local open
|
|
open="$(interrupted | paste -sd' ' -)"
|
|
[ -z "$open" ] || check_refuse interrupted-run "activation $open started and has no final line; Jason or Mos settles it (status shows it)"
|
|
# Row 233: root can't read the stack's state, so the plan lists any rollback set still waiting
|
|
# for units, whatever the log says.
|
|
local m sets=""
|
|
local -a markers
|
|
mapfile -t markers < <(compgen -G "$STATE/stack-rollback/*/AWAITING-UNITS" || true)
|
|
for m in "${markers[@]}"; do sets+=" $(basename "$(dirname "$m")")"; done
|
|
[ -z "$sets" ] || check_refuse open-activation "rollback set(s)$sets hold AWAITING-UNITS; root finishes them with confirm or revert first"
|
|
}
|
|
|
|
# A3's checks before `started`. Root has stopped the units and installs them after this run, so
|
|
# there is no Console flag, port, drop-in or claude pin of this script's to check.
|
|
ext_precheck() {
|
|
check_target
|
|
instance_busy
|
|
read_units || refuse unit-flags-drift "cannot read $BUS_UNIT or $CONSOLE_UNIT"
|
|
flags_drift
|
|
units_down
|
|
}
|
|
|
|
# r6 A3 (N4): each unit reads inactive, or failed as stop_both accepts, and the store is closed,
|
|
# so nothing below runs against a store a live bus holds.
|
|
units_down() {
|
|
local u s
|
|
for u in "$CONSOLE_UNIT" "$BUS_UNIT"; do
|
|
s="$(sc is-active "$u" 2>/dev/null || true)"
|
|
case "$s" in inactive|failed) ;; *) refuse units-running "$u is ${s:-unknown}; root stops the units first" ;; esac
|
|
done
|
|
stopped || refuse units-running "the store under $DATA/bus is still open (writer.lock or an open file)"
|
|
}
|
|
|
|
# A3's capture into $RS, which run_external made: the pointer, the wrapper and its mode, and for each
|
|
# unit its show output and the sha256 of its unit file. It reads root's units and makes no directory.
|
|
capture_units() {
|
|
local u frag
|
|
[ ! -f "$POINTER" ] || cp -p "$POINTER" "$RS/pointer" || return 1
|
|
[ -f "$BIN/$WRAPPER_NAME" ] || { say "capture: no wrapper $BIN/$WRAPPER_NAME"; return 1; }
|
|
cp -p "$BIN/$WRAPPER_NAME" "$RS/wrapper" || return 1
|
|
stat -c %a "$BIN/$WRAPPER_NAME" > "$RS/WRAPPER_MODE" || return 1
|
|
: > "$RS/UNITS.sha256" || return 1
|
|
for u in "$BUS_UNIT" "$CONSOLE_UNIT"; do
|
|
show "$u" > "$RS/show-$u.txt" || return 1
|
|
frag="$(sc show "$u" -p FragmentPath 2>/dev/null | sed -n 's/^FragmentPath=//p')" && [ -n "$frag" ] || { say "capture: no FragmentPath for $u"; return 1; }
|
|
sha256sum "$frag" >> "$RS/UNITS.sha256" || return 1
|
|
done
|
|
sums "$RS" > "$RS/SHA256SUMS" || return 1
|
|
say "captured the rollback set: $RS"
|
|
}
|
|
|
|
# A failure after the capture: FAILED holds the code, nothing is logged or cleaned up, exit 1.
|
|
# Root runs R1-R4, and revert is the one cleanup path (r6 A3).
|
|
ext_fail() {
|
|
trap - INT TERM HUP
|
|
{ echo "$1" > "$RS/FAILED"; } 2>/dev/null || true
|
|
say "failed: $1"
|
|
echo "stack-release: $OP failed with $1; $ACT stays open for root's revert and confirm --rollback. Rollback set: $RS" >&2
|
|
exit 1
|
|
}
|
|
|
|
# A3 for target $TO. $1 is the store step: migrate or restore. The marker goes in first, so every
|
|
# failure after it leaves what revert needs. Writes no pointer and starts nothing.
|
|
run_external() {
|
|
log_line result started
|
|
{ mkdir -p -m 0700 "$STATE/stack-rollback" && mkdir -m 0700 "$RS"; } 2>/dev/null || refuse marker-failed "could not make the rollback set $RS; nothing was changed"
|
|
{ echo "$TO" > "$RS/AWAITING-UNITS"; } 2>/dev/null || { rm -f "$RS/AWAITING-UNITS"; refuse marker-failed "could not write $RS/AWAITING-UNITS; nothing was changed"; }
|
|
STEPS="$RS/steps.log"
|
|
capture_units || { rm -f "$RS/AWAITING-UNITS"; refuse capture-failed "could not write the rollback set $RS; nothing was changed"; }
|
|
trap 'ext_fail signal' INT TERM HUP
|
|
if [ -e "$STORE" ]; then
|
|
if store_unclean; then say "$UNCLEAN is present; the store is left as it is for Jason or Mos"; ext_fail store-unclean; fi
|
|
STORE_SCHEMA="$(store_read version)" && [ -n "$STORE_SCHEMA" ] || ext_fail schema-unreadable
|
|
say "stopped store schema: $STORE_SCHEMA"
|
|
if [ "$1" = migrate ]; then
|
|
if [ "$STORE_SCHEMA" != "$T_SCHEMA" ]; then
|
|
in_chain "$STORE_SCHEMA" || ext_fail store-newer
|
|
[ "$MIGRATE" = 1 ] || ext_fail migration-needs-flag
|
|
fi
|
|
backup_store || ext_fail backup
|
|
echo "$BACKUP" > "$RS/BACKUP" || ext_fail backup
|
|
if [ "$STORE_SCHEMA" != "$T_SCHEMA" ]; then
|
|
# MIGRATING and TAKEN before the migration, so revert restores a partial one (r6 B2).
|
|
: > "$RS/MIGRATING" || ext_fail migrate-failed
|
|
TAKEN="$ACT"
|
|
node "$COPIES/$TO/packages/bus/src/migrate-schema.mjs" "$DATA" >> "$STEPS" 2>&1 || ext_fail migrate-failed
|
|
say "migrated the store to $T_SCHEMA"
|
|
fi
|
|
else
|
|
[ "$STORE_SCHEMA" = "$P_SCHEMA" ] || ext_fail store-differs
|
|
if [ -n "$RESTORE" ]; then
|
|
local n
|
|
n="$(store_read count "$(cat "$DATA/backups/stack/$RESTORE/STAMP")" "$COUNTED")" || ext_fail schema-unreadable
|
|
[ "${n%%$'\t'*}" = "$COUNT" ] || ext_fail post-backup-events
|
|
echo "$RESTORE" > "$RS/BACKUP" || ext_fail restore
|
|
# RESTORED tells confirm --rollback the store was swapped, so it skips OLD's gate
|
|
# and logs rollbackCode not-attempted (ruling 4, T14b).
|
|
if ! restore_store "$RESTORE"; then
|
|
[ -z "$RESTORED" ] || : > "$RS/RESTORED" || true
|
|
ext_fail restore
|
|
fi
|
|
: > "$RS/RESTORED" || ext_fail restore
|
|
fi
|
|
fi
|
|
elif [ -n "$RESTORE" ]; then
|
|
ext_fail store-differs
|
|
fi
|
|
scripts/stack-copy.sh wrapper "$TO" "${INSTANCE_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" >> "$STEPS" 2>&1 || ext_fail wrapper
|
|
trap - INT TERM HUP
|
|
say "staged $TO"
|
|
echo "stack-release: $OP staged $TO for $BUSINESS as $ACT; root installs and starts the units, then runs confirm $TO $BUSINESS --units external"
|
|
}
|
|
|
|
# r6 "The open activation": exactly one AWAITING-UNITS file, its activation the one the log holds
|
|
# open, holding the copy root passed. Sets O_ACT. Anything else refuses open-activation, and ACT
|
|
# stays unset, so the refused line names no activation and the open one stays open.
|
|
open_activation() {
|
|
local -a markers open
|
|
local m found="" seen=""
|
|
mapfile -t markers < <(compgen -G "$STATE/stack-rollback/*/AWAITING-UNITS" || true)
|
|
mapfile -t open < <(interrupted)
|
|
for m in "${markers[@]}"; do
|
|
seen+=" $(basename "$(dirname "$m")") (copy $(cat "$m" 2>/dev/null || echo unreadable))"
|
|
done
|
|
if [ "${#markers[@]}" = 1 ] && [ "${#open[@]}" = 1 ]; then
|
|
m="$(basename "$(dirname "${markers[0]}")")"
|
|
[ "$m" != "${open[0]}" ] || [ "$(cat "${markers[0]}" 2>/dev/null)" != "$ID" ] || found="$m"
|
|
fi
|
|
[ -n "$found" ] || refuse open-activation "need one AWAITING-UNITS for copy $ID whose activation the log holds open; markers:${seen:- none}; open: ${open[*]:-none}"
|
|
O_ACT="$found"
|
|
}
|
|
|
|
# The op, from, to and start second of O_ACT's `started` line, one per line.
|
|
started_of() {
|
|
node -e '
|
|
const [file,act]=process.argv.slice(1);
|
|
let s;
|
|
for(const l of require("fs").readFileSync(file,"utf8").split("\n")){
|
|
if(!l.trim())continue;
|
|
let o;try{o=JSON.parse(l)}catch{continue}
|
|
if(o.activation===act&&o.result==="started")s=o;
|
|
}
|
|
const at=s&&Math.floor(Date.parse(s.at)/1000);
|
|
if(!s||!["activate","rollback"].includes(s.op)||!Number.isFinite(at))process.exit(1);
|
|
process.stdout.write([s.op,s.from??"",s.to??"",at].join("\n")+"\n");' -- "$LOG" "$O_ACT"
|
|
}
|
|
|
|
# A5, and R4 with --rollback. The lines it writes carry the open activation's op, from, to and
|
|
# activation, and the backup A3 recorded, so from_log reads them as a dragon-lin run's. Every check
|
|
# runs before ACT is set.
|
|
do_confirm() {
|
|
lock_only
|
|
open_activation
|
|
local s
|
|
local -a f
|
|
s="$(started_of)" || refuse open-activation "the log's started line for $O_ACT is unreadable"
|
|
mapfile -t f <<< "$s"
|
|
[ "${f[2]}" = "$ID" ] || refuse open-activation "$O_ACT started for ${f[2]}, not $ID"
|
|
if read_pointer; then [ "$P_BUSINESS" = "$BUSINESS" ] || refuse business-mismatch "the pointer is for $P_BUSINESS"; fi
|
|
if [ -n "$ROLLBACK_OLD" ]; then
|
|
[ "$ROLLBACK_OLD" = "${f[1]}" ] || refuse rollback-mismatch "$ROLLBACK_OLD is not the copy $O_ACT started from, ${f[1]:-none}"
|
|
else
|
|
[ ! -e "$STATE/stack-rollback/$O_ACT/FAILED" ] || refuse activation-failed "$O_ACT has a FAILED file; root runs revert and confirm --rollback"
|
|
target_schema "$ID" || refuse copy-schema-unknown "cannot read SCHEMA_VERSION from $COPIES/$ID/packages/bus/src/store.mjs"
|
|
fi
|
|
OP="${f[0]}" FROM="${f[1]}" TO="${f[2]}" FLOOR="${f[3]}" ACT="$O_ACT" RS="$STATE/stack-rollback/$O_ACT"
|
|
STEPS="$RS/steps.log"
|
|
BACKUP="$(cat "$RS/BACKUP" 2>/dev/null || true)"
|
|
if [ -z "$ROLLBACK_OLD" ]; then
|
|
read_units || ext_fail units-unreadable
|
|
P_CONSOLE=("${U_CONSOLE[@]}")
|
|
gate "$TO" "$U_PM" || ext_fail "$GATE"
|
|
write_pointer "$TO" "$U_PM" 0 "$U_CLAUDE" "$T_SCHEMA" "${U_CONSOLE[@]}"
|
|
log_line result ok
|
|
rm -f "$RS/AWAITING-UNITS"
|
|
echo "stack-release: $OP ok: $TO is live for $BUSINESS"
|
|
return
|
|
fi
|
|
local code rcode=""
|
|
if [ "$CAUSE" = A4 ]; then code=units-start
|
|
else code="$(head -n 1 "$RS/FAILED" 2>/dev/null || true)"; [ -n "$code" ] || code=unrecorded
|
|
fi
|
|
if [ -e "$RS/RESTORED" ]; then rcode=not-attempted
|
|
elif ! read_units; then rcode=units-unreadable
|
|
else
|
|
P_CONSOLE=("${U_CONSOLE[@]}")
|
|
gate "$ROLLBACK_OLD" "$U_PM" || rcode="$GATE"
|
|
fi
|
|
if [ -z "$rcode" ]; then
|
|
local r=failed
|
|
[ "$CAUSE" != A5 ] || r=gate-failed
|
|
log_line result "$r" code "$code" rollback ok
|
|
rm -f "$RS/AWAITING-UNITS" "$RS/FAILED"
|
|
echo "stack-release: $code; rolled back to $ROLLBACK_OLD, which passed the gate" >&2
|
|
exit 1
|
|
fi
|
|
say "rollback-failed: run $code, rollback $rcode"
|
|
log_line result rollback-failed code "$code" rollback failed rollbackCode "$rcode"
|
|
rm -f "$RS/AWAITING-UNITS" "$RS/FAILED"
|
|
echo "stack-release: rollback-failed: the run failed with $code and the rollback with $rcode; root stops the units. Rollback set: $RS${BACKUP:+, backup $DATA/backups/stack/$BACKUP}" >&2
|
|
exit 2
|
|
}
|
|
|
|
revert_fail() {
|
|
say "revert failed: $1"
|
|
echo "stack-release: revert failed: $1; $O_ACT stays open. Rollback set: $RS" >&2
|
|
exit 1
|
|
}
|
|
|
|
# The store already holds backup $1 when every file checks against its SHA256SUMS and the listing
|
|
# matches, as after a revert that restored it.
|
|
store_matches() {
|
|
local o="$DATA/backups/stack/$1"
|
|
[ -d "$DATA/bus" ] && (cd "$DATA/bus" && sha256sum -c --quiet "$o/SHA256SUMS") >/dev/null 2>&1 &&
|
|
[ "$(ls -1A "$DATA/bus")" = "$(ls -1A "$o/bus")" ]
|
|
}
|
|
|
|
# R2: puts the wrapper back and, after a migration, the backup taken before it. Each part can run
|
|
# again, so it serves an A3 that stopped partway. A rollback's --restore swap is confirm's to
|
|
# report, not revert's to undo. Writes no log line.
|
|
do_revert() {
|
|
lock_only
|
|
open_activation
|
|
RS="$STATE/stack-rollback/$O_ACT" STEPS="$STATE/stack-rollback/$O_ACT/steps.log"
|
|
[[ "$O_ACT" =~ ^([0-9]{8}T[0-9]{6}Z)-[0-9a-f]{12}(-[1-9])?$ ]] || revert_fail "$O_ACT is not an activation id"
|
|
POST="$DATA/bus.post-attempt-${BASH_REMATCH[1]}${BASH_REMATCH[2]}"
|
|
[ -f "$RS/wrapper" ] && [ -f "$RS/WRAPPER_MODE" ] || revert_fail "the rollback set has no saved wrapper"
|
|
local tmp
|
|
tmp="$(mktemp "$BIN/.$WRAPPER_NAME.XXXXXX")" && cp -p "$RS/wrapper" "$tmp" && chmod "$(cat "$RS/WRAPPER_MODE")" "$tmp" && mv -f "$tmp" "$BIN/$WRAPPER_NAME" ||
|
|
revert_fail "could not put the wrapper back"
|
|
[ "$(sha256sum < "$BIN/$WRAPPER_NAME" | cut -d' ' -f1)" = "$(sum_of "$RS" wrapper)" ] &&
|
|
[ "$(stat -c %a "$BIN/$WRAPPER_NAME")" = "$(cat "$RS/WRAPPER_MODE")" ] || revert_fail "the wrapper does not match the rollback set"
|
|
say "put back the wrapper from $RS"
|
|
if [ -e "$RS/MIGRATING" ]; then
|
|
TAKEN="$(cat "$RS/BACKUP" 2>/dev/null || true)"
|
|
[ -n "$TAKEN" ] || revert_fail "MIGRATING is set and BACKUP names no backup"
|
|
if store_matches "$TAKEN"; then say "the store already matches backup $TAKEN"
|
|
else restore_store "$TAKEN" || revert_fail "could not restore backup $TAKEN"
|
|
fi
|
|
fi
|
|
echo "stack-release: revert ok: $O_ACT's wrapper$([ -e "$RS/MIGRATING" ] && echo ' and store are' || echo ' is') back; root installs the old units next"
|
|
}
|
|
|
|
# A2f: gates copy ID with no open activation, on a start at or after --since. Takes no lock and
|
|
# writes nothing, so it refuses nothing: 0 on a pass, 1 otherwise.
|
|
do_gate() {
|
|
FLOOR="$SINCE"
|
|
read_units || { echo "stack-release: gate: cannot read $BUS_UNIT or $CONSOLE_UNIT" >&2; exit 1; }
|
|
P_CONSOLE=("${U_CONSOLE[@]}")
|
|
gate "$ID" "$U_PM" || exit 1
|
|
}
|
|
|
|
# --- pm-on (row 169 item 3) ---
|
|
|
|
# Stops the bus alone and waits for it and the stopped check, within --timeout. The Console runs on.
|
|
stop_bus() {
|
|
sc stop "$BUS_UNIT" || return 1
|
|
local deadline=$((SECONDS + TIMEOUT)) s
|
|
while :; do
|
|
s="$(sc is-active "$BUS_UNIT" 2>/dev/null || true)"
|
|
case "$s" in inactive|failed) if stopped; then say "stopped $BUS_UNIT"; return 0; fi ;; esac
|
|
[ "$SECONDS" -lt "$deadline" ] || return 1
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
start_bus() {
|
|
START_EPOCH="$(date +%s)" && START_MS="$(date +%s%3N)" || return 1
|
|
sc start "$BUS_UNIT"
|
|
}
|
|
|
|
# mos-0848 point 3: put the phase-1 bus unit back (no --pm), restart it and rerun the phase-1 gate.
|
|
# Passing leaves the bus live without its PM and the pointer pending (exit 1); failing stops both
|
|
# units (exit 2). Neither touches the copy or the store. $1 is the code pm-on failed with.
|
|
pm_on_fail() {
|
|
trap - INT TERM HUP
|
|
local code="$1" why="$GATE_WHY" pcode=""
|
|
say "pm-on failed: $code${why:+ ($why)}; putting the phase-1 bus unit back"
|
|
GATE_WHY=""
|
|
if ! stop_bus; then pcode=stop-timeout
|
|
elif ! put_back; then pcode=put-back
|
|
elif ! start_bus; then pcode=start
|
|
elif ! gate "$P_COPY" 0; then pcode="$GATE"
|
|
fi
|
|
if [ -z "$pcode" ]; then
|
|
log_line result pm-on-failed code "$code" why "${why:-@null}"
|
|
echo "stack-release: pm-on failed with $code${why:+ ($why)}; the bus runs $P_COPY without its PM again and passed the phase-1 gate, and the pointer stays pending. Fix harness with plugins set, then run pm-on again. Rollback set: $RS" >&2
|
|
exit 1
|
|
fi
|
|
local pwhy="$GATE_WHY"
|
|
sc stop "$CONSOLE_UNIT" || true
|
|
sc stop "$BUS_UNIT" || true
|
|
say "pm-on-down: pm-on $code, phase 1 $pcode"
|
|
log_line result pm-on-down code "$code" why "${why:-@null}" phase1 "$pcode" phase1Why "${pwhy:-@null}"
|
|
echo "stack-release: pm-on-down: pm-on failed with $code${why:+ ($why)} and the phase-1 bus with $pcode${pwhy:+ ($pwhy)}; both units are stopped. Rollback set: $RS" >&2
|
|
exit 2
|
|
}
|
|
|
|
# Phase 2: the bus unit alone, on the pointer's copy with --pm, gated with pm_launched.
|
|
do_pm_on() {
|
|
lock
|
|
[ "$UNITS" = user ] || refuse external-pm "under --units external the PM is a flag in root's bus unit; changing it is a re-install with its own go"
|
|
pointer_for_business
|
|
[ "$P_PENDING" = 1 ] || refuse not-pending "the pointer is not pending; pm-on follows activate --pm-after-harness"
|
|
local out
|
|
out="$(plugins harness "$P_COPY" 2>&1)" || refuse harness-unset "$P_COPY's loader refuses the business file: $out"
|
|
[ "$out" != none ] || refuse harness-unset "the business file sets no harness; Jason sets it with plugins set first"
|
|
node -e 'const h=JSON.parse(process.argv[1]);process.exit(Array.isArray(h)&&h.length>0?0:1)' -- "$out" 2>/dev/null || refuse harness-unset "the business file's harness is $out, not a non-empty list"
|
|
instance_busy
|
|
drop_in_override
|
|
read_units || refuse unit-flags-drift "cannot read $BUS_UNIT or $CONSOLE_UNIT"
|
|
[ -n "$U_CLAUDE" ] || refuse no-claude-pin "$BUS_UNIT has no MOSAIC_CLAUDE_BIN"
|
|
flags_drift
|
|
TO="$P_COPY"
|
|
new_activation
|
|
log_line result started
|
|
capture || refuse capture-failed "could not write the rollback set $RS; nothing was stopped"
|
|
trap 'pm_on_fail signal' INT TERM HUP
|
|
stop_bus || pm_on_fail stop-timeout
|
|
scripts/bus-service.sh install "${INSTANCE_FLAGS[@]}" --copy "$P_COPY" --copies "$COPIES" --pm --claude "$P_CLAUDE" >> "$STEPS" 2>&1 || pm_on_fail install-bus
|
|
say "installed $BUS_UNIT for $P_COPY with --pm"
|
|
start_bus || pm_on_fail start
|
|
gate "$P_COPY" 1 || pm_on_fail "$GATE"
|
|
trap - INT TERM HUP
|
|
write_pointer "$P_COPY" 1 0 "$P_CLAUDE" "$P_SCHEMA" "${P_CONSOLE[@]}"
|
|
log_line result ok pm on
|
|
echo "stack-release: pm-on ok: the PM is live on $P_COPY for $BUSINESS"
|
|
}
|
|
|
|
do_status() {
|
|
echo "pointer: $POINTER"
|
|
if read_pointer; then sed 's/^/ /' "$POINTER"; else echo " none"; fi
|
|
[ "$P_PENDING" = 0 ] || echo "pm: pending (activate --pm-after-harness); next: Jason's plugins set of harness, then stack-release.sh pm-on $BUSINESS"
|
|
echo "log (last five lines):"
|
|
if [ -f "$LOG" ]; then tail -n 5 "$LOG" | sed 's/^/ /'; else echo " none"; fi
|
|
local open
|
|
open="$(interrupted | paste -sd' ' -)"
|
|
[ -z "$open" ] || echo "interrupted run: $open (started, no final line; Jason or Mos settles it)"
|
|
if [ "$UNITS" = external ]; then
|
|
local m
|
|
local -a markers
|
|
mapfile -t markers < <(compgen -G "$STATE/stack-rollback/*/AWAITING-UNITS" || true)
|
|
for m in "${markers[@]}"; do
|
|
echo "awaiting units: $(basename "$(dirname "$m")") (copy $(cat "$m" 2>/dev/null || echo unreadable))"
|
|
done
|
|
fi
|
|
echo "check:"
|
|
if scripts/stack-copy.sh check "${INSTANCE_FLAGS[@]}" "${UNITS_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>&1 | sed 's/^/ /'; then echo " result: pass"; else echo " result: FAIL"; fi
|
|
[ -n "$P_COPY" ] || return 0
|
|
echo "bus status:"
|
|
"$COPIES/$P_COPY/scripts/mosaic" bus status 2>&1 | sed 's/^/ /' || true
|
|
if read_units; then
|
|
local d=""
|
|
[ "$U_PM" = "$P_PM" ] || d+=" pm"
|
|
[ "$U_CLAUDE" = "$P_CLAUDE" ] || d+=" claude"
|
|
[ "${P_CONSOLE[*]}" = "${U_CONSOLE[*]}" ] || d+=" console"
|
|
echo "unit flags: ${d:+differ from the pointer:}${d:-match the pointer}"
|
|
else
|
|
echo "unit flags: cannot read the units"
|
|
fi
|
|
if git -C "$REPO" merge-base --is-ancestor "$P_COPY" refs/heads/refactor 2>/dev/null; then
|
|
echo "drift: $(git -C "$REPO" rev-list --count "$P_COPY..refs/heads/refactor") commit(s) on refactor since the pointer's copy"
|
|
else
|
|
echo "drift: the pointer's copy is not an ancestor of refactor"
|
|
fi
|
|
}
|
|
|
|
# --- The rehearsal (D2, row 163) ---
|
|
|
|
# rehearse runs the new copy's bus host (--no-launch) and Console on a scratch root built from a
|
|
# backup, under <dataRoot>/rehearsals/<stamp>/, and gates them as activate does. It never writes the
|
|
# live store, the live config files, the backup, the pointer, the log or a unit. Nothing prunes
|
|
# the rehearsal directory. Design, rulings (mos-0961, mos-0962, mos-0963) and the deviations from
|
|
# the brief: agents/sage/work/queue-163/BUILD.md.
|
|
|
|
# One PASS or FAIL line per check, on stdout and in steps.log. Lines from before the directory
|
|
# exists wait in R_EARLY and go in first.
|
|
r_line() {
|
|
echo "$*"
|
|
if [ -z "$STEPS" ]; then R_EARLY+=("$(now) $*"); else echo "$(now) $*" >> "$STEPS" || true; fi
|
|
}
|
|
r_pass() { r_line "PASS $1${2:+: $2}"; }
|
|
r_fail() { r_line "FAIL $1${2:+: $2}"; R_RC=1; }
|
|
r_abort() { r_fail "$@"; r_end; }
|
|
|
|
r_alive() { local s; s="$(cat "/proc/$1/stat" 2>/dev/null)" || return 1; s="${s##*) }"; [ "${s:0:1}" != Z ]; }
|
|
r_gone() {
|
|
local deadline=$((SECONDS + TIMEOUT))
|
|
while r_alive "$1"; do [ "$SECONDS" -lt "$deadline" ] || return 1; sleep 1; done
|
|
}
|
|
|
|
# Stops the Console, then the host, with SIGTERM within --timeout; one still there gets SIGKILL
|
|
# and a FAIL line. The host has to leave the store's writer.lock behind it.
|
|
r_stop() {
|
|
local name pid
|
|
for name in console host; do
|
|
if [ "$name" = console ]; then pid="$R_CONSOLE_PID"; R_CONSOLE_PID=""; else pid="$R_HOST_PID"; R_HOST_PID=""; fi
|
|
[ -n "$pid" ] || continue
|
|
kill -TERM "$pid" 2>/dev/null || true
|
|
if r_gone "$pid"; then r_pass "$name-stopped" "pid $pid"
|
|
else kill -KILL "$pid" 2>/dev/null || true; r_fail "$name-stopped" "pid $pid was still running ${TIMEOUT}s after SIGTERM; sent SIGKILL"
|
|
fi
|
|
wait "$pid" 2>/dev/null || true
|
|
done
|
|
[ -n "$R_STARTED" ] || return 0
|
|
R_STARTED=""
|
|
if [ -e "$R/data/bus/writer.lock" ]; then r_fail host-released-store "$R/data/bus/writer.lock is still there"; else r_pass host-released-store; fi
|
|
}
|
|
|
|
r_tree_sums() { (cd "$1" && find . -type f -print0 | sort -z | xargs -0 -r sha256sum); }
|
|
|
|
# The last checks, then the last line. The backup and the live config files are compared with the
|
|
# sums taken before the rehearsal directory was made.
|
|
r_end() {
|
|
trap - INT TERM HUP
|
|
r_stop
|
|
if [ -n "$R" ]; then
|
|
if [ "$(r_tree_sums "$R_BK")" = "$(cat "$R/backup.before")" ]; then r_pass backup-unchanged; else r_fail backup-unchanged "$R_BK differs from $R/backup.before"; fi
|
|
if [ "$(sha256sum "$LIVE_CFG" "$LIVE_BIZ" 2>&1)" = "$(cat "$R/live.before")" ]; then r_pass live-config-unchanged; else r_fail live-config-unchanged "$LIVE_CFG or $LIVE_BIZ differs from $R/live.before"; fi
|
|
fi
|
|
if [ "$R_RC" = 0 ]; then r_line REHEARSAL_PASS; else r_line REHEARSAL_FAIL; fi
|
|
[ -z "$R" ] || echo "rehearsal directory: $R"
|
|
exit "$R_RC"
|
|
}
|
|
|
|
r_port_free() {
|
|
node -e 'const s=require("node:net").createServer();s.once("error",()=>process.exit(1));s.listen(Number(process.argv[1]),"127.0.0.1",()=>s.close(()=>process.exit(0)));' -- "$1"
|
|
}
|
|
|
|
# The scratch config: the live one with dataRoot set to the scratch data root.
|
|
R_CONFIG_JS='
|
|
const fs=require("node:fs");const [file,data]=process.argv.slice(1);
|
|
const c=JSON.parse(fs.readFileSync(file,"utf8"));c.dataRoot=data;process.stdout.write(JSON.stringify(c,null,2)+"\n");'
|
|
|
|
# The scratch business file (mos-0962, mos-0963): the live file without its tracker.* variables,
|
|
# business and role, and with every credential reference, the roles and tracker.sync, moved to its
|
|
# own 0600 placeholder file. The service and its rotateBy or expires date stay. The tracker key
|
|
# stays because the loader requires it. Mode `normalize` prints the live file as this writes it,
|
|
# for the diff.
|
|
R_BUSINESS_JS='
|
|
const fs=require("node:fs"),path=require("node:path");const [mode,live,out,ph]=process.argv.slice(1);
|
|
const b=JSON.parse(fs.readFileSync(live,"utf8"));
|
|
if(mode==="normalize"){process.stdout.write(JSON.stringify(b,null,2)+"\n");process.exit(0)}
|
|
const DATE={gitea:"rotateBy",vikunja:"expires"};
|
|
const strip=(vars)=>{if(vars&&typeof vars==="object")for(const k of Object.keys(vars))if(k.startsWith("tracker."))delete vars[k]};
|
|
const swap=(name,refs)=>{
|
|
if(!refs||typeof refs!=="object")return;
|
|
for(const s of Object.keys(refs)){
|
|
const file=path.join(ph,`${name}-${s}`),old=refs[s],ref={file};
|
|
fs.writeFileSync(file,"rehearsal-placeholder-token",{mode:0o600,flag:"wx"});
|
|
if(old&&typeof old==="object"&&Object.hasOwn(DATE,s)&&old[DATE[s]]!==undefined)ref[DATE[s]]=old[DATE[s]];
|
|
refs[s]=ref;
|
|
}
|
|
};
|
|
strip(b.vars);
|
|
for(const [name,r] of Object.entries(b.roles??{})){strip(r?.vars);swap(name,r?.credentials)}
|
|
swap("@sync",b.tracker?.sync?.credentials);
|
|
fs.writeFileSync(out,JSON.stringify(b,null,2)+"\n",{mode:0o600,flag:"wx"});'
|
|
|
|
# The guard, before the host starts. It reads the scratch config through the new copy's own
|
|
# loadSystem and bootConfig, as the host will, and prints one line per check:
|
|
# data-root-live the data root is not the live one (mos-0961)
|
|
# sessions-live no sessions.json there names a running process (mos-0961)
|
|
# data-root-scratch the data root is the scratch one
|
|
# notify-off the notifier config is {"notifyVersion":1,"binding":null}, 0600 (mos-0962)
|
|
# tracker-configured bootConfig sets no trackers (mos-0962)
|
|
# credential-ref-live every credential reference is a file under placeholders/ (mos-0963)
|
|
# Exit 0 when all pass, 1 on the first FAIL.
|
|
R_GUARD_JS='
|
|
const fs=require("node:fs"),path=require("node:path");const {pathToFileURL}=require("node:url");
|
|
const [copy,id,live,scratch,ph]=process.argv.slice(1);
|
|
const real=(p)=>{try{return fs.realpathSync(p)}catch{return null}};
|
|
const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,"utf8");return s[s.lastIndexOf(")")+2]!=="Z"}catch{return false}};
|
|
const pass=(n,w)=>console.log(`PASS ${n}${w?": "+w:""}`);
|
|
const fail=(n,w)=>{console.log(`FAIL ${n}: ${w}`);process.exit(1)};
|
|
(async()=>{
|
|
let c,system,config;
|
|
try{c=await import(pathToFileURL(`${copy}/packages/cli/src/config.mjs`).href);system=c.loadSystem({env:process.env})}
|
|
catch(e){fail("scratch-config",String(e?.message??e).split("\n")[0])}
|
|
const root=real(system.dataRoot);
|
|
if(root===null)fail("data-root-scratch",`${system.dataRoot} does not exist`);
|
|
if(root===real(live))fail("data-root-live",`the scratch config names dataRoot ${system.dataRoot}, which resolves to the live data root ${live}`);
|
|
pass("data-root-live",`${system.dataRoot} is not the live data root`);
|
|
const sf=path.join(root,"bus-host","sessions.json");
|
|
if(fs.existsSync(sf)){
|
|
let doc;try{doc=JSON.parse(fs.readFileSync(sf,"utf8"))}catch{fail("sessions-live",`${sf} is unreadable`)}
|
|
if(!Array.isArray(doc?.sessions))fail("sessions-live",`${sf} is malformed`);
|
|
const running=doc.sessions.flatMap((s)=>[s?.pid,s?.runnerPid]).filter((p)=>Number.isSafeInteger(p)&&p>0&&alive(p));
|
|
if(running.length)fail("sessions-live",`${sf} names running pid(s) ${running.join(", ")}`);
|
|
}
|
|
pass("sessions-live","no running session named in the data root");
|
|
if(root!==real(scratch))fail("data-root-scratch",`the scratch config resolves to ${system.dataRoot}, not ${scratch}`);
|
|
pass("data-root-scratch");
|
|
const nf=path.join(root,"notify",id,"notify.json");
|
|
let st,doc;try{st=fs.lstatSync(nf);doc=JSON.parse(fs.readFileSync(nf,"utf8"))}catch{fail("notify-off",`${nf} is missing or not JSON`)}
|
|
if(!st.isFile()||(st.mode&0o777)!==0o600||JSON.stringify(doc)!==JSON.stringify({notifyVersion:1,binding:null}))fail("notify-off",`${nf} is not {"notifyVersion":1,"binding":null} at mode 0600`);
|
|
pass("notify-off","binding null, so no Discord bridge");
|
|
try{config=c.bootConfig({system,businessId:id,env:process.env})}catch(e){fail("boot-config",String(e?.message??e).split("\n")[0])}
|
|
if(config.trackers!==undefined)fail("tracker-configured",`bootConfig sets trackers for ${Object.keys(config.trackers).join(", ")}`);
|
|
pass("tracker-configured","bootConfig sets no trackers");
|
|
const b=config.businesses?.[id],phr=real(ph),refs=[];
|
|
for(const [role,r] of Object.entries(b?.roles??{}))for(const [s,ref] of Object.entries(r?.credentials??{}))refs.push([`${role} ${s}`,ref]);
|
|
for(const [s,ref] of Object.entries(b?.tracker?.sync?.credentials??{}))refs.push([`@sync ${s}`,ref]);
|
|
for(const [what,ref] of refs){
|
|
if(ref?.env!==undefined)fail("credential-ref-live",`${what} is env ${ref.env}`);
|
|
const f=typeof ref?.file==="string"?real(ref.file):null;
|
|
if(f===null||phr===null||path.dirname(f)!==phr)fail("credential-ref-live",`${what} is ${ref?.file}, not a file in ${ph}`);
|
|
}
|
|
pass("credential-ref-live",`${refs.length} reference(s), each a placeholder`);
|
|
})().catch((e)=>fail("guard",String(e?.message??e).split("\n")[0]));'
|
|
|
|
# Opens a data root's store with a copy's Store, as the bus does at start, with no migration
|
|
# (de352dfaf's verify-open.mjs). Prints OPEN_OK <version> or REFUSED <message>.
|
|
R_OPEN_JS='
|
|
const {pathToFileURL}=require("node:url");const [copy,root]=process.argv.slice(1);
|
|
(async()=>{
|
|
const {Store}=await import(pathToFileURL(`${copy}/packages/bus/src/store.mjs`).href);
|
|
let s;
|
|
try{s=new Store(root);process.stdout.write(`OPEN_OK ${s.get("SELECT value FROM meta WHERE key=?","schema_version").value}\n`)}
|
|
catch(e){process.stdout.write(`REFUSED ${e.message}\n`)}
|
|
finally{s?.close()}
|
|
})();'
|
|
r_open() { node -e "$R_OPEN_JS" -- "$1" "$2" 2>&1 || true; }
|
|
|
|
# Every process this rehearsal starts gets only HOME, PATH and the scratch MOSAIC_CONFIG.
|
|
r_env() { env -i HOME="$HOME" PATH="$PATH" MOSAIC_CONFIG="$R/config/config.json" "$@"; }
|
|
|
|
# The de352dfaf checks, when the backup's schema is not the new copy's.
|
|
r_migration() {
|
|
local copy="$COPIES/$TO" old="$COPIES/$P_COPY" v out rc
|
|
v="$(node -e "$STORE_JS" -- version "$R/data/bus/bus.sqlite" 2>&1)" || r_abort store-unreadable "$v"
|
|
# Row 233 (sage-1653): which store the rehearsal ran against, for root's step log. Output only:
|
|
# the rehearsal's own steps.log has the backup's path on its backup line.
|
|
echo "BACKUP_STORE $R_BACKUP schema $v"
|
|
if [ "$v" = "$T_SCHEMA" ]; then r_pass migration "not needed: the backup is already $v"; return 0; fi
|
|
in_chain "$v" || r_abort store-newer "the backup is $v, which $TO's migration chain does not start from"
|
|
(umask 077; mkdir "$R/refuse" "$R/refuse/bus" && cp "$R_BK"/bus/* "$R/refuse/bus/") || r_abort scratch-build "cannot copy the backup to $R/refuse"
|
|
out="$(r_open "$copy" "$R/refuse")"
|
|
[ "$out" = "REFUSED schema-mismatch" ] || r_abort new-refuses-old-store "$TO opened the $v store: $out"
|
|
r_pass new-refuses-old-store "$TO refuses the $v store at a normal open"
|
|
rc=0; out="$(r_env node "$copy/packages/bus/src/migrate-schema.mjs" "$R/data" 2>&1)" || rc=$?
|
|
printf '%s\n' "$out" >> "$R/migrate.log"
|
|
[ "$rc" = 0 ] && [ "$(tail -n 1 <<< "$out")" = "bus schema $T_SCHEMA ready" ] || r_abort migrate "migrate-schema.mjs exit $rc: $(tail -n 1 <<< "$out")"
|
|
r_pass migrate "$v to $T_SCHEMA"
|
|
out="$(r_open "$copy" "$R/data")"
|
|
[ "$out" = "OPEN_OK $T_SCHEMA" ] || r_abort new-opens-migrated "$out"
|
|
r_pass new-opens-migrated
|
|
rc=0; out="$(r_env node "$copy/packages/bus/src/migrate-schema.mjs" "$R/data" 2>&1)" || rc=$?
|
|
printf '%s\n' "$out" >> "$R/migrate.log"
|
|
[ "$rc" = 0 ] && [ "$(tail -n 1 <<< "$out")" = "bus schema $T_SCHEMA ready" ] || r_abort migrate-again "second migrate-schema.mjs exit $rc: $(tail -n 1 <<< "$out")"
|
|
r_pass migrate-again "a second run is a no-op"
|
|
out="$(r_open "$old" "$R/data")"
|
|
[ "$out" = "REFUSED schema-mismatch" ] || r_abort old-refuses-migrated "the pointer's copy $P_COPY opened the $T_SCHEMA store: $out"
|
|
r_pass old-refuses-migrated "$P_COPY refuses the $T_SCHEMA store"
|
|
}
|
|
|
|
# One pass of D1's gate checks but the PM's, against the processes this run started. Sets R_GATE
|
|
# to the first failing check and R_WHY to what it saw.
|
|
r_gate_once() {
|
|
local copy="$COPIES/$TO" out
|
|
R_GATE=host-up R_WHY=""
|
|
r_alive "$R_HOST_PID" || { R_WHY="the host (pid $R_HOST_PID) exited; see $R/host.log"; return 1; }
|
|
grep -q "^bus host up: business $BUSINESS, .*, launches disabled\$" "$R/host.log" || { R_WHY="no \"bus host up ... launches disabled\" line in $R/host.log"; return 1; }
|
|
R_GATE=bus-status
|
|
out="$(r_env "$copy/scripts/mosaic" bus status --json 2>/dev/null)" || { R_WHY="bus status failed"; return 1; }
|
|
R_WHY="$(node -e '
|
|
const [b,main,since]=process.argv.slice(2),s=JSON.parse(process.argv[1]);
|
|
const why=!s.host?"no host":s.host.business!==b?`host for ${s.host.business}`:s.host.live!==true?"host not live":String(s.host.pid)!==main?`host pid ${s.host.pid} is not ${main}, the host this run started`:!s.boot||s.boot.ok!==true?"boot status not ok":s.boot.business!==b?`boot status for ${s.boot.business}`:!(Date.parse(s.boot.at)>=Number(since))?"boot status is from before this start":"";
|
|
process.stdout.write(why);' -- "$out" "$BUSINESS" "$R_HOST_PID" "$START_MS" 2>&1)" || { R_WHY="bus status unreadable"; return 1; }
|
|
[ -z "$R_WHY" ] || return 1
|
|
R_GATE=console-up
|
|
r_alive "$R_CONSOLE_PID" || { R_WHY="the Console (pid $R_CONSOLE_PID) exited; see $R/console.log"; return 1; }
|
|
grep -qxF "Mosaic Console: http://127.0.0.1:$R_PORT/" "$R/console.log" || { R_WHY="no \"Mosaic Console: http://127.0.0.1:$R_PORT/\" line in $R/console.log"; return 1; }
|
|
R_GATE=healthz
|
|
node -e '
|
|
fetch(`http://127.0.0.1:${process.argv[1]}/healthz`,{signal:AbortSignal.timeout(5000)}).then((r)=>process.exit(r.status===200?0:1),()=>process.exit(1));' -- "$R_PORT" || return 1
|
|
R_GATE=copy-match
|
|
tr '\0' ' ' < "/proc/$R_HOST_PID/cmdline" | grep -qF " $copy/packages/cli/src/cli.mjs bus start $BUSINESS --no-launch" || { R_WHY="the host's command line is not $copy's cli.mjs bus start $BUSINESS --no-launch"; return 1; }
|
|
tr '\0' ' ' < "/proc/$R_CONSOLE_PID/cmdline" | grep -qF " $copy/packages/webui/src/cli.mjs serve " || { R_WHY="the Console's command line is not $copy's webui cli.mjs serve"; return 1; }
|
|
R_GATE=""
|
|
}
|
|
|
|
r_gate() {
|
|
local deadline=$((SECONDS + TIMEOUT))
|
|
until r_gate_once; do
|
|
[ "$SECONDS" -lt "$deadline" ] || { r_fail "$R_GATE" "${R_WHY:-no answer within ${TIMEOUT}s}"; return 1; }
|
|
sleep 1
|
|
done
|
|
r_pass gate "host-up, bus-status, console-up, healthz and copy-match on $TO"
|
|
}
|
|
|
|
# No role session: the host wrote no sessions.json and no launches directory or log.
|
|
r_no_session() {
|
|
local found=""
|
|
[ ! -e "$R/data/bus-host/sessions.json" ] || found+=" bus-host/sessions.json"
|
|
[ ! -e "$R/data/launches" ] || found+=" launches"
|
|
if [ -z "$found" ]; then r_pass no-session "no sessions.json and no launches under $R/data"; else r_fail no-session "found$found under $R/data"; fi
|
|
}
|
|
|
|
do_rehearse() {
|
|
TO="$ID"
|
|
trap 'r_fail signal "the rehearsal was interrupted"; r_end' INT TERM HUP
|
|
read_pointer || r_abort no-pointer "no $POINTER; the pointer's copy is the old copy the checks use"
|
|
[ "$P_BUSINESS" = "$BUSINESS" ] || r_abort business-mismatch "the pointer is for $P_BUSINESS"
|
|
# Row 233 (sage-1650, sage-1653): last is the newest backup the log records. With none, exit 3
|
|
# and no directory, the one rehearse refusal stack-upgrade.sh reads as no-backup.
|
|
if [ "$R_BACKUP" = last ]; then
|
|
R_BACKUP="$(from_log newest)" || { r_fail no-backup "the log records no backup"; R_RC=3; r_end; }
|
|
[[ "$R_BACKUP" =~ ^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}(-[1-9])?$ ]] || r_abort backup-unreadable "the log's newest backup is not an activation id"
|
|
r_pass backup-last "the log's newest backup is $R_BACKUP"
|
|
fi
|
|
copy_exists "$TO" 2>/dev/null || r_abort copy-missing "no copy $TO in $COPIES"
|
|
copy_ok "$TO" 2>/dev/null || r_abort copy-unreachable "$TO is not reachable from refactor and origin/refactor"
|
|
target_schema "$TO" || r_abort copy-schema-unknown "cannot read SCHEMA_VERSION and its chain from $COPIES/$TO/packages/bus/src/store.mjs"
|
|
# mos-0961: an older copy's host has no --no-launch and would start the PM on its own.
|
|
grep -qF -- '"--no-launch"' "$COPIES/$TO/packages/cli/src/cli.mjs" 2>/dev/null || r_abort copy-without-no-launch "$TO's bus host has no --no-launch flag"
|
|
r_pass copy "$TO, schema $T_SCHEMA"
|
|
local i
|
|
local -a reserved=(7332 7333 "$(console_port)")
|
|
for ((i = 0; i < ${#P_CONSOLE[@]}; i++)); do [ "${P_CONSOLE[i]}" != --tester-port ] || reserved+=("${P_CONSOLE[i + 1]:-}"); done
|
|
for i in "${reserved[@]}"; do [ "$((10#$R_PORT))" != "$i" ] || r_abort port-reserved "--port $R_PORT is a live Console port (7332, 7333 and the pointer's: ${reserved[*]:2})"; done
|
|
if i="$(instance_port_holder "$UNIT_DIR" "$R_PORT")"; then r_abort port-reserved "--port $R_PORT is $UNIT_DIR/$i's Console port"; fi
|
|
r_port_free "$R_PORT" || r_abort port-in-use "127.0.0.1:$R_PORT is in use; the rehearsal takes no other port"
|
|
r_pass port "127.0.0.1:$R_PORT is free"
|
|
case "$R_BACKUP" in /*) R_BK="$R_BACKUP" ;; *) R_BK="$DATA/backups/stack/$R_BACKUP" ;; esac
|
|
[ -f "$R_BK/bus/bus.sqlite" ] && [ -f "$R_BK/SHA256SUMS" ] || r_abort backup-missing "$R_BK has no bus/bus.sqlite and SHA256SUMS"
|
|
(cd "$R_BK/bus" && sha256sum -c --quiet ../SHA256SUMS) >/dev/null 2>&1 || r_abort backup-sums "$R_BK/bus does not match its SHA256SUMS"
|
|
r_pass backup "$R_BK matches its SHA256SUMS"
|
|
LIVE_CFG="$(realpath -m "${MOSAIC_CONFIG:-$HOME/.config/mosaic-dev/config.json}")"
|
|
LIVE_BIZ="$(dirname "$LIVE_CFG")/businesses/$BUSINESS.json"
|
|
[ -f "$LIVE_BIZ" ] || r_abort business-missing "no $LIVE_BIZ"
|
|
|
|
local stamp before live
|
|
before="$(r_tree_sums "$R_BK")" && live="$(sha256sum "$LIVE_CFG" "$LIVE_BIZ" 2>&1)" || r_abort sums "cannot read $R_BK or the live config files"
|
|
stamp="$(date -u +%Y%m%dT%H%M%SZ)-${TO:0:12}"
|
|
(umask 077; mkdir -p "$DATA/rehearsals" && mkdir "$DATA/rehearsals/$stamp") || r_abort rehearsal-dir "cannot make $DATA/rehearsals/$stamp (one rehearsal per second and copy)"
|
|
R="$DATA/rehearsals/$stamp" STEPS="$DATA/rehearsals/$stamp/steps.log"
|
|
printf '%s\n' "${R_EARLY[@]}" > "$STEPS" && printf '%s\n' "$before" > "$R/backup.before" && printf '%s\n' "$live" > "$R/live.before" ||
|
|
r_abort scratch-build "cannot write the sums to $R"
|
|
(
|
|
umask 077
|
|
mkdir "$R/config" "$R/config/businesses" "$R/placeholders" "$R/data" "$R/data/bus" "$R/data/notify" "$R/data/notify/$BUSINESS" &&
|
|
cp "$R_BK"/bus/* "$R/data/bus/" &&
|
|
node -e "$R_CONFIG_JS" -- "$LIVE_CFG" "$R/data" > "$R/config/config.json" &&
|
|
node -e "$R_BUSINESS_JS" -- write "$LIVE_BIZ" "$R/config/businesses/$BUSINESS.json" "$R/placeholders" &&
|
|
printf '{"notifyVersion":1,"binding":null}\n' > "$R/data/notify/$BUSINESS/notify.json"
|
|
) 2>> "$STEPS" || r_abort scratch-build "cannot build the scratch root under $R; see $STEPS"
|
|
local rc=0
|
|
diff -u --label "$LIVE_BIZ (as JSON.stringify writes it)" --label "$R/config/businesses/$BUSINESS.json" \
|
|
<(node -e "$R_BUSINESS_JS" -- normalize "$LIVE_BIZ") "$R/config/businesses/$BUSINESS.json" > "$R/business.diff" || rc=$?
|
|
[ "$rc" -le 1 ] || r_abort scratch-build "cannot write $R/business.diff"
|
|
r_pass scratch-root "$R/data from the backup, config and business file under $R/config, business.diff beside them"
|
|
|
|
local out
|
|
rc=0; out="$(r_env node -e "$R_GUARD_JS" -- "$COPIES/$TO" "$BUSINESS" "$DATA" "$R/data" "$R/placeholders" 2>&1)" || rc=$?
|
|
while IFS= read -r i; do r_line "$i"; done <<< "$out"
|
|
[ "$rc" = 0 ] || { R_RC=1; r_end; }
|
|
|
|
r_migration
|
|
|
|
START_MS="$(date +%s%3N)"
|
|
R_STARTED=1
|
|
# env, not r_env: a function in the background is a subshell, and $! has to be the process
|
|
# itself (env execs scripts/mosaic, which execs node), for the bus-status pid check and the stop.
|
|
env -i HOME="$HOME" PATH="$PATH" MOSAIC_CONFIG="$R/config/config.json" "$COPIES/$TO/scripts/mosaic" bus start "$BUSINESS" --no-launch > "$R/host.log" 2>&1 &
|
|
R_HOST_PID=$!
|
|
env -i HOME="$HOME" PATH="$PATH" MOSAIC_CONFIG="$R/config/config.json" node "$COPIES/$TO/packages/webui/src/cli.mjs" serve --port "$R_PORT" --business "$BUSINESS" > "$R/console.log" 2>&1 &
|
|
R_CONSOLE_PID=$!
|
|
if r_gate; then r_no_session; fi
|
|
r_end
|
|
}
|
|
|
|
# --- Dispatch ---
|
|
|
|
# Row 190: --instance NAME may come anywhere; instance.sh takes it out and sets the names.
|
|
instance_args "$@"
|
|
set -- ${INSTANCE_ARGS[@]+"${INSTANCE_ARGS[@]}"}
|
|
[ $# -gt 0 ] || usage
|
|
OP="$1"; shift
|
|
ID=""
|
|
case "$OP" in
|
|
activate|rehearse) [ $# -ge 2 ] || usage; ID="$1"; BUSINESS="$2"; shift 2; [[ "$ID" =~ ^[0-9a-f]{40}$ ]] || usage ;;
|
|
adopt|pm-on|rollback|status) [ $# -ge 1 ] || usage; BUSINESS="$1"; shift ;;
|
|
confirm|revert|gate) [ $# -ge 2 ] || usage; ID="$1"; BUSINESS="$2"; shift 2; [[ "$ID" =~ ^[0-9a-f]{40}$ ]] || usage ;;
|
|
-h|--help) usage 0 ;;
|
|
*) usage ;;
|
|
esac
|
|
[[ "$BUSINESS" =~ ^[a-z0-9][a-z0-9-]*$ ]] || usage
|
|
parse_opts "$@"
|
|
# Row 232: the stack-copy.sh check flags and the systemctl the instance-busy message names.
|
|
UNITS_FLAGS=()
|
|
SC_NAME="systemctl --user"
|
|
if [ "$UNITS" = external ]; then UNITS_FLAGS=(--units external); SC_NAME=systemctl; fi
|
|
COPIES_DIR="$COPIES"
|
|
instance_select
|
|
load_config
|
|
DATA="$MOSAIC_DATA_ROOT"
|
|
STATE="$DATA/state"
|
|
POINTER="$STATE/stack-active.json"
|
|
LOG="$STATE/stack-activation-log.jsonl"
|
|
STORE="$DATA/bus/bus.sqlite"
|
|
BUS_UNIT="${BUS_TEMPLATE%.service}$BUSINESS.service"
|
|
FROM="" TO="" ACT="" RS="" POST="" BACKUP="" TAKEN="" RESTORED="" STEPS="" COUNT="" UNCLEAN="" FAIL=""
|
|
STORE_SCHEMA="" START_EPOCH="" START_MS=0 GATE="" GATE_WHY="" S_CHANGED=""
|
|
P_COPY="" P_PM=0 P_PENDING=0 P_SCHEMA="" P_CLAUDE="" P_CONSOLE=() RUN_PM=0 RUN_PENDING=0 RUN_CONSOLE=() DROP_TESTER="" PROOF_PLAN="" PROOF_STEP="" SEND_PLAN="" SEND_STEP="" PLAN_REFUSE=()
|
|
BASE_CONSOLE=() SERVE_FLAGS=() SSO_PLAN="" SSO_STEP="" SSO_DROP="" SSO_NO_TARGET=""
|
|
R="" R_RC=0 R_EARLY=() R_BK="" R_HOST_PID="" R_CONSOLE_PID="" R_STARTED="" LIVE_CFG="" LIVE_BIZ=""
|
|
O_ACT="" FLOOR=0
|
|
case "$OP" in
|
|
adopt) do_adopt ;;
|
|
activate) do_activate ;;
|
|
pm-on) do_pm_on ;;
|
|
rollback) do_rollback ;;
|
|
status) do_status ;;
|
|
rehearse) do_rehearse ;;
|
|
confirm) do_confirm ;;
|
|
revert) do_revert ;;
|
|
gate) do_gate ;;
|
|
esac
|