stack: several Mosaic Stack instances on one host (row 190, #1688)

--instance NAME on the shell entry points through scripts/instance.sh;
per-instance config, data root and marker, units, wrapper and ports;
the claim checks in mosaic-config.mjs; the guard protects every
instance's config and data root. Brief blob 4e8f1e0528, design
agents/darkwing/work/queue-190/DESIGN.md rev 3 (Filbert approved).
This commit is contained in:
jason.woltje committed 2026-10-20 16:30:34 -05:00
1 parent 176a6bf4b9
commit ab1d2ee9e6
24 files changed
+1490 -212

No files matched your search

+67 -31
View File
@@ -24,11 +24,11 @@ set -euo pipefail
cd "$(dirname "$0")/.."
REPO="$(pwd)"
source scripts/common.sh
source scripts/instance.sh
COPIES="${XDG_DATA_HOME:-$HOME/.local/share}/mosaic-stack/copies"
BIN="$HOME/.npm-global/bin"
UNIT_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user"
CONSOLE_UNIT=mosaic-webui.service
TIMEOUT=60 WANT_PM=0 AFTER_HARNESS=0 MIGRATE=0 PLAN=0 RESTORE="" EVENTS="" TESTER_PORT=""
PROOF_HEADER="" PROOF_FILE="" NO_PROOF=0 NO_SEND=0 R_BACKUP="" R_PORT=""
@@ -41,7 +41,9 @@ usage: stack-release.sh adopt BUSINESS
stack-release.sh rollback BUSINESS [--restore ACTIVATION [--events N]] [--timeout S] [--plan]
stack-release.sh status BUSINESS
stack-release.sh rehearse ID BUSINESS --backup ACTIVATION|PATH --port N [--timeout S]
common options: --copies DIR, --bin DIR
common options: --copies DIR, --bin DIR, --instance NAME
--instance NAME (row 190) runs on stack instance NAME: its config, data root, units and the wrapper
mosaic-NAME. Without it, the default instance, as before.
--events N is the total of the per-table counts the post-backup-events refusal prints: events and
every table in COUNTED.
USAGE
@@ -69,7 +71,10 @@ refuse() {
# Reject tunnel/main ports and anything other than a whole high port (exit 4 via usage).
valid_tester_port() {
[[ "$1" =~ ^[0-9]{1,5}$ ]] && [ "$((10#$1))" -ge 1024 ] && [ "$((10#$1))" -le 65535 ] &&
[ "$((10#$1))" -ne 7332 ] && [ "$((10#$1))" -ne "$2" ]
[ "$((10#$1))" -ne 7332 ] && [ "$((10#$1))" -ne "$2" ] || return 1
# Row 190: a named instance's Console never takes 7330, 7332 or 7333.
local r
for r in "${INSTANCE_RESERVED[@]}"; do [ "$((10#$1))" -ne "$r" ] || return 1; done
}
parse_opts() {
@@ -416,16 +421,17 @@ capture() {
mkdir -m 0700 "$RS" 2>/dev/null || return 1
STEPS="$RS/steps.log"
local u d
for u in [email protected] "$CONSOLE_UNIT"; do
for u in "$BUS_TEMPLATE" "$CONSOLE_UNIT"; do
[ -f "$UNIT_DIR/$u" ] || { say "capture: no $UNIT_DIR/$u"; return 1; }
cp -p "$UNIT_DIR/$u" "$RS/$u" || return 1
done
for d in [email protected] "mosaic-bus@$BUSINESS.service.d" "$CONSOLE_UNIT.d"; do
# This instance's own directories only; the shared prefix ones are never captured (row 190).
for d in "$BUS_TEMPLATE.d" "$BUS_UNIT.d" "$CONSOLE_UNIT.d"; do
[ ! -d "$UNIT_DIR/$d" ] || cp -pr "$UNIT_DIR/$d" "$RS/$d" || return 1
done
[ -f "$BIN/mosaic" ] || { say "capture: no wrapper $BIN/mosaic"; return 1; }
cp -p "$BIN/mosaic" "$RS/wrapper" || return 1
stat -c %a "$BIN/mosaic" > "$RS/WRAPPER_MODE" || return 1
[ -f "$BIN/$WRAPPER_NAME" ] || { say "capture: no wrapper $BIN/$WRAPPER_NAME"; return 1; }
cp -p "$BIN/$WRAPPER_NAME" "$RS/wrapper" || return 1
stat -c %a "$BIN/$WRAPPER_NAME" > "$RS/WRAPPER_MODE" || return 1
for u in "$BUS_UNIT" "$CONSOLE_UNIT"; do show "$u" > "$RS/show-$u.txt" || return 1; done
sums "$RS" > "$RS/SHA256SUMS" || return 1
say "captured the rollback set: $RS"
@@ -458,9 +464,9 @@ install_stack() {
local id="$1" pm="$2" claude="$3"; shift 3
local -a pmflag=()
[ "$pm" = 0 ] || pmflag=(--pm)
scripts/bus-service.sh install --copy "$id" --copies "$COPIES" "${pmflag[@]}" --claude "$claude" >> "$STEPS" 2>&1 || { FAIL=install-bus; return 1; }
scripts/webui-service.sh install --copy "$id" --copies "$COPIES" "$@" >> "$STEPS" 2>&1 || { FAIL=install-console; return 1; }
scripts/stack-copy.sh wrapper "$id" --copies "$COPIES" --bin "$BIN" >> "$STEPS" 2>&1 || { FAIL=wrapper; return 1; }
scripts/bus-service.sh install "${INSTANCE_FLAGS[@]}" --copy "$id" --copies "$COPIES" "${pmflag[@]}" --claude "$claude" >> "$STEPS" 2>&1 || { FAIL=install-bus; return 1; }
scripts/webui-service.sh install "${INSTANCE_FLAGS[@]}" --copy "$id" --copies "$COPIES" "$@" >> "$STEPS" 2>&1 || { FAIL=install-console; return 1; }
scripts/stack-copy.sh wrapper "$id" "${INSTANCE_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" >> "$STEPS" 2>&1 || { FAIL=wrapper; return 1; }
say "installed $id"
}
@@ -486,7 +492,7 @@ process.stdout.write(why);' -- "$out" "$BUSINESS" "$main" "$START_MS" 2>&1)" ||
node -e '
fetch(`http://127.0.0.1:${process.argv[1]}/healthz`,{signal:AbortSignal.timeout(5000)}).then((r)=>process.exit(r.status===200?0:1),()=>process.exit(1));' -- "$(console_port)" || return 1
GATE=copy-check
out="$(scripts/stack-copy.sh check --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>/dev/null)" || return 1
out="$(scripts/stack-copy.sh check "${INSTANCE_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>/dev/null)" || return 1
# check passes when the three agree on one copy; this run needs it to be the target (row 166 item 3).
out="$(sed -n 's/^bus: //p' <<< "$out")"
[ "$out" = "$copy" ] || { GATE_WHY="the bus, Console and CLI agree on $out, not $copy"; return 1; }
@@ -533,10 +539,10 @@ gate() {
# Puts back the captured unit files, drop-ins and wrapper, checked against SHA256SUMS.
put_back() {
local u f rel tmp
for u in [email protected] "$CONSOLE_UNIT"; do
for u in "$BUS_TEMPLATE" "$CONSOLE_UNIT"; do
tmp="$(mktemp "$UNIT_DIR/.$u.XXXXXX")" && cp -p "$RS/$u" "$tmp" && mv -f "$tmp" "$UNIT_DIR/$u" || return 1
done
for u in [email protected] "mosaic-bus@$BUSINESS.service.d" "$CONSOLE_UNIT.d"; do
for u in "$BUS_TEMPLATE.d" "$BUS_UNIT.d" "$CONSOLE_UNIT.d"; do
[ -d "$RS/$u" ] || continue
while IFS= read -r -d '' f; do
rel="${f#"$RS/$u/"}"
@@ -545,12 +551,12 @@ put_back() {
tmp="$(mktemp "$UNIT_DIR/$u/.restore.XXXXXX")" && cp -p "$f" "$tmp" && mv -f "$tmp" "$UNIT_DIR/$u/$rel" || return 1
done < <(find "$RS/$u" -type f -print0)
done
tmp="$(mktemp "$BIN/.mosaic.XXXXXX")" && cp -p "$RS/wrapper" "$tmp" && chmod "$(cat "$RS/WRAPPER_MODE")" "$tmp" && mv -f "$tmp" "$BIN/mosaic" || return 1
for u in [email protected] "$CONSOLE_UNIT"; do
tmp="$(mktemp "$BIN/.$WRAPPER_NAME.XXXXXX")" && cp -p "$RS/wrapper" "$tmp" && chmod "$(cat "$RS/WRAPPER_MODE")" "$tmp" && mv -f "$tmp" "$BIN/$WRAPPER_NAME" || return 1
for u in "$BUS_TEMPLATE" "$CONSOLE_UNIT"; do
[ "$(sha256sum < "$UNIT_DIR/$u" | cut -d' ' -f1)" = "$(sum_of "$RS" "$u")" ] || return 1
done
[ "$(sha256sum < "$BIN/mosaic" | cut -d' ' -f1)" = "$(sum_of "$RS" wrapper)" ] || return 1
[ "$(stat -c %a "$BIN/mosaic")" = "$(cat "$RS/WRAPPER_MODE")" ] || return 1
[ "$(sha256sum < "$BIN/$WRAPPER_NAME" | cut -d' ' -f1)" = "$(sum_of "$RS" wrapper)" ] || return 1
[ "$(stat -c %a "$BIN/$WRAPPER_NAME")" = "$(cat "$RS/WRAPPER_MODE")" ] || return 1
systemctl --user daemon-reload || return 1
say "put back the unit files and the wrapper from $RS"
}
@@ -607,9 +613,9 @@ lock() {
instance_busy() {
local out unit load state rest
out="$(systemctl --user list-units --all --no-legend --plain 'mosaic-bus@*.service' 2>/dev/null)" || refuse instance-busy "systemctl --user list-units failed, so the bus state is unknown"
out="$(systemctl --user list-units --all --no-legend --plain "$BUS_GLOB" 2>/dev/null)" || refuse instance-busy "systemctl --user list-units failed, so the bus state is unknown"
while read -r unit load state rest; do
[[ "$unit" == mosaic-bus@*.service ]] || continue
[[ "$unit" == $BUS_GLOB ]] || continue
if [ "$unit" = "$BUS_UNIT" ]; then
case "$state" in activating|deactivating|reloading) refuse instance-busy "$unit is $state" ;; esac
else
@@ -621,14 +627,17 @@ instance_busy() {
# install_guard's drop-in scan (bus-service.sh), done before anything stops. A path it can't read
# refuses here, as install_guard would refuse it after the stop (row 166 item 1).
drop_in_override() {
local paths directory dropdir file content
local paths directory dropdir file content glob
local -a dropdirs
paths="$(systemd-analyze --user unit-paths 2>/dev/null)" && [ -n "$paths" ] || refuse drop-in-override "systemd unit paths are unknown"
while IFS= read -r directory; do
[ -n "$directory" ] || continue
case "$directory" in /*) ;; *) refuse drop-in-override "invalid systemd unit path $directory" ;; esac
[ -e "$directory" ] || [ -L "$directory" ] || continue
[ -d "$directory" ] && [ -r "$directory" ] && [ -x "$directory" ] || refuse drop-in-override "cannot inspect unit path $directory"
for dropdir in "$directory"/mosaic-bus@*.service.d "$directory"/mosaic-.service.d "$directory"/service.d; do
dropdirs=()
for glob in "${BUS_DROPIN_GLOBS[@]}"; do dropdirs+=("$directory"/$glob); done
for dropdir in "${dropdirs[@]}"; do
[ -e "$dropdir" ] || [ -L "$dropdir" ] || continue
[ -d "$dropdir" ] && [ -r "$dropdir" ] && [ -x "$dropdir" ] || refuse drop-in-override "cannot inspect drop-ins $dropdir"
for file in "$dropdir"/*.conf; do
@@ -684,6 +693,7 @@ console_for_target() {
[ "$PLAN" = 1 ] || usage
local other=""
[ "$main" = 7332 ] || other=" and the Console's --port $main"
[ "${#INSTANCE_RESERVED[@]}" = 0 ] || other+=" and instance $MOSAIC_INSTANCE's reserved ${INSTANCE_RESERVED[*]}"
check_refuse tester-port-invalid "--tester-port $TESTER_PORT is not a whole port from 1024 to 65535 other than 7332$other"
fi
[ "$supported" = 1 ] || check_refuse tester-port-target "$TO's Console does not support --tester-port"
@@ -813,10 +823,30 @@ proof_file_problem() {
[ "$uid" = "$(id -u)" ] || echo "is owned by uid $uid, not $(id -u), the uid this script runs as"
}
# Row 190: the target Console's --port (7332 when absent) and --tester-port against every other
# installed Console unit in UNIT_DIR, the default's and each named instance's.
ports_unclaimed() {
local main=7332 i holder
local -a ports=()
for ((i = 0; i < ${#RUN_CONSOLE[@]}; i++)); do
case "${RUN_CONSOLE[i]}" in
--port) main="${RUN_CONSOLE[i + 1]:-7332}" ;;
--tester-port) ports+=("${RUN_CONSOLE[i + 1]:-}") ;;
esac
done
for i in "$main" "${ports[@]}"; do
[[ "$i" =~ ^[0-9]{1,5}$ ]] || continue
if holder="$(instance_port_holder "$UNIT_DIR" "$i")"; then
check_refuse port-claimed "port $i is taken by $UNIT_DIR/$holder"
fi
done
}
# The item 3 checks after the pointer, shared by activate and rollback, for target $TO.
precheck() {
check_target
console_for_target
ports_unclaimed
instance_busy
drop_in_override
read_units || refuse unit-flags-drift "cannot read $BUS_UNIT or $CONSOLE_UNIT"
@@ -857,7 +887,7 @@ do_adopt() {
lock
[ ! -f "$POINTER" ] || { read_pointer; FROM="$P_COPY"; refuse already-adopted "$POINTER exists (copy $P_COPY)"; }
local out
out="$(scripts/stack-copy.sh check --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>&1)" || refuse check-failed "stack-copy.sh check did not pass:"$'\n'"$out"
out="$(scripts/stack-copy.sh check "${INSTANCE_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>&1)" || refuse check-failed "stack-copy.sh check did not pass:"$'\n'"$out"
TO="$(sed -n 's/^bus: //p' <<< "$out")"
TO="${TO#"$COPIES/"}"
read_units || refuse check-failed "cannot read $BUS_UNIT or $CONSOLE_UNIT"
@@ -970,12 +1000,13 @@ plan() {
plan_commits
[ "$P_PENDING" = 0 ] || echo "pm: the pointer is pending (activate --pm-after-harness); the bus runs without its PM"
[ "$RUN_PENDING" = 0 ] || echo "pm: the bus starts without --pm and the PM stays pending; next: Jason's plugins set of harness, then pm-on"
echo "bus: scripts/bus-service.sh install --copy $TO --copies $COPIES$([ "$RUN_PM" = 0 ] || echo ' --pm') --claude $P_CLAUDE"
local flags="${INSTANCE_FLAGS[*]:+${INSTANCE_FLAGS[*]} }"
echo "bus: scripts/bus-service.sh install $flags--copy $TO --copies $COPIES$([ "$RUN_PM" = 0 ] || echo ' --pm') --claude $P_CLAUDE"
[ -z "$DROP_TESTER" ] || echo "drop $DROP_TESTER; the target Console does not support it"
echo "$PROOF_PLAN"
echo "$SEND_PLAN"
echo "Console: scripts/webui-service.sh install --copy $TO --copies $COPIES${RUN_CONSOLE[*]:+ ${RUN_CONSOLE[*]}}"
echo "CLI: scripts/stack-copy.sh wrapper $TO --copies $COPIES --bin $BIN"
echo "Console: scripts/webui-service.sh install $flags--copy $TO --copies $COPIES${RUN_CONSOLE[*]:+ ${RUN_CONSOLE[*]}}"
echo "CLI: scripts/stack-copy.sh wrapper $TO $flags--copies $COPIES --bin $BIN"
local r
for r in "${PLAN_REFUSE[@]}"; do echo "refuses $r"; done
[ "${#PLAN_REFUSE[@]}" = 0 ] || exit 3
@@ -1047,7 +1078,7 @@ do_activate() {
pointer_for_business
run_pm
[ "$P_PENDING" = 0 ] || check_refuse pm-pending "the pointer is pending (activate --pm-after-harness); run pm-on or rollback first"
if [ "$PLAN" = 1 ]; then check_target; console_for_target; plugin_checks; plan; exit 0; fi
if [ "$PLAN" = 1 ]; then check_target; console_for_target; ports_unclaimed; plugin_checks; plan; exit 0; fi
precheck
plugin_checks
new_activation
@@ -1059,7 +1090,7 @@ do_rollback() {
pointer_for_business
TO="$(from_log previous)" || refuse no-previous "the log names no earlier copy than $P_COPY"
run_pm
if [ "$PLAN" = 1 ]; then check_target; console_for_target; plugin_checks; plan; exit 0; fi
if [ "$PLAN" = 1 ]; then check_target; console_for_target; ports_unclaimed; plugin_checks; plan; exit 0; fi
precheck
plugin_checks
[ "$P_SCHEMA" = "$T_SCHEMA" ] || [ -n "$RESTORE" ] || refuse schema-differs "the store is at $P_SCHEMA and $TO expects $T_SCHEMA; pass --restore ACTIVATION"
@@ -1153,7 +1184,7 @@ do_pm_on() {
capture || refuse capture-failed "could not write the rollback set $RS; nothing was stopped"
trap 'pm_on_fail signal' INT TERM HUP
stop_bus || pm_on_fail stop-timeout
scripts/bus-service.sh install --copy "$P_COPY" --copies "$COPIES" --pm --claude "$P_CLAUDE" >> "$STEPS" 2>&1 || pm_on_fail install-bus
scripts/bus-service.sh install "${INSTANCE_FLAGS[@]}" --copy "$P_COPY" --copies "$COPIES" --pm --claude "$P_CLAUDE" >> "$STEPS" 2>&1 || pm_on_fail install-bus
say "installed $BUS_UNIT for $P_COPY with --pm"
start_bus || pm_on_fail start
gate "$P_COPY" 1 || pm_on_fail "$GATE"
@@ -1173,7 +1204,7 @@ do_status() {
open="$(interrupted | paste -sd' ' -)"
[ -z "$open" ] || echo "interrupted run: $open (started, no final line; Jason or Mos settles it)"
echo "check:"
if scripts/stack-copy.sh check --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>&1 | sed 's/^/ /'; then echo " result: pass"; else echo " result: FAIL"; fi
if scripts/stack-copy.sh check "${INSTANCE_FLAGS[@]}" --copies "$COPIES" --bin "$BIN" --business "$BUSINESS" 2>&1 | sed 's/^/ /'; then echo " result: pass"; else echo " result: FAIL"; fi
[ -n "$P_COPY" ] || return 0
echo "bus status:"
"$COPIES/$P_COPY/scripts/mosaic" bus status 2>&1 | sed 's/^/ /' || true
@@ -1437,6 +1468,7 @@ do_rehearse() {
local -a reserved=(7332 7333 "$(console_port)")
for ((i = 0; i < ${#P_CONSOLE[@]}; i++)); do [ "${P_CONSOLE[i]}" != --tester-port ] || reserved+=("${P_CONSOLE[i + 1]:-}"); done
for i in "${reserved[@]}"; do [ "$((10#$R_PORT))" != "$i" ] || r_abort port-reserved "--port $R_PORT is a live Console port (7332, 7333 and the pointer's: ${reserved[*]:2})"; done
if i="$(instance_port_holder "$UNIT_DIR" "$R_PORT")"; then r_abort port-reserved "--port $R_PORT is $UNIT_DIR/$i's Console port"; fi
r_port_free "$R_PORT" || r_abort port-in-use "127.0.0.1:$R_PORT is in use; the rehearsal takes no other port"
r_pass port "127.0.0.1:$R_PORT is free"
case "$R_BACKUP" in /*) R_BK="$R_BACKUP" ;; *) R_BK="$DATA/backups/stack/$R_BACKUP" ;; esac
@@ -1489,6 +1521,9 @@ do_rehearse() {
# --- Dispatch ---
# Row 190: --instance NAME may come anywhere; instance.sh takes it out and sets the names.
instance_args "$@"
set -- ${INSTANCE_ARGS[@]+"${INSTANCE_ARGS[@]}"}
[ $# -gt 0 ] || usage
OP="$1"; shift
ID=""
@@ -1501,13 +1536,14 @@ esac
[[ "$BUSINESS" =~ ^[a-z0-9][a-z0-9-]*$ ]] || usage
parse_opts "$@"
COPIES_DIR="$COPIES"
instance_select
load_config
DATA="$MOSAIC_DATA_ROOT"
STATE="$DATA/state"
POINTER="$STATE/stack-active.json"
LOG="$STATE/stack-activation-log.jsonl"
STORE="$DATA/bus/bus.sqlite"
BUS_UNIT="mosaic-bus@$BUSINESS.service"
BUS_UNIT="${BUS_TEMPLATE%.service}$BUSINESS.service"
FROM="" TO="" ACT="" RS="" POST="" BACKUP="" TAKEN="" RESTORED="" STEPS="" COUNT="" UNCLEAN="" FAIL=""
STORE_SCHEMA="" START_EPOCH="" START_MS=0 GATE="" GATE_WHY="" S_CHANGED=""
P_COPY="" P_PM=0 P_PENDING=0 P_SCHEMA="" P_CLAUDE="" P_CONSOLE=() RUN_PM=0 RUN_PENDING=0 RUN_CONSOLE=() DROP_TESTER="" PROOF_PLAN="" PROOF_STEP="" SEND_PLAN="" SEND_STEP="" PLAN_REFUSE=()