mosaicstack/stack
25
0 Fork 1
Code Issues 404 Pull Requests 13 Packages Projects Releases 15 Wiki Activity

fix(cli): M3 review round: re-pin migration, rehearsal provider (#1873)

Browse Source 
An installed unit must equal the render but for the copy id, so a
re-pin that adds --sso refused render-refused. case2-unit gains --check
(every render check, nothing written) and --check-new (a first install's
checks). stack-upgrade runs both checks before anything stops, and
--next-args reads root's <instance>.args.next: equal before -- but for
--copy, different after it. After the stage it moves the installed unit
files into a new <instance>.replaced.XXXXXX directory and renders a
first install. A failure moves them back unchanged and restarts the old
copy, and confirm swaps the args file. The run sheet gives the commands,
the rollback past the migration included.

A stamped rehearsal now starts an isolated provider on 127.0.0.1 port 0
with a certificate and keys made for the run. It serves discovery and a
JWKS for the live provider ids and answers 403 to authorize and token.
Only the scratch Console trusts its certificate. The gate requires each
provider ready (sso-ready); provider-discovery requires discovery and
JWKS fetches and no code or token request. The provider stops with the
run, and by itself once the run's shell is gone.

The system-path test helper asserts the log's from copy again. The
pinned-copy sign-in requests are bounded, and cleanup waits for each
Console to exit.
This commit is contained in:
jason.woltje committed 2026-11-10 21:38:17 -06:00
1 parent db822c5b9d
commit 9acc378555
10 files changed
+766 -116
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/case2-unit.sh
+20 -5
View File
@@ -5,9 +5,15 @@
# mosaic-proxy.service under --proxy, from the existing templates, which it
# doesn't change.
#
# case2-unit.sh --user NAME --home DIR --business ID --copy ID --node PATH
# [--profile factory|library] [--proxy] [--pm] [--claude PATH]
# -- SERVE FLAGS
# case2-unit.sh [--check|--check-new] --user NAME --home DIR --business ID
# --copy ID --node PATH [--profile factory|library] [--proxy]
# [--pm] [--claude PATH] -- SERVE FLAGS
#
# --check runs every check a render runs, the installed units' included, then
# exits 0 without writing, enabling or starting anything. --check-new does the
# same as a first install would, without the installed units' checks; it is for
# stack-upgrade.sh --next-args, which moves the installed units aside before its
# render (docs/guides/host-run-sheet.md section 7). Neither writes a file.
#
# --org ID is the same option as --business ID (rename design r2), in each of
# two scopes: this script's own flags before --, and the serve flags after it.
@@ -85,7 +91,7 @@ PROXY_UNIT=mosaic-proxy.service
BUS_UNIT=[email protected]
WEBUI_UNIT=mosaic-webui.service
usage() { echo 'usage: case2-unit.sh --user NAME --home DIR --business ID --copy ID --node PATH [--profile factory|library] [--proxy] [--pm] [--claude PATH] -- SERVE FLAGS' >&2; exit 4; }
usage() { echo 'usage: case2-unit.sh [--check|--check-new] --user NAME --home DIR --business ID --copy ID --node PATH [--profile factory|library] [--proxy] [--pm] [--claude PATH] -- SERVE FLAGS' >&2; exit 4; }
die() { echo "case2-unit: $*" >&2; exit 1; }
# A value the unit carries as itself: no space, %, $, quote, backslash, & or |.
plain() { case "$1" in *[!A-Za-z0-9/._+@-]*) return 1 ;; esac; }
@@ -116,7 +122,8 @@ serve_refused() {
*) die "serve $2" ;;
esac
}
PROFILE=library USER_NAME='' HOME_DIR='' BUSINESS='' COPY_ID='' NODE='' PROXY=0 PM=0 CLAUDE='' CLAUDE_SET=0
PROFILE=library USER_NAME='' HOME_DIR='' BUSINESS='' COPY_ID='' NODE='' PROXY=0 PM=0 CLAUDE='' CLAUDE_SET=0 CHECK=''
case "${1-}" in --check|--check-new) CHECK="$1"; shift ;; esac
while [ $# -gt 0 ]; do
case "$1" in
--business|--org) BUSINESS="${2-}"; selector_take outer id outer_refused business_id_ok "$1" "${@:2:1}"; shift 2 ;;
@@ -720,6 +727,7 @@ else
|| die "installed $PROXY_UNIT requires --proxy; nothing was changed"
fi
for pair in "${PAIRS[@]}"; do
[ "$CHECK" != --check-new ] || break
IFS=: read -r kind name var <<<"$pair"
path="$UNIT_DIR/$name"
[ ! -L "$path" ] || die "refusing: $path is a symlink"
@@ -737,6 +745,13 @@ read -r -a ids <<<"$installed_ids"
for old in "${ids[@]}"; do
[ "$old" = "${ids[0]}" ] || die "the installed units name different copies, ${ids[0]} and $old"
done
if [ "$CHECK" = --check ]; then
echo "case2-unit: check passed: the units for copy $COPY_ID render, and the installed ones match them; nothing was written"
exit 0
elif [ "$CHECK" = --check-new ]; then
echo "case2-unit: check passed: the units for copy $COPY_ID render as a first install; the installed ones were not compared; nothing was written"
exit 0
fi
# `systemctl start` leaves a running unit on the copy it started from, so
# note which units run before the write (row 195 owns the restart).
scripts/stack-release.sh
+137 -21
View File
@@ -2041,12 +2041,16 @@ r_gone() {
while r_alive "$1"; do [ "$SECONDS" -lt "$deadline" ] || return 1; sleep 1; done
}
# Stops the Console, then the host, with SIGTERM within --timeout; one still there gets SIGKILL
# and a FAIL line. The host has to leave the store's writer.lock behind it.
# Stops the Console, the host, then the rehearsal provider, with SIGTERM within --timeout; one
# still there gets SIGKILL and a FAIL line. The host has to leave the store's writer.lock behind it.
r_stop() {
local name pid
for name in console host; do
if [ "$name" = console ]; then pid="$R_CONSOLE_PID"; R_CONSOLE_PID=""; else pid="$R_HOST_PID"; R_HOST_PID=""; fi
for name in console host provider; do
case "$name" in
console) pid="$R_CONSOLE_PID"; R_CONSOLE_PID="" ;;
host) pid="$R_HOST_PID"; R_HOST_PID="" ;;
provider) pid="$R_PROVIDER_PID"; R_PROVIDER_PID="" ;;
esac
[ -n "$pid" ] || continue
kill -TERM "$pid" 2>/dev/null || true
if r_gone "$pid"; then r_pass "$name-stopped" "pid $pid"
@@ -2080,20 +2084,77 @@ r_port_free() {
}
# The scratch config: the live one with dataRoot set to the scratch data root. M3 (#1873): a
# console.sso keeps its provider ids, client ids, algorithms and subjects, but the public origin and
# every issuer move to rehearsal.invalid, a name that never resolves (RFC 6761). The scratch Console
# reaches no real provider: discovery fails as unreachable and the Console starts degraded, after
# every other startup check. No subject is added; the live config's own are the only ones.
# console.sso keeps its provider ids, client ids, algorithms and subjects; the public origin moves to
# rehearsal.invalid, a name that never resolves (RFC 6761). For a stamped copy, whose Console signs
# in, every issuer moves to the rehearsal's own provider (R_PROVIDER_JS, the origin argument). An
# unstamped copy's Console runs without --sso, so its issuers move to rehearsal.invalid too. No
# subject is added; the live config's own are the only ones.
R_CONFIG_JS='
const fs=require("node:fs");const [file,data]=process.argv.slice(1);
const fs=require("node:fs");const [file,data,op]=process.argv.slice(1);
const c=JSON.parse(fs.readFileSync(file,"utf8"));c.dataRoot=data;
const sso=c.console?.sso;
if(sso&&typeof sso==="object"){
sso.publicOrigin="https://rehearsal.invalid";
for(const p of Array.isArray(sso.providers)?sso.providers:[])if(p&&typeof p==="object")p.issuer=`https://rehearsal.invalid/${encodeURIComponent(String(p.id))}`;
for(const p of Array.isArray(sso.providers)?sso.providers:[])if(p&&typeof p==="object")p.issuer=`${op||"https://rehearsal.invalid"}/${encodeURIComponent(String(p.id))}${op?"/":""}`;
}
process.stdout.write(JSON.stringify(c,null,2)+"\n");'
# M3 (#1873): the rehearsal's own OpenID provider, for a stamped copy's --sso Console. An https
# server on 127.0.0.1, port 0, with a certificate openssl makes for this run in DIR. Its TLS key is
# read into memory and its file removed before the server listens; cert.pem stays for the scratch
# Console's NODE_EXTRA_CA_CERTS. Per configured provider, at /<id>/, it serves discovery and a JWKS
# holding one public key made in memory for the first of the provider's algorithms it can make.
# Authorization, token and every other route answer 403 or 404: it issues no code, no token and no
# sign-in. Each request is a line in requests.log. It writes origin when listening and exits when
# PARENT (the rehearsal) is gone, on SIGTERM, or on an error.
R_PROVIDER_JS='
const fs=require("node:fs"),path=require("node:path"),https=require("node:https"),crypto=require("node:crypto"),{spawnSync}=require("node:child_process");
const [dir,cfg,parent]=process.argv.slice(1);
const die=(m)=>{console.error(`rehearsal provider: ${m}`);process.exit(1)};
const sso=JSON.parse(fs.readFileSync(cfg,"utf8")).console?.sso;
if(!sso||!Array.isArray(sso.providers)||!sso.providers.length)die("the live config has no console.sso providers");
const GEN={RS:["rsa",{modulusLength:2048}],PS:["rsa",{modulusLength:2048}],ES256:["ec",{namedCurve:"P-256"}],ES384:["ec",{namedCurve:"P-384"}],ES512:["ec",{namedCurve:"P-521"}],Ed25519:["ed25519",{}],EdDSA:["ed25519",{}]};
const keyFor=(alg)=>{
const g=GEN[alg]??GEN[String(alg).slice(0,2)]??(/^ML-DSA-(44|65|87)$/.test(alg)?[alg.toLowerCase(),{}]:null);
if(!g)return null;
try{const {publicKey}=crypto.generateKeyPairSync(...g);return {...publicKey.export({format:"jwk"}),kid:`rehearsal-${crypto.randomBytes(4).toString("hex")}`,use:"sig",alg}}catch{return null}
};
const providers=new Map();
for(const p of sso.providers){
const id=String(p?.id),algs=Array.isArray(p?.algs)?p.algs:[];
let jwk=null;for(const a of algs){jwk=keyFor(a);if(jwk)break}
if(!jwk)die(`provider ${id}: none of ${algs.join(", ")||"no algorithms"} is one this provider can make a key for`);
providers.set(encodeURIComponent(id),jwk);
}
const tls=path.join(dir,"tls-key.pem"),certFile=path.join(dir,"cert.pem");
const r=spawnSync("openssl",["req","-x509","-newkey","ec","-pkeyopt","ec_paramgen_curve:P-256","-nodes","-days","1","-subj","/CN=127.0.0.1","-addext","subjectAltName=IP:127.0.0.1","-keyout",tls,"-out",certFile],{encoding:"utf8"});
let key,cert;
try{if(r.status!==0){const why=String(r.stderr??"").trim().split("\n").pop();throw new Error(`openssl exit ${r.status??r.error?.code}${why?`: ${why}`:""}`)};key=fs.readFileSync(tls);cert=fs.readFileSync(certFile);fs.chmodSync(certFile,0o600)}
catch(e){die(`cannot make the certificate: ${e.message}`)}
finally{fs.rmSync(tls,{force:true})}
const log=(q,status)=>fs.appendFileSync(path.join(dir,"requests.log"),`${q.method} ${q.url} ${status}\n`);
let origin;
const server=https.createServer({key,cert,requestTimeout:5000,headersTimeout:5000},(q,res)=>{
const send=(status,body)=>{log(q,status);res.writeHead(status,{"content-type":"application/json","cache-control":"no-store"});res.end(JSON.stringify(body))};
if(q.method!=="GET")return send(405,{error:"GET only"});
const m=/^\/([^/?#]+)\/(\.well-known\/openid-configuration|jwks|authorize|token)$/.exec(q.url);
const jwk=m?providers.get(m[1]):undefined;
if(!jwk)return send(404,{error:"not found"});
const issuer=`${origin}/${m[1]}/`;
if(m[2]==="jwks")return send(200,{keys:[jwk]});
if(m[2]!==".well-known/openid-configuration")return send(403,{error:"the rehearsal provider issues no code and no token"});
send(200,{issuer,authorization_endpoint:`${issuer}authorize`,token_endpoint:`${issuer}token`,jwks_uri:`${issuer}jwks`,response_types_supported:["code"],subject_types_supported:["public"],id_token_signing_alg_values_supported:[jwk.alg],code_challenge_methods_supported:["S256"]});
});
server.on("error",(e)=>die(e.message));
server.listen(0,"127.0.0.1",()=>{
origin=`https://127.0.0.1:${server.address().port}`;
fs.writeFileSync(path.join(dir,"origin.tmp"),`${origin}\n`,{mode:0o600});fs.renameSync(path.join(dir,"origin.tmp"),path.join(dir,"origin"));
console.log(`rehearsal provider: ${origin}, ${providers.size} provider(s)`);
});
const stop=()=>{server.close();server.closeAllConnections();process.exit(0)};
process.on("SIGTERM",stop);process.on("SIGINT",stop);process.on("SIGHUP",stop);
setInterval(()=>{if(!fs.existsSync(`/proc/${parent}`))stop()},1000);'
# M3 (#1873): one 0600 placeholder per console.sso client secret slot, in the scratch config
# directory where readSlot looks. Never the live slot: the rehearsal reads no secret.
R_SLOTS_JS='
@@ -2145,12 +2206,13 @@ fs.writeFileSync(out,JSON.stringify(b,null,2)+"\n",{mode:0o600,flag:"wx"});'
# notify-off the notifier config is {"notifyVersion":1,"binding":null}, 0600 (mos-0962)
# tracker-configured bootConfig sets no trackers (mos-0962)
# credential-ref-live every credential reference is a file under placeholders/ (mos-0963)
# sso-isolated every console.sso origin and issuer is rehearsal.invalid and every client
# secret slot is a scratch placeholder (M3, #1873)
# sso-isolated the console.sso origin is rehearsal.invalid, every issuer is the
# rehearsal provider's (OP, a stamped copy) or on rehearsal.invalid, and every
# client secret slot is a scratch placeholder (M3, #1873)
# Exit 0 when all pass, 1 on the first FAIL.
R_GUARD_JS='
const fs=require("node:fs"),path=require("node:path");const {pathToFileURL}=require("node:url");
const [copy,id,live,scratch,ph]=process.argv.slice(1);
const [copy,id,live,scratch,ph,op]=process.argv.slice(1);
const real=(p)=>{try{return fs.realpathSync(p)}catch{return null}};
const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,"utf8");return s[s.lastIndexOf(")")+2]!=="Z"}catch{return false}};
const pass=(n,w)=>console.log(`PASS ${n}${w?": "+w:""}`);
@@ -2195,11 +2257,13 @@ const fail=(n,w)=>{console.log(`FAIL ${n}: ${w}`);process.exit(1)};
if(host(sso.publicOrigin)!=="rehearsal.invalid")fail("sso-isolated",`console.sso.publicOrigin is ${sso.publicOrigin}, not https://rehearsal.invalid`);
const sd=real(path.join(path.dirname(path.resolve(process.env.MOSAIC_CONFIG)),"secrets","mosaic-stack"));
for(const p of sso.providers){
if(host(p.issuer)!=="rehearsal.invalid")fail("sso-isolated",`console.sso provider ${p.id} issuer is ${p.issuer}, not on rehearsal.invalid`);
const want=op?`${op}/${encodeURIComponent(String(p.id))}/`:null;
if(op&&p.issuer!==want)fail("sso-isolated",`console.sso provider ${p.id} issuer is ${p.issuer}, not ${want} at the rehearsal provider`);
if(!op&&host(p.issuer)!=="rehearsal.invalid")fail("sso-isolated",`console.sso provider ${p.id} issuer is ${p.issuer}, not on rehearsal.invalid`);
const f=real(path.join(path.dirname(path.resolve(process.env.MOSAIC_CONFIG)),"secrets","mosaic-stack",p.clientSecret.slot));
if(f===null||sd===null||path.dirname(f)!==sd||fs.readFileSync(f,"utf8")!=="rehearsal-placeholder-secret\n")fail("sso-isolated",`console.sso provider ${p.id} has no scratch placeholder secret`);
}
pass("sso-isolated",`${sso.providers.length} provider(s), each at rehearsal.invalid with a placeholder secret`);
pass("sso-isolated",`${sso.providers.length} provider(s), each ${op?`at the rehearsal provider ${op}`:"at rehearsal.invalid"} with a placeholder secret`);
})().catch((e)=>fail("guard",String(e?.message??e).split("\n")[0]));'
# Opens a data root's store with a copy's Store, as the bus does at start, with no migration
@@ -2267,6 +2331,18 @@ process.stdout.write(why);' -- "$out" "$BUSINESS" "$R_HOST_PID" "$START_MS" 2>&1
R_GATE=healthz
node -e '
fetch(`http://127.0.0.1:${process.argv[1]}/healthz`,{signal:AbortSignal.timeout(5000)}).then((r)=>process.exit(r.status===200?0:1),()=>process.exit(1));' -- "$R_PORT" || return 1
# M3 (#1873): a stamped copy's Console must have found every configured provider at the
# rehearsal provider, not started degraded.
if [ -n "$R_PROVIDER" ]; then
R_GATE=sso-ready
R_WHY="$(node -e '
const ids=JSON.parse(require("node:fs").readFileSync(process.argv[2],"utf8")).console.sso.providers.map((p)=>String(p.id));
fetch(`http://127.0.0.1:${process.argv[1]}/healthz`,{signal:AbortSignal.timeout(5000)}).then((r)=>r.json()).then((h)=>{
const s=h?.sso&&typeof h.sso==="object"?h.sso:{};
process.stdout.write(ids.filter((id)=>s[id]!=="ready").map((id)=>`provider ${id} is ${s[id]??"missing"}, not ready`).join("; "));
},(e)=>process.stdout.write(`healthz unreadable: ${e.message}`));' -- "$R_PORT" "$R/config/config.json" 2>&1)" || R_WHY="${R_WHY:-healthz unreadable}"
[ -z "$R_WHY" ] || return 1
fi
R_GATE=copy-match
tr '\0' ' ' < "/proc/$R_HOST_PID/cmdline" | grep -qF " $copy/packages/cli/src/cli.mjs bus start $BUSINESS --no-launch" || { R_WHY="the host's command line is not $copy's cli.mjs bus start $BUSINESS --no-launch"; return 1; }
tr '\0' ' ' < "/proc/$R_CONSOLE_PID/cmdline" | grep -qF " $copy/packages/webui/src/cli.mjs serve " || { R_WHY="the Console's command line is not $copy's webui cli.mjs serve"; return 1; }
@@ -2279,7 +2355,43 @@ r_gate() {
[ "$SECONDS" -lt "$deadline" ] || { r_fail "$R_GATE" "${R_WHY:-no answer within ${TIMEOUT}s}"; return 1; }
sleep 1
done
r_pass gate "host-up, bus-status, console-up, healthz and copy-match on $TO"
r_pass gate "host-up, bus-status, console-up, healthz${R_PROVIDER:+, sso-ready} and copy-match on $TO"
}
# M3 (#1873): starts R_PROVIDER_JS for a stamped copy, in $R/provider, and waits for its origin.
r_provider() {
local deadline=$((SECONDS + TIMEOUT))
mkdir -m 700 "$R/provider" || r_abort provider-start "cannot make $R/provider"
env -i HOME="$HOME" PATH="$PATH" node -e "$R_PROVIDER_JS" -- "$R/provider" "$LIVE_CFG" "$$" > "$R/provider/provider.log" 2>&1 &
R_PROVIDER_PID=$!
until [ -s "$R/provider/origin" ]; do
r_alive "$R_PROVIDER_PID" || { wait "$R_PROVIDER_PID" 2>/dev/null || true; R_PROVIDER_PID=""; r_abort provider-start "the rehearsal provider exited: $(tail -n 1 "$R/provider/provider.log" 2>/dev/null)"; }
[ "$SECONDS" -lt "$deadline" ] || r_abort provider-start "the rehearsal provider wrote no origin within ${TIMEOUT}s"
sleep 0.2
done
R_PROVIDER="$(cat "$R/provider/origin")"
[[ "$R_PROVIDER" =~ ^https://127\.0\.0\.1:[0-9]{1,5}$ ]] || r_abort provider-start "the rehearsal provider's origin $R_PROVIDER is not https on 127.0.0.1"
[ ! -e "$R/provider/tls-key.pem" ] || r_abort provider-start "the rehearsal provider left its TLS key at $R/provider/tls-key.pem"
r_pass provider-start "pid $R_PROVIDER_PID at $R_PROVIDER: discovery and a JWKS from keys made for this run, no code and no token; its certificate $R/provider/cert.pem is the trust the scratch Console alone gets"
}
# M3 (#1873): the Console fetched each provider's discovery and JWKS from the rehearsal provider and
# asked it for no code and no token.
r_provider_seen() {
local why
why="$(node -e '
const fs=require("node:fs");const [cfg,log]=process.argv.slice(1);
const lines=fs.existsSync(log)?fs.readFileSync(log,"utf8").split("\n").filter(Boolean):[];
const why=[];
for(const p of JSON.parse(fs.readFileSync(cfg,"utf8")).console.sso.providers){
const seg=encodeURIComponent(String(p.id));
for(const what of [".well-known/openid-configuration","jwks"])if(!lines.includes(`GET /${seg}/${what} 200`))why.push(`no GET /${seg}/${what} 200`);
}
for(const l of lines)if(/\/(authorize|token)\b/.test(l))why.push(`a request for ${l}`);
process.stdout.write(why.join("; "));' -- "$R/config/config.json" "$R/provider/requests.log" 2>&1)" || why="${why:-cannot read $R/provider/requests.log}"
if [ -z "$why" ]; then r_pass provider-discovery "the Console fetched each provider's discovery and JWKS from $R_PROVIDER and asked for no code or token"
else r_fail provider-discovery "$why; see $R/provider/requests.log"
fi
}
# No role session: the host wrote no sessions.json and no launches directory or log.
@@ -2343,11 +2455,12 @@ do_rehearse() {
R="$DATA/rehearsals/$stamp" STEPS="$DATA/rehearsals/$stamp/steps.log"
printf '%s\n' "${R_EARLY[@]}" > "$STEPS" && printf '%s\n' "$before" > "$R/backup.before" && printf '%s\n' "$live" > "$R/live.before" ||
r_abort scratch-build "cannot write the sums to $R"
[ -z "$R_STAMPED" ] || r_provider
(
umask 077
mkdir "$R/config" "$R/config/businesses" "$R/placeholders" "$R/data" "$R/data/bus" "$R/data/notify" "$R/data/notify/$BUSINESS" &&
cp "$R_BK"/bus/* "$R/data/bus/" &&
node -e "$R_CONFIG_JS" -- "$LIVE_CFG" "$R/data" > "$R/config/config.json" &&
node -e "$R_CONFIG_JS" -- "$LIVE_CFG" "$R/data" "$R_PROVIDER" > "$R/config/config.json" &&
node -e "$R_BUSINESS_JS" -- write "$LIVE_BIZ" "$R/config/businesses/$BUSINESS.json" "$R/placeholders" &&
printf '{"notifyVersion":1,"binding":null}\n' > "$R/data/notify/$BUSINESS/notify.json" &&
{ ! node -e 'process.exit(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).console?.sso?0:1)' -- "$R/config/config.json" ||
@@ -2364,7 +2477,7 @@ do_rehearse() {
[ -z "$R_STAMPED" ] || r_pass deployment "$TO is stamped: $R/data/.deployed from its writeDeployment, the live accounts.json copied to $R/config, the Console starts with --sso"
local out
rc=0; out="$(r_env node -e "$R_GUARD_JS" -- "$COPIES/$TO" "$BUSINESS" "$DATA" "$R/data" "$R/placeholders" 2>&1)" || rc=$?
rc=0; out="$(r_env node -e "$R_GUARD_JS" -- "$COPIES/$TO" "$BUSINESS" "$DATA" "$R/data" "$R/placeholders" "$R_PROVIDER" 2>&1)" || rc=$?
while IFS= read -r i; do r_line "$i"; done <<< "$out"
[ "$rc" = 0 ] || { R_RC=1; r_end; }
@@ -2379,9 +2492,12 @@ do_rehearse() {
local -a tester_rehearsal=()
if grep -q 'MOSAIC_TESTER_SOCKET_V1' "$COPIES/$TO/packages/webui/src/cli.mjs"; then tester_rehearsal=(--tester-socket); fi
[ -z "$R_STAMPED" ] || tester_rehearsal+=(--sso)
env -i HOME="$HOME" PATH="$PATH" MOSAIC_CONFIG="$R/config/config.json" node "$COPIES/$TO/packages/webui/src/cli.mjs" serve --port "$R_PORT" --business "$BUSINESS" "${tester_rehearsal[@]}" > "$R/console.log" 2>&1 &
# M3 (#1873): the rehearsal provider's certificate is trusted by the scratch Console only.
local -a r_ca=()
[ -z "$R_PROVIDER" ] || r_ca=(NODE_EXTRA_CA_CERTS="$R/provider/cert.pem")
env -i HOME="$HOME" PATH="$PATH" MOSAIC_CONFIG="$R/config/config.json" "${r_ca[@]}" node "$COPIES/$TO/packages/webui/src/cli.mjs" serve --port "$R_PORT" --business "$BUSINESS" "${tester_rehearsal[@]}" > "$R/console.log" 2>&1 &
R_CONSOLE_PID=$!
if r_gate; then r_no_session; fi
if r_gate; then r_no_session; [ -z "$R_PROVIDER" ] || r_provider_seen; fi
r_end
}
@@ -2419,7 +2535,7 @@ FROM="" TO="" ACT="" RS="" POST="" BACKUP="" TAKEN="" RESTORED="" STEPS="" COUNT
STORE_SCHEMA="" START_EPOCH="" START_MS=0 GATE="" GATE_WHY="" S_CHANGED=""
P_COPY="" P_PM=0 P_PENDING=0 P_SCHEMA="" P_CLAUDE="" P_CONSOLE=() RUN_PM=0 RUN_PENDING=0 RUN_CONSOLE=() DROP_TESTER="" PROOF_PLAN="" PROOF_STEP="" SEND_PLAN="" SEND_STEP="" PLAN_REFUSE=()
BASE_CONSOLE=() SERVE_FLAGS=() SSO_PLAN="" SSO_STEP="" SSO_DROP="" SSO_NO_TARGET=""
R="" R_RC=0 R_EARLY=() R_BK="" R_HOST_PID="" R_CONSOLE_PID="" R_STARTED="" LIVE_CFG="" LIVE_BIZ="" LIVE_FILES=() R_STAMPED=""
R="" R_RC=0 R_EARLY=() R_BK="" R_HOST_PID="" R_CONSOLE_PID="" R_PROVIDER_PID="" R_PROVIDER="" R_STARTED="" LIVE_CFG="" LIVE_BIZ="" LIVE_FILES=() R_STAMPED=""
O_ACT="" FLOOR=0
case "$OP" in
adopt) do_adopt ;;
scripts/stack-upgrade.sh
+128 -18
View File
@@ -6,8 +6,9 @@
# agents/darkwing/work/queue-233/BUILD.md.
#
# stack-upgrade.sh upgrade --instance dev --bundle FILE --sha SHA [--migrate] [--no-rehearsal]
# [--rehearse-port N] [--timeout S]
# stack-upgrade.sh rollback --instance dev --to ID [--restore ACT [--events N]] [--timeout S]
# [--rehearse-port N] [--next-args] [--timeout S]
# stack-upgrade.sh rollback --instance dev --to ID [--restore ACT [--events N]] [--next-args]
# [--timeout S]
#
# Root runs it from a root-owned checkout of the reviewed commit, never from a copy, because the
# stack's user can write the copies. The unit values come from /etc/mosaic-stack/<instance>.args,
@@ -20,6 +21,20 @@
# output but exit codes and the rehearsal's BACKUP_STORE line (sage-1653), which it copies into its
# step log after a match against a fixed pattern and decides nothing on.
#
# Before anything stops, case2-unit.sh --check renders the installed copy from the args file and
# compares the installed units with it, then checks the target's render the same way; either
# refusal is render-refused, with the units left running (#1873 M3).
#
# --next-args changes the serve flags, which a re-render otherwise refuses (row 195's installed-unit
# match): /etc/mosaic-stack/<instance>.args.next, root's with no group or other write, must equal
# the args file before `--` but for --copy's value, and differ from it after `--`. The target
# renders from it through case2-unit.sh --check-new before anything stops. After A3, root moves the
# installed unit files into a new <instance>.replaced.XXXXXX directory beside the args file, and A4
# renders from the next args as a first install. A failure from there moves the replaced files back
# unchanged, restarts them on the old copy and gates it, with no render. After confirm, the next
# args become the args file, and the replaced directory keeps the old unit files and args file for
# a later rollback with --next-args (docs/guides/host-run-sheet.md section 7).
#
# --no-rehearsal is for a stack whose log records no backup. The rehearsal runs either way: exit 3
# from it (no-backup) goes on only with the flag, and exit 0 with the flag refuses
# rehearsal-required (sage-1650, sage-1652).
@@ -47,8 +62,8 @@ REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
usage() {
cat >&2 <<'USAGE'
usage: stack-upgrade.sh upgrade --instance dev --bundle FILE --sha SHA [--migrate] [--no-rehearsal] [--rehearse-port N] [--timeout S]
stack-upgrade.sh rollback --instance dev --to ID [--restore ACT [--events N]] [--timeout S]
usage: stack-upgrade.sh upgrade --instance dev --bundle FILE --sha SHA [--migrate] [--no-rehearsal] [--rehearse-port N] [--next-args] [--timeout S]
stack-upgrade.sh rollback --instance dev --to ID [--restore ACT [--events N]] [--next-args] [--timeout S]
USAGE
exit 4
}
@@ -70,7 +85,7 @@ ARGV=("$@")
[ $# -gt 0 ] || usage
OP="$1"; shift
case "$OP" in upgrade|rollback) ;; *) usage ;; esac
INSTANCE="" BUNDLE="" SHA="" MIGRATE=0 NO_REHEARSAL=0 R_PORT=7339 TIMEOUT=60 TO="" RESTORE="" EVENTS=""
INSTANCE="" BUNDLE="" SHA="" MIGRATE=0 NO_REHEARSAL=0 R_PORT=7339 TIMEOUT=60 TO="" RESTORE="" EVENTS="" NEXT=0 ASIDE="" MOVED=() REPLACED_NOTE=""
while [ $# -gt 0 ]; do
case "$1" in
--instance) [ $# -ge 2 ] || usage; INSTANCE="$2"; shift 2 ;;
@@ -79,6 +94,7 @@ while [ $# -gt 0 ]; do
--migrate) [ "$OP" = upgrade ] || usage; MIGRATE=1; shift ;;
--no-rehearsal) [ "$OP" = upgrade ] || usage; NO_REHEARSAL=1; shift ;;
--rehearse-port) [ "$OP" = upgrade ] && [ $# -ge 2 ] && [[ "$2" =~ ^[1-9][0-9]{3,4}$ ]] && [ "$2" -le 65535 ] || usage; R_PORT="$2"; shift 2 ;;
--next-args) NEXT=1; shift ;;
--timeout) [ $# -ge 2 ] && [[ "$2" =~ ^[1-9][0-9]{0,4}$ ]] || usage; TIMEOUT="$2"; shift 2 ;;
--to) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ ^[0-9a-f]{40}$ ]] || usage; TO="$2"; shift 2 ;;
--restore) [ "$OP" = rollback ] && [ $# -ge 2 ] && [[ "$2" =~ $ID_RE ]] || usage; RESTORE="$2"; shift 2 ;;
@@ -125,8 +141,8 @@ logl "start rc=0 argv:$(printf ' %q' "$0" "${ARGV[@]}")"
# The selector scopes: selector.sh defines functions only, from this root-owned checkout, as RENDER is.
. "$REPO/scripts/selector.sh" || refuse checkout-unreadable "cannot read $REPO/scripts/selector.sh"
args_business_ok() { [[ "$1" =~ ^[a-z0-9][a-z0-9-]*$ ]]; }
args_outer_refused() { refuse args-unreadable "$ETC/$INSTANCE.args before --: $2"; }
args_serve_refused() { refuse args-unreadable "$ETC/$INSTANCE.args after --: $2"; }
args_outer_refused() { refuse args-unreadable "$ARGS_FILE before --: $2"; }
args_serve_refused() { refuse args-unreadable "$ARGS_FILE after --: $2"; }
# I2's file: one case2-unit.sh argument per line, owned by root with no group or other write. Sets
# ARGS (the lines), S_USER, S_HOME, BUSINESS, NODE, ARGS_COPY, ARGS_PROFILE and ARGS_PROXY, and the paths derived
@@ -140,7 +156,8 @@ args_serve_refused() { refuse args-unreadable "$ETC/$INSTANCE.args after --: $2"
# says which of them take a value, so a value is never taken for a selector; a token that grammar
# lacks refuses, as the render would.
read_args() {
local f="$ETC/$INSTANCE.args" st i sep copies=0 v profile=library served
ARGS_FILE="${1:-$ETC/$INSTANCE.args}"
local f="$ARGS_FILE" st i sep copies=0 v profile=library served
[ -f "$f" ] && [ ! -L "$f" ] || refuse args-unreadable "no regular file $f"
st="$(stat -c '%u %a' "$f")" || refuse args-unreadable "cannot stat $f"
[ "${st% *}" = "$(id -u)" ] && (( (8#${st#* } & 8#022) == 0 )) || refuse args-unreadable "$f must be $(id -un)'s with no group or other write (it is uid ${st% *}, mode ${st#* })"
@@ -224,17 +241,82 @@ legacy_console() {
done
}
# RENDER_ARGS: the args file's lines with copy $1 in place of --copy's value.
# RENDER_ARGS: the lines of array $2 (default ARGS) with copy $1 in place of --copy's value.
args_with() {
local i past=""
local -n lines_="${2:-ARGS}"
RENDER_ARGS=()
for ((i = 0; i < ${#ARGS[@]}; i++)); do
RENDER_ARGS+=("${ARGS[i]}")
[ "${ARGS[i]}" != -- ] || past=1
if [ -z "$past" ] && [ "${ARGS[i]}" = --copy ]; then RENDER_ARGS+=("$1"); i=$((i + 1)); fi
for ((i = 0; i < ${#lines_[@]}; i++)); do
RENDER_ARGS+=("${lines_[i]}")
[ "${lines_[i]}" != -- ] || past=1
if [ -z "$past" ] && [ "${lines_[i]}" = --copy ]; then RENDER_ARGS+=("$1"); i=$((i + 1)); fi
done
}
# OLD_ARGS: the args file's lines, which the installed units were rendered from. Under --next-args,
# ARGS becomes the next file's lines, which must equal OLD_ARGS before -- but for --copy's value and
# differ after it.
read_next() {
OLD_ARGS=("${ARGS[@]}")
[ "$NEXT" = 1 ] || return 0
local f="$ETC/$INSTANCE.args.next" a b i copy="$ARGS_COPY"
unset SELECTOR_SEEN_outer SELECTOR_VALUE_outer SELECTOR_SEEN_serve SELECTOR_VALUE_serve
read_args "$f"
# read_old checks the installed units against the args file's copy, not the next file's.
ARGS_COPY="$copy"
args_with COPY OLD_ARGS; a=("${RENDER_ARGS[@]}")
args_with COPY; b=("${RENDER_ARGS[@]}")
for ((i = 0; i < ${#a[@]} && i < ${#b[@]}; i++)); do
[ "${a[i]}" = "${b[i]}" ] || refuse next-args-differ "$f must equal $ETC/$INSTANCE.args before -- but for --copy's value; line $((i + 1)) differs"
[ "${a[i]}" != -- ] || break
done
[ "${a[*]}" != "${b[*]}" ] || [ "${#a[@]}" != "${#b[@]}" ] ||
refuse next-args-same "$f has the same serve flags as $ETC/$INSTANCE.args; run without --next-args"
}
# Before anything stops: the installed units against OLD's render from the args file, so a rollback
# render-old works, then copy $1's render, against the installed units or, under --next-args, as a
# first install from the next args.
render_checks() {
args_with "$OLD" OLD_ARGS
step render-old-check "$RENDER" --check "${RENDER_ARGS[@]}" ||
refuse render-refused "case2-unit.sh --check refuses $OLD's units from $ETC/$INSTANCE.args, so a rollback could not render them; its message is above"
args_with "$1"
if [ "$NEXT" = 1 ]; then
step render-check "$RENDER" --check-new "${RENDER_ARGS[@]}" ||
refuse render-refused "case2-unit.sh --check-new refuses $1's units from $ETC/$INSTANCE.args.next; its message is above"
else
step render-check "$RENDER" --check "${RENDER_ARGS[@]}" ||
refuse render-refused "case2-unit.sh --check refuses $1's units from $ETC/$INSTANCE.args; its message is above. A re-pin that changes the serve flags takes --next-args (docs/guides/host-run-sheet.md section 7)"
fi
}
# --next-args, after A3: the installed unit files into a new directory beside the args file, so A4
# renders as a first install. MOVED lists the files moved, for units_back.
units_aside() {
local u rc=0
local -a files=([email protected] "$CON_UNIT")
[ "$ARGS_PROXY" = 0 ] || files+=("$PROXY_UNIT")
ASIDE="$(mktemp -d "$ETC/$INSTANCE.replaced.XXXXXX")" && chmod 0700 "$ASIDE" || rc=1
if [ "$rc" = 0 ]; then
for u in "${files[@]}"; do
mv -T -- "$UNIT_DIR/$u" "$ASIDE/$u" || { rc=1; break; }
MOVED+=("$u")
done
fi
logl "units-aside rc=$rc dir: ${ASIDE:-none} moved: ${MOVED[*]}"
return "$rc"
}
# The moved files back, unchanged, over whatever A4 wrote, then the replaced directory removed.
units_back() {
local u rc=0
for u in "${MOVED[@]}"; do mv -fT -- "$ASIDE/$u" "$UNIT_DIR/$u" || rc=1; done
[ "$rc" != 0 ] || rmdir -- "$ASIDE" || rc=1
logl "units-back rc=$rc dir: $ASIDE moved: ${MOVED[*]}"
return "$rc"
}
# The new args file, written beside the old one before anything stops, moved into place only after
# confirm passes. $1 is the copy.
prepare_args() {
@@ -334,9 +416,25 @@ rollback_to_old() {
stop_wait || { stop_units; finish 2 rollback-failed "$cause failed and the units did not stop for the rollback; $STOPPED_UNITS are stopped"; }
step revert "${AS_USER[@]}" "$SR" revert "$id" "$BUSINESS" --units external --copies "$COPIES_DIR" ||
{ stop_units; finish 2 rollback-failed "$cause failed and revert failed; $STOPPED_UNITS are stopped"; }
args_with "$OLD"
step render-old "$RENDER" "${RENDER_ARGS[@]}" ||
{ stop_units; finish 2 rollback-failed "$cause failed and the render of $OLD failed; $STOPPED_UNITS are stopped"; }
if [ -n "$ASIDE" ]; then
# --next-args: the replaced files back as they were, then the start render-old would do.
units_back ||
{ stop_units; finish 2 rollback-failed "$cause failed and the replaced units in $ASIDE did not all move back; $STOPPED_UNITS are stopped"; }
step daemon-reload "$SYSTEMCTL" daemon-reload ||
{ stop_units; finish 2 rollback-failed "$cause failed and daemon-reload failed after the replaced units moved back; $STOPPED_UNITS are stopped"; }
if [ "$ARGS_PROXY" = 1 ]; then
step start-proxy "$SYSTEMCTL" start "$PROXY_UNIT" ||
{ stop_units; finish 2 rollback-failed "$cause failed and the proxy did not start for $OLD; $STOPPED_UNITS are stopped"; }
fi
step start-bus "$SYSTEMCTL" start "$BUS_UNIT" ||
{ stop_units; finish 2 rollback-failed "$cause failed and the bus did not start for $OLD; $STOPPED_UNITS are stopped"; }
step start-console "$SYSTEMCTL" start "$CON_UNIT" ||
{ stop_units; finish 2 rollback-failed "$cause failed and the Console did not start for $OLD; $STOPPED_UNITS are stopped"; }
else
args_with "$OLD" OLD_ARGS
step render-old "$RENDER" "${RENDER_ARGS[@]}" ||
{ stop_units; finish 2 rollback-failed "$cause failed and the render of $OLD failed; $STOPPED_UNITS are stopped"; }
fi
# confirm --rollback exits 1 when OLD passed its gate and 2 when it failed; anything else is a
# refusal, with OLD's gate never run.
local rc=0
@@ -360,18 +458,27 @@ run_steps() {
*) rollback_to_old "$id" A3 ;;
esac
# A4
if [ "$NEXT" = 1 ]; then units_aside || rollback_to_old "$id" A4; fi
args_with "$id"
step render "$RENDER" "${RENDER_ARGS[@]}" || rollback_to_old "$id" A4
# A5
step confirm "${AS_USER[@]}" "$SR" confirm "$id" "$BUSINESS" --units external --copies "$COPIES_DIR" --timeout "$TIMEOUT" ||
rollback_to_old "$id" A5
if [ "$NEXT" = 1 ]; then
# The old args file joins the replaced units before the next one takes its place.
cp -p -- "$ETC/$INSTANCE.args" "$ASIDE/$INSTANCE.args" && rm -f -- "$ETC/$INSTANCE.args.next" &&
logl "next-args rc=0 replaced: $ASIDE" ||
{ logl "next-args rc=1 replaced: $ASIDE"; finish 5 args-failed "$id is live and confirmed, but the old args file did not reach $ASIDE; $ETC/$INSTANCE.args still names $OLD; copy it there, then put $NEW_ARGS in its place and remove $ETC/$INSTANCE.args.next"; }
fi
if mv -f "$NEW_ARGS" "$ETC/$INSTANCE.args"; then logl "args rc=0 copy: $id"
else logl "args rc=1 copy: $id"; finish 5 args-failed "$id is live and confirmed, but $ETC/$INSTANCE.args still names $OLD; put $NEW_ARGS in its place"
fi
REPLACED_NOTE="${ASIDE:+; the replaced units and args file are in $ASIDE}"
}
do_upgrade() {
read_args
read_next
legacy_console
# Before A1: the bundle into the stack's user's clone, then the copy from it (I1).
step fetch "${AS_USER[@]}" git -C "$CLONE" fetch -q "$BUNDLE" refs/heads/refactor:refs/remotes/origin/refactor ||
@@ -390,9 +497,10 @@ do_upgrade() {
local -a migrate=()
[ "$MIGRATE" = 0 ] || migrate=(--migrate)
plan activate "$SHA" "$BUSINESS" "${migrate[@]}"
render_checks "$SHA"
rehearse
run_steps "$SHA" activate "$SHA" "$BUSINESS" "${migrate[@]}"
finish 0 ok "upgrade ok: $SHA is live for $BUSINESS"
finish 0 ok "upgrade ok: $SHA is live for $BUSINESS$REPLACED_NOTE"
}
# The rehearsal against the log's newest backup, before anything stops. Its output goes to the
@@ -419,6 +527,7 @@ rehearse() {
# check: the target is an older copy than this checkout, and no rehearsal.
do_rollback() {
read_args
read_next
legacy_console
[ -x "$COPIES_DIR/$TO/scripts/mosaic" ] || refuse copy-missing "no $COPIES_DIR/$TO/scripts/mosaic"
hostcheck
@@ -427,8 +536,9 @@ do_rollback() {
[ -z "$RESTORE" ] || restore=(--restore "$RESTORE")
[ -z "$EVENTS" ] || restore+=(--events "$EVENTS")
plan rollback "$BUSINESS" --to "$TO" "${restore[@]}"
render_checks "$TO"
run_steps "$TO" rollback "$BUSINESS" --to "$TO" "${restore[@]}"
finish 0 ok "rollback ok: $TO is live for $BUSINESS"
finish 0 ok "rollback ok: $TO is live for $BUSINESS$REPLACED_NOTE"
}
case "$OP" in