fix(cli): M3 review round: re-pin migration, rehearsal provider (#1873)

An installed unit must equal the render but for the copy id, so a
re-pin that adds --sso refused render-refused. case2-unit gains --check
(every render check, nothing written) and --check-new (a first install's
checks). stack-upgrade runs both checks before anything stops, and
--next-args reads root's <instance>.args.next: equal before -- but for
--copy, different after it. After the stage it moves the installed unit
files into a new <instance>.replaced.XXXXXX directory and renders a
first install. A failure moves them back unchanged and restarts the old
copy, and confirm swaps the args file. The run sheet gives the commands,
the rollback past the migration included.

A stamped rehearsal now starts an isolated provider on 127.0.0.1 port 0
with a certificate and keys made for the run. It serves discovery and a
JWKS for the live provider ids and answers 403 to authorize and token.
Only the scratch Console trusts its certificate. The gate requires each
provider ready (sso-ready); provider-discovery requires discovery and
JWKS fetches and no code or token request. The provider stops with the
run, and by itself once the run's shell is gone.

The system-path test helper asserts the log's from copy again. The
pinned-copy sign-in requests are bounded, and cleanup waits for each
Console to exit.
This commit is contained in:
jason.woltje committed 2026-11-10 21:38:17 -06:00
1 parent db822c5b9d
commit 9acc378555
10 files changed
+766 -116

No files matched your search

+137 -21
View File
@@ -2041,12 +2041,16 @@ r_gone() {
while r_alive "$1"; do [ "$SECONDS" -lt "$deadline" ] || return 1; sleep 1; done
}
# Stops the Console, then the host, with SIGTERM within --timeout; one still there gets SIGKILL
# and a FAIL line. The host has to leave the store's writer.lock behind it.
# Stops the Console, the host, then the rehearsal provider, with SIGTERM within --timeout; one
# still there gets SIGKILL and a FAIL line. The host has to leave the store's writer.lock behind it.
r_stop() {
local name pid
for name in console host; do
if [ "$name" = console ]; then pid="$R_CONSOLE_PID"; R_CONSOLE_PID=""; else pid="$R_HOST_PID"; R_HOST_PID=""; fi
for name in console host provider; do
case "$name" in
console) pid="$R_CONSOLE_PID"; R_CONSOLE_PID="" ;;
host) pid="$R_HOST_PID"; R_HOST_PID="" ;;
provider) pid="$R_PROVIDER_PID"; R_PROVIDER_PID="" ;;
esac
[ -n "$pid" ] || continue
kill -TERM "$pid" 2>/dev/null || true
if r_gone "$pid"; then r_pass "$name-stopped" "pid $pid"
@@ -2080,20 +2084,77 @@ r_port_free() {
}
# The scratch config: the live one with dataRoot set to the scratch data root. M3 (#1873): a
# console.sso keeps its provider ids, client ids, algorithms and subjects, but the public origin and
# every issuer move to rehearsal.invalid, a name that never resolves (RFC 6761). The scratch Console
# reaches no real provider: discovery fails as unreachable and the Console starts degraded, after
# every other startup check. No subject is added; the live config's own are the only ones.
# console.sso keeps its provider ids, client ids, algorithms and subjects; the public origin moves to
# rehearsal.invalid, a name that never resolves (RFC 6761). For a stamped copy, whose Console signs
# in, every issuer moves to the rehearsal's own provider (R_PROVIDER_JS, the origin argument). An
# unstamped copy's Console runs without --sso, so its issuers move to rehearsal.invalid too. No
# subject is added; the live config's own are the only ones.
R_CONFIG_JS='
const fs=require("node:fs");const [file,data]=process.argv.slice(1);
const fs=require("node:fs");const [file,data,op]=process.argv.slice(1);
const c=JSON.parse(fs.readFileSync(file,"utf8"));c.dataRoot=data;
const sso=c.console?.sso;
if(sso&&typeof sso==="object"){
sso.publicOrigin="https://rehearsal.invalid";
for(const p of Array.isArray(sso.providers)?sso.providers:[])if(p&&typeof p==="object")p.issuer=`https://rehearsal.invalid/${encodeURIComponent(String(p.id))}`;
for(const p of Array.isArray(sso.providers)?sso.providers:[])if(p&&typeof p==="object")p.issuer=`${op||"https://rehearsal.invalid"}/${encodeURIComponent(String(p.id))}${op?"/":""}`;
}
process.stdout.write(JSON.stringify(c,null,2)+"\n");'
# M3 (#1873): the rehearsal's own OpenID provider, for a stamped copy's --sso Console. An https
# server on 127.0.0.1, port 0, with a certificate openssl makes for this run in DIR. Its TLS key is
# read into memory and its file removed before the server listens; cert.pem stays for the scratch
# Console's NODE_EXTRA_CA_CERTS. Per configured provider, at /<id>/, it serves discovery and a JWKS
# holding one public key made in memory for the first of the provider's algorithms it can make.
# Authorization, token and every other route answer 403 or 404: it issues no code, no token and no
# sign-in. Each request is a line in requests.log. It writes origin when listening and exits when
# PARENT (the rehearsal) is gone, on SIGTERM, or on an error.
R_PROVIDER_JS='
const fs=require("node:fs"),path=require("node:path"),https=require("node:https"),crypto=require("node:crypto"),{spawnSync}=require("node:child_process");
const [dir,cfg,parent]=process.argv.slice(1);
const die=(m)=>{console.error(`rehearsal provider: ${m}`);process.exit(1)};
const sso=JSON.parse(fs.readFileSync(cfg,"utf8")).console?.sso;
if(!sso||!Array.isArray(sso.providers)||!sso.providers.length)die("the live config has no console.sso providers");
const GEN={RS:["rsa",{modulusLength:2048}],PS:["rsa",{modulusLength:2048}],ES256:["ec",{namedCurve:"P-256"}],ES384:["ec",{namedCurve:"P-384"}],ES512:["ec",{namedCurve:"P-521"}],Ed25519:["ed25519",{}],EdDSA:["ed25519",{}]};
const keyFor=(alg)=>{
const g=GEN[alg]??GEN[String(alg).slice(0,2)]??(/^ML-DSA-(44|65|87)$/.test(alg)?[alg.toLowerCase(),{}]:null);
if(!g)return null;
try{const {publicKey}=crypto.generateKeyPairSync(...g);return {...publicKey.export({format:"jwk"}),kid:`rehearsal-${crypto.randomBytes(4).toString("hex")}`,use:"sig",alg}}catch{return null}
};
const providers=new Map();
for(const p of sso.providers){
const id=String(p?.id),algs=Array.isArray(p?.algs)?p.algs:[];
let jwk=null;for(const a of algs){jwk=keyFor(a);if(jwk)break}
if(!jwk)die(`provider ${id}: none of ${algs.join(", ")||"no algorithms"} is one this provider can make a key for`);
providers.set(encodeURIComponent(id),jwk);
}
const tls=path.join(dir,"tls-key.pem"),certFile=path.join(dir,"cert.pem");
const r=spawnSync("openssl",["req","-x509","-newkey","ec","-pkeyopt","ec_paramgen_curve:P-256","-nodes","-days","1","-subj","/CN=127.0.0.1","-addext","subjectAltName=IP:127.0.0.1","-keyout",tls,"-out",certFile],{encoding:"utf8"});
let key,cert;
try{if(r.status!==0){const why=String(r.stderr??"").trim().split("\n").pop();throw new Error(`openssl exit ${r.status??r.error?.code}${why?`: ${why}`:""}`)};key=fs.readFileSync(tls);cert=fs.readFileSync(certFile);fs.chmodSync(certFile,0o600)}
catch(e){die(`cannot make the certificate: ${e.message}`)}
finally{fs.rmSync(tls,{force:true})}
const log=(q,status)=>fs.appendFileSync(path.join(dir,"requests.log"),`${q.method} ${q.url} ${status}\n`);
let origin;
const server=https.createServer({key,cert,requestTimeout:5000,headersTimeout:5000},(q,res)=>{
const send=(status,body)=>{log(q,status);res.writeHead(status,{"content-type":"application/json","cache-control":"no-store"});res.end(JSON.stringify(body))};
if(q.method!=="GET")return send(405,{error:"GET only"});
const m=/^\/([^/?#]+)\/(\.well-known\/openid-configuration|jwks|authorize|token)$/.exec(q.url);
const jwk=m?providers.get(m[1]):undefined;
if(!jwk)return send(404,{error:"not found"});
const issuer=`${origin}/${m[1]}/`;
if(m[2]==="jwks")return send(200,{keys:[jwk]});
if(m[2]!==".well-known/openid-configuration")return send(403,{error:"the rehearsal provider issues no code and no token"});
send(200,{issuer,authorization_endpoint:`${issuer}authorize`,token_endpoint:`${issuer}token`,jwks_uri:`${issuer}jwks`,response_types_supported:["code"],subject_types_supported:["public"],id_token_signing_alg_values_supported:[jwk.alg],code_challenge_methods_supported:["S256"]});
});
server.on("error",(e)=>die(e.message));
server.listen(0,"127.0.0.1",()=>{
origin=`https://127.0.0.1:${server.address().port}`;
fs.writeFileSync(path.join(dir,"origin.tmp"),`${origin}\n`,{mode:0o600});fs.renameSync(path.join(dir,"origin.tmp"),path.join(dir,"origin"));
console.log(`rehearsal provider: ${origin}, ${providers.size} provider(s)`);
});
const stop=()=>{server.close();server.closeAllConnections();process.exit(0)};
process.on("SIGTERM",stop);process.on("SIGINT",stop);process.on("SIGHUP",stop);
setInterval(()=>{if(!fs.existsSync(`/proc/${parent}`))stop()},1000);'
# M3 (#1873): one 0600 placeholder per console.sso client secret slot, in the scratch config
# directory where readSlot looks. Never the live slot: the rehearsal reads no secret.
R_SLOTS_JS='
@@ -2145,12 +2206,13 @@ fs.writeFileSync(out,JSON.stringify(b,null,2)+"\n",{mode:0o600,flag:"wx"});'
# notify-off the notifier config is {"notifyVersion":1,"binding":null}, 0600 (mos-0962)
# tracker-configured bootConfig sets no trackers (mos-0962)
# credential-ref-live every credential reference is a file under placeholders/ (mos-0963)
# sso-isolated every console.sso origin and issuer is rehearsal.invalid and every client
# secret slot is a scratch placeholder (M3, #1873)
# sso-isolated the console.sso origin is rehearsal.invalid, every issuer is the
# rehearsal provider's (OP, a stamped copy) or on rehearsal.invalid, and every
# client secret slot is a scratch placeholder (M3, #1873)
# Exit 0 when all pass, 1 on the first FAIL.
R_GUARD_JS='
const fs=require("node:fs"),path=require("node:path");const {pathToFileURL}=require("node:url");
const [copy,id,live,scratch,ph]=process.argv.slice(1);
const [copy,id,live,scratch,ph,op]=process.argv.slice(1);
const real=(p)=>{try{return fs.realpathSync(p)}catch{return null}};
const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,"utf8");return s[s.lastIndexOf(")")+2]!=="Z"}catch{return false}};
const pass=(n,w)=>console.log(`PASS ${n}${w?": "+w:""}`);
@@ -2195,11 +2257,13 @@ const fail=(n,w)=>{console.log(`FAIL ${n}: ${w}`);process.exit(1)};
if(host(sso.publicOrigin)!=="rehearsal.invalid")fail("sso-isolated",`console.sso.publicOrigin is ${sso.publicOrigin}, not https://rehearsal.invalid`);
const sd=real(path.join(path.dirname(path.resolve(process.env.MOSAIC_CONFIG)),"secrets","mosaic-stack"));
for(const p of sso.providers){
if(host(p.issuer)!=="rehearsal.invalid")fail("sso-isolated",`console.sso provider ${p.id} issuer is ${p.issuer}, not on rehearsal.invalid`);
const want=op?`${op}/${encodeURIComponent(String(p.id))}/`:null;
if(op&&p.issuer!==want)fail("sso-isolated",`console.sso provider ${p.id} issuer is ${p.issuer}, not ${want} at the rehearsal provider`);
if(!op&&host(p.issuer)!=="rehearsal.invalid")fail("sso-isolated",`console.sso provider ${p.id} issuer is ${p.issuer}, not on rehearsal.invalid`);
const f=real(path.join(path.dirname(path.resolve(process.env.MOSAIC_CONFIG)),"secrets","mosaic-stack",p.clientSecret.slot));
if(f===null||sd===null||path.dirname(f)!==sd||fs.readFileSync(f,"utf8")!=="rehearsal-placeholder-secret\n")fail("sso-isolated",`console.sso provider ${p.id} has no scratch placeholder secret`);
}
pass("sso-isolated",`${sso.providers.length} provider(s), each at rehearsal.invalid with a placeholder secret`);
pass("sso-isolated",`${sso.providers.length} provider(s), each ${op?`at the rehearsal provider ${op}`:"at rehearsal.invalid"} with a placeholder secret`);
})().catch((e)=>fail("guard",String(e?.message??e).split("\n")[0]));'
# Opens a data root's store with a copy's Store, as the bus does at start, with no migration
@@ -2267,6 +2331,18 @@ process.stdout.write(why);' -- "$out" "$BUSINESS" "$R_HOST_PID" "$START_MS" 2>&1
R_GATE=healthz
node -e '
fetch(`http://127.0.0.1:${process.argv[1]}/healthz`,{signal:AbortSignal.timeout(5000)}).then((r)=>process.exit(r.status===200?0:1),()=>process.exit(1));' -- "$R_PORT" || return 1
# M3 (#1873): a stamped copy's Console must have found every configured provider at the
# rehearsal provider, not started degraded.
if [ -n "$R_PROVIDER" ]; then
R_GATE=sso-ready
R_WHY="$(node -e '
const ids=JSON.parse(require("node:fs").readFileSync(process.argv[2],"utf8")).console.sso.providers.map((p)=>String(p.id));
fetch(`http://127.0.0.1:${process.argv[1]}/healthz`,{signal:AbortSignal.timeout(5000)}).then((r)=>r.json()).then((h)=>{
const s=h?.sso&&typeof h.sso==="object"?h.sso:{};
process.stdout.write(ids.filter((id)=>s[id]!=="ready").map((id)=>`provider ${id} is ${s[id]??"missing"}, not ready`).join("; "));
},(e)=>process.stdout.write(`healthz unreadable: ${e.message}`));' -- "$R_PORT" "$R/config/config.json" 2>&1)" || R_WHY="${R_WHY:-healthz unreadable}"
[ -z "$R_WHY" ] || return 1
fi
R_GATE=copy-match
tr '\0' ' ' < "/proc/$R_HOST_PID/cmdline" | grep -qF " $copy/packages/cli/src/cli.mjs bus start $BUSINESS --no-launch" || { R_WHY="the host's command line is not $copy's cli.mjs bus start $BUSINESS --no-launch"; return 1; }
tr '\0' ' ' < "/proc/$R_CONSOLE_PID/cmdline" | grep -qF " $copy/packages/webui/src/cli.mjs serve " || { R_WHY="the Console's command line is not $copy's webui cli.mjs serve"; return 1; }
@@ -2279,7 +2355,43 @@ r_gate() {
[ "$SECONDS" -lt "$deadline" ] || { r_fail "$R_GATE" "${R_WHY:-no answer within ${TIMEOUT}s}"; return 1; }
sleep 1
done
r_pass gate "host-up, bus-status, console-up, healthz and copy-match on $TO"
r_pass gate "host-up, bus-status, console-up, healthz${R_PROVIDER:+, sso-ready} and copy-match on $TO"
}
# M3 (#1873): starts R_PROVIDER_JS for a stamped copy, in $R/provider, and waits for its origin.
r_provider() {
local deadline=$((SECONDS + TIMEOUT))
mkdir -m 700 "$R/provider" || r_abort provider-start "cannot make $R/provider"
env -i HOME="$HOME" PATH="$PATH" node -e "$R_PROVIDER_JS" -- "$R/provider" "$LIVE_CFG" "$$" > "$R/provider/provider.log" 2>&1 &
R_PROVIDER_PID=$!
until [ -s "$R/provider/origin" ]; do
r_alive "$R_PROVIDER_PID" || { wait "$R_PROVIDER_PID" 2>/dev/null || true; R_PROVIDER_PID=""; r_abort provider-start "the rehearsal provider exited: $(tail -n 1 "$R/provider/provider.log" 2>/dev/null)"; }
[ "$SECONDS" -lt "$deadline" ] || r_abort provider-start "the rehearsal provider wrote no origin within ${TIMEOUT}s"
sleep 0.2
done
R_PROVIDER="$(cat "$R/provider/origin")"
[[ "$R_PROVIDER" =~ ^https://127\.0\.0\.1:[0-9]{1,5}$ ]] || r_abort provider-start "the rehearsal provider's origin $R_PROVIDER is not https on 127.0.0.1"
[ ! -e "$R/provider/tls-key.pem" ] || r_abort provider-start "the rehearsal provider left its TLS key at $R/provider/tls-key.pem"
r_pass provider-start "pid $R_PROVIDER_PID at $R_PROVIDER: discovery and a JWKS from keys made for this run, no code and no token; its certificate $R/provider/cert.pem is the trust the scratch Console alone gets"
}
# M3 (#1873): the Console fetched each provider's discovery and JWKS from the rehearsal provider and
# asked it for no code and no token.
r_provider_seen() {
local why
why="$(node -e '
const fs=require("node:fs");const [cfg,log]=process.argv.slice(1);
const lines=fs.existsSync(log)?fs.readFileSync(log,"utf8").split("\n").filter(Boolean):[];
const why=[];
for(const p of JSON.parse(fs.readFileSync(cfg,"utf8")).console.sso.providers){
const seg=encodeURIComponent(String(p.id));
for(const what of [".well-known/openid-configuration","jwks"])if(!lines.includes(`GET /${seg}/${what} 200`))why.push(`no GET /${seg}/${what} 200`);
}
for(const l of lines)if(/\/(authorize|token)\b/.test(l))why.push(`a request for ${l}`);
process.stdout.write(why.join("; "));' -- "$R/config/config.json" "$R/provider/requests.log" 2>&1)" || why="${why:-cannot read $R/provider/requests.log}"
if [ -z "$why" ]; then r_pass provider-discovery "the Console fetched each provider's discovery and JWKS from $R_PROVIDER and asked for no code or token"
else r_fail provider-discovery "$why; see $R/provider/requests.log"
fi
}
# No role session: the host wrote no sessions.json and no launches directory or log.
@@ -2343,11 +2455,12 @@ do_rehearse() {
R="$DATA/rehearsals/$stamp" STEPS="$DATA/rehearsals/$stamp/steps.log"
printf '%s\n' "${R_EARLY[@]}" > "$STEPS" && printf '%s\n' "$before" > "$R/backup.before" && printf '%s\n' "$live" > "$R/live.before" ||
r_abort scratch-build "cannot write the sums to $R"
[ -z "$R_STAMPED" ] || r_provider
(
umask 077
mkdir "$R/config" "$R/config/businesses" "$R/placeholders" "$R/data" "$R/data/bus" "$R/data/notify" "$R/data/notify/$BUSINESS" &&
cp "$R_BK"/bus/* "$R/data/bus/" &&
node -e "$R_CONFIG_JS" -- "$LIVE_CFG" "$R/data" > "$R/config/config.json" &&
node -e "$R_CONFIG_JS" -- "$LIVE_CFG" "$R/data" "$R_PROVIDER" > "$R/config/config.json" &&
node -e "$R_BUSINESS_JS" -- write "$LIVE_BIZ" "$R/config/businesses/$BUSINESS.json" "$R/placeholders" &&
printf '{"notifyVersion":1,"binding":null}\n' > "$R/data/notify/$BUSINESS/notify.json" &&
{ ! node -e 'process.exit(JSON.parse(require("node:fs").readFileSync(process.argv[1],"utf8")).console?.sso?0:1)' -- "$R/config/config.json" ||
@@ -2364,7 +2477,7 @@ do_rehearse() {
[ -z "$R_STAMPED" ] || r_pass deployment "$TO is stamped: $R/data/.deployed from its writeDeployment, the live accounts.json copied to $R/config, the Console starts with --sso"
local out
rc=0; out="$(r_env node -e "$R_GUARD_JS" -- "$COPIES/$TO" "$BUSINESS" "$DATA" "$R/data" "$R/placeholders" 2>&1)" || rc=$?
rc=0; out="$(r_env node -e "$R_GUARD_JS" -- "$COPIES/$TO" "$BUSINESS" "$DATA" "$R/data" "$R/placeholders" "$R_PROVIDER" 2>&1)" || rc=$?
while IFS= read -r i; do r_line "$i"; done <<< "$out"
[ "$rc" = 0 ] || { R_RC=1; r_end; }
@@ -2379,9 +2492,12 @@ do_rehearse() {
local -a tester_rehearsal=()
if grep -q 'MOSAIC_TESTER_SOCKET_V1' "$COPIES/$TO/packages/webui/src/cli.mjs"; then tester_rehearsal=(--tester-socket); fi
[ -z "$R_STAMPED" ] || tester_rehearsal+=(--sso)
env -i HOME="$HOME" PATH="$PATH" MOSAIC_CONFIG="$R/config/config.json" node "$COPIES/$TO/packages/webui/src/cli.mjs" serve --port "$R_PORT" --business "$BUSINESS" "${tester_rehearsal[@]}" > "$R/console.log" 2>&1 &
# M3 (#1873): the rehearsal provider's certificate is trusted by the scratch Console only.
local -a r_ca=()
[ -z "$R_PROVIDER" ] || r_ca=(NODE_EXTRA_CA_CERTS="$R/provider/cert.pem")
env -i HOME="$HOME" PATH="$PATH" MOSAIC_CONFIG="$R/config/config.json" "${r_ca[@]}" node "$COPIES/$TO/packages/webui/src/cli.mjs" serve --port "$R_PORT" --business "$BUSINESS" "${tester_rehearsal[@]}" > "$R/console.log" 2>&1 &
R_CONSOLE_PID=$!
if r_gate; then r_no_session; fi
if r_gate; then r_no_session; [ -z "$R_PROVIDER" ] || r_provider_seen; fi
r_end
}
@@ -2419,7 +2535,7 @@ FROM="" TO="" ACT="" RS="" POST="" BACKUP="" TAKEN="" RESTORED="" STEPS="" COUNT
STORE_SCHEMA="" START_EPOCH="" START_MS=0 GATE="" GATE_WHY="" S_CHANGED=""
P_COPY="" P_PM=0 P_PENDING=0 P_SCHEMA="" P_CLAUDE="" P_CONSOLE=() RUN_PM=0 RUN_PENDING=0 RUN_CONSOLE=() DROP_TESTER="" PROOF_PLAN="" PROOF_STEP="" SEND_PLAN="" SEND_STEP="" PLAN_REFUSE=()
BASE_CONSOLE=() SERVE_FLAGS=() SSO_PLAN="" SSO_STEP="" SSO_DROP="" SSO_NO_TARGET=""
R="" R_RC=0 R_EARLY=() R_BK="" R_HOST_PID="" R_CONSOLE_PID="" R_STARTED="" LIVE_CFG="" LIVE_BIZ="" LIVE_FILES=() R_STAMPED=""
R="" R_RC=0 R_EARLY=() R_BK="" R_HOST_PID="" R_CONSOLE_PID="" R_PROVIDER_PID="" R_PROVIDER="" R_STARTED="" LIVE_CFG="" LIVE_BIZ="" LIVE_FILES=() R_STAMPED=""
O_ACT="" FLOOR=0
case "$OP" in
adopt) do_adopt ;;