// Row 235 (#1741, #436 row b): the host grant executor (r6 X2, X3 failure table, X5) over a scratch sshd // on 127.0.0.1 that this test starts. The sshd logs in only as the test's own user, so the installer // and the granted agent share it. The installer's `list`, `add` and `remove` are stubs on the sshd's // PATH that edit a second authorized-keys file, so the new key's login works only once `add` has run. // The host writer itself is tested in host-tools.test.mjs (G16). Nothing runs as root and nothing // reaches another host. import test from 'node:test'; import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import { createServer } from 'node:net'; import { appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; import { tmpdir, userInfo } from 'node:os'; import { join } from 'node:path'; import { Broker, registrySha256, writeGrantEvent } from '../src/broker.mjs'; import { Credentials, parseHostKey } from '../src/credentials.mjs'; import { definitionSha256, fingerprint, grantKeys, grantTag, hostGrant, hostReference, sweepHostKeys } from '../src/host-grant.mjs'; import { hostExecutor, runHost } from '../src/host-run.mjs'; import { Store } from '../src/store.mjs'; import { grantDecision, hostRecord } from './host-fixtures.mjs'; import { ephemeralRange, installSleepStub, pickPort, startScratchSshd } from './scratch-sshd.mjs'; const SSHD = '/usr/bin/sshd'; const USER = userInfo().username; const AT = '2026-10-21T00:00:00.000Z'; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const sha256 = (x) => createHash('sha256').update(x).digest('hex'); const dayFrom = (n) => new Date(Date.now() + n * 86400000).toISOString().slice(0, 10); const keygen = (file) => execFileSync('ssh-keygen', ['-q', '-t', 'ed25519', '-N', '', '-C', 'test', '-f', file]); const pubLine = (file) => readFileSync(file + '.pub', 'utf8').trim().split(' ').slice(0, 2).join(' '); const readOr = (file, fallback) => (existsSync(file) ? readFileSync(file, 'utf8') : fallback); const options = [{ key: 'yes', text: 'Yes' }, { key: 'no', text: 'No' }]; const none = { authority: { withinRole: ['message.send'], crossRole: [] } }; const DEFINITIONS = { 'chief-of-staff': { roleVersion: 2, credentials: [] }, pm: { roleVersion: 2, credentials: [] }, ops: { roleVersion: 2, credentials: [{ service: 'host' }] }, }; async function until(fn, ms = 10000) { const end = Date.now() + ms; while (!fn()) { if (Date.now() > end) throw new Error('timed out'); await sleep(20); } } // The stubs read flag files in the test dir: fail- exits 1, noop-add exits 0 and writes nothing, // hold-add waits for `release` (then exits 1 if `skip` is there), and hold-remove waits for `release`. // Both also stop when the test dir is gone, since a held stub on the sshd side outlives its test (#1750). // wrap.sh logs every request. The sudo stub logs its arguments to sudo.log and runs the command as the // test's user. async function sshd(t, dir, options) { if (!existsSync(SSHD)) throw new Error(`${SSHD} is missing: the executor tests need it and fail, never skip`); keygen(join(dir, 'hostkey')); keygen(join(dir, 'installer')); writeFileSync(join(dir, 'authorized_keys'), readFileSync(join(dir, 'installer.pub')), { mode: 0o600 }); mkdirSync(join(dir, 'fake')); installSleepStub(dir); const stub = (name, body) => writeFileSync(join(dir, 'fake', name), `#!/bin/sh\nD='${dir}'\n${body}\n`, { mode: 0o755 }); stub('list', '[ -e "$D/fail-list" ] && exit 1\n[ -e "$D/granted" ] && grep " mosaic-[0-9a-f]*$" "$D/granted"\nexit 0'); stub( 'add', 'if [ -e "$D/hold-add" ]; then : > "$D/holding"; while [ ! -e "$D/release" ] && [ -d "$D" ]; do sleep 0.05; done; [ -e "$D/skip" ] && exit 1; fi\n' + '[ -e "$D/fail-add" ] && exit 1\n[ -e "$D/noop-add" ] && exit 0\n' + 'grep -q " $4\\$" "$D/granted" 2>/dev/null && exit 0\necho "restrict $2 $3 $4" >> "$D/granted"', ); stub('sudo', 'printf "%s\\n" "$*" >> "$D/sudo.log"\nshift 2\nexec "$@"'); stub('remove', 'if [ -e "$D/hold-remove" ]; then : > "$D/holding-remove"; while [ ! -e "$D/release" ] && [ -d "$D" ]; do sleep 0.05; done; fi\n' + '[ -e "$D/fail-remove" ] && exit 1\n[ -e "$D/granted" ] || exit 0\ngrep -v " $2\\$" "$D/granted" > "$D/granted.tmp"\nmv "$D/granted.tmp" "$D/granted"'); writeFileSync( join(dir, 'wrap.sh'), `#!/bin/sh\nprintf '%s\\n' "$SSH_ORIGINAL_COMMAND" >> '${dir}/requests.log'\nPATH=${dir}/fake:/usr/bin:/bin; export PATH\nexec /bin/sh -c "$SSH_ORIGINAL_COMMAND"\n`, { mode: 0o755 }, ); return startScratchSshd(t, dir, [ 'ListenAddress 127.0.0.1', `HostKey ${dir}/hostkey`, `AuthorizedKeysFile ${dir}/authorized_keys ${dir}/granted`, `PidFile ${dir}/sshd.pid`, 'UsePAM no', 'StrictModes no', 'PasswordAuthentication no', 'KbdInteractiveAuthentication no', `ForceCommand ${dir}/wrap.sh`, ], options); } test('row 280: an occupied TCP port is attempted, refused, and retried with our sshd', async t => { const dir = mkdtempSync(join(tmpdir(), 'sshd280-')); t.after(() => rmSync(dir, { recursive: true, force: true })); const server = createServer(socket => socket.end()); await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); }); t.after(() => new Promise(resolve => server.close(resolve))); const occupied = server.address().port; const port = await sshd(t, dir, { firstPort: occupied }); assert.match(readFileSync(join(dir, 'sshd-0.conf'), 'utf8'), new RegExp(`^Port ${occupied}\\n`)); assert.equal(readFileSync(join(dir, 'sshd-0.log'), 'utf8').includes(`Server listening on 127.0.0.1 port ${occupied}.`), false); assert.notEqual(port, occupied); assert.ok([1, 2, 3, 4].some(n => existsSync(join(dir, `sshd-${n}.log`)) && readFileSync(join(dir, `sshd-${n}.log`), 'utf8').includes(`Server listening on 127.0.0.1 port ${port}.`))); }); test('row 280: every injected picker sample is unprivileged and outside the ephemeral range', () => { const [lo, hi] = ephemeralRange(); for (const sample of [0, 0.25, 0.5, 0.6384, 0.75, 0.999999]) { const port = pickPort(() => sample); assert.ok(port >= 1024 && port <= 65535); assert.ok(port < lo || port > hi, `sample ${sample}: ${port} inside ${lo}-${hi}`); } }); async function setup(t, opts = {}) { const dir = mkdtempSync(join(tmpdir(), 'g235x-')); t.after(() => rmSync(dir, { recursive: true, force: true })); const port = await sshd(t, dir); const configDir = join(dir, 'cfg'); const keyDir = join(configDir, 'secrets', 'hosts', 'demo', 'web1'); mkdirSync(keyDir, { recursive: true, mode: 0o700 }); for (const d of [configDir, join(configDir, 'secrets'), join(configDir, 'secrets', 'hosts'), join(configDir, 'secrets', 'hosts', 'demo'), keyDir]) chmodSync(d, 0o700); // opts.enroll: no host record and no installer key, so the test enrolls the host itself (G22). opts.wrap // changes the host API the broker gets. if (!opts.enroll) writeFileSync(join(keyDir, 'installer'), readFileSync(join(dir, 'installer')), { mode: 0o600 }); const runRoot = join(configDir, 'secrets', 'run'); const storeRoot = join(configDir, 'secrets', 'agents'); mkdirSync(join(dir, 'data'), { mode: 0o700 }); const store = new Store(join(dir, 'data')); const credentials = new Credentials({ references: {}, storeRoot, dataRoot: join(dir, 'data') }); t.after(() => { credentials.close(); store.close(); }); const business = { id: 'demo', human: 'jason', arbiters: { technical: 'pm', delivery: 'pm' }, roles: { chief: { definition: 'chief-of-staff', ...none }, pm: { definition: 'pm', ...none }, ops: { definition: 'ops', ...none }, ...(opts.assistant ? { assistant: { definition: 'assistant', ...none } } : {}), }, definitions: { ...structuredClone(DEFINITIONS), ...(opts.assistant ? { assistant: { roleVersion: 2, credentials: [] } } : {}) }, vars: { 'bus.tester': true }, }; const broker = new Broker({ store, businesses: { demo: business }, secretCheck: (value) => credentials.assertClean(value), hostGrant: (opts.wrap ?? ((h) => h))(hostGrant({ configDir, runRoot, installerUser: USER, scan: async () => [pubLine(join(dir, 'hostkey'))] })), hostKeys: grantKeys({ credentials, storeRoot }), }); const caps = {}; for (const role of ['chief', 'pm']) { caps[role] = broker.bindLaunch({ business: 'demo', role, run: role + '-run', harness: 'pi', address: role }); broker.request(caps[role], { verb: 'role.claim' }); } caps.human = broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }); const entry = { address: '127.0.0.1', port, user: USER, hostKeys: [pubLine(join(dir, 'hostkey'))] }; if (!opts.enroll) hostRecord(store, { host: 'web1', entry }); const call = (who, verb, args = {}) => broker.request(caps[who], { verb, args }); const raise = (grant = {}) => call('chief', 'decision.raise', { action: 'credential.mint', question: 'Grant ops web1?', options, recommendation: 'yes', blocking: false, grant: { method: 'host', agent: 'ops', host: 'web1', commands: [['systemctl', 'status']], rotateBy: dayFrom(30), ...grant }, }); const grantEvents = () => store .all("SELECT body FROM events WHERE business='demo' AND kind='action.allowed' AND actor_run='grant' ORDER BY seq") .map((e) => JSON.parse(e.body)) .filter((b) => b.action?.startsWith('grant.')); const ended = (decision) => grantEvents().find((e) => e.grantDecision === decision && ['grant.applied', 'grant.failed'].includes(e.action)); // The human's cli yes starts the executor; this waits for its end. const approve = async (grant) => { const d = await raise(grant); call('human', 'decision.resolve', { id: d.id, choice: 'yes' }); await until(() => ended(d.id)); return { id: d.id, end: ended(d.id) }; }; // A decision put in by SQL, with R4's body built here so one field can be broken. const fixtureGrant = (over = {}, close) => { const commands = over.commands ?? [['uptime']]; const grant = { method: 'host', agent: 'ops', host: 'web1', commands, commandsSha256: sha256(JSON.stringify(commands)), rotateBy: dayFrom(30), registrySha256: registrySha256({ ...entry, sudo: true }), definitionSha256: definitionSha256(DEFINITIONS.ops), ...over, }; return grantDecision(store, { grant, ...(close !== undefined ? { close } : {}) }); }; const tag = grantTag('demo', 'ops', 'web1'); const keyFile = join(storeRoot, 'demo', 'ops', 'host.web1'); const record = () => store.get("SELECT * FROM host_grants WHERE business='demo' AND agent='ops' AND host='web1'"); const requests = () => readOr(join(dir, 'requests.log'), '').split('\n').filter(Boolean); const granted = () => readOr(join(dir, 'granted'), '').split('\n').filter(Boolean); const state = () => credentials.status().find((e) => e.instance === 'demo/ops' && e.service === 'host.web1')?.state; // The executor's notices, apart from the close notice the resolve sends the chief. const messages = (decision) => store.all("SELECT to_role, body FROM messages WHERE business=? AND decision=? AND from_role='_bus' AND instr(body, ?) ORDER BY rowid", 'demo', decision, `(decision ${decision})`); const flag = (name) => writeFileSync(join(dir, name), ''); return { caps, dir, port, entry, store, broker, credentials, call, raise, approve, fixtureGrant, grantEvents, ended, tag, keyFile, record, requests, granted, state, messages, flag, runRoot, storeRoot, configDir, business }; } // A restart as runtime.mjs does it, on the same store: the key sweep, Credentials with a reference per // record, a new broker, reconcileGrants before the socket opens, then reconcileHosts. `busy` is the // slot between the two. async function restart(t, f, business = f.business) { const granted = new Set(); const references = {}; for (const g of f.store.all('SELECT business, agent, host, rotate_by FROM host_grants')) { granted.add(`${g.business}/${g.agent}/${g.host}`); (references[`${g.business}/${g.agent}`] ??= {})['host.' + g.host] = hostReference(f.storeRoot, g.business, g.agent, g.host, g.rotate_by); } const swept = sweepHostKeys(f.storeRoot, ['demo'], granted); const credentials = new Credentials({ references, storeRoot: f.storeRoot, dataRoot: join(f.dir, 'data') }); t.after(() => credentials.close()); const broker = new Broker({ store: f.store, businesses: { demo: business }, secretCheck: (value) => credentials.assertClean(value), hostGrant: hostGrant({ configDir: f.configDir, runRoot: f.runRoot, installerUser: USER }), hostKeys: grantKeys({ credentials, storeRoot: f.storeRoot }), }); const now = broker.reconcileGrants(); const busy = credentials.busy('demo/ops', 'host.web1'); const later = await broker.reconcileHosts(); const state = credentials.status().find((e) => e.instance === 'demo/ops' && e.service === 'host.web1')?.state; return { swept, credentials, broker, now, busy, later, state }; } // The business after the human moved ops's instance to the creative `research` definition, which has // ops's content, so the grant's definitionSha256 still matches and only the creative check refuses. const creativeOps = (business) => ({ ...structuredClone(business), roles: { ...structuredClone(business.roles), ops: { ...structuredClone(business.roles.ops), definition: 'research' } }, definitions: { ...structuredClone(business.definitions), research: structuredClone(DEFINITIONS.ops) }, }); // A decision the human approved and a killed bus child spent, with its grant.step events up to `steps`. function interrupted(f, steps, over = {}) { const id = f.fixtureGrant(over); writeGrantEvent(f.store, AT, 'demo', { operation: 'grant.host', decision: id }, id); for (const [step, extra = {}] of steps) writeGrantEvent(f.store, AT, 'demo', { action: 'grant.step', grantDecision: id, agent: 'ops', host: 'web1', user: USER, tag: f.tag, step, ...extra }, 'web1'); return id; } // The line an interrupted add left: a key nobody holds, under the grant's tag. function leftLine(f) { keygen(join(f.dir, 'orphan')); writeFileSync(join(f.dir, 'granted'), `restrict ${pubLine(join(f.dir, 'orphan'))} ${f.tag}\n`); } test('G7: an Approve installs the tagged line, logs in with the new key, and writes the key file, record and grant.applied', async (t) => { const f = await setup(t); const { id, end } = await f.approve(); assert.equal(end.action, 'grant.applied'); const key = parseHostKey(readFileSync(f.keyFile, 'utf8')); assert.deepEqual(end, { action: 'grant.applied', grantDecision: id, agent: 'ops', host: 'web1', user: USER, tag: f.tag, fingerprint: key.fingerprint, commandsSha256: sha256(JSON.stringify([['systemctl', 'status']])), }); assert.equal(f.tag.length, 39); assert.deepEqual(f.requests(), [`list ${USER}`, `add ${USER} ${key.publicLine} ${f.tag}`, 'true']); assert.deepEqual(f.granted(), [`restrict ${key.publicLine} ${f.tag}`]); assert.equal(statSync(f.keyFile).mode & 0o777, 0o600); assert.deepEqual({ ...f.record() }, { business: 'demo', agent: 'ops', host: 'web1', commands: '[["systemctl","status"]]', rotate_by: dayFrom(30), tag: f.tag, decision: id, }); assert.equal(f.state(), 'valid'); assert.deepEqual( f.grantEvents().filter((e) => e.grantDecision === id).map((e) => e.action + (e.step ? ' ' + e.step : '')), ['grant.step install', 'grant.step commit', 'grant.applied'], ); assert.equal(f.grantEvents().find((e) => e.step === 'commit').previous, null); assert.deepEqual(f.messages(id).map((m) => m.to_role), ['human', 'chief']); // The key file logs in as the user, and the slot is free again. const out = await runHost({ root: f.runRoot, hostId: 'web1', entry: f.entry, key, timeout: 10, command: 'true' }); assert.equal(out.exit, 0); f.credentials.release('demo/ops', 'host.web1', f.credentials.claim('demo/ops', 'host.web1')); }); test('G7: a commands change on a live grant updates the record in one transaction and keeps the key and line', async (t) => { const f = await setup(t); const first = await f.approve(); const before = readFileSync(f.keyFile, 'utf8'); const asked = f.requests().length; const { id, end } = await f.approve({ commands: [['uptime'], ['systemctl', 'status']] }); assert.equal(end.action, 'grant.applied'); assert.equal(end.fingerprint, null); assert.equal(f.requests().length, asked); assert.equal(readFileSync(f.keyFile, 'utf8'), before); assert.equal(f.record().commands, '[["uptime"],["systemctl","status"]]'); assert.equal(f.record().decision, id); assert.deepEqual(f.grantEvents().find((e) => e.grantDecision === id && e.step === 'commit').previous, [['systemctl', 'status']]); assert.equal(f.grantEvents().some((e) => e.grantDecision === id && e.step === 'install'), false); assert.equal(f.state(), 'valid'); assert.notEqual(first.id, id); }); test('G8: a failure at install or verify removes the line and leaves no record, key file or reference', async (t) => { for (const flag of ['fail-list', 'fail-add', 'noop-add']) { const f = await setup(t); f.flag(flag); const { id, end } = await f.approve(); assert.equal(end.code, 'host-failed', flag); assert.equal(end.tag, f.tag); // An install that didn't return keeps the sweep hint, since its add may still be running (row 249). // Here the hint is false, and a sweep drops it (G24). noop-add fails at verify, after install returned. const returned = flag === 'noop-add'; assert.equal(end.hostLineLeft, returned ? undefined : f.tag, flag); assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`, flag); assert.deepEqual(f.granted(), []); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.equal(f.state(), undefined); assert.deepEqual(f.grantEvents().filter((e) => e.grantDecision === id).map((e) => e.action), ['grant.step', 'grant.failed']); assert.match(f.messages(id)[0].body, returned ? /failed: host-failed\.$/ : /Run: mosaic hosts sweep web1$/); } }); test('G8: a failed remove adds hostLineLeft and tells the human to sweep', async (t) => { const f = await setup(t); f.flag('noop-add'); f.flag('fail-remove'); const { id, end } = await f.approve(); assert.equal(end.code, 'host-failed'); assert.equal(end.hostLineLeft, f.tag); assert.match(f.messages(id)[0].body, new RegExp(`tagged ${f.tag} may be left on the host\\. Run: mosaic hosts sweep web1$`)); }); test('G8: a failure in step 5 before commit unlinks the key file, drops the reference and removes the line', async (t) => { const f = await setup(t); // A file where the agent's store dir goes makes issue refuse credential-file. mkdirSync(f.storeRoot, { recursive: true, mode: 0o700 }); writeFileSync(join(f.storeRoot, 'demo'), ''); const { id, end } = await f.approve(); assert.equal(end.code, 'credential-file'); assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`); assert.deepEqual(f.granted(), []); assert.equal(f.record(), undefined); assert.equal(f.state(), undefined); assert.deepEqual(f.grantEvents().filter((e) => e.grantDecision === id).map((e) => e.action + (e.step ? ' ' + e.step : '')), [ 'grant.step install', 'grant.step commit', 'grant.failed', ]); }); // A store fault at commit (a full disk, an I/O error, a busy timeout): the record's INSERT throws once, // after issue wrote the key file. The undo unlinks it, drops the reference and removes the line. test('G8: a store fault in the commit transaction ends host-failed with no record, key file, reference or line', async (t) => { const f = await setup(t); const run = f.store.run.bind(f.store); let faults = 0; f.store.run = (sql, ...args) => { if (sql.startsWith('INSERT INTO host_grants') && faults++ === 0) throw Object.assign(new Error('database or disk is full'), { code: 'SQLITE_FULL' }); return run(sql, ...args); }; const { id, end } = await f.approve(); assert.equal(faults, 1); assert.equal(end.code, 'host-failed'); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.equal(f.state(), undefined); assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`); assert.deepEqual(f.granted(), []); assert.deepEqual(f.grantEvents().filter((e) => e.grantDecision === id).map((e) => e.action + (e.step ? ' ' + e.step : '')), [ 'grant.step install', 'grant.step commit', 'grant.failed', ]); }); test('G8: a failed step 1 check spends the decision and runs nothing remote', async (t) => { const f = await setup(t); const d = await f.raise(); f.store.run("UPDATE hosts SET state='pending-install' WHERE business='demo' AND host='web1'"); f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' }); await until(() => f.ended(d.id)); assert.equal(f.ended(d.id).code, 'host-not-ready'); assert.deepEqual(f.requests(), []); assert.equal(await f.broker.runHostGrant('demo', d.id), 'decision-consumed'); }); test('G9: a registry, definition or commands change between raise and Approve ends decision-mismatch before any remote step', async (t) => { const f = await setup(t); for (const over of [ { registrySha256: registrySha256({ ...f.entry, port: f.entry.port + 1, sudo: true }) }, { registrySha256: registrySha256({ ...f.entry, sudo: false }) }, { definitionSha256: definitionSha256({ ...DEFINITIONS.ops, roleVersion: 3 }) }, { commandsSha256: sha256(JSON.stringify([['uptime'], ['reboot']])) }, ]) { const id = f.fixtureGrant(over); assert.equal(await f.broker.runHostGrant('demo', id), 'decision-mismatch'); assert.equal(f.ended(id).code, 'decision-mismatch'); } // The control passes step 1 and goes remote. const id = f.fixtureGrant(); assert.equal(await f.broker.runHostGrant('demo', id), 'applied'); assert.deepEqual(f.requests().slice(0, 1), [`list ${USER}`]); assert.equal(f.requests().length, 3); }); test('G9: an agent no longer a role or no longer declaring host ends with its code, spent', async (t) => { const f = await setup(t); for (const [over, code] of [[{ agent: 'nobody' }, 'unknown-role'], [{ agent: 'pm' }, 'host-not-declared'], [{ host: 'web9' }, 'host-unknown']]) { const id = f.fixtureGrant(over); assert.equal(await f.broker.runHostGrant('demo', id), code); assert.equal(f.ended(id).code, code); assert.equal(await f.broker.runHostGrant('demo', id), 'decision-consumed'); } assert.deepEqual(f.requests(), []); }); test('G10: a second start refuses decision-consumed, and an unresolved or agent-resolved decision not-authorized, writing nothing', async (t) => { const f = await setup(t); const { id } = await f.approve(); const count = () => f.store.get("SELECT count(*) AS n FROM events WHERE business='demo'").n; // Each fixture decision writes one events row of its own, so the count starts after both. const fixtures = count(); const open = f.fixtureGrant({}, null); const byAgent = f.fixtureGrant({}, { op: 'resolved', by: 'pm', choice: 'yes', via: 'broker' }); assert.equal(count(), fixtures + 2); const before = count(); assert.equal(await f.broker.runHostGrant('demo', id), 'decision-consumed'); assert.equal(await f.broker.runHostGrant('demo', open), 'not-authorized'); assert.equal(await f.broker.runHostGrant('demo', byAgent), 'not-authorized'); assert.equal(count(), before); // The enrollment executor won't take a host grant either. assert.equal(await f.broker.runEnrollment('demo', f.fixtureGrant()), 'not-authorized'); }); test('G15: no event, message or decision holds the private key', async (t) => { const f = await setup(t); await f.approve(); const pem = readFileSync(f.keyFile, 'utf8'); const body = pem.split('\n').slice(1, -2).join(''); const text = [ ...f.store.all('SELECT body FROM events').map((r) => r.body), ...f.store.all('SELECT body FROM messages').map((r) => r.body), ...f.store.all('SELECT question FROM decisions').map((r) => r.question), ].join('\n'); assert.ok(body.length > 64); for (const line of pem.split('\n').slice(1, -2).filter((l) => l.length >= 20)) assert.equal(text.includes(line), false); assert.equal(text.includes('PRIVATE KEY'), false); assert.equal(readOr(join(f.dir, 'requests.log'), '').includes('PRIVATE KEY'), false); }); test('G23: a revoke during install ends grant.failed revoked with no key file, no grant and no line', async (t) => { const f = await setup(t); f.flag('hold-add'); const d = await f.raise(); f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' }); await until(() => existsSync(join(f.dir, 'holding'))); assert.deepEqual(f.credentials.abortClaim('demo/ops', 'host.web1'), { pending: true }); await until(() => f.ended(d.id)); // `skip` makes the held add exit without writing. A killed session's add can still write; the row 249 // tests below cover that. f.flag('skip'); f.flag('release'); assert.equal(f.ended(d.id).code, 'revoked'); assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`); assert.deepEqual(f.granted(), []); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.equal(f.state(), undefined); assert.equal(f.credentials.busy('demo/ops', 'host.web1'), false); }); // Row 249. An abort and a timeout kill only the local ssh, and sshd sends a non-pty command no SIGHUP, // so the held add goes on. Released after the undo's remove, it writes its line and exits 0. const lateAdd = async (f, id, code) => { await until(() => f.ended(id)); const end = f.ended(id); assert.equal(end.code, code); assert.equal(end.hostLineLeft, f.tag); assert.match(f.messages(id)[0].body, new RegExp(`tagged ${f.tag} may be left on the host\\. Run: mosaic hosts sweep web1$`)); assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`); f.flag('release'); await until(() => f.granted().some((l) => l.endsWith(' ' + f.tag))); assert.equal(f.record(), undefined); assert.deepEqual((await f.call('human', 'hosts.status'))[0].hostLineLeft, [f.tag]); await refusesWith(f.call('human', 'hosts.remove', { host: 'web1' }), 'host-line-left'); assert.deepEqual(await f.call('human', 'hosts.sweep', { host: 'web1' }), { host: 'web1', removed: [f.tag], hostLineLeft: [] }); assert.deepEqual(f.granted(), []); assert.deepEqual(await f.call('human', 'hosts.remove', { host: 'web1' }), { host: 'web1', removed: true }); }; test('G23 (row 249): an add that writes after an abort\'s remove leaves hostLineLeft until the sweep', async (t) => { const f = await setup(t); f.flag('hold-add'); const d = await f.raise(); f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' }); await until(() => existsSync(join(f.dir, 'holding'))); assert.deepEqual(f.credentials.abortClaim('demo/ops', 'host.web1'), { pending: true }); await lateAdd(f, d.id, 'revoked'); }); // The timeout's kill without its 30 s wait: install gets its own signal, which SIGKILLs the local ssh // group as host-run.mjs's timer does, and the call then fails as a timed-out request does. test('G23 (row 249): an add that writes after a timed-out install\'s remove leaves hostLineLeft until the sweep', async (t) => { const f = await setup(t, { wrap: (hg) => ({ ...hg, install: async (business, host, record, tag, line) => { const timer = new AbortController(); const out = hg.install(business, host, record, tag, line, timer.signal); await until(() => existsSync(join(f.dir, 'holding'))); timer.abort(); await out.catch(() => {}); throw new Error('timed out'); }, }), }); f.flag('hold-add'); const d = await f.raise(); f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' }); await lateAdd(f, d.id, 'host-failed'); }); test('G23: a revoke recorded after the raise and before the start ends step 1 revoked, and a held slot ends host-busy', async (t) => { const f = await setup(t); const revoked = f.fixtureGrant(); writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', by: 'jason' }, 'web1'); assert.equal(await f.broker.runHostGrant('demo', revoked), 'revoked'); // A revoke of another agent or host doesn't count, and one before the raise doesn't either. writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', agent: 'pm', host: 'web1', tag: f.tag, hostLine: 'pending', by: 'jason' }, 'web1'); const busy = f.fixtureGrant(); const signal = f.credentials.claim('demo/ops', 'host.web1'); assert.equal(await f.broker.runHostGrant('demo', busy), 'host-busy'); assert.equal(f.ended(busy).code, 'host-busy'); assert.match(f.messages(busy)[0].body, /failed: host-busy\.$/); assert.equal(await f.broker.runHostGrant('demo', busy), 'decision-consumed'); f.credentials.release('demo/ops', 'host.web1', signal); assert.deepEqual(f.requests(), []); assert.equal(await f.broker.runHostGrant('demo', f.fixtureGrant()), 'applied'); }); test('G24: names at 64 characters with . and _ give distinct tags of the fixed length', () => { const long = (c) => ('a' + c).repeat(32); const tags = [ grantTag('demo', long('.'), 'web1'), grantTag('demo', long('_'), 'web1'), grantTag('demo.' + 'x'.repeat(59), 'ops', 'web1'), grantTag('demo', 'ops', 'web1'), grantTag('demo', 'ops', 'web2'), grantTag('other', 'ops', 'web1'), ]; assert.equal(new Set(tags).size, tags.length); for (const tag of tags) assert.match(tag, /^mosaic-[0-9a-f]{32}$/); }); test('G11: a restart after install removes the line, then ends interrupted with the tag, holding the slot until then', async (t) => { const f = await setup(t); const id = interrupted(f, [['install']]); leftLine(f); const r = await restart(t, f); assert.deepEqual(r.now, []); assert.equal(r.busy, true); assert.deepEqual(r.later, [id]); assert.equal(r.credentials.busy('demo/ops', 'host.web1'), false); assert.deepEqual(f.requests(), [`remove ${USER} ${f.tag}`]); assert.deepEqual(f.granted(), []); assert.deepEqual(f.ended(id), { action: 'grant.failed', grantDecision: id, agent: 'ops', host: 'web1', code: 'interrupted', tag: f.tag }); assert.deepEqual(f.messages(id).map((m) => m.to_role), ['human', 'chief']); assert.match(f.messages(id)[0].body, /failed: interrupted\.$/); assert.equal(f.record(), undefined); assert.equal(r.state, undefined); // It never runs late, and a second start finds it ended. assert.equal(await r.broker.runHostGrant('demo', id), 'decision-consumed'); const again = await restart(t, f); assert.deepEqual([again.now, again.later], [[], []]); }); test('G11: a restart after a new key\'s commit, with the key file written, ends with no key file, record, reference or line', async (t) => { const f = await setup(t); const id = interrupted(f, [['install'], ['commit', { previous: null }]]); leftLine(f); mkdirSync(join(f.storeRoot, 'demo', 'ops'), { recursive: true, mode: 0o700 }); chmodSync(f.storeRoot, 0o700); chmodSync(join(f.storeRoot, 'demo'), 0o700); keygen(f.keyFile); const r = await restart(t, f); assert.deepEqual(r.swept, [{ instance: 'demo/ops', service: 'host.web1' }]); assert.equal(existsSync(f.keyFile), false); // ssh-keygen's .pub isn't a host. name, so the sweep leaves it. assert.equal(existsSync(f.keyFile + '.pub'), true); assert.deepEqual(r.later, [id]); assert.deepEqual(f.granted(), []); assert.equal(f.record(), undefined); assert.equal(r.state, undefined); assert.equal(f.ended(id).code, 'interrupted'); }); test('G11: a restart after a commands change\'s commit leaves the record as it was and asks nothing of the host', async (t) => { const f = await setup(t); const first = await f.approve(); const before = { record: { ...f.record() }, key: readFileSync(f.keyFile, 'utf8'), granted: f.granted(), asked: f.requests().length }; const id = interrupted(f, [['commit', { previous: [['systemctl', 'status']] }]], { commands: [['uptime']] }); const r = await restart(t, f); assert.deepEqual(r.now, [id]); assert.equal(r.busy, false); assert.deepEqual(r.later, []); assert.deepEqual({ ...f.record() }, before.record); assert.equal(f.record().decision, first.id); assert.equal(readFileSync(f.keyFile, 'utf8'), before.key); assert.deepEqual(f.granted(), before.granted); assert.equal(f.requests().length, before.asked); assert.equal(r.state, 'valid'); assert.deepEqual(f.ended(id), { action: 'grant.failed', grantDecision: id, agent: 'ops', host: 'web1', code: 'interrupted', tag: f.tag }); // The applied grant isn't touched. assert.equal(f.ended(first.id).action, 'grant.applied'); }); test('G11 (B2): a restart before step 1, spent or not yet spent, ends interrupted and nothing runs later', async (t) => { const f = await setup(t); const spent = interrupted(f, []); const open = f.fixtureGrant(); const r = await restart(t, f); assert.deepEqual(r.now, [spent, open]); assert.deepEqual(r.later, []); assert.deepEqual(f.requests(), []); for (const id of [spent, open]) { assert.equal(f.ended(id).code, 'interrupted'); assert.equal(f.ended(id).tag, f.tag); assert.equal(f.messages(id).length, 2); assert.equal(await r.broker.runHostGrant('demo', id), 'decision-consumed'); } // The open one was spent once, by the reconcile. assert.equal(f.store.get("SELECT count(*) AS n FROM events WHERE kind='action.allowed' AND actor_run='grant' AND json_extract(body,'$.operation')='grant.host' AND json_extract(body,'$.decision')=?", open).n, 1); assert.equal(f.record(), undefined); }); test('G11: a remote undo that fails ends interrupted with hostLineLeft and the sweep hint', async (t) => { const f = await setup(t); const id = interrupted(f, [['install']]); leftLine(f); f.flag('fail-remove'); const r = await restart(t, f); assert.deepEqual(r.later, [id]); assert.deepEqual(f.ended(id), { action: 'grant.failed', grantDecision: id, agent: 'ops', host: 'web1', code: 'interrupted', tag: f.tag, hostLineLeft: f.tag }); assert.match(f.messages(id)[0].body, new RegExp(`tagged ${f.tag} may be left on the host\\. Run: mosaic hosts sweep web1$`)); assert.equal(f.granted().length, 1); assert.equal(r.credentials.busy('demo/ops', 'host.web1'), false); // The same with the host record gone: nothing to reach, so the line is left. const g = await setup(t); const id2 = interrupted(g, [['install']]); g.store.transaction(() => { g.store.run("DELETE FROM hosts WHERE business='demo' AND host='web1'"); writeGrantEvent(g.store, AT, 'demo', { action: 'grant.host.removed', host: 'web1', by: 'jason' }, 'web1'); }); const r2 = await restart(t, g); assert.deepEqual(r2.later, [id2]); assert.equal(g.ended(id2).hostLineLeft, g.tag); assert.deepEqual(g.requests(), []); }); test('G11: a restart leaves a finished grant alone, and the reconcile adds nothing for it', async (t) => { const f = await setup(t); const { id } = await f.approve(); const events = f.grantEvents().length; const asked = f.requests().length; const r = await restart(t, f); assert.deepEqual([r.swept, r.now, r.later], [[], [], []]); assert.equal(f.grantEvents().length, events); assert.equal(f.requests().length, asked); assert.equal(existsSync(f.keyFile), true); assert.equal(r.state, 'valid'); assert.equal(f.ended(id).action, 'grant.applied'); }); test('N1 (G27): the start sweep unlinks host. files with no record in loaded businesses only, and refuses a store dir that is not private', (t) => { const root = mkdtempSync(join(tmpdir(), 'g235s-')); t.after(() => rmSync(root, { recursive: true, force: true })); const storeRoot = join(root, 'agents'); assert.deepEqual(sweepHostKeys(storeRoot, ['demo'], new Set()), []); assert.deepEqual(sweepHostKeys(null, ['demo'], new Set()), []); const put = (business, agent, name) => { const d = join(storeRoot, business, agent); mkdirSync(d, { recursive: true, mode: 0o700 }); for (const x of [storeRoot, join(storeRoot, business), d]) chmodSync(x, 0o700); writeFileSync(join(d, name), 'k', { mode: 0o600 }); return join(d, name); }; const kept = put('demo', 'ops', 'host.web1'); const gone = put('demo', 'ops', 'host.web2'); const goneToo = put('demo', 'pm', 'host.web1'); const other = [put('demo', 'ops', 'gitea'), put('demo', 'ops', 'host.Web3'), put('demo', 'ops', 'host.'), put('other', 'ops', 'host.web1')]; const swept = sweepHostKeys(storeRoot, ['demo'], new Set(['demo/ops/web1'])); assert.deepEqual(swept.sort((a, b) => a.instance.localeCompare(b.instance) || a.service.localeCompare(b.service)), [ { instance: 'demo/ops', service: 'host.web2' }, { instance: 'demo/pm', service: 'host.web1' }, ]); assert.equal(existsSync(kept), true); assert.equal(existsSync(gone), false); assert.equal(existsSync(goneToo), false); for (const file of other) assert.equal(existsSync(file), true, file); chmodSync(join(storeRoot, 'demo', 'ops'), 0o750); assert.throws(() => sweepHostKeys(storeRoot, ['demo'], new Set()), /credential-file/); assert.equal(existsSync(kept), true); }); // Row 235 step 5 (r6 section 6): an assistant thread in the business, for G26. function assistant(f) { const project = f.call('human', 'project.create', { name: 'hosts' }).id; const thread = f.call('human', 'thread.create', { project, name: 'main' }).id; f.caps.assistant = f.broker.bindLaunch({ business: 'demo', role: 'assistant', run: 'a-run', harness: 'pi', thread }); f.broker.request(f.caps.assistant, { verb: 'role.claim' }); } const refusesWith = async (p, code) => assert.rejects(Promise.resolve().then(() => p), (e) => (assert.equal(e.code, code), true)); const revokeBodies = (f) => f.grantEvents().filter((e) => ['grant.revoked', 'grant.revoke.closed'].includes(e.action)); test('G12: the human\'s hosts.revoke deletes the record, the key file, the reference and the line, with a closed marker', async (t) => { const f = await setup(t); await f.approve(); const asked = f.requests().length; const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'done' }); assert.deepEqual(r, { agent: 'ops', host: 'web1', tag: f.tag, granted: true, keyFile: 'removed', hostLine: 'removed' }); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.notEqual(f.state(), 'valid'); assert.deepEqual(f.granted(), []); assert.deepEqual(f.requests().slice(asked), [`remove ${USER} ${f.tag}`]); const [open, closed] = revokeBodies(f); assert.equal(typeof open.revoke, 'string'); assert.deepEqual(open, { action: 'grant.revoked', revoke: open.revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', granted: true, reason: 'done', by: 'jason' }); assert.deepEqual(closed, { action: 'grant.revoke.closed', revoke: open.revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'removed', by: 'jason' }); const [row] = await f.call('human', 'hosts.status'); assert.deepEqual([row.revoking, row.hostLineLeft], [[], []]); // A restart finds the revoke closed and does nothing for it. const again = await restart(t, f); assert.deepEqual([again.swept, again.now, again.later], [[], [], []]); assert.equal(again.state, undefined); }); test('G12 (N4): a revoke with no record still removes the computed tag\'s line, with granted false; bad args refuse', async (t) => { const f = await setup(t); leftLine(f); const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'stale line' }); assert.deepEqual(r, { agent: 'ops', host: 'web1', tag: f.tag, granted: false, keyFile: 'removed', hostLine: 'removed' }); assert.deepEqual(f.granted(), []); for (const args of [{ agent: 'ops', host: 'web1' }, { agent: 'ops', host: 'web1', reason: '' }, { agent: 'ops', host: 'web1', reason: 'x'.repeat(501) }, { agent: 'Ops!', host: 'web1', reason: 'r' }, { agent: 'ops', host: 'Web1', reason: 'r' }, { agent: 'ops', host: 'web1', reason: 'r', extra: 1 }]) await refusesWith(f.call('human', 'hosts.revoke', args), 'invalid-request'); await refusesWith(f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web9', reason: 'r' }), 'host-unknown'); assert.equal(revokeBodies(f).length, 2); }); test('G13: a revoke during install aborts the executor, which ends revoked, and the line goes', async (t) => { const f = await setup(t); f.flag('hold-add'); const d = await f.raise(); f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' }); await until(() => existsSync(join(f.dir, 'holding'))); const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'stop' }); assert.equal(r.granted, false); await until(() => f.ended(d.id)); f.flag('skip'); f.flag('release'); assert.equal(f.ended(d.id).code, 'revoked'); assert.deepEqual(f.granted(), []); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.equal(f.credentials.busy('demo/ops', 'host.web1'), false); // A decision raised before the revoke and started after it ends revoked at step 1. const late = await f.raise(); writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', revoke: 'x', agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', by: 'jason' }, 'web1'); f.call('human', 'decision.resolve', { id: late.id, choice: 'yes' }); await until(() => f.ended(late.id)); assert.equal(f.ended(late.id).code, 'revoked'); }); test('G26: the human, the assistant and the tester revoke; the chief and a pm refuse; sweep is the human\'s or the tester\'s', async (t) => { const f = await setup(t, { assistant: true }); assistant(f); f.caps.tester = f.broker.bindTester({ business: 'demo' }); f.caps.reader = f.broker.bindReader({ business: 'demo' }); const args = { agent: 'ops', host: 'web1', reason: 'r' }; await refusesWith(f.call('chief', 'hosts.revoke', args), 'revoke-refused'); await refusesWith(f.call('pm', 'hosts.revoke', args), 'revoke-refused'); await refusesWith(f.call('reader', 'hosts.revoke', args), 'read-only'); for (const who of ['chief', 'pm', 'assistant']) await refusesWith(f.call(who, 'hosts.sweep', { host: 'web1' }), 'human-required'); assert.equal(revokeBodies(f).length, 0); assert.equal((await f.call('assistant', 'hosts.revoke', args)).agent, 'ops'); assert.equal((await f.call('tester', 'hosts.revoke', args)).agent, 'ops'); assert.deepEqual(revokeBodies(f).filter((e) => e.action === 'grant.revoked').map((e) => e.by), ['assistant', 'tester']); assert.deepEqual((await f.call('tester', 'hosts.sweep', { host: 'web1' })).removed, []); }); // Row 281 (#1786, d6 section 8, T8): hosts.revoke shares its steps with a fire's step G now. G12 pins its // records and their order; this pins the tester's dev-only refusal and that a verb revoke starts no job. test('row 281 T8: a tester hosts.revoke refuses tester-dev-only once bus.tester is off, and a human revoke writes no job records', async (t) => { const f = await setup(t); await f.approve(); f.caps.tester = f.broker.bindTester({ business: 'demo' }); f.broker.reloadBusiness('demo', { ...f.business, vars: { 'bus.tester': 'yes' } }); const args = { agent: 'ops', host: 'web1', reason: 'r' }; await refusesWith(f.call('tester', 'hosts.revoke', args), 'tester-dev-only'); assert.equal(revokeBodies(f).length, 0); assert.ok(f.record()); assert.equal((await f.call('human', 'hosts.revoke', args)).granted, true); assert.deepEqual(revokeBodies(f).map((e) => [e.action, e.fire, e.delete]), [['grant.revoked', undefined, undefined], ['grant.revoke.closed', undefined, undefined]]); const jobs = f.store.all("SELECT body FROM events WHERE business='demo' AND kind='action.allowed'").map((e) => JSON.parse(e.body).action) .filter((a) => /^(fire|grants)\./.test(a)); assert.deepEqual(jobs, []); }); test('G27: a restart after step 1 sweeps the key file and runs step 3 once the bus is up; status shows the revoke until then', async (t) => { const f = await setup(t); await f.approve(); const revoke = 'b6d0f0c4-6c3e-4c35-9d55-1f3c2b8f0a11'; // Step 1 as hosts.revoke writes it, then the bus dies. f.store.transaction(() => { f.store.run("DELETE FROM host_grants WHERE business='demo' AND agent='ops' AND host='web1'"); writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', granted: true, reason: 'r', by: 'jason' }, 'web1'); }); assert.deepEqual((await f.call('human', 'hosts.status'))[0].revoking, [f.tag]); const r = await restart(t, f); assert.deepEqual(r.swept, [{ instance: 'demo/ops', service: 'host.web1' }]); assert.equal(existsSync(f.keyFile), false); assert.deepEqual(r.now, []); assert.deepEqual(r.later, [revoke]); assert.deepEqual(f.granted(), []); assert.equal(r.state, undefined); assert.deepEqual(revokeBodies(f).at(-1), { action: 'grant.revoke.closed', revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'removed', by: 'jason' }); assert.deepEqual((await r.broker.request(r.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }), { verb: 'hosts.status', args: {} }))[0].revoking, []); const again = await restart(t, f); assert.deepEqual([again.swept, again.later], [[], []]); }); test('G27: after step 2 with the host down, the restart closes the marker with hostLineLeft; a marker without a revoke id is not reopened', async (t) => { const f = await setup(t); await f.approve(); const revoke = 'c1d2e3f4-0000-4000-8000-000000000001'; f.store.transaction(() => { f.store.run("DELETE FROM host_grants WHERE business='demo' AND agent='ops' AND host='web1'"); writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', granted: true, reason: 'r', by: 'jason' }, 'web1'); }); rmSync(f.keyFile); writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', agent: 'pm', host: 'web1', tag: f.tag, hostLine: 'pending', by: 'jason' }, 'web1'); f.flag('fail-remove'); const r = await restart(t, f); assert.deepEqual(r.swept, []); assert.deepEqual(r.later, [revoke]); assert.equal(revokeBodies(f).at(-1).hostLineLeft, f.tag); assert.equal(f.granted().length, 1); const human = r.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }); assert.deepEqual((await r.broker.request(human, { verb: 'hosts.status', args: {} }))[0].hostLineLeft, [f.tag]); // The sweep then removes it and clears the left line. rmSync(join(f.dir, 'fail-remove')); assert.deepEqual(await r.broker.request(human, { verb: 'hosts.sweep', args: { host: 'web1' } }), { host: 'web1', removed: [f.tag], hostLineLeft: [] }); assert.deepEqual(f.granted(), []); assert.deepEqual((await r.broker.request(human, { verb: 'hosts.status', args: {} }))[0].hostLineLeft, []); }); test('G27: a key file a revoke could not unlink is reported left and goes at the next start', async (t) => { const f = await setup(t); await f.approve(); const agentDir = join(f.storeRoot, 'demo', 'ops'); // setup's t.after removes the scratch dir before any hook added here runs, so the body restores the mode. chmodSync(agentDir, 0o500); try { const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' }); assert.equal(r.keyFile, 'left'); assert.equal(r.hostLine, 'removed'); assert.equal(existsSync(f.keyFile), true); assert.notEqual(f.state(), 'valid'); } finally { chmodSync(agentDir, 0o700); } const again = await restart(t, f); assert.deepEqual(again.swept, [{ instance: 'demo/ops', service: 'host.web1' }]); assert.equal(existsSync(f.keyFile), false); assert.deepEqual(again.later, []); }); test('G28: a commands change, then a revoke, then a restart leave no record, key, reference or line', async (t) => { const f = await setup(t); await f.approve(); await f.approve({ commands: [['uptime']] }); assert.equal(f.record().commands, '[["uptime"]]'); await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' }); const r = await restart(t, f); assert.deepEqual([r.swept, r.now, r.later], [[], [], []]); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.equal(r.state, undefined); assert.deepEqual(f.granted(), []); }); test('G24 (B4): hosts.sweep removes only this business\'s recorded tags no live grant holds, and records what it left', async (t) => { const f = await setup(t); await f.approve(); const live = f.granted(); const pmTag = grantTag('demo', 'pm', 'web1'); const otherTag = grantTag('other', 'ops', 'web1'); const unrecorded = 'mosaic-' + 'f'.repeat(32); keygen(join(f.dir, 'k2')); const line = (tag) => `restrict ${pubLine(join(f.dir, 'k2'))} ${tag}`; writeFileSync(join(f.dir, 'granted'), [...live, line(pmTag), line(otherTag), line(unrecorded), ''].join('\n')); writeGrantEvent(f.store, AT, 'demo', { action: 'grant.step', grantDecision: 'x', agent: 'pm', host: 'web1', user: USER, tag: pmTag, step: 'install' }, 'web1'); writeGrantEvent(f.store, AT, 'demo', { action: 'grant.failed', grantDecision: 'x', agent: 'pm', host: 'web1', code: 'host-failed', tag: pmTag, hostLineLeft: pmTag }, 'web1'); // A left tag no longer on the host drops out at the sweep. writeGrantEvent(f.store, AT, 'demo', { action: 'grant.failed', grantDecision: 'y', agent: 'pm', host: 'web1', code: 'host-failed', tag: 'mosaic-' + '1'.repeat(32), hostLineLeft: 'mosaic-' + '1'.repeat(32) }, 'web1'); assert.equal((await f.call('human', 'hosts.status'))[0].hostLineLeft.length, 2); f.flag('fail-list'); await refusesWith(f.call('human', 'hosts.sweep', { host: 'web1' }), 'host-failed'); assert.equal(f.grantEvents().some((e) => e.action === 'grant.host.swept'), false); rmSync(join(f.dir, 'fail-list')); await refusesWith(f.call('human', 'hosts.sweep', { host: 'web9' }), 'host-unknown'); const asked = f.requests().length; assert.deepEqual(await f.call('human', 'hosts.sweep', { host: 'web1' }), { host: 'web1', removed: [pmTag], hostLineLeft: [] }); assert.deepEqual(f.requests().slice(asked), [`list ${USER}`, `remove ${USER} ${pmTag}`]); assert.deepEqual(f.granted(), [...live, line(otherTag), line(unrecorded)]); assert.deepEqual(f.grantEvents().at(-1), { action: 'grant.host.swept', host: 'web1', removed: [pmTag], left: [], by: 'jason' }); assert.deepEqual((await f.call('human', 'hosts.status'))[0].hostLineLeft, []); // A later failure adds to the cleared list. writeGrantEvent(f.store, AT, 'demo', { action: 'grant.failed', grantDecision: 'z', agent: 'pm', host: 'web1', code: 'host-failed', tag: pmTag, hostLineLeft: pmTag }, 'web1'); assert.deepEqual((await f.call('human', 'hosts.status'))[0].hostLineLeft, [pmTag]); assert.equal(f.state(), 'valid'); }); test('G12: the local steps finish before the host step, so a host that hangs leaves the agent no access', async (t) => { const f = await setup(t); await f.approve(); f.flag('hold-remove'); const revoking = f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' }); await until(() => existsSync(join(f.dir, 'holding-remove'))); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.notEqual(f.state(), 'valid'); assert.deepEqual((await f.call('human', 'hosts.status'))[0].revoking, [f.tag]); f.flag('release'); assert.equal((await revoking).hostLine, 'removed'); assert.deepEqual(f.granted(), []); }); // X5: the executor claims the slot in the same tick as its spend, so a revoke right after the start // call returns finds the claim and aborts it. test('G23 (X5): a revoke right after the spend reaches the executor\'s claim, so the grant ends revoked', async (t) => { const f = await setup(t); const id = f.fixtureGrant(); const run = f.broker.runHostGrant('demo', id); const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' }); assert.equal(r.granted, false); assert.equal(await run, 'revoked'); assert.equal(f.ended(id).code, 'revoked'); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.deepEqual(f.requests(), [`remove ${USER} ${f.tag}`]); }); // X5: the abort check after verify. The revoke lands once the login has worked and before the commit, // so only that check stops the record and key file. test('G23 (X5): a revoke that lands as verify returns ends revoked with no record, key file or line', async (t) => { let revoking; const f = await setup(t, { wrap: (hg) => ({ ...hg, verify: async (...args) => { const out = await hg.verify(...args); revoking = f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' }); return out; }, }), }); const { id, end } = await f.approve(); assert.equal(end.code, 'revoked'); assert.equal((await revoking).granted, false); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.equal(f.state(), undefined); assert.deepEqual(f.granted(), []); // The check stops the grant before step 5, so no commit step is logged. assert.deepEqual(f.grantEvents().filter((e) => e.grantDecision === id && e.action === 'grant.step').map((e) => e.step), ['install']); }); test('G13: a revoke whose host remove fails still drops the record and key, and reports the line left for a sweep', async (t) => { const f = await setup(t); await f.approve(); f.flag('fail-remove'); const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' }); assert.deepEqual(r, { agent: 'ops', host: 'web1', tag: f.tag, granted: true, keyFile: 'removed', hostLineLeft: f.tag }); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.notEqual(f.state(), 'valid'); assert.equal(f.granted().length, 1); assert.equal(revokeBodies(f).at(-1).hostLineLeft, f.tag); const [row] = await f.call('human', 'hosts.status'); assert.deepEqual([row.revoking, row.hostLineLeft], [[], [f.tag]]); // The marker is closed, so a restart doesn't run step 3 again. const again = await restart(t, f); assert.deepEqual(again.later, []); }); // G22 (B1) on one bus with no restart. Enrollment refuses this machine's address, so the chief enrolls // 192.0.2.10 through the fake keyscan. The test then points the record at the scratch sshd and lets the // enrolled installer key log in there. Everything after that is the real path. test('G22: enroll, check, grant, host.run, a commands change the next host.run uses, and a revoke that kills a run in flight', async (t) => { const f = await setup(t, { enroll: true }); const enroll = await f.call('chief', 'decision.raise', { action: 'credential.mint', question: 'Enroll web1?', options, recommendation: 'yes', blocking: false, grant: { method: 'host-enroll', host: 'web1', address: '192.0.2.10', port: 22, user: USER, sudo: true, fingerprints: [fingerprint(pubLine(join(f.dir, 'hostkey')))] }, }); f.call('human', 'decision.resolve', { id: enroll.id, choice: 'yes' }); const enrolled = () => f.grantEvents().find((e) => e.grantDecision === enroll.id && ['grant.host.enrolled', 'grant.failed'].includes(e.action)); await until(enrolled); assert.equal(enrolled().action, 'grant.host.enrolled'); const installer = parseHostKey(readFileSync(join(f.configDir, 'secrets', 'hosts', 'demo', 'web1', 'installer'), 'utf8')); appendFileSync(join(f.dir, 'authorized_keys'), installer.publicLine + '\n'); f.store.run("UPDATE hosts SET address='127.0.0.1', port=? WHERE business='demo' AND host='web1'", f.port); await refusesWith(f.raise(), 'host-not-ready'); assert.equal((await f.call('chief', 'hosts.check', { host: 'web1' })).state, 'ready'); assert.equal((await f.approve({ commands: [['echo']] })).end.action, 'grant.applied'); f.caps.ops = f.broker.bindLaunch({ business: 'demo', role: 'ops', run: 'ops-run', harness: 'pi', address: 'ops' }); f.broker.request(f.caps.ops, { verb: 'role.claim' }); const executor = hostExecutor({ credentials: f.credentials, root: join(f.dir, 'host-run') }); const run = async (argv) => { const d = await f.call('ops', 'decision.raise', { action: 'host.sudo', target: 'web1', argv, question: 'Run it?', options, recommendation: 'yes', blocking: false }); f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' }); return f.broker.requestHost(f.caps.ops, { verb: 'host.run', args: { host: 'web1', argv, decision: d.id } }, executor); }; const out = await run(['echo', 'hi']); assert.deepEqual([out.exit, out.stdout], [0, 'hi\n']); await refusesWith(run(['sleep', '30']), 'command-not-granted'); assert.equal((await f.approve({ commands: [['echo'], ['sleep']] })).end.action, 'grant.applied'); // The handler goes on at creation: the revoke rejects the run while the test still awaits the revoke. const slow = refusesWith(run(['sleep', '30']), 'credential-unavailable'); await until(() => readOr(join(f.dir, 'sudo.log'), '').includes('sleep 30')); const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'done' }); assert.deepEqual([r.granted, r.keyFile, r.hostLine], [true, 'removed', 'removed']); await slow; // The record goes at step 1, so the next call refuses at the record check, before the key (B1 names // credential-unavailable; BUILD.md records the difference). await refusesWith(f.broker.requestHost(f.caps.ops, { verb: 'host.run', args: { host: 'web1', argv: ['echo', 'hi'] } }, executor), 'host-not-granted'); assert.equal(f.record(), undefined); assert.equal(existsSync(f.keyFile), false); assert.deepEqual(f.granted(), []); }); // The decision goes in after the restart, so the reconcile doesn't end it first. test('mos-1163: a grant whose agent became a creative role after the raise ends host-creative-role at step 1, spent, with nothing remote', async (t) => { const f = await setup(t); const r = await restart(t, f, creativeOps(f.business)); const id = f.fixtureGrant(); assert.equal(await r.broker.runHostGrant('demo', id), 'host-creative-role'); assert.equal(f.ended(id).code, 'host-creative-role'); assert.equal(await r.broker.runHostGrant('demo', id), 'decision-consumed'); assert.deepEqual(f.requests(), []); assert.equal(f.record(), undefined); }); test('sage-1908: once the agent of an applied grant is a creative role, host.run refuses host-creative-role, runs nothing remote and keeps the record', async (t) => { const f = await setup(t); assert.equal((await f.approve()).end.action, 'grant.applied'); const r = await restart(t, f, creativeOps(f.business)); const human = r.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }); const project = r.broker.request(human, { verb: 'project.create', args: { name: 'host grant check' } }).id; const thread = r.broker.request(human, { verb: 'thread.create', args: { project, name: 'ops', instance: 'ops' } }).id; const cap = r.broker.bindLaunch({ business: 'demo', role: 'ops', run: 'ops-run', harness: 'pi', address: 'ops', thread }); r.broker.request(cap, { verb: 'role.claim' }); const executor = hostExecutor({ credentials: r.credentials, root: join(f.dir, 'host-run') }); const before = f.requests().length; await refusesWith(r.broker.requestHost(cap, { verb: 'host.run', args: { host: 'web1', argv: ['systemctl', 'status'] } }, executor), 'host-creative-role'); assert.equal(f.requests().length, before); assert.equal(readOr(join(f.dir, 'sudo.log'), '').includes('systemctl'), false); assert.notEqual(f.record(), undefined); });