1062 lines
59 KiB
JavaScript
1062 lines
59 KiB
JavaScript
// Row 235 (#1741, #436 row b): the host grant executor (r6 X2, X3 failure table, X5) over a scratch sshd
|
|
// on 127.0.0.1 that this test starts. The sshd logs in only as the test's own user, so the installer
|
|
// and the granted agent share it. The installer's `list`, `add` and `remove` are stubs on the sshd's
|
|
// PATH that edit a second authorized-keys file, so the new key's login works only once `add` has run.
|
|
// The host writer itself is tested in host-tools.test.mjs (G16). Nothing runs as root and nothing
|
|
// reaches another host.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { createHash } from 'node:crypto';
|
|
import { createServer } from 'node:net';
|
|
import { appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir, userInfo } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { Broker, registrySha256, writeGrantEvent } from '../src/broker.mjs';
|
|
import { Credentials, parseHostKey } from '../src/credentials.mjs';
|
|
import { definitionSha256, fingerprint, grantKeys, grantTag, hostGrant, hostReference, sweepHostKeys } from '../src/host-grant.mjs';
|
|
import { hostExecutor, runHost } from '../src/host-run.mjs';
|
|
import { Store } from '../src/store.mjs';
|
|
import { grantDecision, hostRecord } from './host-fixtures.mjs';
|
|
|
|
import { ephemeralRange, installSleepStub, pickPort, startScratchSshd } from './scratch-sshd.mjs';
|
|
|
|
const SSHD = '/usr/bin/sshd';
|
|
const USER = userInfo().username;
|
|
const AT = '2026-10-21T00:00:00.000Z';
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
const sha256 = (x) => createHash('sha256').update(x).digest('hex');
|
|
const dayFrom = (n) => new Date(Date.now() + n * 86400000).toISOString().slice(0, 10);
|
|
const keygen = (file) => execFileSync('ssh-keygen', ['-q', '-t', 'ed25519', '-N', '', '-C', 'test', '-f', file]);
|
|
const pubLine = (file) => readFileSync(file + '.pub', 'utf8').trim().split(' ').slice(0, 2).join(' ');
|
|
const readOr = (file, fallback) => (existsSync(file) ? readFileSync(file, 'utf8') : fallback);
|
|
const options = [{ key: 'yes', text: 'Yes' }, { key: 'no', text: 'No' }];
|
|
const none = { authority: { withinRole: ['message.send'], crossRole: [] } };
|
|
const DEFINITIONS = {
|
|
'chief-of-staff': { roleVersion: 2, credentials: [] },
|
|
pm: { roleVersion: 2, credentials: [] },
|
|
ops: { roleVersion: 2, credentials: [{ service: 'host' }] },
|
|
};
|
|
|
|
async function until(fn, ms = 10000) {
|
|
const end = Date.now() + ms;
|
|
while (!fn()) {
|
|
if (Date.now() > end) throw new Error('timed out');
|
|
await sleep(20);
|
|
}
|
|
}
|
|
|
|
// The stubs read flag files in the test dir: fail-<verb> exits 1, noop-add exits 0 and writes nothing,
|
|
// hold-add waits for `release` (then exits 1 if `skip` is there), and hold-remove waits for `release`.
|
|
// Both also stop when the test dir is gone, since a held stub on the sshd side outlives its test (#1750).
|
|
// wrap.sh logs every request. The sudo stub logs its arguments to sudo.log and runs the command as the
|
|
// test's user.
|
|
async function sshd(t, dir, options) {
|
|
if (!existsSync(SSHD)) throw new Error(`${SSHD} is missing: the executor tests need it and fail, never skip`);
|
|
keygen(join(dir, 'hostkey'));
|
|
keygen(join(dir, 'installer'));
|
|
writeFileSync(join(dir, 'authorized_keys'), readFileSync(join(dir, 'installer.pub')), { mode: 0o600 });
|
|
mkdirSync(join(dir, 'fake'));
|
|
installSleepStub(dir);
|
|
const stub = (name, body) => writeFileSync(join(dir, 'fake', name), `#!/bin/sh\nD='${dir}'\n${body}\n`, { mode: 0o755 });
|
|
stub('list', '[ -e "$D/fail-list" ] && exit 1\n[ -e "$D/granted" ] && grep " mosaic-[0-9a-f]*$" "$D/granted"\nexit 0');
|
|
stub(
|
|
'add',
|
|
'if [ -e "$D/hold-add" ]; then : > "$D/holding"; while [ ! -e "$D/release" ] && [ -d "$D" ]; do sleep 0.05; done; [ -e "$D/skip" ] && exit 1; fi\n' +
|
|
'[ -e "$D/fail-add" ] && exit 1\n[ -e "$D/noop-add" ] && exit 0\n' +
|
|
'grep -q " $4\\$" "$D/granted" 2>/dev/null && exit 0\necho "restrict $2 $3 $4" >> "$D/granted"',
|
|
);
|
|
stub('sudo', 'printf "%s\\n" "$*" >> "$D/sudo.log"\nshift 2\nexec "$@"');
|
|
stub('remove', 'if [ -e "$D/hold-remove" ]; then : > "$D/holding-remove"; while [ ! -e "$D/release" ] && [ -d "$D" ]; do sleep 0.05; done; fi\n' +
|
|
'[ -e "$D/fail-remove" ] && exit 1\n[ -e "$D/granted" ] || exit 0\ngrep -v " $2\\$" "$D/granted" > "$D/granted.tmp"\nmv "$D/granted.tmp" "$D/granted"');
|
|
writeFileSync(
|
|
join(dir, 'wrap.sh'),
|
|
`#!/bin/sh\nprintf '%s\\n' "$SSH_ORIGINAL_COMMAND" >> '${dir}/requests.log'\nPATH=${dir}/fake:/usr/bin:/bin; export PATH\nexec /bin/sh -c "$SSH_ORIGINAL_COMMAND"\n`,
|
|
{ mode: 0o755 },
|
|
);
|
|
return startScratchSshd(t, dir, [
|
|
'ListenAddress 127.0.0.1', `HostKey ${dir}/hostkey`,
|
|
`AuthorizedKeysFile ${dir}/authorized_keys ${dir}/granted`, `PidFile ${dir}/sshd.pid`,
|
|
'UsePAM no', 'StrictModes no', 'PasswordAuthentication no', 'KbdInteractiveAuthentication no',
|
|
`ForceCommand ${dir}/wrap.sh`,
|
|
], options);
|
|
}
|
|
|
|
|
|
test('row 280: an occupied TCP port is attempted, refused, and retried with our sshd', async t => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'sshd280-'));
|
|
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
const server = createServer(socket => socket.end());
|
|
await new Promise((resolve, reject) => {
|
|
server.once('error', reject);
|
|
server.listen(0, '127.0.0.1', resolve);
|
|
});
|
|
t.after(() => new Promise(resolve => server.close(resolve)));
|
|
const occupied = server.address().port;
|
|
const port = await sshd(t, dir, { firstPort: occupied });
|
|
assert.match(readFileSync(join(dir, 'sshd-0.conf'), 'utf8'), new RegExp(`^Port ${occupied}\\n`));
|
|
assert.equal(readFileSync(join(dir, 'sshd-0.log'), 'utf8').includes(`Server listening on 127.0.0.1 port ${occupied}.`), false);
|
|
assert.notEqual(port, occupied);
|
|
assert.ok([1, 2, 3, 4].some(n => existsSync(join(dir, `sshd-${n}.log`)) &&
|
|
readFileSync(join(dir, `sshd-${n}.log`), 'utf8').includes(`Server listening on 127.0.0.1 port ${port}.`)));
|
|
});
|
|
|
|
test('row 280: every injected picker sample is unprivileged and outside the ephemeral range', () => {
|
|
const [lo, hi] = ephemeralRange();
|
|
for (const sample of [0, 0.25, 0.5, 0.6384, 0.75, 0.999999]) {
|
|
const port = pickPort(() => sample);
|
|
assert.ok(port >= 1024 && port <= 65535);
|
|
assert.ok(port < lo || port > hi, `sample ${sample}: ${port} inside ${lo}-${hi}`);
|
|
}
|
|
});
|
|
|
|
async function setup(t, opts = {}) {
|
|
const dir = mkdtempSync(join(tmpdir(), 'g235x-'));
|
|
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
const port = await sshd(t, dir);
|
|
const configDir = join(dir, 'cfg');
|
|
const keyDir = join(configDir, 'secrets', 'hosts', 'demo', 'web1');
|
|
mkdirSync(keyDir, { recursive: true, mode: 0o700 });
|
|
for (const d of [configDir, join(configDir, 'secrets'), join(configDir, 'secrets', 'hosts'), join(configDir, 'secrets', 'hosts', 'demo'), keyDir]) chmodSync(d, 0o700);
|
|
// opts.enroll: no host record and no installer key, so the test enrolls the host itself (G22). opts.wrap
|
|
// changes the host API the broker gets.
|
|
if (!opts.enroll) writeFileSync(join(keyDir, 'installer'), readFileSync(join(dir, 'installer')), { mode: 0o600 });
|
|
const runRoot = join(configDir, 'secrets', 'run');
|
|
const storeRoot = join(configDir, 'secrets', 'agents');
|
|
mkdirSync(join(dir, 'data'), { mode: 0o700 });
|
|
const store = new Store(join(dir, 'data'));
|
|
const credentials = new Credentials({ references: {}, storeRoot, dataRoot: join(dir, 'data') });
|
|
t.after(() => {
|
|
credentials.close();
|
|
store.close();
|
|
});
|
|
const business = {
|
|
id: 'demo',
|
|
human: 'jason',
|
|
arbiters: { technical: 'pm', delivery: 'pm' },
|
|
roles: {
|
|
chief: { definition: 'chief-of-staff', ...none }, pm: { definition: 'pm', ...none }, ops: { definition: 'ops', ...none },
|
|
...(opts.assistant ? { assistant: { definition: 'assistant', ...none } } : {}),
|
|
},
|
|
definitions: { ...structuredClone(DEFINITIONS), ...(opts.assistant ? { assistant: { roleVersion: 2, credentials: [] } } : {}) },
|
|
vars: { 'bus.tester': true },
|
|
};
|
|
const broker = new Broker({
|
|
store,
|
|
businesses: { demo: business },
|
|
secretCheck: (value) => credentials.assertClean(value),
|
|
hostGrant: (opts.wrap ?? ((h) => h))(hostGrant({ configDir, runRoot, installerUser: USER, scan: async () => [pubLine(join(dir, 'hostkey'))] })),
|
|
hostKeys: grantKeys({ credentials, storeRoot }),
|
|
});
|
|
const caps = {};
|
|
for (const role of ['chief', 'pm']) {
|
|
caps[role] = broker.bindLaunch({ business: 'demo', role, run: role + '-run', harness: 'pi', address: role });
|
|
broker.request(caps[role], { verb: 'role.claim' });
|
|
}
|
|
caps.human = broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true });
|
|
const entry = { address: '127.0.0.1', port, user: USER, hostKeys: [pubLine(join(dir, 'hostkey'))] };
|
|
if (!opts.enroll) hostRecord(store, { host: 'web1', entry });
|
|
const call = (who, verb, args = {}) => broker.request(caps[who], { verb, args });
|
|
const raise = (grant = {}) =>
|
|
call('chief', 'decision.raise', {
|
|
action: 'credential.mint', question: 'Grant ops web1?', options, recommendation: 'yes', blocking: false,
|
|
grant: { method: 'host', agent: 'ops', host: 'web1', commands: [['systemctl', 'status']], rotateBy: dayFrom(30), ...grant },
|
|
});
|
|
const grantEvents = () =>
|
|
store
|
|
.all("SELECT body FROM events WHERE business='demo' AND kind='action.allowed' AND actor_run='grant' ORDER BY seq")
|
|
.map((e) => JSON.parse(e.body))
|
|
.filter((b) => b.action?.startsWith('grant.'));
|
|
const ended = (decision) => grantEvents().find((e) => e.grantDecision === decision && ['grant.applied', 'grant.failed'].includes(e.action));
|
|
// The human's cli yes starts the executor; this waits for its end.
|
|
const approve = async (grant) => {
|
|
const d = await raise(grant);
|
|
call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
await until(() => ended(d.id));
|
|
return { id: d.id, end: ended(d.id) };
|
|
};
|
|
// A decision put in by SQL, with R4's body built here so one field can be broken.
|
|
const fixtureGrant = (over = {}, close) => {
|
|
const commands = over.commands ?? [['uptime']];
|
|
const grant = {
|
|
method: 'host', agent: 'ops', host: 'web1', commands, commandsSha256: sha256(JSON.stringify(commands)), rotateBy: dayFrom(30),
|
|
registrySha256: registrySha256({ ...entry, sudo: true }), definitionSha256: definitionSha256(DEFINITIONS.ops), ...over,
|
|
};
|
|
return grantDecision(store, { grant, ...(close !== undefined ? { close } : {}) });
|
|
};
|
|
const tag = grantTag('demo', 'ops', 'web1');
|
|
const keyFile = join(storeRoot, 'demo', 'ops', 'host.web1');
|
|
const record = () => store.get("SELECT * FROM host_grants WHERE business='demo' AND agent='ops' AND host='web1'");
|
|
const requests = () => readOr(join(dir, 'requests.log'), '').split('\n').filter(Boolean);
|
|
const granted = () => readOr(join(dir, 'granted'), '').split('\n').filter(Boolean);
|
|
const state = () => credentials.status().find((e) => e.instance === 'demo/ops' && e.service === 'host.web1')?.state;
|
|
// The executor's notices, apart from the close notice the resolve sends the chief.
|
|
const messages = (decision) =>
|
|
store.all("SELECT to_role, body FROM messages WHERE business=? AND decision=? AND from_role='_bus' AND instr(body, ?) ORDER BY rowid", 'demo', decision, `(decision ${decision})`);
|
|
const flag = (name) => writeFileSync(join(dir, name), '');
|
|
return { caps, dir, port, entry, store, broker, credentials, call, raise, approve, fixtureGrant, grantEvents, ended, tag, keyFile, record, requests, granted, state, messages, flag, runRoot, storeRoot, configDir, business };
|
|
}
|
|
|
|
// A restart as runtime.mjs does it, on the same store: the key sweep, Credentials with a reference per
|
|
// record, a new broker, reconcileGrants before the socket opens, then reconcileHosts. `busy` is the
|
|
// slot between the two.
|
|
async function restart(t, f, business = f.business) {
|
|
const granted = new Set();
|
|
const references = {};
|
|
for (const g of f.store.all('SELECT business, agent, host, rotate_by FROM host_grants')) {
|
|
granted.add(`${g.business}/${g.agent}/${g.host}`);
|
|
(references[`${g.business}/${g.agent}`] ??= {})['host.' + g.host] = hostReference(f.storeRoot, g.business, g.agent, g.host, g.rotate_by);
|
|
}
|
|
const swept = sweepHostKeys(f.storeRoot, ['demo'], granted);
|
|
const credentials = new Credentials({ references, storeRoot: f.storeRoot, dataRoot: join(f.dir, 'data') });
|
|
t.after(() => credentials.close());
|
|
const broker = new Broker({
|
|
store: f.store,
|
|
businesses: { demo: business },
|
|
secretCheck: (value) => credentials.assertClean(value),
|
|
hostGrant: hostGrant({ configDir: f.configDir, runRoot: f.runRoot, installerUser: USER }),
|
|
hostKeys: grantKeys({ credentials, storeRoot: f.storeRoot }),
|
|
});
|
|
const now = broker.reconcileGrants();
|
|
const busy = credentials.busy('demo/ops', 'host.web1');
|
|
const later = await broker.reconcileHosts();
|
|
const state = credentials.status().find((e) => e.instance === 'demo/ops' && e.service === 'host.web1')?.state;
|
|
return { swept, credentials, broker, now, busy, later, state };
|
|
}
|
|
|
|
// The business after the human moved ops's instance to the creative `research` definition, which has
|
|
// ops's content, so the grant's definitionSha256 still matches and only the creative check refuses.
|
|
const creativeOps = (business) => ({
|
|
...structuredClone(business),
|
|
roles: { ...structuredClone(business.roles), ops: { ...structuredClone(business.roles.ops), definition: 'research' } },
|
|
definitions: { ...structuredClone(business.definitions), research: structuredClone(DEFINITIONS.ops) },
|
|
});
|
|
|
|
// A decision the human approved and a killed bus child spent, with its grant.step events up to `steps`.
|
|
function interrupted(f, steps, over = {}) {
|
|
const id = f.fixtureGrant(over);
|
|
writeGrantEvent(f.store, AT, 'demo', { operation: 'grant.host', decision: id }, id);
|
|
for (const [step, extra = {}] of steps)
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.step', grantDecision: id, agent: 'ops', host: 'web1', user: USER, tag: f.tag, step, ...extra }, 'web1');
|
|
return id;
|
|
}
|
|
|
|
// The line an interrupted add left: a key nobody holds, under the grant's tag.
|
|
function leftLine(f) {
|
|
keygen(join(f.dir, 'orphan'));
|
|
writeFileSync(join(f.dir, 'granted'), `restrict ${pubLine(join(f.dir, 'orphan'))} ${f.tag}\n`);
|
|
}
|
|
|
|
test('G7: an Approve installs the tagged line, logs in with the new key, and writes the key file, record and grant.applied', async (t) => {
|
|
const f = await setup(t);
|
|
const { id, end } = await f.approve();
|
|
assert.equal(end.action, 'grant.applied');
|
|
const key = parseHostKey(readFileSync(f.keyFile, 'utf8'));
|
|
assert.deepEqual(end, {
|
|
action: 'grant.applied', grantDecision: id, agent: 'ops', host: 'web1', user: USER, tag: f.tag,
|
|
fingerprint: key.fingerprint, commandsSha256: sha256(JSON.stringify([['systemctl', 'status']])),
|
|
});
|
|
assert.equal(f.tag.length, 39);
|
|
assert.deepEqual(f.requests(), [`list ${USER}`, `add ${USER} ${key.publicLine} ${f.tag}`, 'true']);
|
|
assert.deepEqual(f.granted(), [`restrict ${key.publicLine} ${f.tag}`]);
|
|
assert.equal(statSync(f.keyFile).mode & 0o777, 0o600);
|
|
assert.deepEqual({ ...f.record() }, {
|
|
business: 'demo', agent: 'ops', host: 'web1', commands: '[["systemctl","status"]]', rotate_by: dayFrom(30), tag: f.tag, decision: id,
|
|
});
|
|
assert.equal(f.state(), 'valid');
|
|
assert.deepEqual(
|
|
f.grantEvents().filter((e) => e.grantDecision === id).map((e) => e.action + (e.step ? ' ' + e.step : '')),
|
|
['grant.step install', 'grant.step commit', 'grant.applied'],
|
|
);
|
|
assert.equal(f.grantEvents().find((e) => e.step === 'commit').previous, null);
|
|
assert.deepEqual(f.messages(id).map((m) => m.to_role), ['human', 'chief']);
|
|
// The key file logs in as the user, and the slot is free again.
|
|
const out = await runHost({ root: f.runRoot, hostId: 'web1', entry: f.entry, key, timeout: 10, command: 'true' });
|
|
assert.equal(out.exit, 0);
|
|
f.credentials.release('demo/ops', 'host.web1', f.credentials.claim('demo/ops', 'host.web1'));
|
|
});
|
|
|
|
test('G7: a commands change on a live grant updates the record in one transaction and keeps the key and line', async (t) => {
|
|
const f = await setup(t);
|
|
const first = await f.approve();
|
|
const before = readFileSync(f.keyFile, 'utf8');
|
|
const asked = f.requests().length;
|
|
const { id, end } = await f.approve({ commands: [['uptime'], ['systemctl', 'status']] });
|
|
assert.equal(end.action, 'grant.applied');
|
|
assert.equal(end.fingerprint, null);
|
|
assert.equal(f.requests().length, asked);
|
|
assert.equal(readFileSync(f.keyFile, 'utf8'), before);
|
|
assert.equal(f.record().commands, '[["uptime"],["systemctl","status"]]');
|
|
assert.equal(f.record().decision, id);
|
|
assert.deepEqual(f.grantEvents().find((e) => e.grantDecision === id && e.step === 'commit').previous, [['systemctl', 'status']]);
|
|
assert.equal(f.grantEvents().some((e) => e.grantDecision === id && e.step === 'install'), false);
|
|
assert.equal(f.state(), 'valid');
|
|
assert.notEqual(first.id, id);
|
|
});
|
|
|
|
test('G8: a failure at install or verify removes the line and leaves no record, key file or reference', async (t) => {
|
|
for (const flag of ['fail-list', 'fail-add', 'noop-add']) {
|
|
const f = await setup(t);
|
|
f.flag(flag);
|
|
const { id, end } = await f.approve();
|
|
assert.equal(end.code, 'host-failed', flag);
|
|
assert.equal(end.tag, f.tag);
|
|
// An install that didn't return keeps the sweep hint, since its add may still be running (row 249).
|
|
// Here the hint is false, and a sweep drops it (G24). noop-add fails at verify, after install returned.
|
|
const returned = flag === 'noop-add';
|
|
assert.equal(end.hostLineLeft, returned ? undefined : f.tag, flag);
|
|
assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`, flag);
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.equal(f.state(), undefined);
|
|
assert.deepEqual(f.grantEvents().filter((e) => e.grantDecision === id).map((e) => e.action), ['grant.step', 'grant.failed']);
|
|
assert.match(f.messages(id)[0].body, returned ? /failed: host-failed\.$/ : /Run: mosaic hosts sweep web1$/);
|
|
}
|
|
});
|
|
|
|
test('G8: a failed remove adds hostLineLeft and tells the human to sweep', async (t) => {
|
|
const f = await setup(t);
|
|
f.flag('noop-add');
|
|
f.flag('fail-remove');
|
|
const { id, end } = await f.approve();
|
|
assert.equal(end.code, 'host-failed');
|
|
assert.equal(end.hostLineLeft, f.tag);
|
|
assert.match(f.messages(id)[0].body, new RegExp(`tagged ${f.tag} may be left on the host\\. Run: mosaic hosts sweep web1$`));
|
|
});
|
|
|
|
test('G8: a failure in step 5 before commit unlinks the key file, drops the reference and removes the line', async (t) => {
|
|
const f = await setup(t);
|
|
// A file where the agent's store dir goes makes issue refuse credential-file.
|
|
mkdirSync(f.storeRoot, { recursive: true, mode: 0o700 });
|
|
writeFileSync(join(f.storeRoot, 'demo'), '');
|
|
const { id, end } = await f.approve();
|
|
assert.equal(end.code, 'credential-file');
|
|
assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`);
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(f.state(), undefined);
|
|
assert.deepEqual(f.grantEvents().filter((e) => e.grantDecision === id).map((e) => e.action + (e.step ? ' ' + e.step : '')), [
|
|
'grant.step install', 'grant.step commit', 'grant.failed',
|
|
]);
|
|
});
|
|
|
|
// A store fault at commit (a full disk, an I/O error, a busy timeout): the record's INSERT throws once,
|
|
// after issue wrote the key file. The undo unlinks it, drops the reference and removes the line.
|
|
test('G8: a store fault in the commit transaction ends host-failed with no record, key file, reference or line', async (t) => {
|
|
const f = await setup(t);
|
|
const run = f.store.run.bind(f.store);
|
|
let faults = 0;
|
|
f.store.run = (sql, ...args) => {
|
|
if (sql.startsWith('INSERT INTO host_grants') && faults++ === 0) throw Object.assign(new Error('database or disk is full'), { code: 'SQLITE_FULL' });
|
|
return run(sql, ...args);
|
|
};
|
|
const { id, end } = await f.approve();
|
|
assert.equal(faults, 1);
|
|
assert.equal(end.code, 'host-failed');
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.equal(f.state(), undefined);
|
|
assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`);
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.deepEqual(f.grantEvents().filter((e) => e.grantDecision === id).map((e) => e.action + (e.step ? ' ' + e.step : '')), [
|
|
'grant.step install', 'grant.step commit', 'grant.failed',
|
|
]);
|
|
});
|
|
|
|
test('G8: a failed step 1 check spends the decision and runs nothing remote', async (t) => {
|
|
const f = await setup(t);
|
|
const d = await f.raise();
|
|
f.store.run("UPDATE hosts SET state='pending-install' WHERE business='demo' AND host='web1'");
|
|
f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
await until(() => f.ended(d.id));
|
|
assert.equal(f.ended(d.id).code, 'host-not-ready');
|
|
assert.deepEqual(f.requests(), []);
|
|
assert.equal(await f.broker.runHostGrant('demo', d.id), 'decision-consumed');
|
|
});
|
|
|
|
test('G9: a registry, definition or commands change between raise and Approve ends decision-mismatch before any remote step', async (t) => {
|
|
const f = await setup(t);
|
|
for (const over of [
|
|
{ registrySha256: registrySha256({ ...f.entry, port: f.entry.port + 1, sudo: true }) },
|
|
{ registrySha256: registrySha256({ ...f.entry, sudo: false }) },
|
|
{ definitionSha256: definitionSha256({ ...DEFINITIONS.ops, roleVersion: 3 }) },
|
|
{ commandsSha256: sha256(JSON.stringify([['uptime'], ['reboot']])) },
|
|
]) {
|
|
const id = f.fixtureGrant(over);
|
|
assert.equal(await f.broker.runHostGrant('demo', id), 'decision-mismatch');
|
|
assert.equal(f.ended(id).code, 'decision-mismatch');
|
|
}
|
|
// The control passes step 1 and goes remote.
|
|
const id = f.fixtureGrant();
|
|
assert.equal(await f.broker.runHostGrant('demo', id), 'applied');
|
|
assert.deepEqual(f.requests().slice(0, 1), [`list ${USER}`]);
|
|
assert.equal(f.requests().length, 3);
|
|
});
|
|
|
|
test('G9: an agent no longer a role or no longer declaring host ends with its code, spent', async (t) => {
|
|
const f = await setup(t);
|
|
for (const [over, code] of [[{ agent: 'nobody' }, 'unknown-role'], [{ agent: 'pm' }, 'host-not-declared'], [{ host: 'web9' }, 'host-unknown']]) {
|
|
const id = f.fixtureGrant(over);
|
|
assert.equal(await f.broker.runHostGrant('demo', id), code);
|
|
assert.equal(f.ended(id).code, code);
|
|
assert.equal(await f.broker.runHostGrant('demo', id), 'decision-consumed');
|
|
}
|
|
assert.deepEqual(f.requests(), []);
|
|
});
|
|
|
|
test('G10: a second start refuses decision-consumed, and an unresolved or agent-resolved decision not-authorized, writing nothing', async (t) => {
|
|
const f = await setup(t);
|
|
const { id } = await f.approve();
|
|
const count = () => f.store.get("SELECT count(*) AS n FROM events WHERE business='demo'").n;
|
|
// Each fixture decision writes one events row of its own, so the count starts after both.
|
|
const fixtures = count();
|
|
const open = f.fixtureGrant({}, null);
|
|
const byAgent = f.fixtureGrant({}, { op: 'resolved', by: 'pm', choice: 'yes', via: 'broker' });
|
|
assert.equal(count(), fixtures + 2);
|
|
const before = count();
|
|
assert.equal(await f.broker.runHostGrant('demo', id), 'decision-consumed');
|
|
assert.equal(await f.broker.runHostGrant('demo', open), 'not-authorized');
|
|
assert.equal(await f.broker.runHostGrant('demo', byAgent), 'not-authorized');
|
|
assert.equal(count(), before);
|
|
// The enrollment executor won't take a host grant either.
|
|
assert.equal(await f.broker.runEnrollment('demo', f.fixtureGrant()), 'not-authorized');
|
|
});
|
|
|
|
test('G15: no event, message or decision holds the private key', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
const pem = readFileSync(f.keyFile, 'utf8');
|
|
const body = pem.split('\n').slice(1, -2).join('');
|
|
const text = [
|
|
...f.store.all('SELECT body FROM events').map((r) => r.body),
|
|
...f.store.all('SELECT body FROM messages').map((r) => r.body),
|
|
...f.store.all('SELECT question FROM decisions').map((r) => r.question),
|
|
].join('\n');
|
|
assert.ok(body.length > 64);
|
|
for (const line of pem.split('\n').slice(1, -2).filter((l) => l.length >= 20)) assert.equal(text.includes(line), false);
|
|
assert.equal(text.includes('PRIVATE KEY'), false);
|
|
assert.equal(readOr(join(f.dir, 'requests.log'), '').includes('PRIVATE KEY'), false);
|
|
});
|
|
|
|
test('G23: a revoke during install ends grant.failed revoked with no key file, no grant and no line', async (t) => {
|
|
const f = await setup(t);
|
|
f.flag('hold-add');
|
|
const d = await f.raise();
|
|
f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
await until(() => existsSync(join(f.dir, 'holding')));
|
|
assert.deepEqual(f.credentials.abortClaim('demo/ops', 'host.web1'), { pending: true });
|
|
await until(() => f.ended(d.id));
|
|
// `skip` makes the held add exit without writing. A killed session's add can still write; the row 249
|
|
// tests below cover that.
|
|
f.flag('skip');
|
|
f.flag('release');
|
|
assert.equal(f.ended(d.id).code, 'revoked');
|
|
assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`);
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.equal(f.state(), undefined);
|
|
assert.equal(f.credentials.busy('demo/ops', 'host.web1'), false);
|
|
});
|
|
|
|
// Row 249. An abort and a timeout kill only the local ssh, and sshd sends a non-pty command no SIGHUP,
|
|
// so the held add goes on. Released after the undo's remove, it writes its line and exits 0.
|
|
const lateAdd = async (f, id, code) => {
|
|
await until(() => f.ended(id));
|
|
const end = f.ended(id);
|
|
assert.equal(end.code, code);
|
|
assert.equal(end.hostLineLeft, f.tag);
|
|
assert.match(f.messages(id)[0].body, new RegExp(`tagged ${f.tag} may be left on the host\\. Run: mosaic hosts sweep web1$`));
|
|
assert.equal(f.requests().at(-1), `remove ${USER} ${f.tag}`);
|
|
f.flag('release');
|
|
await until(() => f.granted().some((l) => l.endsWith(' ' + f.tag)));
|
|
assert.equal(f.record(), undefined);
|
|
assert.deepEqual((await f.call('human', 'hosts.status'))[0].hostLineLeft, [f.tag]);
|
|
await refusesWith(f.call('human', 'hosts.remove', { host: 'web1' }), 'host-line-left');
|
|
assert.deepEqual(await f.call('human', 'hosts.sweep', { host: 'web1' }), { host: 'web1', removed: [f.tag], hostLineLeft: [] });
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.deepEqual(await f.call('human', 'hosts.remove', { host: 'web1' }), { host: 'web1', removed: true });
|
|
};
|
|
|
|
test('G23 (row 249): an add that writes after an abort\'s remove leaves hostLineLeft until the sweep', async (t) => {
|
|
const f = await setup(t);
|
|
f.flag('hold-add');
|
|
const d = await f.raise();
|
|
f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
await until(() => existsSync(join(f.dir, 'holding')));
|
|
assert.deepEqual(f.credentials.abortClaim('demo/ops', 'host.web1'), { pending: true });
|
|
await lateAdd(f, d.id, 'revoked');
|
|
});
|
|
|
|
// The timeout's kill without its 30 s wait: install gets its own signal, which SIGKILLs the local ssh
|
|
// group as host-run.mjs's timer does, and the call then fails as a timed-out request does.
|
|
test('G23 (row 249): an add that writes after a timed-out install\'s remove leaves hostLineLeft until the sweep', async (t) => {
|
|
const f = await setup(t, {
|
|
wrap: (hg) => ({
|
|
...hg,
|
|
install: async (business, host, record, tag, line) => {
|
|
const timer = new AbortController();
|
|
const out = hg.install(business, host, record, tag, line, timer.signal);
|
|
await until(() => existsSync(join(f.dir, 'holding')));
|
|
timer.abort();
|
|
await out.catch(() => {});
|
|
throw new Error('timed out');
|
|
},
|
|
}),
|
|
});
|
|
f.flag('hold-add');
|
|
const d = await f.raise();
|
|
f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
await lateAdd(f, d.id, 'host-failed');
|
|
});
|
|
|
|
test('G23: a revoke recorded after the raise and before the start ends step 1 revoked, and a held slot ends host-busy', async (t) => {
|
|
const f = await setup(t);
|
|
const revoked = f.fixtureGrant();
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', by: 'jason' }, 'web1');
|
|
assert.equal(await f.broker.runHostGrant('demo', revoked), 'revoked');
|
|
// A revoke of another agent or host doesn't count, and one before the raise doesn't either.
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', agent: 'pm', host: 'web1', tag: f.tag, hostLine: 'pending', by: 'jason' }, 'web1');
|
|
const busy = f.fixtureGrant();
|
|
const signal = f.credentials.claim('demo/ops', 'host.web1');
|
|
assert.equal(await f.broker.runHostGrant('demo', busy), 'host-busy');
|
|
assert.equal(f.ended(busy).code, 'host-busy');
|
|
assert.match(f.messages(busy)[0].body, /failed: host-busy\.$/);
|
|
assert.equal(await f.broker.runHostGrant('demo', busy), 'decision-consumed');
|
|
f.credentials.release('demo/ops', 'host.web1', signal);
|
|
assert.deepEqual(f.requests(), []);
|
|
assert.equal(await f.broker.runHostGrant('demo', f.fixtureGrant()), 'applied');
|
|
});
|
|
|
|
test('G24: names at 64 characters with . and _ give distinct tags of the fixed length', () => {
|
|
const long = (c) => ('a' + c).repeat(32);
|
|
const tags = [
|
|
grantTag('demo', long('.'), 'web1'),
|
|
grantTag('demo', long('_'), 'web1'),
|
|
grantTag('demo.' + 'x'.repeat(59), 'ops', 'web1'),
|
|
grantTag('demo', 'ops', 'web1'),
|
|
grantTag('demo', 'ops', 'web2'),
|
|
grantTag('other', 'ops', 'web1'),
|
|
];
|
|
assert.equal(new Set(tags).size, tags.length);
|
|
for (const tag of tags) assert.match(tag, /^mosaic-[0-9a-f]{32}$/);
|
|
});
|
|
|
|
test('G11: a restart after install removes the line, then ends interrupted with the tag, holding the slot until then', async (t) => {
|
|
const f = await setup(t);
|
|
const id = interrupted(f, [['install']]);
|
|
leftLine(f);
|
|
const r = await restart(t, f);
|
|
assert.deepEqual(r.now, []);
|
|
assert.equal(r.busy, true);
|
|
assert.deepEqual(r.later, [id]);
|
|
assert.equal(r.credentials.busy('demo/ops', 'host.web1'), false);
|
|
assert.deepEqual(f.requests(), [`remove ${USER} ${f.tag}`]);
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.deepEqual(f.ended(id), { action: 'grant.failed', grantDecision: id, agent: 'ops', host: 'web1', code: 'interrupted', tag: f.tag });
|
|
assert.deepEqual(f.messages(id).map((m) => m.to_role), ['human', 'chief']);
|
|
assert.match(f.messages(id)[0].body, /failed: interrupted\.$/);
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(r.state, undefined);
|
|
// It never runs late, and a second start finds it ended.
|
|
assert.equal(await r.broker.runHostGrant('demo', id), 'decision-consumed');
|
|
const again = await restart(t, f);
|
|
assert.deepEqual([again.now, again.later], [[], []]);
|
|
});
|
|
|
|
test('G11: a restart after a new key\'s commit, with the key file written, ends with no key file, record, reference or line', async (t) => {
|
|
const f = await setup(t);
|
|
const id = interrupted(f, [['install'], ['commit', { previous: null }]]);
|
|
leftLine(f);
|
|
mkdirSync(join(f.storeRoot, 'demo', 'ops'), { recursive: true, mode: 0o700 });
|
|
chmodSync(f.storeRoot, 0o700);
|
|
chmodSync(join(f.storeRoot, 'demo'), 0o700);
|
|
keygen(f.keyFile);
|
|
const r = await restart(t, f);
|
|
assert.deepEqual(r.swept, [{ instance: 'demo/ops', service: 'host.web1' }]);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
// ssh-keygen's .pub isn't a host.<id> name, so the sweep leaves it.
|
|
assert.equal(existsSync(f.keyFile + '.pub'), true);
|
|
assert.deepEqual(r.later, [id]);
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(r.state, undefined);
|
|
assert.equal(f.ended(id).code, 'interrupted');
|
|
});
|
|
|
|
test('G11: a restart after a commands change\'s commit leaves the record as it was and asks nothing of the host', async (t) => {
|
|
const f = await setup(t);
|
|
const first = await f.approve();
|
|
const before = { record: { ...f.record() }, key: readFileSync(f.keyFile, 'utf8'), granted: f.granted(), asked: f.requests().length };
|
|
const id = interrupted(f, [['commit', { previous: [['systemctl', 'status']] }]], { commands: [['uptime']] });
|
|
const r = await restart(t, f);
|
|
assert.deepEqual(r.now, [id]);
|
|
assert.equal(r.busy, false);
|
|
assert.deepEqual(r.later, []);
|
|
assert.deepEqual({ ...f.record() }, before.record);
|
|
assert.equal(f.record().decision, first.id);
|
|
assert.equal(readFileSync(f.keyFile, 'utf8'), before.key);
|
|
assert.deepEqual(f.granted(), before.granted);
|
|
assert.equal(f.requests().length, before.asked);
|
|
assert.equal(r.state, 'valid');
|
|
assert.deepEqual(f.ended(id), { action: 'grant.failed', grantDecision: id, agent: 'ops', host: 'web1', code: 'interrupted', tag: f.tag });
|
|
// The applied grant isn't touched.
|
|
assert.equal(f.ended(first.id).action, 'grant.applied');
|
|
});
|
|
|
|
test('G11 (B2): a restart before step 1, spent or not yet spent, ends interrupted and nothing runs later', async (t) => {
|
|
const f = await setup(t);
|
|
const spent = interrupted(f, []);
|
|
const open = f.fixtureGrant();
|
|
const r = await restart(t, f);
|
|
assert.deepEqual(r.now, [spent, open]);
|
|
assert.deepEqual(r.later, []);
|
|
assert.deepEqual(f.requests(), []);
|
|
for (const id of [spent, open]) {
|
|
assert.equal(f.ended(id).code, 'interrupted');
|
|
assert.equal(f.ended(id).tag, f.tag);
|
|
assert.equal(f.messages(id).length, 2);
|
|
assert.equal(await r.broker.runHostGrant('demo', id), 'decision-consumed');
|
|
}
|
|
// The open one was spent once, by the reconcile.
|
|
assert.equal(f.store.get("SELECT count(*) AS n FROM events WHERE kind='action.allowed' AND actor_run='grant' AND json_extract(body,'$.operation')='grant.host' AND json_extract(body,'$.decision')=?", open).n, 1);
|
|
assert.equal(f.record(), undefined);
|
|
});
|
|
|
|
test('G11: a remote undo that fails ends interrupted with hostLineLeft and the sweep hint', async (t) => {
|
|
const f = await setup(t);
|
|
const id = interrupted(f, [['install']]);
|
|
leftLine(f);
|
|
f.flag('fail-remove');
|
|
const r = await restart(t, f);
|
|
assert.deepEqual(r.later, [id]);
|
|
assert.deepEqual(f.ended(id), { action: 'grant.failed', grantDecision: id, agent: 'ops', host: 'web1', code: 'interrupted', tag: f.tag, hostLineLeft: f.tag });
|
|
assert.match(f.messages(id)[0].body, new RegExp(`tagged ${f.tag} may be left on the host\\. Run: mosaic hosts sweep web1$`));
|
|
assert.equal(f.granted().length, 1);
|
|
assert.equal(r.credentials.busy('demo/ops', 'host.web1'), false);
|
|
// The same with the host record gone: nothing to reach, so the line is left.
|
|
const g = await setup(t);
|
|
const id2 = interrupted(g, [['install']]);
|
|
g.store.transaction(() => {
|
|
g.store.run("DELETE FROM hosts WHERE business='demo' AND host='web1'");
|
|
writeGrantEvent(g.store, AT, 'demo', { action: 'grant.host.removed', host: 'web1', by: 'jason' }, 'web1');
|
|
});
|
|
const r2 = await restart(t, g);
|
|
assert.deepEqual(r2.later, [id2]);
|
|
assert.equal(g.ended(id2).hostLineLeft, g.tag);
|
|
assert.deepEqual(g.requests(), []);
|
|
});
|
|
|
|
test('G11: a restart leaves a finished grant alone, and the reconcile adds nothing for it', async (t) => {
|
|
const f = await setup(t);
|
|
const { id } = await f.approve();
|
|
const events = f.grantEvents().length;
|
|
const asked = f.requests().length;
|
|
const r = await restart(t, f);
|
|
assert.deepEqual([r.swept, r.now, r.later], [[], [], []]);
|
|
assert.equal(f.grantEvents().length, events);
|
|
assert.equal(f.requests().length, asked);
|
|
assert.equal(existsSync(f.keyFile), true);
|
|
assert.equal(r.state, 'valid');
|
|
assert.equal(f.ended(id).action, 'grant.applied');
|
|
});
|
|
|
|
test('N1 (G27): the start sweep unlinks host.<id> files with no record in loaded businesses only, and refuses a store dir that is not private', (t) => {
|
|
const root = mkdtempSync(join(tmpdir(), 'g235s-'));
|
|
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
const storeRoot = join(root, 'agents');
|
|
assert.deepEqual(sweepHostKeys(storeRoot, ['demo'], new Set()), []);
|
|
assert.deepEqual(sweepHostKeys(null, ['demo'], new Set()), []);
|
|
const put = (business, agent, name) => {
|
|
const d = join(storeRoot, business, agent);
|
|
mkdirSync(d, { recursive: true, mode: 0o700 });
|
|
for (const x of [storeRoot, join(storeRoot, business), d]) chmodSync(x, 0o700);
|
|
writeFileSync(join(d, name), 'k', { mode: 0o600 });
|
|
return join(d, name);
|
|
};
|
|
const kept = put('demo', 'ops', 'host.web1');
|
|
const gone = put('demo', 'ops', 'host.web2');
|
|
const goneToo = put('demo', 'pm', 'host.web1');
|
|
const other = [put('demo', 'ops', 'gitea'), put('demo', 'ops', 'host.Web3'), put('demo', 'ops', 'host.'), put('other', 'ops', 'host.web1')];
|
|
const swept = sweepHostKeys(storeRoot, ['demo'], new Set(['demo/ops/web1']));
|
|
assert.deepEqual(swept.sort((a, b) => a.instance.localeCompare(b.instance) || a.service.localeCompare(b.service)), [
|
|
{ instance: 'demo/ops', service: 'host.web2' },
|
|
{ instance: 'demo/pm', service: 'host.web1' },
|
|
]);
|
|
assert.equal(existsSync(kept), true);
|
|
assert.equal(existsSync(gone), false);
|
|
assert.equal(existsSync(goneToo), false);
|
|
for (const file of other) assert.equal(existsSync(file), true, file);
|
|
chmodSync(join(storeRoot, 'demo', 'ops'), 0o750);
|
|
assert.throws(() => sweepHostKeys(storeRoot, ['demo'], new Set()), /credential-file/);
|
|
assert.equal(existsSync(kept), true);
|
|
});
|
|
|
|
// Row 235 step 5 (r6 section 6): an assistant thread in the business, for G26.
|
|
function assistant(f) {
|
|
const project = f.call('human', 'project.create', { name: 'hosts' }).id;
|
|
const thread = f.call('human', 'thread.create', { project, name: 'main' }).id;
|
|
f.caps.assistant = f.broker.bindLaunch({ business: 'demo', role: 'assistant', run: 'a-run', harness: 'pi', thread });
|
|
f.broker.request(f.caps.assistant, { verb: 'role.claim' });
|
|
}
|
|
const refusesWith = async (p, code) => assert.rejects(Promise.resolve().then(() => p), (e) => (assert.equal(e.code, code), true));
|
|
const revokeBodies = (f) => f.grantEvents().filter((e) => ['grant.revoked', 'grant.revoke.closed'].includes(e.action));
|
|
|
|
test('G12: the human\'s hosts.revoke deletes the record, the key file, the reference and the line, with a closed marker', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
const asked = f.requests().length;
|
|
const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'done' });
|
|
assert.deepEqual(r, { agent: 'ops', host: 'web1', tag: f.tag, granted: true, keyFile: 'removed', hostLine: 'removed' });
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.notEqual(f.state(), 'valid');
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.deepEqual(f.requests().slice(asked), [`remove ${USER} ${f.tag}`]);
|
|
const [open, closed] = revokeBodies(f);
|
|
assert.equal(typeof open.revoke, 'string');
|
|
assert.deepEqual(open, { action: 'grant.revoked', revoke: open.revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', granted: true, reason: 'done', by: 'jason' });
|
|
assert.deepEqual(closed, { action: 'grant.revoke.closed', revoke: open.revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'removed', by: 'jason' });
|
|
const [row] = await f.call('human', 'hosts.status');
|
|
assert.deepEqual([row.revoking, row.hostLineLeft], [[], []]);
|
|
// A restart finds the revoke closed and does nothing for it.
|
|
const again = await restart(t, f);
|
|
assert.deepEqual([again.swept, again.now, again.later], [[], [], []]);
|
|
assert.equal(again.state, undefined);
|
|
});
|
|
|
|
test('G12 (N4): a revoke with no record still removes the computed tag\'s line, with granted false; bad args refuse', async (t) => {
|
|
const f = await setup(t);
|
|
leftLine(f);
|
|
const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'stale line' });
|
|
assert.deepEqual(r, { agent: 'ops', host: 'web1', tag: f.tag, granted: false, keyFile: 'removed', hostLine: 'removed' });
|
|
assert.deepEqual(f.granted(), []);
|
|
for (const args of [{ agent: 'ops', host: 'web1' }, { agent: 'ops', host: 'web1', reason: '' }, { agent: 'ops', host: 'web1', reason: 'x'.repeat(501) },
|
|
{ agent: 'Ops!', host: 'web1', reason: 'r' }, { agent: 'ops', host: 'Web1', reason: 'r' }, { agent: 'ops', host: 'web1', reason: 'r', extra: 1 }])
|
|
await refusesWith(f.call('human', 'hosts.revoke', args), 'invalid-request');
|
|
await refusesWith(f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web9', reason: 'r' }), 'host-unknown');
|
|
assert.equal(revokeBodies(f).length, 2);
|
|
});
|
|
|
|
test('G13: a revoke during install aborts the executor, which ends revoked, and the line goes', async (t) => {
|
|
const f = await setup(t);
|
|
f.flag('hold-add');
|
|
const d = await f.raise();
|
|
f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
await until(() => existsSync(join(f.dir, 'holding')));
|
|
const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'stop' });
|
|
assert.equal(r.granted, false);
|
|
await until(() => f.ended(d.id));
|
|
f.flag('skip');
|
|
f.flag('release');
|
|
assert.equal(f.ended(d.id).code, 'revoked');
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.equal(f.credentials.busy('demo/ops', 'host.web1'), false);
|
|
// A decision raised before the revoke and started after it ends revoked at step 1.
|
|
const late = await f.raise();
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', revoke: 'x', agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', by: 'jason' }, 'web1');
|
|
f.call('human', 'decision.resolve', { id: late.id, choice: 'yes' });
|
|
await until(() => f.ended(late.id));
|
|
assert.equal(f.ended(late.id).code, 'revoked');
|
|
});
|
|
|
|
test('G26: the human, the assistant and the tester revoke; the chief and a pm refuse; sweep is the human\'s or the tester\'s', async (t) => {
|
|
const f = await setup(t, { assistant: true });
|
|
assistant(f);
|
|
f.caps.tester = f.broker.bindTester({ business: 'demo' });
|
|
f.caps.reader = f.broker.bindReader({ business: 'demo' });
|
|
const args = { agent: 'ops', host: 'web1', reason: 'r' };
|
|
await refusesWith(f.call('chief', 'hosts.revoke', args), 'revoke-refused');
|
|
await refusesWith(f.call('pm', 'hosts.revoke', args), 'revoke-refused');
|
|
await refusesWith(f.call('reader', 'hosts.revoke', args), 'read-only');
|
|
for (const who of ['chief', 'pm', 'assistant']) await refusesWith(f.call(who, 'hosts.sweep', { host: 'web1' }), 'human-required');
|
|
assert.equal(revokeBodies(f).length, 0);
|
|
assert.equal((await f.call('assistant', 'hosts.revoke', args)).agent, 'ops');
|
|
assert.equal((await f.call('tester', 'hosts.revoke', args)).agent, 'ops');
|
|
assert.deepEqual(revokeBodies(f).filter((e) => e.action === 'grant.revoked').map((e) => e.by), ['assistant', 'tester']);
|
|
assert.deepEqual((await f.call('tester', 'hosts.sweep', { host: 'web1' })).removed, []);
|
|
});
|
|
|
|
// Row 281 (#1786, d6 section 8, T8): hosts.revoke shares its steps with a fire's step G now. G12 pins its
|
|
// records and their order; this pins the tester's dev-only refusal and that a verb revoke starts no job.
|
|
test('row 281 T8: a tester hosts.revoke refuses tester-dev-only once bus.tester is off, and a human revoke writes no job records', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
f.caps.tester = f.broker.bindTester({ business: 'demo' });
|
|
f.broker.reloadBusiness('demo', { ...f.business, vars: { 'bus.tester': 'yes' } });
|
|
const args = { agent: 'ops', host: 'web1', reason: 'r' };
|
|
await refusesWith(f.call('tester', 'hosts.revoke', args), 'tester-dev-only');
|
|
assert.equal(revokeBodies(f).length, 0);
|
|
assert.ok(f.record());
|
|
assert.equal((await f.call('human', 'hosts.revoke', args)).granted, true);
|
|
assert.deepEqual(revokeBodies(f).map((e) => [e.action, e.fire, e.delete]), [['grant.revoked', undefined, undefined], ['grant.revoke.closed', undefined, undefined]]);
|
|
const jobs = f.store.all("SELECT body FROM events WHERE business='demo' AND kind='action.allowed'").map((e) => JSON.parse(e.body).action)
|
|
.filter((a) => /^(fire|grants)\./.test(a));
|
|
assert.deepEqual(jobs, []);
|
|
});
|
|
|
|
test('G27: a restart after step 1 sweeps the key file and runs step 3 once the bus is up; status shows the revoke until then', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
const revoke = 'b6d0f0c4-6c3e-4c35-9d55-1f3c2b8f0a11';
|
|
// Step 1 as hosts.revoke writes it, then the bus dies.
|
|
f.store.transaction(() => {
|
|
f.store.run("DELETE FROM host_grants WHERE business='demo' AND agent='ops' AND host='web1'");
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', granted: true, reason: 'r', by: 'jason' }, 'web1');
|
|
});
|
|
assert.deepEqual((await f.call('human', 'hosts.status'))[0].revoking, [f.tag]);
|
|
const r = await restart(t, f);
|
|
assert.deepEqual(r.swept, [{ instance: 'demo/ops', service: 'host.web1' }]);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.deepEqual(r.now, []);
|
|
assert.deepEqual(r.later, [revoke]);
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.equal(r.state, undefined);
|
|
assert.deepEqual(revokeBodies(f).at(-1), { action: 'grant.revoke.closed', revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'removed', by: 'jason' });
|
|
assert.deepEqual((await r.broker.request(r.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }), { verb: 'hosts.status', args: {} }))[0].revoking, []);
|
|
const again = await restart(t, f);
|
|
assert.deepEqual([again.swept, again.later], [[], []]);
|
|
});
|
|
|
|
test('G27: after step 2 with the host down, the restart closes the marker with hostLineLeft; a marker without a revoke id is not reopened', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
const revoke = 'c1d2e3f4-0000-4000-8000-000000000001';
|
|
f.store.transaction(() => {
|
|
f.store.run("DELETE FROM host_grants WHERE business='demo' AND agent='ops' AND host='web1'");
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', revoke, agent: 'ops', host: 'web1', tag: f.tag, hostLine: 'pending', granted: true, reason: 'r', by: 'jason' }, 'web1');
|
|
});
|
|
rmSync(f.keyFile);
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.revoked', agent: 'pm', host: 'web1', tag: f.tag, hostLine: 'pending', by: 'jason' }, 'web1');
|
|
f.flag('fail-remove');
|
|
const r = await restart(t, f);
|
|
assert.deepEqual(r.swept, []);
|
|
assert.deepEqual(r.later, [revoke]);
|
|
assert.equal(revokeBodies(f).at(-1).hostLineLeft, f.tag);
|
|
assert.equal(f.granted().length, 1);
|
|
const human = r.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true });
|
|
assert.deepEqual((await r.broker.request(human, { verb: 'hosts.status', args: {} }))[0].hostLineLeft, [f.tag]);
|
|
// The sweep then removes it and clears the left line.
|
|
rmSync(join(f.dir, 'fail-remove'));
|
|
assert.deepEqual(await r.broker.request(human, { verb: 'hosts.sweep', args: { host: 'web1' } }), { host: 'web1', removed: [f.tag], hostLineLeft: [] });
|
|
assert.deepEqual(f.granted(), []);
|
|
assert.deepEqual((await r.broker.request(human, { verb: 'hosts.status', args: {} }))[0].hostLineLeft, []);
|
|
});
|
|
|
|
test('G27: a key file a revoke could not unlink is reported left and goes at the next start', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
const agentDir = join(f.storeRoot, 'demo', 'ops');
|
|
// setup's t.after removes the scratch dir before any hook added here runs, so the body restores the mode.
|
|
chmodSync(agentDir, 0o500);
|
|
try {
|
|
const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' });
|
|
assert.equal(r.keyFile, 'left');
|
|
assert.equal(r.hostLine, 'removed');
|
|
assert.equal(existsSync(f.keyFile), true);
|
|
assert.notEqual(f.state(), 'valid');
|
|
} finally {
|
|
chmodSync(agentDir, 0o700);
|
|
}
|
|
const again = await restart(t, f);
|
|
assert.deepEqual(again.swept, [{ instance: 'demo/ops', service: 'host.web1' }]);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.deepEqual(again.later, []);
|
|
});
|
|
|
|
test('G28: a commands change, then a revoke, then a restart leave no record, key, reference or line', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
await f.approve({ commands: [['uptime']] });
|
|
assert.equal(f.record().commands, '[["uptime"]]');
|
|
await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' });
|
|
const r = await restart(t, f);
|
|
assert.deepEqual([r.swept, r.now, r.later], [[], [], []]);
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.equal(r.state, undefined);
|
|
assert.deepEqual(f.granted(), []);
|
|
});
|
|
|
|
test('G24 (B4): hosts.sweep removes only this business\'s recorded tags no live grant holds, and records what it left', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
const live = f.granted();
|
|
const pmTag = grantTag('demo', 'pm', 'web1');
|
|
const otherTag = grantTag('other', 'ops', 'web1');
|
|
const unrecorded = 'mosaic-' + 'f'.repeat(32);
|
|
keygen(join(f.dir, 'k2'));
|
|
const line = (tag) => `restrict ${pubLine(join(f.dir, 'k2'))} ${tag}`;
|
|
writeFileSync(join(f.dir, 'granted'), [...live, line(pmTag), line(otherTag), line(unrecorded), ''].join('\n'));
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.step', grantDecision: 'x', agent: 'pm', host: 'web1', user: USER, tag: pmTag, step: 'install' }, 'web1');
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.failed', grantDecision: 'x', agent: 'pm', host: 'web1', code: 'host-failed', tag: pmTag, hostLineLeft: pmTag }, 'web1');
|
|
// A left tag no longer on the host drops out at the sweep.
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.failed', grantDecision: 'y', agent: 'pm', host: 'web1', code: 'host-failed', tag: 'mosaic-' + '1'.repeat(32), hostLineLeft: 'mosaic-' + '1'.repeat(32) }, 'web1');
|
|
assert.equal((await f.call('human', 'hosts.status'))[0].hostLineLeft.length, 2);
|
|
f.flag('fail-list');
|
|
await refusesWith(f.call('human', 'hosts.sweep', { host: 'web1' }), 'host-failed');
|
|
assert.equal(f.grantEvents().some((e) => e.action === 'grant.host.swept'), false);
|
|
rmSync(join(f.dir, 'fail-list'));
|
|
await refusesWith(f.call('human', 'hosts.sweep', { host: 'web9' }), 'host-unknown');
|
|
const asked = f.requests().length;
|
|
assert.deepEqual(await f.call('human', 'hosts.sweep', { host: 'web1' }), { host: 'web1', removed: [pmTag], hostLineLeft: [] });
|
|
assert.deepEqual(f.requests().slice(asked), [`list ${USER}`, `remove ${USER} ${pmTag}`]);
|
|
assert.deepEqual(f.granted(), [...live, line(otherTag), line(unrecorded)]);
|
|
assert.deepEqual(f.grantEvents().at(-1), { action: 'grant.host.swept', host: 'web1', removed: [pmTag], left: [], by: 'jason' });
|
|
assert.deepEqual((await f.call('human', 'hosts.status'))[0].hostLineLeft, []);
|
|
// A later failure adds to the cleared list.
|
|
writeGrantEvent(f.store, AT, 'demo', { action: 'grant.failed', grantDecision: 'z', agent: 'pm', host: 'web1', code: 'host-failed', tag: pmTag, hostLineLeft: pmTag }, 'web1');
|
|
assert.deepEqual((await f.call('human', 'hosts.status'))[0].hostLineLeft, [pmTag]);
|
|
assert.equal(f.state(), 'valid');
|
|
});
|
|
|
|
test('G12: the local steps finish before the host step, so a host that hangs leaves the agent no access', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
f.flag('hold-remove');
|
|
const revoking = f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' });
|
|
await until(() => existsSync(join(f.dir, 'holding-remove')));
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.notEqual(f.state(), 'valid');
|
|
assert.deepEqual((await f.call('human', 'hosts.status'))[0].revoking, [f.tag]);
|
|
f.flag('release');
|
|
assert.equal((await revoking).hostLine, 'removed');
|
|
assert.deepEqual(f.granted(), []);
|
|
});
|
|
|
|
// X5: the executor claims the slot in the same tick as its spend, so a revoke right after the start
|
|
// call returns finds the claim and aborts it.
|
|
test('G23 (X5): a revoke right after the spend reaches the executor\'s claim, so the grant ends revoked', async (t) => {
|
|
const f = await setup(t);
|
|
const id = f.fixtureGrant();
|
|
const run = f.broker.runHostGrant('demo', id);
|
|
const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' });
|
|
assert.equal(r.granted, false);
|
|
assert.equal(await run, 'revoked');
|
|
assert.equal(f.ended(id).code, 'revoked');
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.deepEqual(f.requests(), [`remove ${USER} ${f.tag}`]);
|
|
});
|
|
|
|
// X5: the abort check after verify. The revoke lands once the login has worked and before the commit,
|
|
// so only that check stops the record and key file.
|
|
test('G23 (X5): a revoke that lands as verify returns ends revoked with no record, key file or line', async (t) => {
|
|
let revoking;
|
|
const f = await setup(t, {
|
|
wrap: (hg) => ({
|
|
...hg,
|
|
verify: async (...args) => {
|
|
const out = await hg.verify(...args);
|
|
revoking = f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' });
|
|
return out;
|
|
},
|
|
}),
|
|
});
|
|
const { id, end } = await f.approve();
|
|
assert.equal(end.code, 'revoked');
|
|
assert.equal((await revoking).granted, false);
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.equal(f.state(), undefined);
|
|
assert.deepEqual(f.granted(), []);
|
|
// The check stops the grant before step 5, so no commit step is logged.
|
|
assert.deepEqual(f.grantEvents().filter((e) => e.grantDecision === id && e.action === 'grant.step').map((e) => e.step), ['install']);
|
|
});
|
|
|
|
test('G13: a revoke whose host remove fails still drops the record and key, and reports the line left for a sweep', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approve();
|
|
f.flag('fail-remove');
|
|
const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'r' });
|
|
assert.deepEqual(r, { agent: 'ops', host: 'web1', tag: f.tag, granted: true, keyFile: 'removed', hostLineLeft: f.tag });
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.notEqual(f.state(), 'valid');
|
|
assert.equal(f.granted().length, 1);
|
|
assert.equal(revokeBodies(f).at(-1).hostLineLeft, f.tag);
|
|
const [row] = await f.call('human', 'hosts.status');
|
|
assert.deepEqual([row.revoking, row.hostLineLeft], [[], [f.tag]]);
|
|
// The marker is closed, so a restart doesn't run step 3 again.
|
|
const again = await restart(t, f);
|
|
assert.deepEqual(again.later, []);
|
|
});
|
|
|
|
// G22 (B1) on one bus with no restart. Enrollment refuses this machine's address, so the chief enrolls
|
|
// 192.0.2.10 through the fake keyscan. The test then points the record at the scratch sshd and lets the
|
|
// enrolled installer key log in there. Everything after that is the real path.
|
|
test('G22: enroll, check, grant, host.run, a commands change the next host.run uses, and a revoke that kills a run in flight', async (t) => {
|
|
const f = await setup(t, { enroll: true });
|
|
const enroll = await f.call('chief', 'decision.raise', {
|
|
action: 'credential.mint', question: 'Enroll web1?', options, recommendation: 'yes', blocking: false,
|
|
grant: { method: 'host-enroll', host: 'web1', address: '192.0.2.10', port: 22, user: USER, sudo: true, fingerprints: [fingerprint(pubLine(join(f.dir, 'hostkey')))] },
|
|
});
|
|
f.call('human', 'decision.resolve', { id: enroll.id, choice: 'yes' });
|
|
const enrolled = () => f.grantEvents().find((e) => e.grantDecision === enroll.id && ['grant.host.enrolled', 'grant.failed'].includes(e.action));
|
|
await until(enrolled);
|
|
assert.equal(enrolled().action, 'grant.host.enrolled');
|
|
const installer = parseHostKey(readFileSync(join(f.configDir, 'secrets', 'hosts', 'demo', 'web1', 'installer'), 'utf8'));
|
|
appendFileSync(join(f.dir, 'authorized_keys'), installer.publicLine + '\n');
|
|
f.store.run("UPDATE hosts SET address='127.0.0.1', port=? WHERE business='demo' AND host='web1'", f.port);
|
|
await refusesWith(f.raise(), 'host-not-ready');
|
|
assert.equal((await f.call('chief', 'hosts.check', { host: 'web1' })).state, 'ready');
|
|
assert.equal((await f.approve({ commands: [['echo']] })).end.action, 'grant.applied');
|
|
f.caps.ops = f.broker.bindLaunch({ business: 'demo', role: 'ops', run: 'ops-run', harness: 'pi', address: 'ops' });
|
|
f.broker.request(f.caps.ops, { verb: 'role.claim' });
|
|
const executor = hostExecutor({ credentials: f.credentials, root: join(f.dir, 'host-run') });
|
|
const run = async (argv) => {
|
|
const d = await f.call('ops', 'decision.raise', { action: 'host.sudo', target: 'web1', argv, question: 'Run it?', options, recommendation: 'yes', blocking: false });
|
|
f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
return f.broker.requestHost(f.caps.ops, { verb: 'host.run', args: { host: 'web1', argv, decision: d.id } }, executor);
|
|
};
|
|
const out = await run(['echo', 'hi']);
|
|
assert.deepEqual([out.exit, out.stdout], [0, 'hi\n']);
|
|
await refusesWith(run(['sleep', '30']), 'command-not-granted');
|
|
assert.equal((await f.approve({ commands: [['echo'], ['sleep']] })).end.action, 'grant.applied');
|
|
// The handler goes on at creation: the revoke rejects the run while the test still awaits the revoke.
|
|
const slow = refusesWith(run(['sleep', '30']), 'credential-unavailable');
|
|
await until(() => readOr(join(f.dir, 'sudo.log'), '').includes('sleep 30'));
|
|
const r = await f.call('human', 'hosts.revoke', { agent: 'ops', host: 'web1', reason: 'done' });
|
|
assert.deepEqual([r.granted, r.keyFile, r.hostLine], [true, 'removed', 'removed']);
|
|
await slow;
|
|
// The record goes at step 1, so the next call refuses at the record check, before the key (B1 names
|
|
// credential-unavailable; BUILD.md records the difference).
|
|
await refusesWith(f.broker.requestHost(f.caps.ops, { verb: 'host.run', args: { host: 'web1', argv: ['echo', 'hi'] } }, executor), 'host-not-granted');
|
|
assert.equal(f.record(), undefined);
|
|
assert.equal(existsSync(f.keyFile), false);
|
|
assert.deepEqual(f.granted(), []);
|
|
});
|
|
|
|
// The decision goes in after the restart, so the reconcile doesn't end it first.
|
|
test('mos-1163: a grant whose agent became a creative role after the raise ends host-creative-role at step 1, spent, with nothing remote', async (t) => {
|
|
const f = await setup(t);
|
|
const r = await restart(t, f, creativeOps(f.business));
|
|
const id = f.fixtureGrant();
|
|
assert.equal(await r.broker.runHostGrant('demo', id), 'host-creative-role');
|
|
assert.equal(f.ended(id).code, 'host-creative-role');
|
|
assert.equal(await r.broker.runHostGrant('demo', id), 'decision-consumed');
|
|
assert.deepEqual(f.requests(), []);
|
|
assert.equal(f.record(), undefined);
|
|
});
|
|
|
|
test('sage-1908: once the agent of an applied grant is a creative role, host.run refuses host-creative-role, runs nothing remote and keeps the record', async (t) => {
|
|
const f = await setup(t);
|
|
assert.equal((await f.approve()).end.action, 'grant.applied');
|
|
const r = await restart(t, f, creativeOps(f.business));
|
|
const human = r.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true });
|
|
const project = r.broker.request(human, { verb: 'project.create', args: { name: 'host grant check' } }).id;
|
|
const thread = r.broker.request(human, { verb: 'thread.create', args: { project, name: 'ops', instance: 'ops' } }).id;
|
|
const cap = r.broker.bindLaunch({ business: 'demo', role: 'ops', run: 'ops-run', harness: 'pi', address: 'ops', thread });
|
|
r.broker.request(cap, { verb: 'role.claim' });
|
|
const executor = hostExecutor({ credentials: r.credentials, root: join(f.dir, 'host-run') });
|
|
const before = f.requests().length;
|
|
await refusesWith(r.broker.requestHost(cap, { verb: 'host.run', args: { host: 'web1', argv: ['systemctl', 'status'] } }, executor), 'host-creative-role');
|
|
assert.equal(f.requests().length, before);
|
|
assert.equal(readOr(join(f.dir, 'sudo.log'), '').includes('systemctl'), false);
|
|
assert.notEqual(f.record(), undefined);
|
|
});
|