1022 lines
52 KiB
JavaScript
1022 lines
52 KiB
JavaScript
// Row 281 (#1786, design d6 section 8): a fire or a delete revokes the instance's host and Gitea grants
|
|
// (step G), as a job after the verb answers. T2-T13. The broker runs bare, as chief-of-staff.test.mjs
|
|
// runs it, with a fake organization writer, a fake installer in place of the host's remote steps, and
|
|
// the fake Gitea from fake-gitea-api.mjs on 127.0.0.1. Every token is a fake the fake made; no test
|
|
// reaches git.mosaicstack.dev or any provider, and none creates a real account or token (Q188). Host
|
|
// keys are real local ssh-keygen pairs in the scratch store, and no ssh connection is made. Decisions a
|
|
// test needs outside the human's yes go in by SQL as fixtures.
|
|
import test from 'node:test';
|
|
import {autonomyHost} from '../src/autonomy-host.mjs';
|
|
import { hireServiceFixture, fixtureConsole, approveFixtureFire } from './hire-fixtures.mjs';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { Broker, GRANT_EVIDENCE, registrySha256 } from '../src/broker.mjs';
|
|
import { Credentials } from '../src/credentials.mjs';
|
|
import { adminSha256, definitionSha256 as giteaDefinitionSha256, giteaGrant, giteaKeys, identitySha256 } from '../src/gitea-grant.mjs';
|
|
import { definitionSha256 as hostDefinitionSha256, grantKeys, grantTag, hostGrant } from '../src/host-grant.mjs';
|
|
import { Store } from '../src/store.mjs';
|
|
import { fakeGiteaApi } from './fake-gitea-api.mjs';
|
|
import { grantDecision, hostRecord } from './host-fixtures.mjs';
|
|
|
|
const AT = '2026-10-22T00:00:00.000Z';
|
|
const REV = 'b'.repeat(64);
|
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
const dayFrom = (n) => new Date(Date.now() + n * 86400000).toISOString().slice(0, 10);
|
|
const sha256 = (s) => createHash('sha256').update(s).digest('hex');
|
|
const options = [{ key: 'yes', text: 'Yes' }, { key: 'no', text: 'No' }];
|
|
const none = { authority: { withinRole: ['message.send'], crossRole: [] } };
|
|
const chiefAuthority = JSON.parse(readFileSync(fileURLToPath(new URL('../../../roles/chief-of-staff.json', import.meta.url)), 'utf8')).authority;
|
|
const ENTRY = { address: '127.0.0.1', port: 2222, user: 'deploy', hostKeys: ['ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl'] };
|
|
const HOSTS = ['web1', 'web2', 'web3'];
|
|
const GENERATION = '0b6e7f6e-4c1d-4a8e-9a51-6d2f3c4b5a69';
|
|
async function until(fn, ms = 10000) {
|
|
const end = Date.now() + ms;
|
|
while (!fn()) {
|
|
if (Date.now() > end) throw new Error('timed out');
|
|
await sleep(10);
|
|
}
|
|
}
|
|
const rejects = (fn, code) => assert.rejects(async () => fn(), (e) => (assert.equal(e.code, code), true));
|
|
const code = (fn) => {
|
|
try {
|
|
fn();
|
|
} catch (e) {
|
|
return e.code;
|
|
}
|
|
return null;
|
|
};
|
|
|
|
function makeBusiness(url) {
|
|
const gitea = (name, user) => ({ name, email: `${user}@example.test`, gitea: { url, user } });
|
|
return {
|
|
id: 'demo',
|
|
human: 'jason',
|
|
arbiters: { technical: 'pm', delivery: 'pm' },
|
|
roles: {
|
|
chief: { definition: 'chief-of-staff', authority: chiefAuthority },
|
|
pm: { definition: 'pm', ...none },
|
|
coder: { definition: 'coder', ...none, identity: gitea('Coder', 'demo-coder'), publish: ['app'] },
|
|
helper: { definition: 'coder', ...none, identity: gitea('Helper', 'demo-helper'), publish: ['app'] },
|
|
ops: { definition: 'ops', ...none },
|
|
},
|
|
definitions: {
|
|
'chief-of-staff': { roleVersion: 2, credentials: [] },
|
|
pm: { roleVersion: 2, credentials: [] },
|
|
coder: { roleVersion: 2, credentials: [{ service: 'gitea' }, { service: 'host' }] },
|
|
ops: { roleVersion: 2, credentials: [{ service: 'host' }] },
|
|
},
|
|
projects: { app: { remote: `${url}/acme/app` } },
|
|
launch: { by: 'chief', instances: ['pm', 'coder', 'helper', 'ops'] },
|
|
vars: { 'bus.tester': true },
|
|
};
|
|
}
|
|
|
|
async function setup(t,autonomy=null) {
|
|
const dir = mkdtempSync(join(tmpdir(), 'g281-'));
|
|
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
const gitea = await fakeGiteaApi(t, join(dir, 'tls'));
|
|
gitea.addUser('acme');
|
|
gitea.addRepo('acme', 'app');
|
|
const configDir = join(dir, 'cfg');
|
|
mkdirSync(configDir, { mode: 0o700 });
|
|
const storeRoot = join(configDir, 'secrets', 'agents');
|
|
const runRoot = join(configDir, 'secrets', 'run');
|
|
mkdirSync(join(dir, 'data'), { mode: 0o700 });
|
|
const store = new Store(join(dir, 'data'));
|
|
const credentials = new Credentials({ references: {}, storeRoot, dataRoot: join(dir, 'data') });
|
|
t.after(() => {
|
|
// A test that fails while a host step is held must not leave pause() looping after it ends.
|
|
held.clear();
|
|
failing.clear();
|
|
credentials.close();
|
|
store.close();
|
|
});
|
|
const doc = makeBusiness(gitea.url);
|
|
const grantApi = giteaGrant({ configDir, ca: gitea.ca, timeoutMs: 5000 });
|
|
const meta = { url: gitea.url, account: 'mosaic-grant', rotateBy: dayFrom(60), setAt: AT };
|
|
grantApi.writeAdmin('demo', gitea.adminToken, meta);
|
|
const gKeys = giteaKeys({ credentials, storeRoot });
|
|
const hKeys = grantKeys({ credentials, storeRoot });
|
|
// The fake installer: each host's lines as a set of tags. `held` keeps a host's add or remove from
|
|
// answering until the test deletes the key; `failing` makes every request to a host fail.
|
|
const lines = Object.fromEntries(HOSTS.map((h) => [h, new Set()]));
|
|
const sent = [];
|
|
const held = new Set();
|
|
const failing = new Set();
|
|
const pause = async (key, signal) => {
|
|
while (held.has(key)) {
|
|
if (signal?.aborted) throw new Error('aborted');
|
|
await sleep(5);
|
|
}
|
|
if (signal?.aborted) throw new Error('aborted');
|
|
};
|
|
const installer = {
|
|
...hostGrant({ configDir, runRoot }),
|
|
async install(business, host, record, tag, publicLine, signal) {
|
|
sent.push(`add ${host} ${tag}`);
|
|
await pause(host + ':add', signal);
|
|
if (failing.has(host)) throw new Error('host-failed');
|
|
lines[host].add(tag);
|
|
},
|
|
async verify(host, record, key, signal) {
|
|
if (signal?.aborted) throw new Error('aborted');
|
|
if (failing.has(host)) throw new Error('host-failed');
|
|
},
|
|
async remove(business, host, record, tag) {
|
|
sent.push(`remove ${host} ${tag}`);
|
|
await pause(host + ':remove');
|
|
if (failing.has(host)) throw new Error('host-failed');
|
|
lines[host].delete(tag);
|
|
},
|
|
};
|
|
// The fake organization writer. A fire delists the name and a delete drops the role, each with a
|
|
// reload, as the runtime reloads after a commit. `fireError` makes the fire's write refuse, and
|
|
// `deleteError` the delete's (row 239).
|
|
const org = { log: [], fireError: null, deleteError: null };
|
|
const sw = { state: { version: 1, halted: false, generation: GENERATION } };
|
|
let broker;
|
|
const organization = (business, op, args) => {
|
|
org.log.push([op, args?.name ?? args?.instance ?? null]);
|
|
if (op === 'instance.fire') {
|
|
if (org.fireError) throw org.fireError;
|
|
doc.launch.instances = doc.launch.instances.filter((n) => n !== args.name);
|
|
broker.reloadBusiness('demo', doc);
|
|
return { revision: REV, transaction: 't-' + org.log.length };
|
|
}
|
|
if (op === 'instances.set') {
|
|
if (org.deleteError) throw org.deleteError;
|
|
delete doc.roles[args.instance];
|
|
doc.launch.instances = doc.launch.instances.filter((n) => n !== args.instance);
|
|
broker.reloadBusiness('demo', doc);
|
|
return { revision: REV, transaction: 't-' + org.log.length };
|
|
}
|
|
return { revision: REV };
|
|
};
|
|
const make = () =>
|
|
new Broker({hires:hireServiceFixture,autonomy,
|
|
store,
|
|
businesses: { demo: structuredClone(doc) },
|
|
secretCheck: (value) => credentials.assertClean(value),
|
|
organization,
|
|
processing: { read: () => sw.state },
|
|
hostGrant: installer,
|
|
hostKeys: hKeys,
|
|
giteaGrant: grantApi,
|
|
giteaKeys: gKeys,
|
|
});
|
|
broker = make();
|
|
for (const host of HOSTS) hostRecord(store, { host, entry: ENTRY });
|
|
const caps = {};
|
|
for (const role of ['chief', 'pm']) {
|
|
caps[role] = broker.bindLaunch({ business: 'demo', role, run: role + '-run', harness: 'pi', address: role });
|
|
broker.request(caps[role], { verb: 'role.claim' });
|
|
}
|
|
caps.human = broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true });
|
|
caps.tester = broker.bindTester({ business: 'demo' });
|
|
const call = (who, verb, args = {}) => broker.request(caps[who], { verb, args });
|
|
const grantEvents = () =>
|
|
store.all(`SELECT body FROM events WHERE business='demo' AND ${GRANT_EVIDENCE} ORDER BY seq`).map((e) => JSON.parse(e.body));
|
|
const ended = (decision) => grantEvents().findLast((e) => e.grantDecision === decision && ['grant.applied', 'grant.failed'].includes(e.action));
|
|
const allowed = (action) =>
|
|
store
|
|
.all("SELECT actor_role, subject, body FROM events WHERE business='demo' AND kind='action.allowed' ORDER BY seq")
|
|
.map((e) => ({ ...e, body: JSON.parse(e.body) }))
|
|
.filter((e) => e.body.action === action && e.body.operation !== 'decision.raise');
|
|
// The bus's step G messages to the human: no decision, so no grant notice.
|
|
const inbox = () =>
|
|
store.all("SELECT body FROM messages WHERE business='demo' AND from_role='_bus' AND to_role='human' AND decision IS NULL ORDER BY rowid").map((r) => r.body);
|
|
const raiseHost = (agent, host, commands = [['uptime']]) =>
|
|
call('chief', 'decision.raise', {
|
|
action: 'credential.mint', question: `Grant ${agent} ${host}?`, options, recommendation: 'yes', blocking: false,
|
|
grant: { method: 'host', agent, host, commands, rotateBy: dayFrom(30) },
|
|
});
|
|
const raiseGitea = (agent) =>
|
|
call('chief', 'decision.raise', {
|
|
action: 'credential.mint', question: `Grant ${agent} Gitea?`, options, recommendation: 'yes', blocking: false,
|
|
grant: { method: 'gitea', agent, target: { projects: ['app'] }, rotateBy: dayFrom(30) },
|
|
});
|
|
const approve = async (d) => {
|
|
call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
await until(() => ended(d.id));
|
|
await broker.grantsIdle();
|
|
return ended(d.id);
|
|
};
|
|
const approveHost = async (agent, host) => {
|
|
const end = await approve(await raiseHost(agent, host));
|
|
assert.equal(end.action, 'grant.applied', `${agent} on ${host}`);
|
|
return end;
|
|
};
|
|
const approveGitea = async (agent) => {
|
|
const end = await approve(await raiseGitea(agent));
|
|
assert.equal(end.action, 'grant.applied', `${agent} gitea`);
|
|
return end;
|
|
};
|
|
// A host decision approved and never run, with R4's body built here (T6).
|
|
const fixtureHost = (agent, host, commands = [['uptime']]) =>
|
|
grantDecision(store, {
|
|
grant: {
|
|
method: 'host', agent, host, commands, commandsSha256: sha256(JSON.stringify(commands)), rotateBy: dayFrom(30),
|
|
registrySha256: registrySha256({ ...ENTRY, sudo: true }), definitionSha256: hostDefinitionSha256(doc.definitions[doc.roles[agent].definition]),
|
|
},
|
|
});
|
|
// A Gitea decision approved and never run (T6b).
|
|
const fixtureGitea = (agent) =>
|
|
grantDecision(store, {
|
|
grant: {
|
|
method: 'gitea', agent, target: { projects: ['app'] }, rotateBy: dayFrom(30),
|
|
definitionSha256: giteaDefinitionSha256(doc.definitions[doc.roles[agent].definition]),
|
|
identitySha256: identitySha256(doc.roles[agent], doc.projects),
|
|
adminSha256: adminSha256(grantApi.adminMeta('demo') ?? meta),
|
|
},
|
|
});
|
|
const hireConsole=fixtureConsole(broker);
|
|
const fire = name => approveFixtureFire(broker,hireConsole,name);
|
|
// A delete as the launcher's pass runs it: the human's request, the take, then lifecycleDelete. A
|
|
// refused delete closes the request with its code, as the launcher does.
|
|
const remove = (instance) => {
|
|
const { request } = call('human', 'instances.set', { revision: REV, instance, op: 'delete' });
|
|
assert.equal(broker.lifecycleTake({ business: 'demo', request }).go, true);
|
|
try {
|
|
return { request, ...broker.lifecycleDelete({ business: 'demo', request }) };
|
|
} catch (e) {
|
|
if (typeof e.code !== 'string') throw e;
|
|
broker.lifecycleDone({ business: 'demo', request, refusal: e.code });
|
|
return { request, code: e.code };
|
|
}
|
|
};
|
|
const tag = (agent, host) => grantTag('demo', agent, host);
|
|
const keyFile = (agent, host) => join(storeRoot, 'demo', agent, 'host.' + host);
|
|
const tokenFile = (agent) => join(storeRoot, 'demo', agent, 'gitea');
|
|
const hostRow = (agent, host) => store.get("SELECT * FROM host_grants WHERE business='demo' AND agent=? AND host=?", agent, host);
|
|
const giteaRow = (agent) => store.get("SELECT * FROM gitea_grants WHERE business='demo' AND agent=?", agent);
|
|
const state = (agent, service) => credentials.status().find((e) => e.instance === 'demo/' + agent && e.service === service)?.state;
|
|
const userTokens = (login) => gitea.tokens(login) ?? [];
|
|
const revoked = (agent, where = () => true) => grantEvents().filter((e) => e.action === 'grant.revoked' && e.agent === agent && where(e));
|
|
const closed = (revoke) => grantEvents().filter((e) => e.action === 'grant.revoke.closed' && e.revoke === revoke);
|
|
const noLeak = () => {
|
|
const text = [
|
|
...store.all("SELECT body FROM events WHERE business='demo'").map((r) => r.body),
|
|
...store.all("SELECT body FROM messages WHERE business='demo'").map((r) => r.body),
|
|
JSON.stringify(gitea.requests),
|
|
].join('\n');
|
|
assert.ok(gitea.issued.every((tok) => !text.includes(tok)), 'a token value reached the store or the request log');
|
|
};
|
|
return {
|
|
dir, gitea, store, credentials, doc, org, sw, lines, sent, held, failing, caps, call, make, grantEvents, ended, allowed, inbox,
|
|
raiseHost, raiseGitea, approve, approveHost, approveGitea, fixtureHost, fixtureGitea, fire, remove, tag, keyFile, tokenFile,
|
|
hostRow, giteaRow, state, userTokens, revoked, closed, noLeak,
|
|
get broker() {
|
|
return broker;
|
|
},
|
|
};
|
|
}
|
|
|
|
// Coder's three grants: web1, web2 and Gitea, each through the human's yes and its executor.
|
|
async function threeGrants(f) {
|
|
await f.approveHost('coder', 'web1');
|
|
await f.approveHost('coder', 'web2');
|
|
await f.approveGitea('coder');
|
|
for (const h of ['web1', 'web2']) {
|
|
assert.ok(f.lines[h].has(f.tag('coder', h)));
|
|
assert.ok(existsSync(f.keyFile('coder', h)));
|
|
}
|
|
assert.equal(f.userTokens('demo-coder').length, 1);
|
|
assert.ok(existsSync(f.tokenFile('coder')));
|
|
}
|
|
|
|
// Every grant of coder is gone: records, key and token files, host lines, Gitea tokens, login.
|
|
function allRevoked(f, hosts = ['web1', 'web2']) {
|
|
for (const h of hosts) {
|
|
assert.equal(f.hostRow('coder', h), undefined, h);
|
|
assert.equal(existsSync(f.keyFile('coder', h)), false, h);
|
|
assert.equal(f.lines[h].has(f.tag('coder', h)), false, h);
|
|
assert.notEqual(f.state('coder', 'host.' + h), 'valid', h);
|
|
}
|
|
assert.equal(f.giteaRow('coder'), undefined);
|
|
assert.equal(existsSync(f.tokenFile('coder')), false);
|
|
assert.notEqual(f.state('coder', 'gitea'), 'valid');
|
|
assert.deepEqual(f.userTokens('demo-coder'), []);
|
|
assert.equal(f.gitea.users.get('demo-coder').prohibit_login, true);
|
|
}
|
|
|
|
test('row 281 T2: a fire with host grants on two hosts and a Gitea grant answers with a job, then revokes all three', async (t) => {
|
|
const f = await setup(t);
|
|
await threeGrants(f);
|
|
const out = await f.fire('coder');
|
|
assert.equal(out.done, false);
|
|
assert.match(out.job, UUID);
|
|
assert.deepEqual(out.revoking, ['host:web1', 'host:web2', 'gitea']);
|
|
assert.ok(!('transaction' in out));
|
|
const [revoking, ...more] = f.allowed('fire.revoking');
|
|
assert.equal(more.length, 0);
|
|
assert.equal(revoking.body.job, out.job);
|
|
assert.deepEqual(revoking.body.grants, out.revoking);
|
|
assert.equal(revoking.actor_role, null);
|
|
assert.equal(revoking.subject, 'coder');
|
|
await f.broker.grantsIdle();
|
|
// Each host as hosts.revoke records it, with fire: true, line first so the close says removed.
|
|
for (const h of ['web1', 'web2']) {
|
|
const [r, ...rest] = f.revoked('coder', (e) => e.host === h);
|
|
assert.equal(rest.length, 0, h);
|
|
assert.equal(r.fire, true);
|
|
assert.equal(r.granted, true);
|
|
assert.equal(r.hostLine, 'pending');
|
|
assert.equal(r.tag, f.tag('coder', h));
|
|
const [c] = f.closed(r.revoke);
|
|
assert.equal(c.hostLine, 'removed');
|
|
assert.equal(c.fire, true);
|
|
}
|
|
const [g, ...rest] = f.revoked('coder', (e) => e.method === 'gitea');
|
|
assert.equal(rest.length, 0);
|
|
assert.equal(g.user, 'demo-coder');
|
|
assert.equal(g.granted, true);
|
|
assert.deepEqual(f.closed(g.revoke).map((c) => c.serviceLeft), [[]]);
|
|
allRevoked(f);
|
|
const [done, ...again] = f.allowed('fire.done');
|
|
assert.equal(again.length, 0);
|
|
assert.equal(done.body.job, out.job);
|
|
assert.deepEqual(done.body.grants.map((x) => x.grant), ['host:web1', 'host:web2', 'gitea']);
|
|
assert.deepEqual(done.body.grantsLeft, []);
|
|
assert.match(done.body.transaction, /^t-/);
|
|
assert.equal(done.actor_role, null);
|
|
assert.deepEqual(f.org.log.filter(([op]) => op === 'instance.fire'), [['instance.fire', 'coder']]);
|
|
assert.ok(!f.doc.launch.instances.includes('coder'));
|
|
// No step G message for a job that completed.
|
|
assert.deepEqual(f.inbox(), []);
|
|
f.noLeak();
|
|
});
|
|
|
|
test('row 281 T3 (sage-2575): a delete with grants takes two requests: the first starts the job, the next completes', async (t) => {
|
|
const f = await setup(t);
|
|
await threeGrants(f);
|
|
f.held.add('web1:remove');
|
|
const first = f.remove('coder');
|
|
assert.equal(first.code, 'grants-revoking');
|
|
const [start, ...more] = f.allowed('grants.revoking');
|
|
assert.equal(more.length, 0);
|
|
assert.match(start.body.job, UUID);
|
|
assert.deepEqual(start.body.grants, ['host:web1', 'host:web2', 'gitea']);
|
|
assert.equal(start.body.verb, 'lifecycle.delete');
|
|
assert.equal(start.body.request, first.request);
|
|
assert.equal(start.actor_role, null);
|
|
assert.equal(start.subject, 'coder');
|
|
const job = start.body.job;
|
|
const [message] = f.inbox();
|
|
assert.equal(f.inbox().length, 1);
|
|
for (const s of [job, 'host:web1', 'host:web2', 'gitea', 'Raise the delete again']) assert.ok(message.includes(s), s);
|
|
// Held: a second delete refuses grants-revoking, with no second job and no second message.
|
|
await until(() => f.sent.includes(`remove web1 ${f.tag('coder', 'web1')}`));
|
|
assert.equal(f.remove('coder').code, 'grants-revoking');
|
|
assert.equal(f.allowed('grants.revoking').length, 1);
|
|
assert.equal(f.inbox().length, 1);
|
|
f.held.delete('web1:remove');
|
|
await f.broker.grantsIdle();
|
|
for (const h of ['web1', 'web2']) assert.equal(f.revoked('coder', (e) => e.host === h && e.delete === true).length, 1, h);
|
|
allRevoked(f);
|
|
const [done] = f.allowed('grants.revoked.done');
|
|
assert.equal(done.body.job, job);
|
|
assert.deepEqual(done.body.grants.map((x) => x.grant), ['host:web1', 'host:web2', 'gitea']);
|
|
const end = f.inbox()[1];
|
|
for (const s of [job, 'host:web1', 'host:web2', 'gitea', 'Raise the delete again']) assert.ok(end.includes(s), s);
|
|
assert.equal(f.org.log.filter(([op]) => op === 'instances.set').length, 0);
|
|
// The third request completes as today.
|
|
const third = f.remove('coder');
|
|
assert.equal(third.code, undefined);
|
|
assert.match(third.transaction, /^t-/);
|
|
assert.equal(third.retry, false);
|
|
assert.deepEqual(f.org.log.filter(([op]) => op === 'instances.set'), [['instances.set', 'coder']]);
|
|
assert.equal(f.allowed('grants.revoking').length, 1);
|
|
f.noLeak();
|
|
});
|
|
|
|
test('row 281 T3b (Q2): a delete whose job refuses on host B; the retry refuses grant-revoke-failed and revokes only B and Gitea', async (t) => {
|
|
const f = await setup(t);
|
|
await threeGrants(f);
|
|
f.failing.add('web2');
|
|
assert.equal(f.remove('coder').code, 'grants-revoking');
|
|
await f.broker.grantsIdle();
|
|
const job = f.allowed('grants.revoking')[0].body.job;
|
|
const [refused, ...more] = f.grantEvents().filter((e) => e.action === 'grant.revoke.refused');
|
|
assert.equal(more.length, 0);
|
|
assert.deepEqual(
|
|
{ job: refused.job, agent: refused.agent, grant: refused.grant, reason: refused.reason, code: refused.code, verb: refused.verb },
|
|
{ job, agent: 'coder', grant: 'host:web2', reason: 'host-remove-failed', code: 'grant-revoke-failed', verb: 'lifecycle.delete' },
|
|
);
|
|
const end = f.inbox()[1];
|
|
for (const s of [job, 'coder', 'web2', 'host-remove-failed', 'mosaic hosts revoke coder web2']) assert.ok(end.includes(s), s);
|
|
// Nothing deleted: the writer saw no delete and the role is still there. Host A is revoked, B isn't.
|
|
assert.equal(f.org.log.filter(([op]) => op === 'instances.set').length, 0);
|
|
assert.ok(f.doc.roles.coder);
|
|
assert.equal(f.hostRow('coder', 'web1'), undefined);
|
|
assert.ok(f.hostRow('coder', 'web2'));
|
|
assert.ok(existsSync(f.keyFile('coder', 'web2')));
|
|
assert.ok(f.giteaRow('coder'));
|
|
assert.equal(f.allowed('grants.revoked.done').length, 0);
|
|
// B back: the next delete refuses grant-revoke-failed, names the earlier reason and the new job.
|
|
f.failing.delete('web2');
|
|
assert.equal(f.remove('coder').code, 'grant-revoke-failed');
|
|
const retry = f.allowed('grants.revoking')[1].body;
|
|
assert.deepEqual(retry.grants, ['host:web2', 'gitea']);
|
|
const message = f.inbox()[2];
|
|
for (const s of [job, retry.job, 'host:web2', 'host-remove-failed']) assert.ok(message.includes(s), s);
|
|
assert.equal(f.org.log.filter(([op]) => op === 'instances.set').length, 0);
|
|
await f.broker.grantsIdle();
|
|
const [done] = f.allowed('grants.revoked.done');
|
|
assert.equal(done.body.job, retry.job);
|
|
assert.deepEqual(done.body.grants.map((x) => x.grant), ['host:web2', 'gitea']);
|
|
// Host A was sent one remove only, by the first job.
|
|
assert.equal(f.sent.filter((x) => x === `remove web1 ${f.tag('coder', 'web1')}`).length, 1);
|
|
allRevoked(f);
|
|
assert.match(f.remove('coder').transaction, /^t-/);
|
|
});
|
|
|
|
test('row 281 T3b: B cleared by hand with hosts.revoke; the next delete starts a job for Gitea and the leftover line only', async (t) => {
|
|
const f = await setup(t);
|
|
await threeGrants(f);
|
|
f.failing.add('web2');
|
|
assert.equal(f.remove('coder').code, 'grants-revoking');
|
|
await f.broker.grantsIdle();
|
|
const manual = await f.call('human', 'hosts.revoke', { agent: 'coder', host: 'web2', reason: 'web2 is gone' });
|
|
assert.equal(manual.hostLineLeft, f.tag('coder', 'web2'));
|
|
assert.equal(f.hostRow('coder', 'web2'), undefined);
|
|
// The refused grant isn't in the set any more, so this is a new start, not a retry.
|
|
assert.equal(f.remove('coder').code, 'grants-revoking');
|
|
const second = f.allowed('grants.revoking')[1].body;
|
|
assert.deepEqual(second.grants, ['gitea', 'line:web2']);
|
|
await f.broker.grantsIdle();
|
|
const done = f.allowed('grants.revoked.done')[0].body;
|
|
assert.equal(done.job, second.job);
|
|
assert.deepEqual(done.grants.map((x) => x.grant), ['gitea']);
|
|
assert.deepEqual(done.grantsLeft, [{ host: 'web2', tag: f.tag('coder', 'web2') }]);
|
|
assert.ok(f.inbox().at(-1).includes(`${f.tag('coder', 'web2')} on web2`));
|
|
// The line was tried once and listed: it doesn't hold the delete again.
|
|
assert.match(f.remove('coder').transaction, /^t-/);
|
|
assert.equal(f.allowed('grants.revoking').length, 2);
|
|
});
|
|
|
|
test('row 281 T3c (sage-2575 condition 4): a delete of an instance with no grant completes in one request', async (t) => {
|
|
const f = await setup(t);
|
|
const before = f.store.get("SELECT count(*) AS n FROM events WHERE business='demo'").n;
|
|
const out = f.remove('helper');
|
|
assert.equal(out.code, undefined);
|
|
assert.match(out.transaction, /^t-/);
|
|
assert.equal(out.retry, false);
|
|
assert.deepEqual(f.org.log.filter(([op]) => op === 'instances.set'), [['instances.set', 'helper']]);
|
|
assert.equal(f.allowed('grants.revoking').length, 0);
|
|
assert.deepEqual(f.inbox(), []);
|
|
// The request, the take and the delete's own records: none from step G.
|
|
const after = f.store
|
|
.all("SELECT body FROM events WHERE business='demo' ORDER BY seq")
|
|
.slice(before)
|
|
.map((e) => JSON.parse(e.body).action ?? JSON.parse(e.body).operation ?? null);
|
|
assert.ok(after.every((a) => !String(a).startsWith('grant')), after.join());
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.allowed('grants.revoked.done').length, 0);
|
|
});
|
|
|
|
test('row 281 T4: host B unreachable in a fire: the job refuses, A stays revoked, B and Gitea untouched; a repeat completes', async (t) => {
|
|
const f = await setup(t);
|
|
await threeGrants(f);
|
|
f.failing.add('web2');
|
|
const out = await f.fire('coder');
|
|
await f.broker.grantsIdle();
|
|
const [refused, ...more] = f.grantEvents().filter((e) => e.action === 'grant.revoke.refused');
|
|
assert.equal(more.length, 0);
|
|
assert.deepEqual(
|
|
{ job: refused.job, agent: refused.agent, grant: refused.grant, reason: refused.reason, verb: refused.verb },
|
|
{ job: out.job, agent: 'coder', grant: 'host:web2', reason: 'host-remove-failed', verb: 'instance.fire' },
|
|
);
|
|
const [message] = f.inbox();
|
|
for (const s of [out.job, 'web2', 'host-remove-failed', 'mosaic hosts revoke coder web2', 'Run the fire again']) assert.ok(message.includes(s), s);
|
|
// The file is unchanged and the instance still firing.
|
|
assert.deepEqual(f.org.log.filter(([op]) => op === 'instance.fire'), []);
|
|
assert.ok(f.doc.launch.instances.includes('coder'));
|
|
assert.equal(f.allowed('fire.done').length, 0);
|
|
assert.equal(f.allowed('fire.failed').length, 0);
|
|
await rejects(() => f.raiseHost('coder', 'web3'), 'instance-fired');
|
|
// A revoked; B's record, key and line there (line first: nothing local changed on B).
|
|
assert.equal(f.revoked('coder', (e) => e.host === 'web1').length, 1);
|
|
assert.equal(f.hostRow('coder', 'web1'), undefined);
|
|
assert.equal(f.revoked('coder', (e) => e.host === 'web2').length, 0);
|
|
assert.ok(f.hostRow('coder', 'web2'));
|
|
assert.ok(existsSync(f.keyFile('coder', 'web2')));
|
|
assert.equal(f.state('coder', 'host.web2'), 'valid');
|
|
// Gitea comes after B: untouched.
|
|
assert.equal(f.revoked('coder', (e) => e.method === 'gitea').length, 0);
|
|
assert.ok(f.giteaRow('coder'));
|
|
assert.equal(f.userTokens('demo-coder').length, 1);
|
|
assert.equal(f.state('coder', 'gitea'), 'valid');
|
|
// B back: the repeat revokes only B and Gitea.
|
|
f.failing.delete('web2');
|
|
const repeat = await f.fire('coder');
|
|
assert.deepEqual(repeat.revoking, ['host:web2', 'gitea']);
|
|
await f.broker.grantsIdle();
|
|
const [done] = f.allowed('fire.done');
|
|
assert.equal(done.body.job, repeat.job);
|
|
assert.deepEqual(done.body.grants.map((x) => x.grant), ['host:web2', 'gitea']);
|
|
allRevoked(f);
|
|
assert.deepEqual(f.org.log.filter(([op]) => op === 'instance.fire'), [['instance.fire', 'coder']]);
|
|
});
|
|
|
|
test('row 281 T4b (Q1): a line a manual hosts.revoke left on C is removed by the fire, with its close', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('coder', 'web1');
|
|
await f.approveHost('coder', 'web3');
|
|
f.failing.add('web3');
|
|
const manual = await f.call('human', 'hosts.revoke', { agent: 'coder', host: 'web3', reason: 'web3 is down' });
|
|
const tag3 = f.tag('coder', 'web3');
|
|
assert.equal(manual.hostLineLeft, tag3);
|
|
f.failing.delete('web3');
|
|
const out = await f.fire('coder');
|
|
assert.deepEqual(out.revoking, ['host:web1', 'line:web3']);
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.lines.web3.has(tag3), false);
|
|
const close = f.grantEvents().filter((e) => e.action === 'grant.revoke.closed' && e.host === 'web3' && e.fire === true);
|
|
assert.equal(close.length, 1);
|
|
assert.equal(close[0].hostLine, 'removed');
|
|
assert.equal(close[0].tag, tag3);
|
|
// hosts.revoke's answer has no revoke id; the manual's grant.revoked has it.
|
|
const [manualRevoked] = f.revoked('coder', (e) => e.host === 'web3' && e.fire !== true);
|
|
assert.equal(close[0].revoke, manualRevoked.revoke);
|
|
const [done] = f.allowed('fire.done');
|
|
assert.deepEqual(done.body.grants.map((x) => x.grant), ['host:web1', 'line:web3']);
|
|
assert.deepEqual(done.body.grantsLeft, []);
|
|
});
|
|
|
|
test('row 281 T4b (Q1): with C unreachable, the fire completes and names the line in grantsLeft', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('coder', 'web1');
|
|
await f.approveHost('coder', 'web3');
|
|
f.failing.add('web3');
|
|
await f.call('human', 'hosts.revoke', { agent: 'coder', host: 'web3', reason: 'web3 is down' });
|
|
const tag3 = f.tag('coder', 'web3');
|
|
const out = await f.fire('coder');
|
|
assert.deepEqual(out.revoking, ['host:web1', 'line:web3']);
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.grantEvents().filter((e) => e.action === 'grant.revoke.refused').length, 0);
|
|
const [done] = f.allowed('fire.done');
|
|
assert.equal(done.body.job, out.job);
|
|
assert.deepEqual(done.body.grants.map((x) => x.grant), ['host:web1']);
|
|
assert.deepEqual(done.body.grantsLeft, [{ host: 'web3', tag: tag3 }]);
|
|
assert.ok(f.lines.web3.has(tag3));
|
|
assert.deepEqual(f.org.log.filter(([op]) => op === 'instance.fire'), [['instance.fire', 'coder']]);
|
|
});
|
|
|
|
test("row 281 T5: Gitea's admin token can't be checked: the job refuses with the Gitea grant intact", async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveGitea('coder');
|
|
const off = f.gitea.rule((e) => e.method === 'GET' && e.path === '/api/v1/version', { status: 500, json: {} });
|
|
const out = await f.fire('coder');
|
|
assert.deepEqual(out.revoking, ['gitea']);
|
|
await f.broker.grantsIdle();
|
|
off();
|
|
const [refused] = f.grantEvents().filter((e) => e.action === 'grant.revoke.refused');
|
|
assert.deepEqual([refused.grant, refused.reason, refused.job], ['gitea', 'gitea-admin-unavailable', out.job]);
|
|
assert.ok(f.inbox()[0].includes('mosaic grants sweep gitea'));
|
|
assert.ok(f.giteaRow('coder'));
|
|
assert.ok(existsSync(f.tokenFile('coder')));
|
|
assert.equal(f.state('coder', 'gitea'), 'valid');
|
|
assert.equal(f.userTokens('demo-coder').length, 1);
|
|
assert.equal(f.gitea.users.get('demo-coder').prohibit_login, false);
|
|
assert.equal(f.revoked('coder').length, 0);
|
|
assert.equal(f.allowed('fire.done').length, 0);
|
|
assert.deepEqual(f.org.log.filter(([op]) => op === 'instance.fire'), []);
|
|
});
|
|
|
|
test("row 281 T5b (N1): a failed token DELETE refuses gitea-left; the repeat retries only this agent's entries", async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveGitea('helper');
|
|
await f.approveGitea('coder');
|
|
const tokenDelete = (login) => (e) => e.method === 'DELETE' && e.path.startsWith(`/api/v1/users/${login}/tokens/`);
|
|
// Another agent's left entry, from a manual revoke whose DELETE failed.
|
|
const offHelper = f.gitea.rule(tokenDelete('demo-helper'), { status: 500, json: {} });
|
|
const manual = await f.call('human', 'grants.revoke', { service: 'gitea', agent: 'helper', reason: 'test' });
|
|
assert.equal(manual.serviceLeft.length, 1);
|
|
const helperDeletes = () => f.gitea.requests.filter(tokenDelete('demo-helper')).length;
|
|
const before = helperDeletes();
|
|
const offCoder = f.gitea.rule(tokenDelete('demo-coder'), { status: 500, json: {} });
|
|
const out = await f.fire('coder');
|
|
await f.broker.grantsIdle();
|
|
const [refused] = f.grantEvents().filter((e) => e.action === 'grant.revoke.refused');
|
|
assert.equal(refused.grant, 'gitea');
|
|
// The close's serviceLeft (N10): the token id, and "tokens" for the fire's list that didn't clear.
|
|
assert.match(refused.reason, /^gitea-left:\d+,tokens$/);
|
|
assert.equal(refused.job, out.job);
|
|
assert.equal(f.userTokens('demo-coder').length, 1);
|
|
assert.equal(f.allowed('fire.done').length, 0);
|
|
// Gitea back for coder only. The helper's DELETE still fails, so a business-wide sweep would show.
|
|
offCoder();
|
|
const repeat = await f.fire('coder');
|
|
assert.deepEqual(repeat.revoking, ['gitea']);
|
|
await f.broker.grantsIdle();
|
|
const [done] = f.allowed('fire.done');
|
|
assert.equal(done.body.job, repeat.job);
|
|
assert.deepEqual(f.userTokens('demo-coder'), []);
|
|
const swept = f.grantEvents().filter((e) => e.action === 'grant.swept' && e.job === repeat.job);
|
|
assert.equal(swept.length, 1);
|
|
assert.equal(swept[0].agent, 'coder');
|
|
assert.deepEqual(swept[0].left.map((e) => e.agent), ['helper']);
|
|
// Neither job touched the helper's token.
|
|
assert.equal(helperDeletes(), before);
|
|
assert.equal(f.userTokens('demo-helper').length, 1);
|
|
offHelper();
|
|
});
|
|
|
|
test('row 281 T6 (sage-2574): a host decision approved before the fire and run after it ends revoked, with nothing applied', async (t) => {
|
|
const f = await setup(t);
|
|
const id = f.fixtureHost('coder', 'web1');
|
|
const out = await f.fire('coder');
|
|
assert.deepEqual(out.revoking, ['host:web1']);
|
|
await f.broker.grantsIdle();
|
|
const [r] = f.revoked('coder', (e) => e.host === 'web1');
|
|
assert.equal(r.granted, false);
|
|
assert.equal(r.fire, true);
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
assert.equal(await f.broker.runHostGrant('demo', id), 'revoked');
|
|
assert.equal(f.ended(id).code, 'revoked');
|
|
assert.equal(f.hostRow('coder', 'web1'), undefined);
|
|
assert.equal(existsSync(f.keyFile('coder', 'web1')), false);
|
|
assert.ok(!f.sent.some((x) => x.startsWith('add ')));
|
|
});
|
|
|
|
test('row 281 T6b (sage-2574): a Gitea decision approved before the fire and run after it ends revoked before any Gitea write', async (t) => {
|
|
const f = await setup(t);
|
|
const id = f.fixtureGitea('coder');
|
|
const out = await f.fire('coder');
|
|
assert.deepEqual(out.revoking, ['gitea']);
|
|
await f.broker.grantsIdle();
|
|
const [r] = f.revoked('coder', (e) => e.method === 'gitea');
|
|
assert.deepEqual([r.user, r.granted, r.fire, r.decisions], [null, false, true, [id]]);
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
const calls = f.gitea.requests.length;
|
|
await f.broker.runGiteaGrant('demo', id);
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.ended(id).code, 'revoked');
|
|
assert.equal(f.giteaRow('coder'), undefined);
|
|
assert.equal(existsSync(f.tokenFile('coder')), false);
|
|
// Only step 0's C1 to C4 reach Gitea, all reads; step 1's recheck ends it before any write.
|
|
assert.deepEqual(f.gitea.requests.slice(calls).map((e) => `${e.method} ${e.path}`), [
|
|
'GET /api/v1/version',
|
|
'GET /api/v1/user',
|
|
'GET /api/v1/token',
|
|
'GET /api/v1/user/repos?limit=1&page=1',
|
|
]);
|
|
assert.ok(!f.gitea.requests.some((e) => e.method === 'POST' && e.path.endsWith('/tokens')));
|
|
});
|
|
|
|
test('row 281 T6c: a host executor in flight aborts and ends revoked; a Gitea executor in flight is awaited and leaves no token', async (t) => {
|
|
const f = await setup(t);
|
|
// Host: held at its add.
|
|
f.held.add('web1:add');
|
|
const host = await f.raiseHost('coder', 'web1');
|
|
f.call('human', 'decision.resolve', { id: host.id, choice: 'yes' });
|
|
await until(() => f.sent.some((x) => x.startsWith('add web1 ')));
|
|
// Gitea: held at the token create.
|
|
let go;
|
|
const gate = new Promise((r) => (go = r));
|
|
t.after(() => go());
|
|
let reached = false;
|
|
const off = f.gitea.rule(
|
|
(e) => e.method === 'POST' && e.path === '/api/v1/users/demo-coder/tokens',
|
|
async () => {
|
|
reached = true;
|
|
await gate;
|
|
return { pass: true };
|
|
},
|
|
);
|
|
const gd = await f.raiseGitea('coder');
|
|
f.call('human', 'decision.resolve', { id: gd.id, choice: 'yes' });
|
|
await until(() => reached);
|
|
const out = await f.fire('coder');
|
|
assert.deepEqual(out.revoking, ['host:web1', 'gitea']);
|
|
await until(() => f.ended(host.id));
|
|
assert.equal(f.ended(host.id).code, 'revoked');
|
|
// The job waits on the Gitea executor.
|
|
await sleep(50);
|
|
assert.equal(f.allowed('fire.done').length, 0);
|
|
go();
|
|
await f.broker.grantsIdle();
|
|
off();
|
|
f.held.delete('web1:add');
|
|
assert.equal(f.hostRow('coder', 'web1'), undefined);
|
|
assert.equal(existsSync(f.keyFile('coder', 'web1')), false);
|
|
assert.equal(f.lines.web1.has(f.tag('coder', 'web1')), false);
|
|
assert.ok(f.ended(gd.id));
|
|
assert.deepEqual(f.userTokens('demo-coder'), []);
|
|
assert.equal(f.giteaRow('coder'), undefined);
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
f.noLeak();
|
|
});
|
|
|
|
test('row 281 T6d (B1): undecided host and Gitea decisions are in the set, and approved after the fire both end revoked', async (t) => {
|
|
const f = await setup(t);
|
|
const host = await f.raiseHost('coder', 'web1');
|
|
const gd = await f.raiseGitea('coder');
|
|
const out = await f.fire('coder');
|
|
assert.deepEqual(out.revoking, ['host:web1', 'gitea']);
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
for (const d of [host, gd]) {
|
|
f.call('human', 'decision.resolve', { id: d.id, choice: 'yes' });
|
|
await until(() => f.ended(d.id));
|
|
}
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.ended(host.id).code, 'revoked');
|
|
assert.equal(f.ended(gd.id).code, 'revoked');
|
|
assert.ok(!f.sent.some((x) => x.startsWith('add ')));
|
|
assert.ok(!f.gitea.requests.some((e) => e.method === 'POST' && e.path.endsWith('/tokens')));
|
|
assert.equal(f.hostRow('coder', 'web1'), undefined);
|
|
assert.equal(f.giteaRow('coder'), undefined);
|
|
});
|
|
|
|
test('row 281 T6e (B1): a grant raise refuses instance-fired while firing and after the fire, goes through after a rehire, and refuses grant-revoke-running during a delete job', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('coder', 'web1');
|
|
// Firing with no job: step B refused, so fire.stopped stands with no fire.done.
|
|
f.org.fireError = Object.assign(new Error('refused'), { code: 'credential-revoke-failed' });
|
|
await f.fire('coder');
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.allowed('fire.failed').length, 1);
|
|
await rejects(() => f.raiseHost('coder', 'web2'), 'instance-fired');
|
|
await rejects(() => f.raiseGitea('coder'), 'instance-fired');
|
|
// After fire.done: delisted.
|
|
f.org.fireError = null;
|
|
assert.match((await f.fire('coder')).transaction, /^t-/);
|
|
await rejects(() => f.raiseHost('coder', 'web2'), 'instance-fired');
|
|
await rejects(() => f.raiseGitea('coder'), 'instance-fired');
|
|
// A rehire lists the name again.
|
|
f.doc.launch.instances.push('coder');
|
|
f.broker.reloadBusiness('demo', f.doc);
|
|
assert.match((await f.raiseHost('coder', 'web2')).id, UUID);
|
|
// While a delete's job holds ops.
|
|
await f.approveHost('ops', 'web3');
|
|
f.held.add('web3:remove');
|
|
assert.equal(f.remove('ops').code, 'grants-revoking');
|
|
await rejects(() => f.raiseHost('ops', 'web1'), 'grant-revoke-running');
|
|
f.held.delete('web3:remove');
|
|
await f.broker.grantsIdle();
|
|
});
|
|
|
|
test('row 281 T7: an instance with no grant fires as today, synchronously, with the same events', async (t) => {
|
|
const f = await setup(t);
|
|
const out = await f.fire('helper');
|
|
assert.deepEqual(Object.keys(out).sort(), ['cancelled', 'request', 'revision', 'runs', 'taken', 'transaction']);
|
|
assert.match(out.transaction, /^t-/);
|
|
// Recorded before the verb answered: no job.
|
|
assert.equal(f.allowed('fire.stopped').length, 1);
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
assert.equal(f.allowed('fire.done')[0].body.job, undefined);
|
|
assert.equal(f.allowed('fire.revoking').length, 0);
|
|
assert.equal(f.allowed('role.fire').length, 1);
|
|
assert.deepEqual(f.org.log.map(([op]) => op), ['instance.fireCheck', 'instance.fire']);
|
|
await f.broker.grantsIdle();
|
|
assert.deepEqual(f.inbox(), []);
|
|
assert.equal(f.grantEvents().filter((e) => e.action?.startsWith('grant.revoke')).length, 0);
|
|
});
|
|
|
|
test('row 281 T9 (B3): one step G per agent; the manual verbs refuse while it runs, and sweep names it running', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('coder', 'web1');
|
|
await f.approveGitea('coder');
|
|
f.held.add('web1:remove');
|
|
await f.fire('coder');
|
|
await until(() => f.sent.includes(`remove web1 ${f.tag('coder', 'web1')}`));
|
|
await assert.rejects(f.fire('coder'),{code:'fire-in-progress'});
|
|
assert.equal(f.remove('coder').code, 'grants-revoking');
|
|
assert.equal(f.allowed('grants.revoking').length, 0);
|
|
assert.equal(f.allowed('fire.revoking').length, 1);
|
|
await rejects(() => f.call('human', 'hosts.revoke', { agent: 'coder', host: 'web1', reason: 'r' }), 'grant-revoke-running');
|
|
await rejects(() => f.call('tester', 'hosts.revoke', { agent: 'coder', host: 'web1', reason: 'r' }), 'grant-revoke-running');
|
|
await rejects(() => f.call('human', 'grants.revoke', { service: 'gitea', agent: 'coder', reason: 'r' }), 'grant-revoke-running');
|
|
const sweep = await f.call('human', 'grants.sweep', { service: 'gitea' });
|
|
assert.deepEqual(sweep.running, ['coder']);
|
|
f.held.delete('web1:remove');
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.revoked('coder', (e) => e.host === 'web1').length, 1);
|
|
assert.equal(f.revoked('coder', (e) => e.method === 'gitea').length, 1);
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
allRevoked(f, ['web1']);
|
|
});
|
|
|
|
test('row 281 T10 (B2): the fire and the delete answer while their jobs are held, and the broker answers other verbs', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveGitea('coder');
|
|
await f.approveHost('ops', 'web2');
|
|
let go;
|
|
const gate = new Promise((r) => (go = r));
|
|
t.after(() => go());
|
|
let reached = false;
|
|
const off = f.gitea.rule(
|
|
(e) => e.method === 'DELETE' && e.path.startsWith('/api/v1/users/demo-coder/tokens/'),
|
|
async () => {
|
|
reached = true;
|
|
await gate;
|
|
return { pass: true };
|
|
},
|
|
);
|
|
f.held.add('web2:remove');
|
|
// The verbs answer before the event loop turns, so before any part of a job has run.
|
|
let turned = false;
|
|
setImmediate(() => (turned = true));
|
|
const fired = await f.fire('coder');
|
|
const deleted = f.remove('ops');
|
|
assert.equal(turned, false);
|
|
assert.equal(fired.done, false);
|
|
assert.equal(deleted.code, 'grants-revoking');
|
|
await until(() => reached && f.sent.includes(`remove web2 ${f.tag('ops', 'web2')}`));
|
|
// Held: another session's verb answers.
|
|
assert.ok(f.call('pm', 'message.send', { to: 'chief', body: 'still here' }));
|
|
assert.equal(f.allowed('fire.done').length, 0);
|
|
assert.equal(f.allowed('grants.revoked.done').length, 0);
|
|
go();
|
|
f.held.delete('web2:remove');
|
|
await f.broker.grantsIdle();
|
|
off();
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
assert.equal(f.allowed('grants.revoked.done').length, 1);
|
|
assert.deepEqual(f.userTokens('demo-coder'), []);
|
|
assert.equal(f.hostRow('ops', 'web2'), undefined);
|
|
});
|
|
|
|
test("row 281 T11 (C1): a fire's job completes after the chief's run ends and processing halts", async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('coder', 'web1');
|
|
f.held.add('web1:remove');
|
|
const out = await f.fire('coder');
|
|
await until(() => f.sent.includes(`remove web1 ${f.tag('coder', 'web1')}`));
|
|
f.broker.endLaunch({ business: 'demo', run: 'chief-run', reason: 'test' });
|
|
f.sw.state = { version: 1, halted: true, generation: GENERATION };
|
|
f.held.delete('web1:remove');
|
|
await f.broker.grantsIdle();
|
|
const [done] = f.allowed('fire.done');
|
|
assert.equal(done.body.job, out.job);
|
|
assert.match(done.body.transaction, /^t-/);
|
|
const [used, ...more] = f.allowed('role.fire');
|
|
assert.equal(more.length, 0);
|
|
assert.equal(used.actor_role, null);
|
|
assert.equal(used.body.transaction, done.body.transaction);
|
|
assert.ok(!f.doc.launch.instances.includes('coder'));
|
|
assert.equal(f.allowed('fire.failed').length, 0);
|
|
assert.equal(f.grantEvents().filter((e) => e.action === 'grant.revoke.refused').length, 0);
|
|
});
|
|
|
|
test("row 281 T12 (C1, N7): a step B refusal in the job is fire.failed, and the repeat runs B and C synchronously", async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('coder', 'web1');
|
|
// N7: the writer's own credential-revoke-failed, which step G's key revokes can't clear.
|
|
f.org.fireError = Object.assign(new Error('refused'), { code: 'credential-revoke-failed' });
|
|
const out = await f.fire('coder');
|
|
await f.broker.grantsIdle();
|
|
const [failed, ...more] = f.allowed('fire.failed');
|
|
assert.equal(more.length, 0);
|
|
assert.deepEqual([failed.body.job, failed.body.step, failed.body.code], [out.job, 'B', 'credential-revoke-failed']);
|
|
assert.equal(failed.actor_role, null);
|
|
const [message] = f.inbox();
|
|
for (const s of [out.job, 'host:web1', 'step B', 'credential-revoke-failed']) assert.ok(message.includes(s), s);
|
|
assert.equal(f.grantEvents().filter((e) => e.action === 'grant.revoke.refused').length, 0);
|
|
assert.equal(f.allowed('fire.done').length, 0);
|
|
assert.ok(f.doc.launch.instances.includes('coder'));
|
|
await rejects(() => f.raiseHost('coder', 'web2'), 'instance-fired');
|
|
// The grant was revoked before step B.
|
|
assert.equal(f.hostRow('coder', 'web1'), undefined);
|
|
f.org.fireError = null;
|
|
const repeat = await f.fire('coder');
|
|
assert.equal(repeat.job, undefined);
|
|
assert.match(repeat.transaction, /^t-/);
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
assert.ok(!f.doc.launch.instances.includes('coder'));
|
|
});
|
|
|
|
test('row 281 T13 (N4): a restart records each held job interrupted once, with one message each; an ended job gets none', async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('helper', 'web3');
|
|
await f.approveHost('coder', 'web1');
|
|
await f.approveHost('ops', 'web2');
|
|
// A delete job that ended before the restart.
|
|
assert.equal(f.remove('helper').code, 'grants-revoking');
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.allowed('grants.revoked.done').length, 1);
|
|
f.held.add('web1:remove');
|
|
f.held.add('web2:remove');
|
|
const fired = await f.fire('coder');
|
|
assert.equal(f.remove('ops').code, 'grants-revoking');
|
|
const deleteJob = f.allowed('grants.revoking').at(-1).body.job;
|
|
await until(() => f.sent.includes(`remove web1 ${f.tag('coder', 'web1')}`) && f.sent.includes(`remove web2 ${f.tag('ops', 'web2')}`));
|
|
const restartNotes = () => f.inbox().filter((m) => m.includes('interrupted by a bus restart'));
|
|
const second = f.make();
|
|
second.reconcileGrants();
|
|
const interrupted = f.allowed('grants.job.interrupted');
|
|
assert.deepEqual(
|
|
interrupted.map((e) => [e.body.job, e.body.agent, e.body.verb]).sort(),
|
|
[[fired.job, 'coder', 'instance.fire'], [deleteJob, 'ops', 'lifecycle.delete']].sort(),
|
|
);
|
|
assert.ok(interrupted.every((e) => e.actor_role === null));
|
|
const notes = restartNotes();
|
|
assert.equal(notes.length, 2);
|
|
assert.ok(notes.some((m) => m.includes(fired.job) && m.includes('Run the fire again')));
|
|
assert.ok(notes.some((m) => m.includes(deleteJob) && m.includes('Raise the delete again')));
|
|
// A third broker on the same root sends nothing more.
|
|
f.make().reconcileGrants();
|
|
assert.equal(f.allowed('grants.job.interrupted').length, 2);
|
|
assert.equal(restartNotes().length, 2);
|
|
f.held.clear();
|
|
await f.broker.grantsIdle();
|
|
});
|
|
|
|
test("row 281 T14 (N1, sage-2653): a host removed under an undecided decision stops the fire on host-unknown; the human's no closes it, and the repeat fire has no job", async (t) => {
|
|
const f = await setup(t);
|
|
const d = await f.raiseHost('coder', 'web1');
|
|
// N1: hosts.remove doesn't check open decisions, so the decision outlives its host.
|
|
assert.deepEqual(await f.call('human', 'hosts.remove', { host: 'web1' }), { host: 'web1', removed: true });
|
|
const out = await f.fire('coder');
|
|
assert.match(out.job, UUID);
|
|
assert.deepEqual(out.revoking, ['host:web1']);
|
|
await f.broker.grantsIdle();
|
|
const [refused, ...more] = f.grantEvents().filter((e) => e.action === 'grant.revoke.refused');
|
|
assert.equal(more.length, 0);
|
|
assert.deepEqual([refused.job, refused.grant, refused.reason, refused.verb], [out.job, 'host:web1', 'host-unknown', 'instance.fire']);
|
|
const [message, ...others] = f.inbox();
|
|
assert.equal(others.length, 0);
|
|
for (const s of [out.job, 'web1', 'host-unknown']) assert.ok(message.includes(s), s);
|
|
assert.equal(f.allowed('fire.done').length, 0);
|
|
assert.ok(f.doc.launch.instances.includes('coder'));
|
|
// The human's no: no executor runs, so the closed check alone takes it out of the set (M22).
|
|
f.call('human', 'decision.resolve', { id: d.id, choice: 'no' });
|
|
await f.broker.grantsIdle();
|
|
assert.equal(f.ended(d.id), undefined);
|
|
const repeat = await f.fire('coder');
|
|
assert.equal(repeat.job, undefined);
|
|
assert.match(repeat.transaction, /^t-/);
|
|
assert.equal(f.allowed('fire.done').length, 1);
|
|
assert.equal(f.allowed('fire.done')[0].body.job, undefined);
|
|
assert.ok(!f.doc.launch.instances.includes('coder'));
|
|
assert.equal(f.grantEvents().filter((e) => e.action === 'grant.revoke.refused').length, 1);
|
|
});
|
|
|
|
// Row 239 (#1745): the writer marks an error raised after its own revoke ran (revoked: true), and the
|
|
// bus names it, so the human reads that the tokens may already be revoked. Unmarked, today's code.
|
|
test("row 239 T4: a refusal after the writer's revoke is named -revoked, in the job's fire.failed and notice, a synchronous fire and a delete", async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('coder', 'web1');
|
|
f.org.fireError = Object.assign(new Error('refused'), { code: 'revision-conflict', revoked: true });
|
|
const out = await f.fire('coder');
|
|
await f.broker.grantsIdle();
|
|
const [failed] = f.allowed('fire.failed');
|
|
assert.deepEqual([failed.body.job, failed.body.step, failed.body.code], [out.job, 'B', 'revision-conflict-revoked']);
|
|
const [message] = f.inbox();
|
|
assert.ok(message.includes('(revision-conflict-revoked). Its credential tokens may already be revoked. The instance is still firing.'), message);
|
|
// helper has no grant, so its fire writes synchronously and answers with the code.
|
|
await rejects(() => f.fire('helper'), 'revision-conflict-revoked');
|
|
// Any other marked error: the write may have landed.
|
|
f.org.fireError = Object.assign(new Error('EIO: i/o error'), { code: 'EIO', revoked: true });
|
|
await rejects(() => f.fire('helper'), 'outcome-unknown-revoked');
|
|
f.org.fireError = Object.assign(new Error('refused'), { code: 'revision-conflict' });
|
|
await rejects(() => f.fire('helper'), 'revision-conflict');
|
|
f.org.fireError = null;
|
|
f.org.deleteError = Object.assign(new Error('refused'), { code: 'revision-conflict', revoked: true });
|
|
assert.equal(f.remove('helper').code, 'revision-conflict-revoked');
|
|
f.org.deleteError = Object.assign(new Error('EIO: i/o error'), { code: 'EIO', revoked: true });
|
|
assert.equal(f.remove('helper').code, 'outcome-unknown-revoked');
|
|
f.org.deleteError = Object.assign(new Error('refused'), { code: 'revision-conflict' });
|
|
assert.equal(f.remove('helper').code, 'revision-conflict');
|
|
});
|
|
|
|
test("row 239 T4b: an unmarked step B refusal keeps its code, and the notice says nothing of the tokens", async (t) => {
|
|
const f = await setup(t);
|
|
await f.approveHost('coder', 'web1');
|
|
f.org.fireError = Object.assign(new Error('refused'), { code: 'revision-conflict' });
|
|
const out = await f.fire('coder');
|
|
await f.broker.grantsIdle();
|
|
const [failed] = f.allowed('fire.failed');
|
|
assert.deepEqual([failed.body.job, failed.body.code], [out.job, 'revision-conflict']);
|
|
const [message] = f.inbox();
|
|
assert.ok(message.includes('(revision-conflict). The instance is still firing.'), message);
|
|
assert.ok(!message.includes('may already be revoked'), message);
|
|
});
|
|
|
|
for(const expired of [false,true])test(`AU hire inverse waits for grant removal and refuses late publication (expired: ${expired})`,async t=>{
|
|
const host=autonomyHost({hires:{reverse:async(id,decision,caller)=>caller.fire({id:decision,level:'org',name:'coder',revision:REV,digest:'a'.repeat(64)})},reload:()=>true});
|
|
const f=await setup(t,host);await f.approveHost('coder','web1');
|
|
const decision=grantDecision(f.store,{action:'role.hire',route:'pm',close:{op:'resolved',by:'pm-run',choice:'yes',via:'broker'}});
|
|
const at=new Date().toISOString(),deadline=new Date(Date.now()+3600000).toISOString();
|
|
f.store.run("INSERT INTO events(id,at,business,kind,subject,body) VALUES(?,?,?,'action.allowed',?,?)",decision,at,'demo',decision,JSON.stringify({operation:'autonomy.notice',decision,decisionClass:'authority',reverseUntil:deadline}));
|
|
f.held.add('web1:remove');
|
|
const owner=fixtureConsole(f.broker),pending=f.broker.request(owner,{verb:'decision.reverse.console',args:{id:decision}});
|
|
// Attach the rejection handler before letting the asynchronous revoke complete.
|
|
const outcome=pending.then(value=>({value}),error=>({error}));
|
|
await until(()=>f.sent.some(s=>s.startsWith('remove web1')));
|
|
assert.equal(f.store.get("SELECT count(*) n FROM events WHERE json_extract(body,'$.operation')='autonomy.reverse'").n,0);assert.ok(f.doc.launch.instances.includes('coder'));
|
|
const clock=Date.now;
|
|
try {
|
|
if(expired)Date.now=()=>Date.parse(deadline)+1;
|
|
f.held.delete('web1:remove');const result=await outcome;await f.broker.grantsIdle();
|
|
if(expired){assert.equal(result.error?.code,'reverse-window-closed');assert.ok(f.doc.launch.instances.includes('coder'));}
|
|
else{assert.equal(result.error,undefined);assert.equal(result.value.committed,true);assert.match(result.value.transaction,/^t-/);assert.equal(f.store.get("SELECT count(*) n FROM events WHERE json_extract(body,'$.operation')='autonomy.reverse'").n,1);assert.ok(!f.doc.launch.instances.includes('coder'));}
|
|
}finally{Date.now=clock;f.held.clear();}
|
|
});
|