Compare commits
2 Commits
feat/mos-l
...
17932c3fb1
| Author | SHA1 | Date | |
|---|---|---|---|
|
17932c3fb1 | ||
|
|
83a96b0081 |
@@ -21,12 +21,6 @@ import { LogModule } from '../log/log.module.js';
|
||||
import { CommandsModule } from '../commands/commands.module.js';
|
||||
import { CommandRuntimeApprovalVerifier } from '../commands/runtime-approval-verifier.js';
|
||||
import { GatewayHermesRuntimeTransport } from './hermes-runtime.transport.js';
|
||||
import { ConnectorLeaseRepository } from './connector-lease.repository.js';
|
||||
import {
|
||||
CONNECTOR_LEASE_POLICY,
|
||||
ConnectorLeaseService,
|
||||
DenyConnectorLeasePolicy,
|
||||
} from './connector-lease.service.js';
|
||||
import {
|
||||
AGENT_RUNTIME_PROVIDER_REGISTRY,
|
||||
RUNTIME_APPROVAL_VERIFIER,
|
||||
@@ -52,13 +46,6 @@ export function createGatewayRuntimeProviderRegistry(): AgentRuntimeProviderRegi
|
||||
SkillLoaderService,
|
||||
DurableSessionRepository,
|
||||
DurableSessionService,
|
||||
ConnectorLeaseRepository,
|
||||
DenyConnectorLeasePolicy,
|
||||
{
|
||||
provide: CONNECTOR_LEASE_POLICY,
|
||||
useExisting: DenyConnectorLeasePolicy,
|
||||
},
|
||||
ConnectorLeaseService,
|
||||
{
|
||||
provide: AGENT_RUNTIME_PROVIDER_REGISTRY,
|
||||
useFactory: createGatewayRuntimeProviderRegistry,
|
||||
@@ -91,7 +78,6 @@ export function createGatewayRuntimeProviderRegistry(): AgentRuntimeProviderRegi
|
||||
SkillLoaderService,
|
||||
DurableSessionService,
|
||||
RuntimeProviderService,
|
||||
ConnectorLeaseService,
|
||||
AGENT_RUNTIME_PROVIDER_REGISTRY,
|
||||
],
|
||||
})
|
||||
|
||||
@@ -1,232 +0,0 @@
|
||||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { Test, type TestingModule } from '@nestjs/testing';
|
||||
import {
|
||||
connectorLeaseAuditLog,
|
||||
createPgliteDb,
|
||||
eq,
|
||||
runPgliteMigrations,
|
||||
type DbHandle,
|
||||
} from '@mosaicstack/db';
|
||||
import type { ConnectorExecutionContext, FencedConnectorAdapter } from '@mosaicstack/types';
|
||||
import { DB } from '../database/database.module.js';
|
||||
import { ConnectorLeaseRepository } from './connector-lease.repository.js';
|
||||
import {
|
||||
CONNECTOR_LEASE_POLICY,
|
||||
ConnectorLeaseService,
|
||||
type ConnectorLeasePolicy,
|
||||
} from './connector-lease.service.js';
|
||||
|
||||
const authorize = vi.fn().mockResolvedValue(true);
|
||||
const policy: ConnectorLeasePolicy = { authorize };
|
||||
const context = {
|
||||
actorScope: { userId: 'operator-a', tenantId: 'tenant-a' },
|
||||
correlationId: 'correlation-acquire',
|
||||
};
|
||||
|
||||
describe('gateway connector lease fencing integration', (): void => {
|
||||
let dataDir: string;
|
||||
let handle: DbHandle;
|
||||
let moduleRef: TestingModule;
|
||||
let service: ConnectorLeaseService;
|
||||
|
||||
beforeAll(async (): Promise<void> => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-07-14T17:00:00.000Z'));
|
||||
dataDir = await mkdtemp(join(tmpdir(), 'mosaic-gateway-connector-lease-'));
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
moduleRef = await Test.createTestingModule({
|
||||
providers: [
|
||||
ConnectorLeaseRepository,
|
||||
ConnectorLeaseService,
|
||||
{ provide: DB, useValue: handle.db },
|
||||
{ provide: CONNECTOR_LEASE_POLICY, useValue: policy },
|
||||
],
|
||||
}).compile();
|
||||
service = moduleRef.get(ConnectorLeaseService);
|
||||
});
|
||||
|
||||
afterAll(async (): Promise<void> => {
|
||||
vi.useRealTimers();
|
||||
await moduleRef.close();
|
||||
await handle.close();
|
||||
await rm(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('derives tenant authority at the gateway and validates a grant before side effects', async (): Promise<void> => {
|
||||
const lease = await service.acquire(
|
||||
{
|
||||
logicalAgentId: 'Mos',
|
||||
bindingId: 'operator-chat',
|
||||
connectorId: 'pi-worker-a',
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 60_000,
|
||||
},
|
||||
context,
|
||||
);
|
||||
const grant = await service.issueGrant(
|
||||
{ lease, scopes: ['runtime.send'], ttlMs: 30_000 },
|
||||
{ ...context, correlationId: 'correlation-grant' },
|
||||
);
|
||||
const execute = vi.fn(async (_message: string, leaseContext: ConnectorExecutionContext) => {
|
||||
return leaseContext.leaseEpoch;
|
||||
});
|
||||
const adapter: FencedConnectorAdapter<string, string> = { execute };
|
||||
|
||||
await expect(service.executeGrant(grant, 'runtime.send', 'hello', adapter)).resolves.toBe('1');
|
||||
expect(execute).toHaveBeenCalledOnce();
|
||||
expect(authorize).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: 'grant.issue',
|
||||
requestedScopes: ['runtime.send'],
|
||||
requestedTtlMs: 30_000,
|
||||
}),
|
||||
);
|
||||
expect(execute.mock.calls[0]?.[1]).toMatchObject({
|
||||
identity: { tenantId: 'tenant-a', logicalAgentId: 'mos' },
|
||||
bindingId: 'operator-chat',
|
||||
connectorId: 'pi-worker-a',
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes lease-derived policy subjects before authorization', async (): Promise<void> => {
|
||||
const lease = await service.acquire(
|
||||
{
|
||||
logicalAgentId: 'mos',
|
||||
bindingId: 'operator-chat-policy',
|
||||
connectorId: 'pi-worker-a',
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 60_000,
|
||||
},
|
||||
{ ...context, correlationId: 'correlation-policy-setup' },
|
||||
);
|
||||
const aliasedLease = {
|
||||
...lease,
|
||||
identity: { ...lease.identity, logicalAgentId: ' MOS ' },
|
||||
bindingId: ' Operator-Chat-Policy ',
|
||||
connectorId: ' PI-Worker-A ',
|
||||
scopes: [' Runtime.Send '],
|
||||
leaseEpoch: `00${lease.leaseEpoch}`,
|
||||
};
|
||||
|
||||
await service.heartbeat(aliasedLease, 30_000, {
|
||||
...context,
|
||||
correlationId: 'correlation-policy-heartbeat',
|
||||
});
|
||||
expect(authorize).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
action: 'lease.heartbeat',
|
||||
logicalAgentId: 'mos',
|
||||
bindingId: 'operator-chat-policy',
|
||||
connectorId: 'pi-worker-a',
|
||||
requestedScopes: ['runtime.send'],
|
||||
}),
|
||||
);
|
||||
|
||||
await service.issueGrant(
|
||||
{ lease: aliasedLease, scopes: [' Runtime.Send '], ttlMs: 1_000 },
|
||||
{ ...context, correlationId: 'correlation-policy-grant' },
|
||||
);
|
||||
expect(authorize).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
action: 'grant.issue',
|
||||
logicalAgentId: 'mos',
|
||||
bindingId: 'operator-chat-policy',
|
||||
connectorId: 'pi-worker-a',
|
||||
requestedScopes: ['runtime.send'],
|
||||
}),
|
||||
);
|
||||
|
||||
await service.release(aliasedLease, {
|
||||
...context,
|
||||
correlationId: 'correlation-policy-release',
|
||||
});
|
||||
expect(authorize).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
action: 'lease.release',
|
||||
logicalAgentId: 'mos',
|
||||
bindingId: 'operator-chat-policy',
|
||||
connectorId: 'pi-worker-a',
|
||||
requestedScopes: ['runtime.send'],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('denies stale, forged, expired, cross-tenant, and cross-binding grants before effects', async (): Promise<void> => {
|
||||
const bindingId = 'operator-chat-denials';
|
||||
const current = await service.acquire(
|
||||
{
|
||||
logicalAgentId: 'mos',
|
||||
bindingId,
|
||||
connectorId: 'pi-worker-a',
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 60_000,
|
||||
},
|
||||
{ ...context, correlationId: 'correlation-denial-setup' },
|
||||
);
|
||||
const stale = await service.issueGrant(
|
||||
{ lease: current, scopes: ['runtime.send'], ttlMs: 30_000 },
|
||||
{ ...context, correlationId: 'correlation-stale' },
|
||||
);
|
||||
await service.takeover(
|
||||
{
|
||||
logicalAgentId: 'mos',
|
||||
bindingId,
|
||||
connectorId: 'pi-worker-b',
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 60_000,
|
||||
expectedEpoch: current.leaseEpoch,
|
||||
},
|
||||
{ ...context, correlationId: 'correlation-takeover' },
|
||||
);
|
||||
const adapter = { execute: vi.fn().mockResolvedValue(undefined) };
|
||||
|
||||
await expect(service.executeGrant(stale, 'runtime.send', undefined, adapter)).rejects.toThrow();
|
||||
|
||||
const active = await service.current('mos', bindingId, context);
|
||||
if (!active) throw new Error('active lease fixture is unavailable');
|
||||
const grant = await service.issueGrant(
|
||||
{ lease: active, scopes: ['runtime.send'], ttlMs: 1_000 },
|
||||
{ ...context, correlationId: 'correlation-active' },
|
||||
);
|
||||
await expect(
|
||||
service.executeGrant({ ...grant }, 'runtime.send', undefined, adapter),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
service.executeGrant(
|
||||
{ ...grant, bindingId: 'other-binding' },
|
||||
'runtime.send',
|
||||
undefined,
|
||||
adapter,
|
||||
),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
service.issueGrant(
|
||||
{ lease: active, scopes: ['runtime.send'], ttlMs: 30_000 },
|
||||
{
|
||||
actorScope: { userId: 'operator-b', tenantId: 'tenant-b' },
|
||||
correlationId: 'correlation-cross-tenant',
|
||||
},
|
||||
),
|
||||
).rejects.toThrow();
|
||||
const crossTenantAudit = await handle.db
|
||||
.select()
|
||||
.from(connectorLeaseAuditLog)
|
||||
.where(eq(connectorLeaseAuditLog.correlationId, 'correlation-cross-tenant'));
|
||||
expect(crossTenantAudit).toHaveLength(1);
|
||||
expect(crossTenantAudit[0]).toMatchObject({
|
||||
tenantId: 'tenant-b',
|
||||
logicalAgentId: 'untrusted',
|
||||
bindingId: 'untrusted',
|
||||
connectorId: 'untrusted',
|
||||
reason: 'policy_denied',
|
||||
});
|
||||
|
||||
vi.setSystemTime(new Date('2026-07-14T17:00:02.000Z'));
|
||||
await expect(service.executeGrant(grant, 'runtime.send', undefined, adapter)).rejects.toThrow();
|
||||
expect(adapter.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,76 +0,0 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
connectorLeaseAuditLog,
|
||||
createDb,
|
||||
eq,
|
||||
logicalAgentConnectorLeases,
|
||||
type DbHandle,
|
||||
} from '@mosaicstack/db';
|
||||
import { ConnectorLeaseCoordinator } from '@mosaicstack/agent';
|
||||
import { ConnectorLeaseRepository } from './connector-lease.repository.js';
|
||||
|
||||
const hasPostgres = Boolean(process.env['DATABASE_URL']);
|
||||
const tenantId = `lease-test-${randomUUID()}`;
|
||||
const identity = { tenantId, logicalAgentId: 'mos' } as const;
|
||||
|
||||
describe.skipIf(!hasPostgres)('ConnectorLeaseRepository real PostgreSQL integration', (): void => {
|
||||
let handle: DbHandle;
|
||||
|
||||
beforeAll((): void => {
|
||||
handle = createDb(process.env['DATABASE_URL']);
|
||||
});
|
||||
|
||||
afterAll(async (): Promise<void> => {
|
||||
if (!handle) return;
|
||||
await handle.db
|
||||
.delete(connectorLeaseAuditLog)
|
||||
.where(eq(connectorLeaseAuditLog.tenantId, tenantId));
|
||||
await handle.db
|
||||
.delete(logicalAgentConnectorLeases)
|
||||
.where(eq(logicalAgentConnectorLeases.tenantId, tenantId));
|
||||
await handle.close();
|
||||
});
|
||||
|
||||
it('preserves the exclusive CAS fence across a real pool close/reopen', async (): Promise<void> => {
|
||||
const command = {
|
||||
identity,
|
||||
bindingId: 'operator-chat',
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 60_000,
|
||||
} as const;
|
||||
const firstCoordinator = new ConnectorLeaseCoordinator(new ConnectorLeaseRepository(handle.db));
|
||||
const contenders = await Promise.allSettled([
|
||||
firstCoordinator.acquire({
|
||||
...command,
|
||||
connectorId: 'connector-a',
|
||||
correlationId: 'postgres-acquire-a',
|
||||
}),
|
||||
firstCoordinator.acquire({
|
||||
...command,
|
||||
connectorId: 'connector-b',
|
||||
correlationId: 'postgres-acquire-b',
|
||||
}),
|
||||
]);
|
||||
const acquired = contenders.find((result) => result.status === 'fulfilled');
|
||||
if (!acquired || acquired.status !== 'fulfilled') throw new Error('no lease contender won');
|
||||
expect(contenders.filter((result) => result.status === 'fulfilled')).toHaveLength(1);
|
||||
|
||||
await handle.close();
|
||||
handle = createDb(process.env['DATABASE_URL']);
|
||||
const reopened = new ConnectorLeaseCoordinator(new ConnectorLeaseRepository(handle.db));
|
||||
const persisted = await reopened.current({ identity, bindingId: 'operator-chat' });
|
||||
expect(persisted).toMatchObject({
|
||||
leaseId: acquired.value.leaseId,
|
||||
leaseEpoch: '1',
|
||||
});
|
||||
|
||||
const takeover = await reopened.takeover({
|
||||
...command,
|
||||
connectorId: 'connector-c',
|
||||
correlationId: 'postgres-takeover',
|
||||
expectedEpoch: acquired.value.leaseEpoch,
|
||||
});
|
||||
expect(takeover).toMatchObject({ connectorId: 'connector-c', leaseEpoch: '2' });
|
||||
});
|
||||
});
|
||||
@@ -1,149 +0,0 @@
|
||||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
connectorLeaseAuditLog,
|
||||
createPgliteDb,
|
||||
eq,
|
||||
runPgliteMigrations,
|
||||
type DbHandle,
|
||||
} from '@mosaicstack/db';
|
||||
import { ConnectorLeaseCoordinator, ConnectorLeaseError } from '@mosaicstack/agent';
|
||||
import { ConnectorLeaseRepository } from './connector-lease.repository.js';
|
||||
|
||||
const identity = { tenantId: 'tenant-a', logicalAgentId: 'mos' } as const;
|
||||
|
||||
function acquireCommand(connectorId: string, correlationId: string) {
|
||||
return {
|
||||
identity,
|
||||
bindingId: 'operator-chat',
|
||||
connectorId,
|
||||
scopes: ['runtime.send', 'tool.execute'],
|
||||
ttlMs: 60_000,
|
||||
correlationId,
|
||||
};
|
||||
}
|
||||
|
||||
describe('ConnectorLeaseRepository PostgreSQL semantics', (): void => {
|
||||
let dataDir: string;
|
||||
let handle: DbHandle;
|
||||
let now: Date;
|
||||
let coordinator: ConnectorLeaseCoordinator;
|
||||
|
||||
beforeEach(async (): Promise<void> => {
|
||||
dataDir = await mkdtemp(join(tmpdir(), 'mosaic-connector-lease-'));
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
now = new Date('2026-07-14T17:00:00.000Z');
|
||||
coordinator = new ConnectorLeaseCoordinator(new ConnectorLeaseRepository(handle.db), {
|
||||
now: (): Date => now,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
await handle.close();
|
||||
await rm(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('allows only one concurrent contender to acquire a binding', async (): Promise<void> => {
|
||||
const outcomes = await Promise.allSettled([
|
||||
coordinator.acquire(acquireCommand('connector-a', 'correlation-a')),
|
||||
coordinator.acquire(acquireCommand('connector-b', 'correlation-b')),
|
||||
]);
|
||||
|
||||
expect(outcomes.filter((result) => result.status === 'fulfilled')).toHaveLength(1);
|
||||
const rejected = outcomes.find((result) => result.status === 'rejected');
|
||||
expect(rejected).toMatchObject({
|
||||
reason: { code: 'lease_held' } satisfies Partial<ConnectorLeaseError>,
|
||||
});
|
||||
});
|
||||
|
||||
it('uses compare-and-swap takeover and increments the fencing epoch monotonically', async (): Promise<void> => {
|
||||
const acquired = await coordinator.acquire(acquireCommand('connector-a', 'correlation-a'));
|
||||
const results = await Promise.allSettled([
|
||||
coordinator.takeover({
|
||||
...acquireCommand('connector-b', 'correlation-b'),
|
||||
expectedEpoch: acquired.leaseEpoch,
|
||||
}),
|
||||
coordinator.takeover({
|
||||
...acquireCommand('connector-c', 'correlation-c'),
|
||||
expectedEpoch: acquired.leaseEpoch,
|
||||
}),
|
||||
]);
|
||||
const winner = results.find((result) => result.status === 'fulfilled');
|
||||
|
||||
expect(results.filter((result) => result.status === 'fulfilled')).toHaveLength(1);
|
||||
expect(winner?.status === 'fulfilled' ? winner.value.leaseEpoch : null).toBe('2');
|
||||
expect(results.find((result) => result.status === 'rejected')).toMatchObject({
|
||||
reason: { code: 'cas_mismatch' } satisfies Partial<ConnectorLeaseError>,
|
||||
});
|
||||
});
|
||||
|
||||
it('heartbeats and releases only the current connector epoch', async (): Promise<void> => {
|
||||
const acquired = await coordinator.acquire(acquireCommand('connector-a', 'correlation-a'));
|
||||
now = new Date('2026-07-14T17:00:30.000Z');
|
||||
const renewed = await coordinator.heartbeat({
|
||||
lease: acquired,
|
||||
ttlMs: 120_000,
|
||||
correlationId: 'correlation-renew',
|
||||
});
|
||||
expect(renewed.expiresAt).toBe('2026-07-14T17:02:30.000Z');
|
||||
|
||||
await coordinator.release({ lease: renewed, correlationId: 'correlation-release' });
|
||||
await expect(
|
||||
coordinator.heartbeat({
|
||||
lease: renewed,
|
||||
ttlMs: 120_000,
|
||||
correlationId: 'correlation-stale',
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'lease_released' } satisfies Partial<ConnectorLeaseError>);
|
||||
});
|
||||
|
||||
it('survives close/reopen and requires CAS takeover to recover an expired lease', async (): Promise<void> => {
|
||||
const acquired = await coordinator.acquire(acquireCommand('connector-a', 'correlation-a'));
|
||||
await handle.close();
|
||||
|
||||
now = new Date('2026-07-14T17:02:00.000Z');
|
||||
handle = createPgliteDb(dataDir);
|
||||
await runPgliteMigrations(handle);
|
||||
coordinator = new ConnectorLeaseCoordinator(new ConnectorLeaseRepository(handle.db), {
|
||||
now: (): Date => now,
|
||||
});
|
||||
|
||||
await expect(
|
||||
coordinator.acquire(acquireCommand('connector-b', 'correlation-plain-acquire')),
|
||||
).rejects.toMatchObject({ code: 'takeover_required' } satisfies Partial<ConnectorLeaseError>);
|
||||
const recovered = await coordinator.takeover({
|
||||
...acquireCommand('connector-b', 'correlation-takeover'),
|
||||
expectedEpoch: acquired.leaseEpoch,
|
||||
});
|
||||
expect(recovered).toMatchObject({ connectorId: 'connector-b', leaseEpoch: '2' });
|
||||
});
|
||||
|
||||
it('writes credential-safe lifecycle and rejection audit records', async (): Promise<void> => {
|
||||
const acquired = await coordinator.acquire(acquireCommand('connector-a', 'correlation-a'));
|
||||
await coordinator.heartbeat({
|
||||
lease: acquired,
|
||||
ttlMs: 60_000,
|
||||
correlationId: 'correlation-renew',
|
||||
});
|
||||
await expect(
|
||||
coordinator.acquire(acquireCommand('connector-b', 'correlation-reject')),
|
||||
).rejects.toBeInstanceOf(ConnectorLeaseError);
|
||||
|
||||
const rows = await handle.db
|
||||
.select()
|
||||
.from(connectorLeaseAuditLog)
|
||||
.where(eq(connectorLeaseAuditLog.tenantId, identity.tenantId));
|
||||
expect(rows.map((row) => row.event)).toEqual(
|
||||
expect.arrayContaining(['acquire', 'renew', 'reject']),
|
||||
);
|
||||
const serialized = JSON.stringify(rows, (_key: string, value: unknown): unknown =>
|
||||
typeof value === 'bigint' ? value.toString(10) : value,
|
||||
);
|
||||
expect(serialized).not.toContain('tool.execute');
|
||||
expect(serialized).not.toContain('runtime.send');
|
||||
expect(serialized).not.toMatch(/token|secret|credential/i);
|
||||
});
|
||||
});
|
||||
@@ -1,354 +0,0 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import {
|
||||
and,
|
||||
connectorLeaseAuditLog,
|
||||
eq,
|
||||
gt,
|
||||
isNull,
|
||||
logicalAgentConnectorLeases,
|
||||
sql,
|
||||
type Db,
|
||||
} from '@mosaicstack/db';
|
||||
import { ConnectorLeaseError } from '@mosaicstack/agent';
|
||||
import type {
|
||||
ConnectorLease,
|
||||
ConnectorLeaseAcquireMutation,
|
||||
ConnectorLeaseAuditEvent,
|
||||
ConnectorLeaseHeartbeatMutation,
|
||||
ConnectorLeaseRejectReason,
|
||||
ConnectorLeaseReleaseMutation,
|
||||
ConnectorLeaseStore,
|
||||
ConnectorLeaseTakeoverMutation,
|
||||
LogicalAgentBinding,
|
||||
} from '@mosaicstack/types';
|
||||
import { DB } from '../database/database.module.js';
|
||||
|
||||
interface SuccessfulMutation {
|
||||
readonly ok: true;
|
||||
readonly lease: ConnectorLease;
|
||||
}
|
||||
|
||||
interface FailedMutation {
|
||||
readonly ok: false;
|
||||
readonly reason: ConnectorLeaseRejectReason;
|
||||
}
|
||||
|
||||
type MutationResult = SuccessfulMutation | FailedMutation;
|
||||
|
||||
@Injectable()
|
||||
export class ConnectorLeaseRepository implements ConnectorLeaseStore {
|
||||
constructor(@Inject(DB) private readonly db: Db) {}
|
||||
|
||||
async acquire(input: ConnectorLeaseAcquireMutation): Promise<ConnectorLease> {
|
||||
const result: MutationResult = await this.db.transaction(
|
||||
async (tx): Promise<MutationResult> => {
|
||||
const inserted = await tx
|
||||
.insert(logicalAgentConnectorLeases)
|
||||
.values({
|
||||
leaseId: input.leaseId,
|
||||
tenantId: input.identity.tenantId,
|
||||
logicalAgentId: input.identity.logicalAgentId,
|
||||
bindingId: input.bindingId,
|
||||
connectorId: input.connectorId,
|
||||
scopes: [...input.scopes],
|
||||
leaseEpoch: 1n,
|
||||
acquiredAt: new Date(input.now),
|
||||
heartbeatAt: new Date(input.now),
|
||||
expiresAt: new Date(input.expiresAt),
|
||||
updatedAt: new Date(input.now),
|
||||
})
|
||||
.onConflictDoNothing()
|
||||
.returning();
|
||||
const row = inserted[0];
|
||||
if (row) {
|
||||
const lease = toLease(row);
|
||||
await insertAudit(tx, lifecycleAudit(input, lease, 'acquire'));
|
||||
return { ok: true, lease };
|
||||
}
|
||||
|
||||
const current = await findRow(tx, input);
|
||||
if (current && current.expiresAt <= new Date(input.now) && !current.releasedAt) {
|
||||
await insertAudit(tx, lifecycleAudit(input, toLease(current), 'expiry'));
|
||||
}
|
||||
const reason: ConnectorLeaseRejectReason =
|
||||
current && (current.releasedAt || current.expiresAt <= new Date(input.now))
|
||||
? 'takeover_required'
|
||||
: 'lease_held';
|
||||
await insertAudit(tx, rejectionAudit(input, current ? toLease(current) : null, reason));
|
||||
return { ok: false, reason };
|
||||
},
|
||||
);
|
||||
return unwrap(result);
|
||||
}
|
||||
|
||||
async takeover(input: ConnectorLeaseTakeoverMutation): Promise<ConnectorLease> {
|
||||
const result: MutationResult = await this.db.transaction(
|
||||
async (tx): Promise<MutationResult> => {
|
||||
const current = await findRow(tx, input);
|
||||
if (!current || current.leaseEpoch.toString(10) !== input.expectedEpoch) {
|
||||
await insertAudit(
|
||||
tx,
|
||||
rejectionAudit(input, current ? toLease(current) : null, 'cas_mismatch'),
|
||||
);
|
||||
return { ok: false, reason: 'cas_mismatch' };
|
||||
}
|
||||
if (current.expiresAt <= new Date(input.now) && !current.releasedAt) {
|
||||
await insertAudit(tx, lifecycleAudit(input, toLease(current), 'expiry'));
|
||||
}
|
||||
const updated = await tx
|
||||
.update(logicalAgentConnectorLeases)
|
||||
.set({
|
||||
leaseId: input.leaseId,
|
||||
connectorId: input.connectorId,
|
||||
scopes: [...input.scopes],
|
||||
leaseEpoch: sql`${logicalAgentConnectorLeases.leaseEpoch} + 1`,
|
||||
acquiredAt: new Date(input.now),
|
||||
heartbeatAt: new Date(input.now),
|
||||
expiresAt: new Date(input.expiresAt),
|
||||
releasedAt: null,
|
||||
updatedAt: new Date(input.now),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
bindingPredicate(input),
|
||||
eq(logicalAgentConnectorLeases.leaseId, current.leaseId),
|
||||
eq(logicalAgentConnectorLeases.leaseEpoch, BigInt(input.expectedEpoch)),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
const row = updated[0];
|
||||
if (!row) {
|
||||
await insertAudit(tx, rejectionAudit(input, toLease(current), 'cas_mismatch'));
|
||||
return { ok: false, reason: 'cas_mismatch' };
|
||||
}
|
||||
const lease = toLease(row);
|
||||
await insertAudit(tx, lifecycleAudit(input, lease, 'takeover'));
|
||||
return { ok: true, lease };
|
||||
},
|
||||
);
|
||||
return unwrap(result);
|
||||
}
|
||||
|
||||
async heartbeat(input: ConnectorLeaseHeartbeatMutation): Promise<ConnectorLease> {
|
||||
const result: MutationResult = await this.db.transaction(
|
||||
async (tx): Promise<MutationResult> => {
|
||||
const updated = await tx
|
||||
.update(logicalAgentConnectorLeases)
|
||||
.set({
|
||||
heartbeatAt: new Date(input.now),
|
||||
expiresAt: new Date(input.expiresAt),
|
||||
updatedAt: new Date(input.now),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
bindingPredicate(input.lease),
|
||||
eq(logicalAgentConnectorLeases.leaseId, input.lease.leaseId),
|
||||
eq(logicalAgentConnectorLeases.connectorId, input.lease.connectorId),
|
||||
eq(logicalAgentConnectorLeases.leaseEpoch, BigInt(input.lease.leaseEpoch)),
|
||||
isNull(logicalAgentConnectorLeases.releasedAt),
|
||||
gt(logicalAgentConnectorLeases.expiresAt, new Date(input.now)),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
const row = updated[0];
|
||||
if (row) {
|
||||
const lease = toLease(row);
|
||||
await insertAudit(tx, lifecycleAudit(input, lease, 'renew'));
|
||||
return { ok: true, lease };
|
||||
}
|
||||
const current = await findRow(tx, input.lease);
|
||||
const reason = classifyAuthorityFailure(
|
||||
current ? toLease(current) : null,
|
||||
input.lease,
|
||||
input.now,
|
||||
);
|
||||
if (reason === 'lease_expired' && current) {
|
||||
await insertAudit(tx, lifecycleAudit(input, toLease(current), 'expiry'));
|
||||
}
|
||||
await insertAudit(
|
||||
tx,
|
||||
rejectionAudit(
|
||||
{ ...input.lease, correlationId: input.correlationId, now: input.now },
|
||||
current ? toLease(current) : null,
|
||||
reason,
|
||||
),
|
||||
);
|
||||
return { ok: false, reason };
|
||||
},
|
||||
);
|
||||
return unwrap(result);
|
||||
}
|
||||
|
||||
async release(input: ConnectorLeaseReleaseMutation): Promise<void> {
|
||||
const result: MutationResult = await this.db.transaction(
|
||||
async (tx): Promise<MutationResult> => {
|
||||
const updated = await tx
|
||||
.update(logicalAgentConnectorLeases)
|
||||
.set({
|
||||
releasedAt: new Date(input.now),
|
||||
expiresAt: new Date(input.now),
|
||||
updatedAt: new Date(input.now),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
bindingPredicate(input.lease),
|
||||
eq(logicalAgentConnectorLeases.leaseId, input.lease.leaseId),
|
||||
eq(logicalAgentConnectorLeases.connectorId, input.lease.connectorId),
|
||||
eq(logicalAgentConnectorLeases.leaseEpoch, BigInt(input.lease.leaseEpoch)),
|
||||
isNull(logicalAgentConnectorLeases.releasedAt),
|
||||
gt(logicalAgentConnectorLeases.expiresAt, new Date(input.now)),
|
||||
),
|
||||
)
|
||||
.returning();
|
||||
const row = updated[0];
|
||||
if (row) {
|
||||
const lease = toLease(row);
|
||||
await insertAudit(tx, lifecycleAudit(input, lease, 'release'));
|
||||
return { ok: true, lease };
|
||||
}
|
||||
const current = await findRow(tx, input.lease);
|
||||
const reason = classifyAuthorityFailure(
|
||||
current ? toLease(current) : null,
|
||||
input.lease,
|
||||
input.now,
|
||||
);
|
||||
if (reason === 'lease_expired' && current) {
|
||||
await insertAudit(tx, lifecycleAudit(input, toLease(current), 'expiry'));
|
||||
}
|
||||
await insertAudit(
|
||||
tx,
|
||||
rejectionAudit(
|
||||
{ ...input.lease, correlationId: input.correlationId, now: input.now },
|
||||
current ? toLease(current) : null,
|
||||
reason,
|
||||
),
|
||||
);
|
||||
return { ok: false, reason };
|
||||
},
|
||||
);
|
||||
unwrap(result);
|
||||
}
|
||||
|
||||
async findCurrent(binding: LogicalAgentBinding): Promise<ConnectorLease | null> {
|
||||
const row = await findRow(this.db, binding);
|
||||
return row ? toLease(row) : null;
|
||||
}
|
||||
|
||||
async recordAudit(event: ConnectorLeaseAuditEvent): Promise<void> {
|
||||
await insertAudit(this.db, event);
|
||||
}
|
||||
}
|
||||
|
||||
function unwrap(result: MutationResult): ConnectorLease {
|
||||
if (!result.ok) throw new ConnectorLeaseError(result.reason, safeErrorMessage(result.reason));
|
||||
return result.lease;
|
||||
}
|
||||
|
||||
function safeErrorMessage(reason: ConnectorLeaseRejectReason): string {
|
||||
return `Connector lease mutation denied: ${reason}`;
|
||||
}
|
||||
|
||||
function bindingPredicate(binding: LogicalAgentBinding) {
|
||||
return and(
|
||||
eq(logicalAgentConnectorLeases.tenantId, binding.identity.tenantId),
|
||||
eq(logicalAgentConnectorLeases.logicalAgentId, binding.identity.logicalAgentId),
|
||||
eq(logicalAgentConnectorLeases.bindingId, binding.bindingId),
|
||||
);
|
||||
}
|
||||
|
||||
async function findRow(
|
||||
db: Pick<Db, 'select'>,
|
||||
binding: LogicalAgentBinding,
|
||||
): Promise<typeof logicalAgentConnectorLeases.$inferSelect | null> {
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(logicalAgentConnectorLeases)
|
||||
.where(bindingPredicate(binding))
|
||||
.limit(1);
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
function toLease(row: typeof logicalAgentConnectorLeases.$inferSelect): ConnectorLease {
|
||||
return Object.freeze({
|
||||
identity: Object.freeze({ tenantId: row.tenantId, logicalAgentId: row.logicalAgentId }),
|
||||
bindingId: row.bindingId,
|
||||
leaseId: row.leaseId,
|
||||
connectorId: row.connectorId,
|
||||
scopes: Object.freeze([...row.scopes]),
|
||||
leaseEpoch: row.leaseEpoch.toString(10),
|
||||
acquiredAt: row.acquiredAt.toISOString(),
|
||||
heartbeatAt: row.heartbeatAt.toISOString(),
|
||||
expiresAt: row.expiresAt.toISOString(),
|
||||
...(row.releasedAt ? { releasedAt: row.releasedAt.toISOString() } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function classifyAuthorityFailure(
|
||||
current: ConnectorLease | null,
|
||||
claimed: ConnectorLease,
|
||||
now: string,
|
||||
): ConnectorLeaseRejectReason {
|
||||
if (!current) return 'lease_missing';
|
||||
if (current.releasedAt) return 'lease_released';
|
||||
if (new Date(current.expiresAt) <= new Date(now)) return 'lease_expired';
|
||||
if (current.leaseEpoch !== claimed.leaseEpoch) return 'stale_epoch';
|
||||
return 'connector_mismatch';
|
||||
}
|
||||
|
||||
function lifecycleAudit(
|
||||
input: { readonly correlationId: string; readonly now: string },
|
||||
lease: ConnectorLease,
|
||||
event: Exclude<ConnectorLeaseAuditEvent['event'], 'reject'>,
|
||||
): ConnectorLeaseAuditEvent {
|
||||
return {
|
||||
identity: lease.identity,
|
||||
bindingId: lease.bindingId,
|
||||
connectorId: lease.connectorId,
|
||||
leaseId: lease.leaseId,
|
||||
leaseEpoch: lease.leaseEpoch,
|
||||
event,
|
||||
outcome: 'succeeded',
|
||||
correlationId: input.correlationId,
|
||||
occurredAt: input.now,
|
||||
};
|
||||
}
|
||||
|
||||
function rejectionAudit(
|
||||
input: {
|
||||
readonly identity: ConnectorLease['identity'];
|
||||
readonly bindingId: string;
|
||||
readonly connectorId: string;
|
||||
readonly correlationId: string;
|
||||
readonly now: string;
|
||||
},
|
||||
current: ConnectorLease | null,
|
||||
reason: ConnectorLeaseRejectReason,
|
||||
): ConnectorLeaseAuditEvent {
|
||||
return {
|
||||
identity: input.identity,
|
||||
bindingId: input.bindingId,
|
||||
connectorId: input.connectorId,
|
||||
event: 'reject',
|
||||
outcome: 'denied',
|
||||
correlationId: input.correlationId,
|
||||
occurredAt: input.now,
|
||||
...(current ? { leaseId: current.leaseId, leaseEpoch: current.leaseEpoch } : {}),
|
||||
reason,
|
||||
};
|
||||
}
|
||||
|
||||
async function insertAudit(db: Pick<Db, 'insert'>, event: ConnectorLeaseAuditEvent): Promise<void> {
|
||||
await db.insert(connectorLeaseAuditLog).values({
|
||||
tenantId: event.identity.tenantId,
|
||||
logicalAgentId: event.identity.logicalAgentId,
|
||||
bindingId: event.bindingId,
|
||||
connectorId: event.connectorId,
|
||||
...(event.leaseId ? { leaseId: event.leaseId } : {}),
|
||||
...(event.leaseEpoch ? { leaseEpoch: BigInt(event.leaseEpoch) } : {}),
|
||||
event: event.event,
|
||||
outcome: event.outcome,
|
||||
...(event.reason ? { reason: event.reason } : {}),
|
||||
correlationId: event.correlationId,
|
||||
occurredAt: new Date(event.occurredAt),
|
||||
});
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
import { ForbiddenException, Inject, Injectable } from '@nestjs/common';
|
||||
import { ConnectorLeaseCoordinator, normalizeConnectorLease } from '@mosaicstack/agent';
|
||||
import {
|
||||
normalizeConnectorId,
|
||||
normalizeConnectorScopes,
|
||||
normalizeCorrelationId,
|
||||
normalizeLogicalAgentIdentity,
|
||||
normalizeLogicalBindingId,
|
||||
type AcquireConnectorLeaseInput,
|
||||
type ConnectorExecutionGrant,
|
||||
type ConnectorLease,
|
||||
type ConnectorLeaseAuditEvent,
|
||||
type FencedConnectorAdapter,
|
||||
} from '@mosaicstack/types';
|
||||
import type { ActorTenantScope } from '../auth/session-scope.js';
|
||||
import { ConnectorLeaseRepository } from './connector-lease.repository.js';
|
||||
|
||||
export const CONNECTOR_LEASE_POLICY = Symbol('CONNECTOR_LEASE_POLICY');
|
||||
|
||||
export type ConnectorLeasePolicyAction =
|
||||
| 'lease.acquire'
|
||||
| 'lease.takeover'
|
||||
| 'lease.heartbeat'
|
||||
| 'lease.release'
|
||||
| 'lease.read'
|
||||
| 'grant.issue';
|
||||
|
||||
export interface ConnectorLeaseRequestContext {
|
||||
readonly actorScope: ActorTenantScope;
|
||||
readonly correlationId: string;
|
||||
}
|
||||
|
||||
export interface GatewayConnectorLeaseRequest {
|
||||
readonly logicalAgentId: string;
|
||||
readonly bindingId: string;
|
||||
readonly connectorId: string;
|
||||
readonly scopes: readonly string[];
|
||||
readonly ttlMs: number;
|
||||
}
|
||||
|
||||
export interface GatewayConnectorLeaseTakeoverRequest extends GatewayConnectorLeaseRequest {
|
||||
readonly expectedEpoch: string;
|
||||
}
|
||||
|
||||
export interface GatewayConnectorGrantRequest {
|
||||
readonly lease: ConnectorLease;
|
||||
readonly scopes: readonly string[];
|
||||
readonly ttlMs: number;
|
||||
}
|
||||
|
||||
export interface ConnectorLeasePolicySubject {
|
||||
readonly action: ConnectorLeasePolicyAction;
|
||||
readonly actorId: string;
|
||||
readonly tenantId: string;
|
||||
readonly logicalAgentId: string;
|
||||
readonly bindingId: string;
|
||||
readonly connectorId: string;
|
||||
readonly requestedScopes: readonly string[];
|
||||
readonly requestedTtlMs: number | null;
|
||||
}
|
||||
|
||||
export interface ConnectorLeasePolicy {
|
||||
authorize(subject: ConnectorLeasePolicySubject): Promise<boolean>;
|
||||
}
|
||||
|
||||
/** M1 has no concrete cutover policy: unconfigured production use fails closed. */
|
||||
@Injectable()
|
||||
export class DenyConnectorLeasePolicy implements ConnectorLeasePolicy {
|
||||
async authorize(_subject: ConnectorLeasePolicySubject): Promise<boolean> {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Gateway-owned policy surface for durable connector authority and fenced effects. */
|
||||
@Injectable()
|
||||
export class ConnectorLeaseService {
|
||||
private readonly coordinator: ConnectorLeaseCoordinator;
|
||||
|
||||
constructor(
|
||||
@Inject(ConnectorLeaseRepository) private readonly repository: ConnectorLeaseRepository,
|
||||
@Inject(CONNECTOR_LEASE_POLICY) private readonly policy: ConnectorLeasePolicy,
|
||||
) {
|
||||
this.coordinator = new ConnectorLeaseCoordinator(repository);
|
||||
}
|
||||
|
||||
async acquire(
|
||||
request: GatewayConnectorLeaseRequest,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
): Promise<ConnectorLease> {
|
||||
const command = this.command(request, context);
|
||||
await this.assertPolicy('lease.acquire', command, context, command.scopes, command.ttlMs);
|
||||
return this.coordinator.acquire({ ...command, correlationId: this.correlation(context) });
|
||||
}
|
||||
|
||||
async takeover(
|
||||
request: GatewayConnectorLeaseTakeoverRequest,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
): Promise<ConnectorLease> {
|
||||
const command = this.command(request, context);
|
||||
await this.assertPolicy('lease.takeover', command, context, command.scopes, command.ttlMs);
|
||||
return this.coordinator.takeover({
|
||||
...command,
|
||||
expectedEpoch: request.expectedEpoch,
|
||||
correlationId: this.correlation(context),
|
||||
});
|
||||
}
|
||||
|
||||
async heartbeat(
|
||||
lease: ConnectorLease,
|
||||
ttlMs: number,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
): Promise<ConnectorLease> {
|
||||
const normalizedLease = normalizeConnectorLease(lease);
|
||||
await this.assertTenant(normalizedLease, context);
|
||||
await this.assertPolicy(
|
||||
'lease.heartbeat',
|
||||
normalizedLease,
|
||||
context,
|
||||
normalizedLease.scopes,
|
||||
ttlMs,
|
||||
);
|
||||
return this.coordinator.heartbeat({
|
||||
lease: normalizedLease,
|
||||
ttlMs,
|
||||
correlationId: this.correlation(context),
|
||||
});
|
||||
}
|
||||
|
||||
async release(lease: ConnectorLease, context: ConnectorLeaseRequestContext): Promise<void> {
|
||||
const normalizedLease = normalizeConnectorLease(lease);
|
||||
await this.assertTenant(normalizedLease, context);
|
||||
await this.assertPolicy(
|
||||
'lease.release',
|
||||
normalizedLease,
|
||||
context,
|
||||
normalizedLease.scopes,
|
||||
null,
|
||||
);
|
||||
await this.coordinator.release({
|
||||
lease: normalizedLease,
|
||||
correlationId: this.correlation(context),
|
||||
});
|
||||
}
|
||||
|
||||
async current(
|
||||
logicalAgentId: string,
|
||||
bindingId: string,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
): Promise<ConnectorLease | null> {
|
||||
const binding = {
|
||||
identity: normalizeLogicalAgentIdentity({
|
||||
tenantId: context.actorScope.tenantId,
|
||||
logicalAgentId,
|
||||
}),
|
||||
bindingId: normalizeLogicalBindingId(bindingId),
|
||||
connectorId: 'gateway',
|
||||
};
|
||||
await this.assertPolicy('lease.read', binding, context, [], null);
|
||||
return this.coordinator.current(binding);
|
||||
}
|
||||
|
||||
async issueGrant(
|
||||
request: GatewayConnectorGrantRequest,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
): Promise<ConnectorExecutionGrant> {
|
||||
const lease = normalizeConnectorLease(request.lease);
|
||||
await this.assertTenant(lease, context);
|
||||
const scopes = normalizeConnectorScopes(request.scopes);
|
||||
await this.assertPolicy('grant.issue', lease, context, scopes, request.ttlMs);
|
||||
return this.coordinator.issueGrant({
|
||||
lease,
|
||||
scopes,
|
||||
ttlMs: request.ttlMs,
|
||||
correlationId: this.correlation(context),
|
||||
});
|
||||
}
|
||||
|
||||
async executeGrant<TInput, TOutput>(
|
||||
grant: ConnectorExecutionGrant,
|
||||
requiredScope: string,
|
||||
input: TInput,
|
||||
adapter: FencedConnectorAdapter<TInput, TOutput>,
|
||||
): Promise<TOutput> {
|
||||
return this.coordinator.executeGrant(grant, requiredScope, input, adapter);
|
||||
}
|
||||
|
||||
private command(
|
||||
request: GatewayConnectorLeaseRequest,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
): Omit<AcquireConnectorLeaseInput, 'correlationId'> {
|
||||
return {
|
||||
identity: normalizeLogicalAgentIdentity({
|
||||
tenantId: context.actorScope.tenantId,
|
||||
logicalAgentId: request.logicalAgentId,
|
||||
}),
|
||||
bindingId: normalizeLogicalBindingId(request.bindingId),
|
||||
connectorId: normalizeConnectorId(request.connectorId),
|
||||
scopes: normalizeConnectorScopes(request.scopes),
|
||||
ttlMs: request.ttlMs,
|
||||
};
|
||||
}
|
||||
|
||||
private async assertTenant(
|
||||
lease: Pick<ConnectorLease, 'identity' | 'bindingId' | 'connectorId'>,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
): Promise<void> {
|
||||
if (lease.identity.tenantId !== context.actorScope.tenantId) {
|
||||
await this.recordPolicyDenial(
|
||||
{
|
||||
identity: {
|
||||
tenantId: context.actorScope.tenantId,
|
||||
logicalAgentId: 'untrusted',
|
||||
},
|
||||
bindingId: 'untrusted',
|
||||
connectorId: 'untrusted',
|
||||
},
|
||||
context,
|
||||
);
|
||||
throw new ForbiddenException('Connector authority tenant scope denied');
|
||||
}
|
||||
}
|
||||
|
||||
private async assertPolicy(
|
||||
action: ConnectorLeasePolicyAction,
|
||||
subject: Pick<ConnectorLease, 'identity' | 'bindingId' | 'connectorId'>,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
requestedScopes: readonly string[],
|
||||
requestedTtlMs: number | null,
|
||||
): Promise<void> {
|
||||
const allowed = await this.policy.authorize({
|
||||
action,
|
||||
actorId: context.actorScope.userId,
|
||||
tenantId: subject.identity.tenantId,
|
||||
logicalAgentId: subject.identity.logicalAgentId,
|
||||
bindingId: subject.bindingId,
|
||||
connectorId: subject.connectorId,
|
||||
requestedScopes: Object.freeze([...requestedScopes]),
|
||||
requestedTtlMs,
|
||||
});
|
||||
if (!allowed) {
|
||||
await this.recordPolicyDenial(subject, context);
|
||||
throw new ForbiddenException('Connector authority policy denied');
|
||||
}
|
||||
}
|
||||
|
||||
private async recordPolicyDenial(
|
||||
subject: Pick<ConnectorLease, 'identity' | 'bindingId' | 'connectorId'>,
|
||||
context: ConnectorLeaseRequestContext,
|
||||
): Promise<void> {
|
||||
const event: ConnectorLeaseAuditEvent = {
|
||||
identity: subject.identity,
|
||||
bindingId: subject.bindingId,
|
||||
connectorId: subject.connectorId,
|
||||
event: 'reject',
|
||||
outcome: 'denied',
|
||||
reason: 'policy_denied',
|
||||
correlationId: this.correlation(context),
|
||||
occurredAt: new Date().toISOString(),
|
||||
};
|
||||
await this.repository.recordAudit(event);
|
||||
}
|
||||
|
||||
private correlation(context: ConnectorLeaseRequestContext): string {
|
||||
return normalizeCorrelationId(context.correlationId);
|
||||
}
|
||||
}
|
||||
@@ -115,38 +115,6 @@ describe('Hermes runtime provider reachability', (): void => {
|
||||
await app?.close();
|
||||
});
|
||||
|
||||
it('returns gateway denial responses from the actual guarded interaction routes', async (): Promise<void> => {
|
||||
if (!app) throw new Error('Nest application did not initialize');
|
||||
|
||||
const attachDenied = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/interaction/Nova/sessions/session-1/attach',
|
||||
headers: { 'x-correlation-id': 'correlation-1' },
|
||||
payload: { mode: 'read' },
|
||||
});
|
||||
expect(attachDenied.statusCode).toBe(401);
|
||||
|
||||
const sendDenied = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/interaction/Nova/sessions/session-1/send',
|
||||
headers: { cookie: 'session=trusted', 'x-correlation-id': 'correlation-1' },
|
||||
payload: {},
|
||||
});
|
||||
expect(sendDenied.statusCode).toBe(403);
|
||||
expect(sendDenied.json()).toMatchObject({
|
||||
message: 'Content and idempotency key are required',
|
||||
});
|
||||
|
||||
const stopDenied = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/interaction/Nova/sessions/session-1/stop',
|
||||
headers: { cookie: 'session=trusted', 'x-correlation-id': 'correlation-1' },
|
||||
payload: {},
|
||||
});
|
||||
expect(stopDenied.statusCode).toBe(403);
|
||||
expect(stopDenied.json()).toMatchObject({ message: 'Exact-action approval is required' });
|
||||
});
|
||||
|
||||
it('requires authentication and reaches the Hermes provider registered by AgentModule', async (): Promise<void> => {
|
||||
if (!app) throw new Error('Nest application did not initialize');
|
||||
const registry = app.get(AGENT_RUNTIME_PROVIDER_REGISTRY);
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
import 'reflect-metadata';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { FastifyAdapter, type NestFastifyApplication } from '@nestjs/platform-fastify';
|
||||
import { AUTH } from '../auth/auth.tokens.js';
|
||||
import { AuthGuard } from '../auth/auth.guard.js';
|
||||
import { InteractionCoordinationController } from './interaction-coordination.controller.js';
|
||||
import { InteractionCoordinationService } from './interaction-coordination.service.js';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [
|
||||
{
|
||||
provide: AUTH,
|
||||
useValue: {
|
||||
api: {
|
||||
getSession: vi.fn(async ({ headers }: { headers: Headers }) =>
|
||||
headers.get('cookie') === 'session=trusted'
|
||||
? { user: { id: 'operator-1', tenantId: 'tenant-1' }, session: { id: 'session-1' } }
|
||||
: null,
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
AuthGuard,
|
||||
],
|
||||
exports: [AUTH, AuthGuard],
|
||||
})
|
||||
class AuthenticatedRequestModule {}
|
||||
|
||||
describe('InteractionCoordinationController route aliases', (): void => {
|
||||
let app: NestFastifyApplication | undefined;
|
||||
const coordination = {
|
||||
handoff: vi.fn(async () => ({ handoffId: 'handoff-1' })),
|
||||
observe: vi.fn(async () => ({ status: 'running' })),
|
||||
result: vi.fn(async () => ({ status: 'completed' })),
|
||||
};
|
||||
|
||||
beforeAll(async (): Promise<void> => {
|
||||
const moduleRef = await Test.createTestingModule({
|
||||
imports: [AuthenticatedRequestModule],
|
||||
controllers: [InteractionCoordinationController],
|
||||
providers: [{ provide: InteractionCoordinationService, useValue: coordination }],
|
||||
}).compile();
|
||||
app = moduleRef.createNestApplication<NestFastifyApplication>(new FastifyAdapter());
|
||||
await app.init();
|
||||
await app.getHttpAdapter().getInstance().ready();
|
||||
});
|
||||
afterAll(async (): Promise<void> => app?.close());
|
||||
|
||||
it('routes handoff, observe, and result through the same AuthGuard-protected service for both prefixes', async (): Promise<void> => {
|
||||
if (!app) throw new Error('test app was not initialized');
|
||||
for (const prefix of ['/api/coord/interaction', '/api/coord/mos']) {
|
||||
const headers = { cookie: 'session=trusted', 'x-correlation-id': `corr-${prefix}` };
|
||||
expect(
|
||||
(
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: `${prefix}/handoff`,
|
||||
headers,
|
||||
payload: { idempotencyKey: `key-${prefix}`, summary: 'handoff' },
|
||||
})
|
||||
).statusCode,
|
||||
).toBe(201);
|
||||
expect(
|
||||
(await app.inject({ method: 'GET', url: `${prefix}/handoff-1/observe`, headers }))
|
||||
.statusCode,
|
||||
).toBe(200);
|
||||
expect(
|
||||
(await app.inject({ method: 'GET', url: `${prefix}/handoff-1/result`, headers }))
|
||||
.statusCode,
|
||||
).toBe(200);
|
||||
}
|
||||
expect(coordination.handoff).toHaveBeenCalledTimes(2);
|
||||
expect(coordination.observe).toHaveBeenCalledTimes(2);
|
||||
expect(coordination.result).toHaveBeenCalledTimes(2);
|
||||
expect(
|
||||
(
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/coord/interaction/handoff',
|
||||
payload: { idempotencyKey: 'denied', summary: 'x' },
|
||||
})
|
||||
).statusCode,
|
||||
).toBe(401);
|
||||
});
|
||||
});
|
||||
31
docs/PRD.md
31
docs/PRD.md
@@ -175,37 +175,6 @@ Delivery uses five gated milestones: runtime contracts/security; Pi service/stat
|
||||
|
||||
---
|
||||
|
||||
## Mos Runtime Portability Workstream (MOS-PORT)
|
||||
|
||||
### Problem and Objective
|
||||
|
||||
Mos is currently identified partly by a harness-native session and communication process. Replacement/rebinding exists, but no gateway-enforced logical identity or fencing prevents a stale harness from continuing to reply or execute effects after takeover.
|
||||
|
||||
The objective is to make Mos a server-derived logical Mosaic identity whose authority can move safely among runtime connectors. The gateway owns identity, lease, policy, and audit; harnesses remain replaceable adapters.
|
||||
|
||||
### M1 Requirements
|
||||
|
||||
1. `MOS-PORT-ID-001`: Define a normalized logical-agent identity independent of Claude Code, Pi, Codex, tmux, Matrix, and provider-native session IDs.
|
||||
2. `MOS-PORT-LEASE-001`: Persist one exclusive connector lease per tenant/logical-agent/binding with CAS acquisition, monotonic fencing epoch, TTL, heartbeat, explicit release, and takeover.
|
||||
3. `MOS-PORT-FENCE-001`: Bind every connector dispatch/execution grant to the current server-derived tenant, logical identity, binding, connector, scopes, expiry, and lease epoch.
|
||||
4. `MOS-PORT-FENCE-002`: Reject and audit stale, expired, forged, cross-tenant, cross-binding, and unauthorized grants before connector, channel, provider, or tool side effects.
|
||||
5. `MOS-PORT-OBS-001`: Emit credential-safe correlation/audit events for lease acquire, renew, takeover, reject, release, and expiry.
|
||||
6. `MOS-PORT-ARCH-001`: Runtime/provider adapters consume normalized lease context without adding harness-native schemas to Mosaic core.
|
||||
|
||||
### M1 Acceptance Criteria
|
||||
|
||||
1. `AC-MOS-PORT-01`: Two contenders for one binding cannot simultaneously hold current authority under concurrency.
|
||||
2. `AC-MOS-PORT-02`: Successful takeover increments the fencing epoch and every operation from the old epoch fails closed before side effects.
|
||||
3. `AC-MOS-PORT-03`: Gateway/database restart preserves lease and epoch state; expired leases can be recovered only through the authorized takeover path.
|
||||
4. `AC-MOS-PORT-04`: Cross-tenant, cross-agent, cross-binding, forged, and expired lease/grant cases are denied and audited.
|
||||
5. `AC-MOS-PORT-05`: Unit, migration, repository close/reopen, concurrency, abuse, gateway integration, independent security review, CI, and documentation gates pass.
|
||||
|
||||
### Deferred to Later #754 Milestones
|
||||
|
||||
Canonical checkpoint/handoff payloads, exactly-once connector receipts, concrete Claude/Pi/Codex adapters, channel cutover, and full cross-harness failover/rollback E2E are explicitly out of M1 scope.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
### High-Level System Diagram
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
# Documentation Sitemap
|
||||
|
||||
## Mos runtime portability
|
||||
|
||||
- [M1 logical identity and fencing architecture](architecture/mos-runtime-portability-m1.md)
|
||||
- [M1 connector lease operations](guides/mos-connector-lease-operations.md)
|
||||
|
||||
## Tess interaction agent
|
||||
|
||||
### Operator guides
|
||||
|
||||
- [User guide](tess/USER-GUIDE.md) — authorized session, attach, send, stop, and handoff workflows.
|
||||
- [Admin guide](tess/ADMIN-GUIDE.md) — deployment configuration, policy, and approval controls.
|
||||
- [Developer guide](tess/DEVELOPER-GUIDE.md) — provider contracts, scope boundaries, and test workflow.
|
||||
- [Plugin guide](tess/PLUGIN-GUIDE.md) — adapter, redaction, and identity-as-data requirements.
|
||||
- [Operations guide](tess/OPERATIONS-GUIDE.md) — readiness, recovery, and incident-safe procedures.
|
||||
|
||||
### Architecture and security
|
||||
|
||||
- [Architecture](tess/ARCHITECTURE.md)
|
||||
- [Threat model](tess/THREAT-MODEL.md)
|
||||
- [Mos coordination boundary](tess/MOS-COORDINATION.md)
|
||||
- [Hermes runtime adapter design](tess/hermes-runtime-adapter-design.md)
|
||||
- [Operator plugin sketch](tess/M4-003-OPERATOR-PLUGIN-SKETCH.md)
|
||||
|
||||
### API contract
|
||||
|
||||
- [Tess OpenAPI contract](openapi-tess.yaml)
|
||||
|
||||
### Migration and qualification
|
||||
|
||||
- [Migration inventory](tess/M5-MIGRATION-INVENTORY.md)
|
||||
- [Cutover procedure](tess/M5-MIGRATION-CUTOVER.md)
|
||||
- [Rollback procedure](tess/M5-MIGRATION-ROLLBACK.md)
|
||||
- [Retention and deprecation evidence](tess/M5-MIGRATION-RETENTION-DEPRECATION.md)
|
||||
- [Verification matrix](tess/VERIFICATION-MATRIX.md)
|
||||
- [Documentation checklist](tess/M5-003-DOCUMENTATION-CHECKLIST.md)
|
||||
@@ -1,49 +0,0 @@
|
||||
# Mos Runtime Portability M1 — Logical Identity and Fencing
|
||||
|
||||
## Boundary
|
||||
|
||||
M1 separates the logical Mosaic agent from any Claude, Pi, Codex, tmux, Matrix, or provider-native session. The normalized identity is:
|
||||
|
||||
```text
|
||||
(tenant_id, logical_agent_id, binding_id)
|
||||
```
|
||||
|
||||
`logical_agent_id` is a server-owned stable identifier. A connector is a replaceable holder of a lease for one binding; it is not the agent identity.
|
||||
|
||||
## Durable lease model
|
||||
|
||||
PostgreSQL table `logical_agent_connector_leases` has one unique row per identity/binding tuple. The current row records:
|
||||
|
||||
- an opaque lease UUID;
|
||||
- connector ID and normalized allowed scopes;
|
||||
- a positive decimal fencing epoch stored as PostgreSQL `bigint`;
|
||||
- acquired, heartbeat, expiry, release, and update timestamps.
|
||||
|
||||
Initial acquisition is insert-only. An existing active row causes `lease_held`. An expired or released row causes `takeover_required`; ordinary acquisition cannot recover it. Authorized takeover uses compare-and-swap against the expected epoch, rotates the lease UUID, and increments the epoch atomically. Heartbeat and release match the full identity, binding, connector, lease UUID, and epoch.
|
||||
|
||||
The companion `connector_lease_audit_log` is append-only metadata. It stores lifecycle event, outcome/reason, identity/binding/connector, epoch, correlation ID, and timestamp. It deliberately excludes scopes, grant objects, payloads, approval references, tokens, and credentials.
|
||||
|
||||
## Execution grants
|
||||
|
||||
`ConnectorLeaseCoordinator` issues a short-lived internal grant only after rereading the durable current lease. Defense-in-depth caps leases at 5 minutes and grants at 30 seconds by default; constructor options may tighten these limits. A grant is bound to tenant, logical agent, binding, connector, lease UUID, scope subset, expiry, and epoch.
|
||||
|
||||
Validation occurs immediately before adapter invocation and rereads PostgreSQL. The adapter receives only `ConnectorExecutionContext`; harness-native schemas remain behind the adapter. Validation denies:
|
||||
|
||||
- grants not minted by the current gateway process (including cloned/forged objects);
|
||||
- expired grants or leases;
|
||||
- released leases;
|
||||
- stale epochs or replaced connector/lease UUIDs;
|
||||
- missing/cross-tenant/cross-agent/cross-binding leases;
|
||||
- scopes not authorized by both grant and current lease.
|
||||
|
||||
A gateway restart intentionally invalidates process-local grants. The durable lease and epoch survive, and a fresh grant may be issued only after current-lease and gateway-policy validation.
|
||||
|
||||
## Concurrency and side-effect rule
|
||||
|
||||
The database CAS determines the sole current holder. A successful takeover makes every old-epoch validation fail. Connector adapters must consume and propagate the normalized lease epoch/context so downstream effect boundaries can also fence races that occur after gateway validation.
|
||||
|
||||
M1 does not provide exactly-once receipts or a side-effect journal. Those remain later #754 work; callers must not infer exactly-once delivery from lease fencing.
|
||||
|
||||
## Extension boundary
|
||||
|
||||
`ConnectorLeaseService` is the gateway-owned policy surface. Every policy decision receives the normalized requested scopes and TTL (or explicit `null` where no TTL applies), so a concrete policy can enforce least privilege and duration limits. Its production default policy denies every lease/grant operation until a server-configured connector policy is supplied. No M1 HTTP endpoint accepts caller-controlled tenant or logical identity, and no concrete Claude/Pi/Codex adapter or channel cutover is included.
|
||||
@@ -1,43 +0,0 @@
|
||||
# Mos Connector Lease Operations — M1
|
||||
|
||||
## Operational status
|
||||
|
||||
M1 installs the durable schema and gateway policy/adapter boundary. It does **not** activate a connector, expose a lease administration endpoint, or cut over a channel. The default gateway connector-lease policy is deny-all until a later work package supplies an authorized server-side policy and concrete adapter.
|
||||
|
||||
## Events to monitor
|
||||
|
||||
Use correlation IDs to follow `connector_lease_audit_log` events:
|
||||
|
||||
| Event | Meaning |
|
||||
| ---------- | --------------------------------------------------------------------- |
|
||||
| `acquire` | First holder inserted for an unused binding |
|
||||
| `renew` | Current holder heartbeat extended the TTL |
|
||||
| `takeover` | Authorized CAS replaced the holder and incremented epoch |
|
||||
| `release` | Current holder explicitly relinquished authority |
|
||||
| `expiry` | An expired current lease was observed |
|
||||
| `reject` | Policy, CAS, expiry, scope, or fencing validation denied an operation |
|
||||
|
||||
Audit data is metadata-only. Raw grant objects, connector payloads, scopes, tokens, approval references, and credentials must never be added to audit output.
|
||||
|
||||
## Incident checks
|
||||
|
||||
For suspected duplicate/stale connector effects:
|
||||
|
||||
1. Correlate the attempted operation with its `reject`, `takeover`, or `expiry` event.
|
||||
2. Compare the current row's connector ID, lease UUID, epoch, expiry, and release time with the adapter's normalized execution context.
|
||||
3. Treat an old epoch, old lease UUID, expired lease, or released lease as non-authoritative. Do not retry it as the old holder.
|
||||
4. Recovery uses the authorized takeover path with the observed expected epoch. Ordinary acquire is intentionally rejected for expired/released rows.
|
||||
5. If an external effect may already have happened, preserve evidence and do not assume lease fencing provides exactly-once replay safety.
|
||||
|
||||
## Migration and rollback safety
|
||||
|
||||
Migration `0016_salty_morlocks.sql` is additive: it creates two new tables and indexes without modifying existing authorization/session tables. Before rollout, normal database backup and migration verification still apply. Rolling application code back leaves unused additive tables in place; dropping tables is not part of automated rollback because it would destroy lease/audit evidence.
|
||||
|
||||
## Security constraints
|
||||
|
||||
- Tenant comes from authenticated gateway context, never a connector request field.
|
||||
- Logical agent, binding, connector, and scope identifiers use normalized constrained forms.
|
||||
- Takeover requires explicit gateway policy authorization and an expected epoch.
|
||||
- Default defense-in-depth TTL caps are 5 minutes for leases and 30 seconds for grants; policy may enforce stricter limits.
|
||||
- Validation and rejection audit complete before adapter side effects.
|
||||
- Existing authz and exact-action approval controls remain additional required gates; a valid connector lease does not bypass them.
|
||||
@@ -124,7 +124,7 @@ paths:
|
||||
{
|
||||
summary: Submit Mos handoff,
|
||||
parameters: [{ $ref: '#/components/parameters/correlation' }],
|
||||
requestBody: { $ref: '#/components/requestBodies/Handoff' },
|
||||
requestBody: { $ref: '#/components/requestBodies/MosHandoff' },
|
||||
responses: { '200': { description: Receipt } },
|
||||
},
|
||||
}
|
||||
@@ -261,7 +261,7 @@ components:
|
||||
},
|
||||
},
|
||||
}
|
||||
Handoff:
|
||||
MosHandoff:
|
||||
{
|
||||
required: true,
|
||||
content:
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
# Issue #755 — Logical Mos identity and connector lease fencing
|
||||
|
||||
- Task: `MOS-PORT-M1-001`
|
||||
- Branch: `feat/mos-logical-identity-fencing`
|
||||
- Base: `origin/main`
|
||||
- Started: 2026-07-14
|
||||
- Working budget: 38K tokens (task ledger estimate); one implementation lane, bounded to M1.
|
||||
|
||||
## Objective
|
||||
|
||||
Implement the first runtime-portability security boundary: normalized logical-agent identity plus a PostgreSQL-durable exclusive connector lease and server-validated fencing grants.
|
||||
|
||||
## Scope
|
||||
|
||||
- Normalized identity contract independent of harness/provider-native session IDs.
|
||||
- DB migration/schema/repository for one lease per tenant/logical-agent/binding.
|
||||
- CAS acquire/takeover, monotonic epoch, TTL, heartbeat, release, expiry handling.
|
||||
- Server-derived grants bound to tenant, logical agent, binding, connector, scopes, expiry, and lease epoch.
|
||||
- Reject and credential-safely audit stale, expired, forged, unauthorized, cross-tenant, and cross-binding grants before adapter side effects.
|
||||
- Runtime adapter boundary consumes normalized lease context.
|
||||
- Unit, migration, close/reopen, concurrency, abuse, and gateway integration tests.
|
||||
- Required developer/operations documentation for schema and security behavior.
|
||||
|
||||
## Explicit exclusions
|
||||
|
||||
No checkpoint/handoff payloads, exactly-once journal/receipts, concrete Claude/Pi/Codex harness adapter, channel cutover, or full cross-harness failover E2E.
|
||||
|
||||
## Plan (TDD RED → GREEN → REFACTOR)
|
||||
|
||||
1. Map existing contracts, DB/migration conventions, gateway authorization/audit boundaries, and test infrastructure.
|
||||
2. Add failing contract/repository/concurrency/restart/abuse/gateway tests and capture RED evidence.
|
||||
3. Implement the smallest normalized contracts, schema/migration/repository, grant validator, audit sink, and gateway service/adapter boundary needed to pass.
|
||||
4. Refactor for clear invariants and credential-safe observability; rerun focused suites.
|
||||
5. Run package/repo typecheck, lint, format, and appropriate tests.
|
||||
6. Run independent code + security review, remediate, and re-review.
|
||||
7. Inspect the final diff for security/scope drift; commit; queue guard; push; open PR with `Refs #755` and exact verification; stop without merge/issue closure.
|
||||
|
||||
## Constraints and safety notes
|
||||
|
||||
- `docs/tess/TASKS.md` is orchestrator-only and will not be edited.
|
||||
- Existing dirty `.mosaic/orchestrator/mission.json` and `.mosaic/orchestrator/session.lock` are launcher/orchestrator state and will not be staged or altered intentionally.
|
||||
- No client-supplied identity may confer authority.
|
||||
- No credential, token, or raw grant material may be persisted to audit/log output.
|
||||
- Existing authorization checks remain intact; fencing is an additional fail-closed layer.
|
||||
|
||||
## Assumptions resolved from existing architecture
|
||||
|
||||
- `ASSUMPTION:` M1 exposes no public lease endpoint. The gateway service is an internal policy surface with deny-all default policy because concrete connector activation/cutover is explicitly deferred.
|
||||
- `ASSUMPTION:` Fencing epochs use PostgreSQL `bigint` and cross-module decimal strings, preserving JSON portability without JavaScript number precision loss.
|
||||
- `ASSUMPTION:` Process-local grant provenance intentionally fails closed across restart; durable lease/epoch state survives and fresh grants require current policy + lease validation.
|
||||
|
||||
## TDD evidence
|
||||
|
||||
RED observed before implementation:
|
||||
|
||||
- `corepack pnpm --filter @mosaicstack/types exec vitest run src/agent/connector-lease.dto.spec.ts` → failed to load missing `connector-lease.dto.js`.
|
||||
- `corepack pnpm --filter @mosaicstack/agent exec vitest run src/connector-lease.test.ts` → failed to load missing `connector-lease.js`.
|
||||
- Gateway focused tests failed before implementation because the new repository/service boundaries did not exist (workspace dependencies were then built before behavioral GREEN runs).
|
||||
|
||||
GREEN to date:
|
||||
|
||||
- Types contract: 6/6 passed.
|
||||
- Agent grant/fencing unit suite: 5/5 passed.
|
||||
- Gateway PGlite repository + policy/side-effect integration: 7/7 passed; 1 real-PostgreSQL test skipped when `DATABASE_URL` absent.
|
||||
- Real PostgreSQL focused run with configured `DATABASE_URL`: 1/1 passed (credential value not emitted in reports).
|
||||
|
||||
## Documentation checklist
|
||||
|
||||
- [x] `docs/PRD.md` contains current MOS-PORT M1 scope and acceptance criteria.
|
||||
- [x] Developer architecture: `docs/architecture/mos-runtime-portability-m1.md`.
|
||||
- [x] Admin/operations guidance: `docs/guides/mos-connector-lease-operations.md`.
|
||||
- [x] `docs/SITEMAP.md` links both pages.
|
||||
- [x] No user-guide change: M1 exposes no user-facing flow or channel cutover.
|
||||
- [x] No OpenAPI/endpoint-index change: M1 adds no HTTP endpoint.
|
||||
- [x] Migration/restart/rollback safety and credential-safe audit constraints documented.
|
||||
- [x] Canonical source remains in-repo; no external publishing action is in scope.
|
||||
- [x] Independent review confirms documentation matches implementation; implementation-specific findings were remediated.
|
||||
|
||||
## Independent review and remediation
|
||||
|
||||
Codex code/security review ran in multiple rounds. Findings and root-cause remediations:
|
||||
|
||||
1. Policy could not inspect requested scope/TTL → policy subject now receives normalized requested scopes and explicit requested TTL.
|
||||
2. Unbounded authority lifetime → hard defaults cap leases at 5 minutes and grants at 30 seconds; overrides may only tighten; over-limit tests added.
|
||||
3. Cross-tenant denial could audit under submitted tenant → mismatch audit uses authenticated tenant plus sanitized `untrusted` target metadata; integration assertion added.
|
||||
4. Malformed forged grant could break the denial/audit path → runtime-safe shape validation with sanitized fallback audit; malformed-input test added.
|
||||
5. Gateway integration test depended on prior test state → denial test now seeds a unique binding itself; isolated `-t` run passed.
|
||||
6. Reviewer repeatedly identified launcher-generated `.mosaic/orchestrator/*` state; those files remain unstaged and excluded from the implementation commit.
|
||||
|
||||
Latest independent security review: no critical/high/medium/low findings. Final commit-level code review remains to run after the intended diff is committed without launcher state.
|
||||
|
||||
## Verification evidence
|
||||
|
||||
- Focused contracts/fencing: types 6/6; agent 9/9.
|
||||
- Gateway focused PGlite repository/policy integration: 7/7; isolated denial test 1/1.
|
||||
- Real PostgreSQL close/reopen/CAS test: 1/1 with configured `DATABASE_URL`.
|
||||
- Root `corepack pnpm typecheck`: 42/42 Turbo tasks passed.
|
||||
- Root `corepack pnpm lint`: 23/23 Turbo tasks passed.
|
||||
- Root `corepack pnpm format:check`: all matched files passed.
|
||||
- Root `corepack pnpm test`: 42/42 Turbo tasks passed; gateway 616 passed / 12 environment-gated skipped; DB 19 passed / 7 environment-gated skipped; Mosaic 650 passed.
|
||||
|
||||
## Known residual risks
|
||||
|
||||
- Concrete connector policies and Claude/Pi/Codex adapters are intentionally deferred; production policy defaults deny-all.
|
||||
- Gateway pre-side-effect validation cannot make an external system exactly-once. Adapters must propagate/enforce the epoch at downstream effect boundaries; receipts/journaling are later #754 scope.
|
||||
- Migration rollback is additive-only; dropping lease/audit tables is intentionally manual to avoid destroying authority/audit evidence.
|
||||
|
||||
## Commit-level review remediation
|
||||
|
||||
- Commit-level Codex code review found one `should-fix`: heartbeat, release, and grant issuance authorized caller-supplied lease fields before canonical normalization.
|
||||
- TDD RED: the isolated gateway policy-boundary test showed mixed-case/padded logical agent, binding, connector, scope, and epoch values reaching policy unchanged.
|
||||
- Remediation: exported the coordinator's canonical lease normalizer and applied it at the gateway boundary before tenant/policy checks and coordinator dispatch for heartbeat, release, and grant issuance.
|
||||
- GREEN: isolated policy test 1/1; focused types 6/6, agent 9/9, gateway 8/8; root typecheck 42/42, lint 23/23, format check passed, and root tests 42/42 (gateway 617 passed / 12 environment-gated skipped).
|
||||
- Commit-level security review remained clean: no critical/high/medium/low findings.
|
||||
|
||||
## Durable grant-expiry review remediation
|
||||
|
||||
- Final commit review found a second `should-fix`: grant expiry was capped against submitted lease metadata after current-authority validation, rather than the durable lease row.
|
||||
- TDD RED: a crafted same-authority lease with a later submitted expiry produced a grant expiring after the durable row.
|
||||
- Remediation: grant authority fields and expiry now derive from the durable current lease; submitted scopes remain an additional narrowing constraint.
|
||||
- GREEN: focused agent fencing suite 10/10.
|
||||
@@ -18,12 +18,12 @@ Implement a transport-neutral coordination contract allowing a configured intera
|
||||
|
||||
## Design checkpoint — 2026-07-12
|
||||
|
||||
Created `docs/tess/MOS-COORDINATION.md`. Mos approved the design and selected the native in-process `InMemoryInteractionCoordinationPort` for M4. Fleet/tmux remains a documented M5 adapter seam; no Mos-side consumer is built in this task.
|
||||
Created `docs/tess/MOS-COORDINATION.md`. Mos approved the design and selected the native in-process `InMemoryMosCoordinationPort` for M4. Fleet/tmux remains a documented M5 adapter seam; no Mos-side consumer is built in this task.
|
||||
|
||||
## Progress checkpoint — 2026-07-13
|
||||
|
||||
- Implemented `InteractionCoordinationPort` with handoff/observe/result only, an authority-checking client, and deterministic native adapter in `@mosaicstack/coord`.
|
||||
- Implemented the gateway `InteractionCoordinationService`, deriving requester identity from trusted configuration and actor/tenant/correlation from authenticated context.
|
||||
- Implemented `MosCoordinationPort` with handoff/observe/result only, an authority-checking client, and deterministic native adapter in `@mosaicstack/coord`.
|
||||
- Implemented the gateway `MosCoordinationService`, deriving requester identity from trusted configuration and actor/tenant/correlation from authenticated context.
|
||||
- Added contract and gateway boundary tests for configurable identities, native round-trip, unconfigured requester, self-delegation, target drift, and cross-tenant observe/result denial before adapter invocation.
|
||||
- Did not modify `apps/gateway/src/commands/command-authorization.service.ts`.
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ Termination is fail-closed: a runtime approval verifier consumes a one-time, exa
|
||||
|
||||
### Mos Coordination Boundary
|
||||
|
||||
`@mosaicstack/coord` exposes only the transport-neutral `InteractionCoordinationPort`
|
||||
`@mosaicstack/coord` exposes only the transport-neutral `MosCoordinationPort`
|
||||
verbs `handoff`, `observe`, and `result`. Gateway derives the actor, tenant,
|
||||
correlation, and interaction-agent identity from authenticated context plus
|
||||
trusted configuration; callers never provide an orchestration target. It
|
||||
|
||||
@@ -6,7 +6,7 @@ This procedure is evidence-bound. It does not authorize a production cutover unt
|
||||
2. Query the normalized runtime capability surface, not a Hermes API directly. Confirm the session capabilities required for the operation are advertised.
|
||||
3. Query the transitional matrix through `RuntimeProviderService.transitionalCapabilityMatrix` (`apps/gateway/src/agent/runtime-provider-registry.service.ts`). Kanban, skills, memory, tools, and cron must remain `unsupported`; stop rather than route those operations through Hermes.
|
||||
4. Route new memory activity through the Mosaic operator-memory plugin path; there is no landed Hermes memory import.
|
||||
5. Use `InteractionCoordinationService` (`apps/gateway/src/coord/interaction-coordination.service.ts`) for orchestration handoff. The interaction agent does not take configured orchestrator authority.
|
||||
5. Use `MosCoordinationService` for orchestration handoff. Tess does not take Mos authority.
|
||||
6. Record the qualification evidence and only then update an external deployment/channel binding through its separately authorized operational process.
|
||||
|
||||
No claim here authorizes bulk transcript copying, data-schema migration, or enabling an unsupported transitional capability.
|
||||
|
||||
@@ -7,5 +7,5 @@ Hermes is a reference adapter, not a Mosaic core dependency. `packages/agent/src
|
||||
| sessions, hierarchy, streaming, send/attach/terminate | `HermesRuntimeProvider` plus `hermes-runtime-provider.test.ts` | adapted |
|
||||
| Kanban, skills, memory, tools, cron | normalized matrix in `HermesRuntimeProvider.transitionalCapabilityMatrix`; each is `unsupported` and `assertTransitionalCapability` denies before a transport call | deferred / fail-closed |
|
||||
| operator memory | `packages/memory/src/operator-memory-plugin.ts`, constructed by `apps/gateway/src/memory/memory.module.ts` and session-scoped by `apps/gateway/src/agent/agent.service.ts` | native Mosaic path |
|
||||
| orchestration handoff | `InteractionCoordinationService` in `apps/gateway/src/coord/interaction-coordination.service.ts` retains authenticated handoff/observe/result ownership checks | native Mosaic path |
|
||||
| orchestration handoff | `apps/gateway/src/coord/mos-coordination.service.ts` retains authenticated handoff/observe/result ownership checks | native Mosaic path |
|
||||
| transcripts, profiles, preferences | no Hermes importer/schema mapping landed | no automatic migration |
|
||||
|
||||
@@ -6,5 +6,5 @@ Rollback is configuration/binding reversal, not a database rollback: no Hermes s
|
||||
2. Keep the gateway registration and core contracts unchanged unless a reviewed code rollback is required; `AgentRuntimeProviderRegistry` registration is explicit and non-replacing (`packages/agent/src/runtime-provider-registry.ts`).
|
||||
3. Do not replay an unsupported Kanban, skills, memory, tools, or cron operation. The transitional matrix is intentionally fail-closed.
|
||||
4. Preserve Mosaic audit, session, and operator-memory records under their normal scoped retention rules; do not copy them into Hermes as a rollback shortcut.
|
||||
5. For an in-flight coordination request, use the owned handoff observation/result flow in `InteractionCoordinationService` (`apps/gateway/src/coord/interaction-coordination.service.ts`); do not create a second orchestrator path.
|
||||
5. For an in-flight coordination request, use the owned handoff observation/result flow in `MosCoordinationService`; do not create a second orchestrator path.
|
||||
6. Capture the binding reversal, affected scope, correlation IDs, and reason in the approved operational record before retrying a cutover.
|
||||
|
||||
@@ -20,29 +20,29 @@ interface CoordinationScope {
|
||||
readonly requesterAgentId: string; // trusted gateway/configuration data
|
||||
}
|
||||
|
||||
interface HandoffRequest {
|
||||
interface MosHandoffRequest {
|
||||
readonly idempotencyKey: string;
|
||||
readonly summary: string;
|
||||
readonly context?: string;
|
||||
readonly missionId?: string;
|
||||
}
|
||||
|
||||
interface HandoffReceipt {
|
||||
interface MosHandoffReceipt {
|
||||
readonly handoffId: string;
|
||||
readonly targetAgentId: string;
|
||||
readonly status: 'accepted' | 'queued';
|
||||
readonly correlationId: string;
|
||||
}
|
||||
|
||||
interface Handoff {
|
||||
interface MosHandoff {
|
||||
readonly handoffId: string;
|
||||
readonly targetAgentId: string;
|
||||
readonly request: HandoffRequest;
|
||||
readonly request: MosHandoffRequest;
|
||||
readonly scope: CoordinationScope;
|
||||
}
|
||||
|
||||
interface InteractionCoordinationPort {
|
||||
handoff(handoff: Handoff): Promise<HandoffReceipt>;
|
||||
interface MosCoordinationPort {
|
||||
handoff(handoff: MosHandoff): Promise<MosHandoffReceipt>;
|
||||
observe(handoffId: string, scope: CoordinationScope): Promise<CoordinationObservation>;
|
||||
result(handoffId: string, scope: CoordinationScope): Promise<CoordinationResult>;
|
||||
}
|
||||
@@ -58,20 +58,20 @@ and the requester agent from trusted authentication/configuration only.
|
||||
|
||||
## Enforcement point
|
||||
|
||||
`apps/gateway` owns an `InteractionCoordinationService` (`apps/gateway/src/coord/interaction-coordination.service.ts`) boundary that compares the
|
||||
`apps/gateway` owns a `MosCoordinationService` boundary that compares the
|
||||
trusted configured requester/target identities and rejects all of the following
|
||||
before calling a transport: unconfigured requester, self-delegation, target
|
||||
identity drift, cross-tenant observe/result lookup, and attempts to observe or
|
||||
receive a result for a handoff outside the originating tenant. The service exposes handoff, observe,
|
||||
and result only, and delegates delivery to an injected adapter.
|
||||
|
||||
M4 ships a native in-process `InMemoryInteractionCoordinationPort` as the concrete,
|
||||
M4 ships a native in-process `InMemoryMosCoordinationPort` as the concrete,
|
||||
deterministic adapter. It preserves the immutable handoff ID, tenant, requester
|
||||
identity, and correlation ID while demonstrating the handoff → observe → result
|
||||
round trip. It is a queue/port adapter, not a Mos-side consumer.
|
||||
|
||||
A future fleet/tmux adapter is a documented M5 deployment seam and must
|
||||
implement the same `InteractionCoordinationPort`; no channel client or interaction
|
||||
implement the same `MosCoordinationPort`; no channel client or interaction
|
||||
runtime calls a transport directly.
|
||||
|
||||
## Required tests
|
||||
|
||||
@@ -43,4 +43,3 @@
|
||||
| TESS-M5-002 | done | Complete migration inventory, cutover, rollback, retention and deprecation evidence | #711 | coder3 | docs/tess | feat/tess-migration-docs | TESS-M4-V | 18K | TESS-MIG-001. **Mos-DISPATCHED to coder3 2026-07-13** ("M4 complete; advancing to M5") — dispatched AHEAD of TESS-M4-V passing; the M4-V-status-vs-#710-CLOSED dependency reconciliation is pending Mos ruling (tracked, not orchestrator-decided). **DELIVERED as PR #742** — 4 new files docs/tess/M5-MIGRATION-{INVENTORY,CUTOVER,ROLLBACK,RETENTION-DEPRECATION}.md, base main b7b0f508, frozen head b5e9d0e528a50aae2e916cc7202bacc2e8db67dd. Docs-only; tracking-control trio (MISSION-MANIFEST/TASKS/VERIFICATION-MATRIX) UNTOUCHED; command-authz byte-identical a9f829e7; no live creds. **CI pipeline 1773 SUCCESS** (repo 47, refs/pull/742/head, commit==head). **Independent non-author ROR COMPLETE: reviewer VERIFIED APPROVE at exact head b5e9d0e528a50aae2e916cc7202bacc2e8db67dd (Gitea comment 17108)** — evidence claims verified to trace to landed Hermes adapter / capability matrix, gateway registry/reachability, operator-memory scope path, Mos coordination boundary; docs do NOT over-claim transcript/profile import, schema migration, unsupported-capability enablement, production cutover, or deprecation completion. Head independently verified UNMOVED at b5e9d0e528a5 post-ROR (live Gitea), base main, mergeable=true. **#742 MERGED by Mos → main 5789711e. Deliverable docs LANDED. GATE: TESS-M4-V/M5-V verification-gate reconciliation remains Mos-owned/pending (this row was dispatched ahead of M4-V passing).** |
|
||||
| TESS-M5-003 | done | Complete OpenAPI, user/admin/developer/plugin/operations docs and checklist | #711 | codex | docs | feat/tess-docs | TESS-M5-001,TESS-M5-002 | 22K | Documentation hard gate. **DELIVERED as PR #746** (branch feat/tess-docs, base main, 7 docs-only files: docs/openapi-tess.yaml + docs/tess/{ADMIN,DEVELOPER,OPERATIONS,PLUGIN,USER}-GUIDE.md + M5-003-DOCUMENTATION-CHECKLIST.md). 4-round revise-loop (heads 470eb911→c9f69300→7aea94e2→25b9d642) converged: OpenAPI covers interaction routes + SSE /sessions/{sessionId}/stream + Mos coord (/api/coord/mos/handoff,/observe,/result) + memory preferences/insights/search; request-body schemas aligned to real DTOs (Send requires content+idempotencyKey, Stop requires approvalRef, Insight requires only content, MosHandoff body requires idempotencyKey+summary); checklist accurate. **CI pipeline 1786 SUCCESS** (repo 47, refs/pull/746/head, commit==head 25b9d642). **Independent non-author ROR COMPLETE: reviewer VERIFIED APPROVE at exact head 25b9d642b939014b3efd61826e7524cafc6ffc2e (Gitea comment 17170)** — docs-only, tracking-trio untouched, command-authz byte-identical a9f829e7, no false coverage claims. Head verified UNMOVED at 25b9d642 (live Gitea, not worker-reported), base main, mergeable=true. **#746 MERGED by Mos → main bc8016c8314ec3a4b6ebc2fec5d9f276fca3327a. Documentation gate LANDED.** GATE: TESS-M4-V/M5-V verification-gate reconciliation remains Mos-owned/pending. |
|
||||
| TESS-M5-V | not-started | Full baseline, contract, integration, Discord/CLI E2E, security review, recovery drill and rollback qualification | #711 | sonnet | apps/gateway, packages/agent, plugins/discord, packages/mosaic | review/tess-final | TESS-M5-003 | 35K | Maps AC-TESS-01..11 to evidence |
|
||||
| MOS-PORT-M1-001 | in-progress | Implement logical Mos identity, PostgreSQL connector lease, monotonic fencing, server-bound execution grants, audit, migrations, concurrency/restart/abuse/integration tests | #755 | codex | packages/types, packages/agent, packages/db, apps/gateway | feat/mos-logical-identity-fencing | — | 38K | Requirements MOS-PORT-ID-001, MOS-PORT-LEASE-001, MOS-PORT-FENCE-001..002, MOS-PORT-OBS-001, MOS-PORT-ARCH-001. One Sol/Pi worker; TDD; PR-open STOP; worker must not edit this ledger. |
|
||||
|
||||
@@ -33,18 +33,14 @@ Trust boundaries: Discord→plugin, CLI→gateway, plugin→gateway service iden
|
||||
6. Every externally caused operation is replay-safe and correlated.
|
||||
7. Provider capability absence is a denial, not an invitation to shell around it.
|
||||
|
||||
## Closed Prerequisite Findings
|
||||
## Existing Findings That Block Tess
|
||||
|
||||
The original M1 findings below are closed by landed controls and retained for audit traceability.
|
||||
- Command executor lacks server-side enforcement for declared scopes.
|
||||
- Session list/reuse/destroy surfaces are not owner-filtered consistently.
|
||||
- MCP schemas accept caller-supplied user identity.
|
||||
- Discord plugin lacks a complete authenticated service ingress and user/channel allowlists.
|
||||
- Chat/tool persistence lacks mandatory redaction.
|
||||
- Sessions/pending Discord output are in-memory and not restart-safe.
|
||||
- Session GC currently performs globally scoped promotion.
|
||||
|
||||
| Former finding | Closed evidence |
|
||||
| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| Command scope/role enforcement | `apps/gateway/src/commands/command-authorization.service.ts` and its authorization tests enforce the server-side approval boundary. |
|
||||
| Cross-owner session access | Gateway session ownership tests cover server-derived owner and tenant scope. |
|
||||
| Caller-controlled MCP identity | MCP tools derive actor and tenant from authenticated gateway context. |
|
||||
| Missing Discord ingress allowlists | `apps/gateway/src/plugin/plugin.module.ts` requires the guild, channel, and user allowlist environment values; `apps/gateway/src/plugin/discord-ingress.security.spec.ts` exercises denial and configured ingress. |
|
||||
| Missing redaction before persistence/egress | Gateway and log redaction coverage verifies sensitive content is classified before durable storage or channel delivery. |
|
||||
| In-memory-only restart safety | `packages/agent/src/durable-session.test.ts` reconstructs durable identity, inbox/outbox, checkpoints, and handoffs after simulated restart. |
|
||||
| Globally scoped session GC | `apps/gateway/src/gc/session-gc.service.spec.ts` verifies session-only collection and the absence of automatic global collection entry points. |
|
||||
|
||||
These controls remain subject to the runtime's independent review and release qualification gates.
|
||||
These are tracked as M1 security prerequisites and must pass independent security review before Tess ingress is enabled.
|
||||
|
||||
@@ -1,261 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type {
|
||||
ConnectorExecutionContext,
|
||||
ConnectorExecutionGrant,
|
||||
ConnectorLease,
|
||||
ConnectorLeaseAuditEvent,
|
||||
ConnectorLeaseStore,
|
||||
FencedConnectorAdapter,
|
||||
LogicalAgentBinding,
|
||||
} from '@mosaicstack/types';
|
||||
import {
|
||||
ConnectorLeaseCoordinator,
|
||||
MAX_CONNECTOR_GRANT_TTL_MS,
|
||||
MAX_CONNECTOR_LEASE_TTL_MS,
|
||||
} from './connector-lease.js';
|
||||
import type { ConnectorLeaseError } from './connector-lease.js';
|
||||
|
||||
const identity = { tenantId: 'tenant-a', logicalAgentId: 'mos' } as const;
|
||||
const binding: LogicalAgentBinding = { identity, bindingId: 'operator-chat' };
|
||||
const activeLease: ConnectorLease = {
|
||||
...binding,
|
||||
leaseId: '00000000-0000-4000-8000-000000000001',
|
||||
connectorId: 'connector-a',
|
||||
scopes: ['runtime.send', 'tool.execute'],
|
||||
leaseEpoch: '3',
|
||||
acquiredAt: '2026-07-14T17:00:00.000Z',
|
||||
heartbeatAt: '2026-07-14T17:00:00.000Z',
|
||||
expiresAt: '2026-07-14T17:10:00.000Z',
|
||||
};
|
||||
|
||||
class FakeLeaseStore implements ConnectorLeaseStore {
|
||||
lease: ConnectorLease | null = activeLease;
|
||||
readonly audits: ConnectorLeaseAuditEvent[] = [];
|
||||
|
||||
async acquire(): Promise<ConnectorLease> {
|
||||
if (!this.lease) throw new Error('fixture has no lease');
|
||||
return this.lease;
|
||||
}
|
||||
|
||||
async takeover(): Promise<ConnectorLease> {
|
||||
if (!this.lease) throw new Error('fixture has no lease');
|
||||
return this.lease;
|
||||
}
|
||||
|
||||
async heartbeat(): Promise<ConnectorLease> {
|
||||
if (!this.lease) throw new Error('fixture has no lease');
|
||||
return this.lease;
|
||||
}
|
||||
|
||||
async release(): Promise<void> {}
|
||||
|
||||
async findCurrent(): Promise<ConnectorLease | null> {
|
||||
return this.lease;
|
||||
}
|
||||
|
||||
async recordAudit(event: ConnectorLeaseAuditEvent): Promise<void> {
|
||||
this.audits.push(event);
|
||||
}
|
||||
}
|
||||
|
||||
describe('ConnectorLeaseCoordinator fencing', (): void => {
|
||||
it('validates a server-minted grant immediately before invoking an adapter side effect', async (): Promise<void> => {
|
||||
const store = new FakeLeaseStore();
|
||||
const coordinator = new ConnectorLeaseCoordinator(store, {
|
||||
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
|
||||
});
|
||||
const grant = await coordinator.issueGrant({
|
||||
lease: activeLease,
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 30_000,
|
||||
correlationId: 'correlation-1',
|
||||
});
|
||||
const execute = vi.fn(async (_input: string, context: ConnectorExecutionContext) => context);
|
||||
const adapter: FencedConnectorAdapter<string, ConnectorExecutionContext> = { execute };
|
||||
|
||||
const context = await coordinator.executeGrant(grant, 'runtime.send', 'hello', adapter);
|
||||
|
||||
expect(execute).toHaveBeenCalledOnce();
|
||||
expect(context).toMatchObject({
|
||||
identity,
|
||||
bindingId: 'operator-chat',
|
||||
connectorId: 'connector-a',
|
||||
leaseEpoch: '3',
|
||||
scopes: ['runtime.send'],
|
||||
});
|
||||
});
|
||||
|
||||
it('caps grant expiry to the durable current lease instead of submitted metadata', async (): Promise<void> => {
|
||||
const store = new FakeLeaseStore();
|
||||
store.lease = { ...activeLease, expiresAt: '2026-07-14T17:01:05.000Z' };
|
||||
const coordinator = new ConnectorLeaseCoordinator(store, {
|
||||
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
|
||||
});
|
||||
|
||||
const grant = await coordinator.issueGrant({
|
||||
lease: { ...activeLease, expiresAt: '2026-07-14T18:00:00.000Z' },
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 30_000,
|
||||
correlationId: 'correlation-durable-expiry',
|
||||
});
|
||||
|
||||
expect(grant.expiresAt).toBe('2026-07-14T17:01:05.000Z');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['forged clone', (grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({ ...grant })],
|
||||
[
|
||||
'cross-tenant clone',
|
||||
(grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({
|
||||
...grant,
|
||||
identity: { ...grant.identity, tenantId: 'tenant-b' },
|
||||
}),
|
||||
],
|
||||
[
|
||||
'cross-agent clone',
|
||||
(grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({
|
||||
...grant,
|
||||
identity: { ...grant.identity, logicalAgentId: 'other-agent' },
|
||||
}),
|
||||
],
|
||||
[
|
||||
'cross-binding clone',
|
||||
(grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({
|
||||
...grant,
|
||||
bindingId: 'other-binding',
|
||||
}),
|
||||
],
|
||||
[
|
||||
'cross-connector clone',
|
||||
(grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({
|
||||
...grant,
|
||||
connectorId: 'connector-b',
|
||||
}),
|
||||
],
|
||||
])('denies and audits a %s before adapter invocation', async (_label, forge): Promise<void> => {
|
||||
const store = new FakeLeaseStore();
|
||||
const coordinator = new ConnectorLeaseCoordinator(store, {
|
||||
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
|
||||
});
|
||||
const grant = await coordinator.issueGrant({
|
||||
lease: activeLease,
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 30_000,
|
||||
correlationId: 'correlation-forged',
|
||||
});
|
||||
const adapter = { execute: vi.fn().mockResolvedValue(undefined) };
|
||||
|
||||
await expect(
|
||||
coordinator.executeGrant(forge(grant), 'runtime.send', undefined, adapter),
|
||||
).rejects.toMatchObject({ code: 'forged_grant' } satisfies Partial<ConnectorLeaseError>);
|
||||
expect(adapter.execute).not.toHaveBeenCalled();
|
||||
expect(store.audits.at(-1)).toMatchObject({
|
||||
event: 'reject',
|
||||
outcome: 'denied',
|
||||
reason: 'forged_grant',
|
||||
correlationId: 'correlation-forged',
|
||||
});
|
||||
});
|
||||
|
||||
it('denies malformed forged grants with sanitized audit metadata', async (): Promise<void> => {
|
||||
const store = new FakeLeaseStore();
|
||||
const coordinator = new ConnectorLeaseCoordinator(store, {
|
||||
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
|
||||
});
|
||||
const adapter = { execute: vi.fn().mockResolvedValue(undefined) };
|
||||
|
||||
await expect(
|
||||
coordinator.executeGrant(
|
||||
// @ts-expect-error Deliberately exercise malformed runtime input at the trust boundary.
|
||||
{},
|
||||
'runtime.send',
|
||||
undefined,
|
||||
adapter,
|
||||
),
|
||||
).rejects.toMatchObject({ code: 'forged_grant' } satisfies Partial<ConnectorLeaseError>);
|
||||
expect(adapter.execute).not.toHaveBeenCalled();
|
||||
expect(store.audits).toContainEqual(
|
||||
expect.objectContaining({
|
||||
identity: { tenantId: 'untrusted', logicalAgentId: 'untrusted' },
|
||||
bindingId: 'untrusted',
|
||||
connectorId: 'untrusted',
|
||||
correlationId: 'untrusted',
|
||||
reason: 'forged_grant',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects lease and grant TTLs above server-side safety caps', async (): Promise<void> => {
|
||||
const store = new FakeLeaseStore();
|
||||
const coordinator = new ConnectorLeaseCoordinator(store, {
|
||||
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
|
||||
});
|
||||
expect(
|
||||
() =>
|
||||
new ConnectorLeaseCoordinator(store, {
|
||||
maxLeaseTtlMs: MAX_CONNECTOR_LEASE_TTL_MS + 1,
|
||||
}),
|
||||
).toThrow(/no greater than/);
|
||||
|
||||
await expect(
|
||||
coordinator.acquire({
|
||||
identity,
|
||||
bindingId: 'operator-chat',
|
||||
connectorId: 'connector-a',
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: MAX_CONNECTOR_LEASE_TTL_MS + 1,
|
||||
correlationId: 'correlation-ttl',
|
||||
}),
|
||||
).rejects.toThrow(/no greater than/);
|
||||
await expect(
|
||||
coordinator.issueGrant({
|
||||
lease: activeLease,
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: MAX_CONNECTOR_GRANT_TTL_MS + 1,
|
||||
correlationId: 'correlation-ttl',
|
||||
}),
|
||||
).rejects.toThrow(/no greater than/);
|
||||
});
|
||||
|
||||
it('denies stale epoch, expired grant, and unauthorized scope before side effects', async (): Promise<void> => {
|
||||
let now = new Date('2026-07-14T17:01:00.000Z');
|
||||
const store = new FakeLeaseStore();
|
||||
const coordinator = new ConnectorLeaseCoordinator(store, { now: (): Date => now });
|
||||
const stale = await coordinator.issueGrant({
|
||||
lease: activeLease,
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 30_000,
|
||||
correlationId: 'correlation-stale',
|
||||
});
|
||||
store.lease = { ...activeLease, leaseEpoch: '4', connectorId: 'connector-b' };
|
||||
const adapter = { execute: vi.fn().mockResolvedValue(undefined) };
|
||||
|
||||
await expect(
|
||||
coordinator.executeGrant(stale, 'runtime.send', undefined, adapter),
|
||||
).rejects.toMatchObject({ code: 'stale_epoch' } satisfies Partial<ConnectorLeaseError>);
|
||||
|
||||
store.lease = activeLease;
|
||||
const expiring = await coordinator.issueGrant({
|
||||
lease: activeLease,
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 1_000,
|
||||
correlationId: 'correlation-expired',
|
||||
});
|
||||
now = new Date('2026-07-14T17:01:02.000Z');
|
||||
await expect(
|
||||
coordinator.executeGrant(expiring, 'runtime.send', undefined, adapter),
|
||||
).rejects.toMatchObject({ code: 'grant_expired' } satisfies Partial<ConnectorLeaseError>);
|
||||
|
||||
now = new Date('2026-07-14T17:01:00.000Z');
|
||||
const scoped = await coordinator.issueGrant({
|
||||
lease: activeLease,
|
||||
scopes: ['runtime.send'],
|
||||
ttlMs: 30_000,
|
||||
correlationId: 'correlation-scope',
|
||||
});
|
||||
await expect(
|
||||
coordinator.executeGrant(scoped, 'tool.execute', undefined, adapter),
|
||||
).rejects.toMatchObject({ code: 'scope_denied' } satisfies Partial<ConnectorLeaseError>);
|
||||
expect(adapter.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,381 +0,0 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
normalizeConnectorId,
|
||||
normalizeConnectorScope,
|
||||
normalizeConnectorScopes,
|
||||
normalizeCorrelationId,
|
||||
normalizeLeaseEpoch,
|
||||
normalizeLogicalAgentIdentity,
|
||||
normalizeLogicalBindingId,
|
||||
type AcquireConnectorLeaseInput,
|
||||
type ConnectorExecutionContext,
|
||||
type ConnectorExecutionGrant,
|
||||
type ConnectorLease,
|
||||
type ConnectorLeaseAuditEvent,
|
||||
type ConnectorLeaseRejectReason,
|
||||
type ConnectorLeaseStore,
|
||||
type FencedConnectorAdapter,
|
||||
type HeartbeatConnectorLeaseInput,
|
||||
type IssueConnectorExecutionGrantInput,
|
||||
type LogicalAgentBinding,
|
||||
type ReleaseConnectorLeaseInput,
|
||||
type TakeoverConnectorLeaseInput,
|
||||
} from '@mosaicstack/types';
|
||||
|
||||
export const MAX_CONNECTOR_LEASE_TTL_MS = 5 * 60 * 1000;
|
||||
export const MAX_CONNECTOR_GRANT_TTL_MS = 30 * 1000;
|
||||
|
||||
export interface ConnectorLeaseCoordinatorOptions {
|
||||
readonly now?: () => Date;
|
||||
readonly maxLeaseTtlMs?: number;
|
||||
readonly maxGrantTtlMs?: number;
|
||||
}
|
||||
|
||||
export class ConnectorLeaseError extends Error {
|
||||
constructor(
|
||||
readonly code: ConnectorLeaseRejectReason,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = ConnectorLeaseError.name;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runtime-neutral lease coordinator. Durable CAS lives in the store adapter;
|
||||
* grant provenance remains process-local so a restart fails closed and mints
|
||||
* fresh grants from the durable current lease.
|
||||
*/
|
||||
export class ConnectorLeaseCoordinator {
|
||||
private readonly issuedGrants = new WeakSet<object>();
|
||||
private readonly now: () => Date;
|
||||
private readonly maxLeaseTtlMs: number;
|
||||
private readonly maxGrantTtlMs: number;
|
||||
|
||||
constructor(
|
||||
private readonly store: ConnectorLeaseStore,
|
||||
options: ConnectorLeaseCoordinatorOptions = {},
|
||||
) {
|
||||
this.now = options.now ?? (() => new Date());
|
||||
this.maxLeaseTtlMs = normalizeTtlLimit(
|
||||
options.maxLeaseTtlMs ?? MAX_CONNECTOR_LEASE_TTL_MS,
|
||||
MAX_CONNECTOR_LEASE_TTL_MS,
|
||||
'lease',
|
||||
);
|
||||
this.maxGrantTtlMs = normalizeTtlLimit(
|
||||
options.maxGrantTtlMs ?? MAX_CONNECTOR_GRANT_TTL_MS,
|
||||
MAX_CONNECTOR_GRANT_TTL_MS,
|
||||
'grant',
|
||||
);
|
||||
}
|
||||
|
||||
async acquire(input: AcquireConnectorLeaseInput): Promise<ConnectorLease> {
|
||||
const command = normalizeAcquireInput(input, this.maxLeaseTtlMs);
|
||||
const now = this.now();
|
||||
return this.store.acquire({
|
||||
...command,
|
||||
leaseId: randomUUID(),
|
||||
now: now.toISOString(),
|
||||
expiresAt: expiresAt(now, command.ttlMs),
|
||||
});
|
||||
}
|
||||
|
||||
async takeover(input: TakeoverConnectorLeaseInput): Promise<ConnectorLease> {
|
||||
const command = normalizeAcquireInput(input, this.maxLeaseTtlMs);
|
||||
const now = this.now();
|
||||
return this.store.takeover({
|
||||
...command,
|
||||
expectedEpoch: normalizeLeaseEpoch(input.expectedEpoch),
|
||||
leaseId: randomUUID(),
|
||||
now: now.toISOString(),
|
||||
expiresAt: expiresAt(now, command.ttlMs),
|
||||
});
|
||||
}
|
||||
|
||||
async heartbeat(input: HeartbeatConnectorLeaseInput): Promise<ConnectorLease> {
|
||||
const now = this.now();
|
||||
const lease = normalizeConnectorLease(input.lease);
|
||||
const ttlMs = normalizeTtl(input.ttlMs, this.maxLeaseTtlMs, 'lease');
|
||||
return this.store.heartbeat({
|
||||
lease,
|
||||
ttlMs,
|
||||
correlationId: normalizeCorrelationId(input.correlationId),
|
||||
now: now.toISOString(),
|
||||
expiresAt: expiresAt(now, ttlMs),
|
||||
});
|
||||
}
|
||||
|
||||
async release(input: ReleaseConnectorLeaseInput): Promise<void> {
|
||||
await this.store.release({
|
||||
lease: normalizeConnectorLease(input.lease),
|
||||
correlationId: normalizeCorrelationId(input.correlationId),
|
||||
now: this.now().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
async current(binding: LogicalAgentBinding): Promise<ConnectorLease | null> {
|
||||
return this.store.findCurrent(normalizeBinding(binding));
|
||||
}
|
||||
|
||||
async issueGrant(input: IssueConnectorExecutionGrantInput): Promise<ConnectorExecutionGrant> {
|
||||
const now = this.now();
|
||||
const lease = normalizeConnectorLease(input.lease);
|
||||
const scopes = normalizeConnectorScopes(input.scopes);
|
||||
const correlationId = normalizeCorrelationId(input.correlationId);
|
||||
const ttlMs = normalizeTtl(input.ttlMs, this.maxGrantTtlMs, 'grant');
|
||||
const current = await this.store.findCurrent(lease);
|
||||
await this.assertCurrentLease(current, lease, now, correlationId);
|
||||
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
|
||||
if (!isScopeSubset(scopes, lease.scopes) || !isScopeSubset(scopes, current.scopes)) {
|
||||
await this.reject(lease, correlationId, now, 'scope_denied');
|
||||
}
|
||||
const requestedExpiry = new Date(now.getTime() + ttlMs);
|
||||
const leaseExpiry = new Date(current.expiresAt);
|
||||
const grantExpiry = requestedExpiry < leaseExpiry ? requestedExpiry : leaseExpiry;
|
||||
const grant: ConnectorExecutionGrant = Object.freeze({
|
||||
identity: current.identity,
|
||||
bindingId: current.bindingId,
|
||||
leaseId: current.leaseId,
|
||||
connectorId: current.connectorId,
|
||||
scopes,
|
||||
leaseEpoch: current.leaseEpoch,
|
||||
issuedAt: now.toISOString(),
|
||||
expiresAt: grantExpiry.toISOString(),
|
||||
correlationId,
|
||||
});
|
||||
this.issuedGrants.add(grant);
|
||||
return grant;
|
||||
}
|
||||
|
||||
async executeGrant<TInput, TOutput>(
|
||||
grant: ConnectorExecutionGrant,
|
||||
requiredScope: string,
|
||||
input: TInput,
|
||||
adapter: FencedConnectorAdapter<TInput, TOutput>,
|
||||
): Promise<TOutput> {
|
||||
const now = this.now();
|
||||
const normalizedScope = normalizeConnectorScope(requiredScope);
|
||||
if (!this.issuedGrants.has(grant)) {
|
||||
await this.rejectForgedGrant(grant, now);
|
||||
}
|
||||
if (new Date(grant.expiresAt) <= now) {
|
||||
await this.rejectGrant(grant, now, 'grant_expired');
|
||||
}
|
||||
const current = await this.store.findCurrent(normalizeBinding(grant));
|
||||
await this.assertCurrentLease(current, grant, now, grant.correlationId);
|
||||
if (!grant.scopes.includes(normalizedScope) || !current?.scopes.includes(normalizedScope)) {
|
||||
await this.rejectGrant(grant, now, 'scope_denied');
|
||||
}
|
||||
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
|
||||
const context: ConnectorExecutionContext = Object.freeze({
|
||||
identity: current.identity,
|
||||
bindingId: current.bindingId,
|
||||
leaseId: current.leaseId,
|
||||
connectorId: current.connectorId,
|
||||
scopes: Object.freeze([...grant.scopes]),
|
||||
leaseEpoch: current.leaseEpoch,
|
||||
correlationId: grant.correlationId,
|
||||
grantExpiresAt: grant.expiresAt,
|
||||
});
|
||||
return adapter.execute(input, context);
|
||||
}
|
||||
|
||||
private async assertCurrentLease(
|
||||
current: ConnectorLease | null,
|
||||
authority: ConnectorLease | ConnectorExecutionGrant,
|
||||
now: Date,
|
||||
correlationId: string,
|
||||
): Promise<void> {
|
||||
if (!current) await this.reject(authority, correlationId, now, 'lease_missing');
|
||||
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
|
||||
if (current.releasedAt) await this.reject(authority, correlationId, now, 'lease_released');
|
||||
if (new Date(current.expiresAt) <= now) {
|
||||
await this.store.recordAudit(auditEvent(current, correlationId, now, 'expiry', 'succeeded'));
|
||||
await this.reject(authority, correlationId, now, 'lease_expired');
|
||||
}
|
||||
if (current.leaseEpoch !== authority.leaseEpoch) {
|
||||
await this.reject(authority, correlationId, now, 'stale_epoch');
|
||||
}
|
||||
if (current.leaseId !== authority.leaseId || current.connectorId !== authority.connectorId) {
|
||||
await this.reject(authority, correlationId, now, 'connector_mismatch');
|
||||
}
|
||||
}
|
||||
|
||||
private async rejectGrant(
|
||||
grant: ConnectorExecutionGrant,
|
||||
now: Date,
|
||||
reason: ConnectorLeaseRejectReason,
|
||||
): Promise<never> {
|
||||
return this.reject(grant, grant.correlationId, now, reason);
|
||||
}
|
||||
|
||||
private async rejectForgedGrant(grant: unknown, now: Date): Promise<never> {
|
||||
const event = safeForgedGrantAudit(grant, now);
|
||||
await this.store.recordAudit(event);
|
||||
throw new ConnectorLeaseError('forged_grant', safeReasonMessage('forged_grant'));
|
||||
}
|
||||
|
||||
private async reject(
|
||||
authority: LogicalAgentBinding & {
|
||||
readonly connectorId: string;
|
||||
readonly leaseId?: string;
|
||||
readonly leaseEpoch?: string;
|
||||
},
|
||||
correlationId: string,
|
||||
now: Date,
|
||||
reason: ConnectorLeaseRejectReason,
|
||||
): Promise<never> {
|
||||
await this.store.recordAudit(
|
||||
auditEvent(authority, correlationId, now, 'reject', 'denied', reason),
|
||||
);
|
||||
throw new ConnectorLeaseError(reason, safeReasonMessage(reason));
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeAcquireInput(
|
||||
input: AcquireConnectorLeaseInput,
|
||||
maxLeaseTtlMs: number,
|
||||
): AcquireConnectorLeaseInput {
|
||||
return {
|
||||
identity: normalizeLogicalAgentIdentity(input.identity),
|
||||
bindingId: normalizeLogicalBindingId(input.bindingId),
|
||||
connectorId: normalizeConnectorId(input.connectorId),
|
||||
scopes: normalizeConnectorScopes(input.scopes),
|
||||
ttlMs: normalizeTtl(input.ttlMs, maxLeaseTtlMs, 'lease'),
|
||||
correlationId: normalizeCorrelationId(input.correlationId),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeBinding(input: LogicalAgentBinding): LogicalAgentBinding {
|
||||
return {
|
||||
identity: normalizeLogicalAgentIdentity(input.identity),
|
||||
bindingId: normalizeLogicalBindingId(input.bindingId),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeConnectorLease(lease: ConnectorLease): ConnectorLease {
|
||||
const binding = normalizeBinding(lease);
|
||||
return Object.freeze({
|
||||
...binding,
|
||||
leaseId: lease.leaseId,
|
||||
connectorId: normalizeConnectorId(lease.connectorId),
|
||||
scopes: normalizeConnectorScopes(lease.scopes),
|
||||
leaseEpoch: normalizeLeaseEpoch(lease.leaseEpoch),
|
||||
acquiredAt: normalizeTimestamp(lease.acquiredAt, 'lease acquisition'),
|
||||
heartbeatAt: normalizeTimestamp(lease.heartbeatAt, 'lease heartbeat'),
|
||||
expiresAt: normalizeTimestamp(lease.expiresAt, 'lease expiry'),
|
||||
...(lease.releasedAt
|
||||
? { releasedAt: normalizeTimestamp(lease.releasedAt, 'lease release') }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeTtl(ttlMs: number, maximum: number, kind: 'lease' | 'grant'): number {
|
||||
if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0 || ttlMs > maximum) {
|
||||
throw new Error(
|
||||
`Connector ${kind} TTL must be a positive safe integer no greater than ${maximum}ms`,
|
||||
);
|
||||
}
|
||||
return ttlMs;
|
||||
}
|
||||
|
||||
function normalizeTtlLimit(ttlMs: number, hardMaximum: number, kind: 'lease' | 'grant'): number {
|
||||
if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0 || ttlMs > hardMaximum) {
|
||||
throw new Error(
|
||||
`Maximum connector ${kind} TTL must be a positive safe integer no greater than ${hardMaximum}ms`,
|
||||
);
|
||||
}
|
||||
return ttlMs;
|
||||
}
|
||||
|
||||
function normalizeTimestamp(value: string, label: string): string {
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) throw new Error(`${label} timestamp is invalid`);
|
||||
return date.toISOString();
|
||||
}
|
||||
|
||||
function expiresAt(now: Date, ttlMs: number): string {
|
||||
const expiry = new Date(now.getTime() + ttlMs);
|
||||
if (Number.isNaN(expiry.getTime())) throw new Error('Connector lease TTL exceeds date range');
|
||||
return expiry.toISOString();
|
||||
}
|
||||
|
||||
function isScopeSubset(requested: readonly string[], allowed: readonly string[]): boolean {
|
||||
return requested.every((scope) => allowed.includes(scope));
|
||||
}
|
||||
|
||||
function auditEvent(
|
||||
authority: LogicalAgentBinding & {
|
||||
readonly connectorId: string;
|
||||
readonly leaseId?: string;
|
||||
readonly leaseEpoch?: string;
|
||||
},
|
||||
correlationId: string,
|
||||
now: Date,
|
||||
event: ConnectorLeaseAuditEvent['event'],
|
||||
outcome: ConnectorLeaseAuditEvent['outcome'],
|
||||
reason?: ConnectorLeaseRejectReason,
|
||||
): ConnectorLeaseAuditEvent {
|
||||
return {
|
||||
identity: authority.identity,
|
||||
bindingId: authority.bindingId,
|
||||
connectorId: authority.connectorId,
|
||||
correlationId,
|
||||
occurredAt: now.toISOString(),
|
||||
event,
|
||||
outcome,
|
||||
...(authority.leaseId ? { leaseId: authority.leaseId } : {}),
|
||||
...(authority.leaseEpoch ? { leaseEpoch: authority.leaseEpoch } : {}),
|
||||
...(reason ? { reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function safeForgedGrantAudit(grant: unknown, now: Date): ConnectorLeaseAuditEvent {
|
||||
const fallback: ConnectorLeaseAuditEvent = {
|
||||
identity: { tenantId: 'untrusted', logicalAgentId: 'untrusted' },
|
||||
bindingId: 'untrusted',
|
||||
connectorId: 'untrusted',
|
||||
correlationId: 'untrusted',
|
||||
occurredAt: now.toISOString(),
|
||||
event: 'reject',
|
||||
outcome: 'denied',
|
||||
reason: 'forged_grant',
|
||||
};
|
||||
if (typeof grant !== 'object' || grant === null || !('identity' in grant)) return fallback;
|
||||
const identity = grant.identity;
|
||||
if (typeof identity !== 'object' || identity === null) return fallback;
|
||||
if (!('tenantId' in identity) || !('logicalAgentId' in identity)) return fallback;
|
||||
if (!('bindingId' in grant) || !('connectorId' in grant) || !('correlationId' in grant)) {
|
||||
return fallback;
|
||||
}
|
||||
if (
|
||||
typeof identity.tenantId !== 'string' ||
|
||||
typeof identity.logicalAgentId !== 'string' ||
|
||||
typeof grant.bindingId !== 'string' ||
|
||||
typeof grant.connectorId !== 'string' ||
|
||||
typeof grant.correlationId !== 'string'
|
||||
) {
|
||||
return fallback;
|
||||
}
|
||||
try {
|
||||
return {
|
||||
identity: normalizeLogicalAgentIdentity({
|
||||
tenantId: identity.tenantId,
|
||||
logicalAgentId: identity.logicalAgentId,
|
||||
}),
|
||||
bindingId: normalizeLogicalBindingId(grant.bindingId),
|
||||
connectorId: normalizeConnectorId(grant.connectorId),
|
||||
correlationId: normalizeCorrelationId(grant.correlationId),
|
||||
occurredAt: now.toISOString(),
|
||||
event: 'reject',
|
||||
outcome: 'denied',
|
||||
reason: 'forged_grant',
|
||||
};
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function safeReasonMessage(reason: ConnectorLeaseRejectReason): string {
|
||||
return `Connector authority denied: ${reason}`;
|
||||
}
|
||||
@@ -5,4 +5,3 @@ export * from './tmux-fleet-runtime-provider.js';
|
||||
export * from './hermes-runtime-provider.js';
|
||||
export * from './matrix-native-runtime-provider.js';
|
||||
export * from './durable-session.js';
|
||||
export * from './connector-lease.js';
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
CREATE TABLE "connector_lease_audit_log" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
"tenant_id" text NOT NULL,
|
||||
"logical_agent_id" text NOT NULL,
|
||||
"binding_id" text NOT NULL,
|
||||
"connector_id" text NOT NULL,
|
||||
"lease_id" uuid,
|
||||
"lease_epoch" bigint,
|
||||
"event" text NOT NULL,
|
||||
"outcome" text NOT NULL,
|
||||
"reason" text,
|
||||
"correlation_id" text NOT NULL,
|
||||
"occurred_at" timestamp with time zone NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "logical_agent_connector_leases" (
|
||||
"lease_id" uuid PRIMARY KEY NOT NULL,
|
||||
"tenant_id" text NOT NULL,
|
||||
"logical_agent_id" text NOT NULL,
|
||||
"binding_id" text NOT NULL,
|
||||
"connector_id" text NOT NULL,
|
||||
"scopes" jsonb NOT NULL,
|
||||
"lease_epoch" bigint NOT NULL,
|
||||
"acquired_at" timestamp with time zone NOT NULL,
|
||||
"heartbeat_at" timestamp with time zone NOT NULL,
|
||||
"expires_at" timestamp with time zone NOT NULL,
|
||||
"released_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX "connector_lease_audit_binding_occurred_idx" ON "connector_lease_audit_log" USING btree ("tenant_id","logical_agent_id","binding_id","occurred_at" DESC NULLS LAST);--> statement-breakpoint
|
||||
CREATE INDEX "connector_lease_audit_correlation_idx" ON "connector_lease_audit_log" USING btree ("correlation_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "logical_agent_connector_lease_binding_idx" ON "logical_agent_connector_leases" USING btree ("tenant_id","logical_agent_id","binding_id");--> statement-breakpoint
|
||||
CREATE INDEX "logical_agent_connector_lease_expiry_idx" ON "logical_agent_connector_leases" USING btree ("expires_at");--> statement-breakpoint
|
||||
CREATE INDEX "logical_agent_connector_lease_connector_idx" ON "logical_agent_connector_leases" USING btree ("connector_id");
|
||||
File diff suppressed because it is too large
Load Diff
@@ -113,13 +113,6 @@
|
||||
"when": 1783942610000,
|
||||
"tag": "0015_interaction_checkpoint_payload_digest",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 16,
|
||||
"version": "7",
|
||||
"when": 1784050648841,
|
||||
"tag": "0016_salty_morlocks",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import {
|
||||
uniqueIndex,
|
||||
real,
|
||||
integer,
|
||||
bigint,
|
||||
customType,
|
||||
} from 'drizzle-orm/pg-core';
|
||||
|
||||
@@ -488,68 +487,6 @@ export const agentLogs = pgTable(
|
||||
],
|
||||
);
|
||||
|
||||
// ─── Logical agent connector authority ──────────────────────────────────────
|
||||
// One durable row is the current authority for a tenant/logical-agent/binding.
|
||||
// Runtime-native session identifiers never enter these core tables.
|
||||
|
||||
export const logicalAgentConnectorLeases = pgTable(
|
||||
'logical_agent_connector_leases',
|
||||
{
|
||||
leaseId: uuid('lease_id').primaryKey(),
|
||||
tenantId: text('tenant_id').notNull(),
|
||||
logicalAgentId: text('logical_agent_id').notNull(),
|
||||
bindingId: text('binding_id').notNull(),
|
||||
connectorId: text('connector_id').notNull(),
|
||||
scopes: jsonb('scopes').notNull().$type<string[]>(),
|
||||
leaseEpoch: bigint('lease_epoch', { mode: 'bigint' }).notNull(),
|
||||
acquiredAt: timestamp('acquired_at', { withTimezone: true }).notNull(),
|
||||
heartbeatAt: timestamp('heartbeat_at', { withTimezone: true }).notNull(),
|
||||
expiresAt: timestamp('expires_at', { withTimezone: true }).notNull(),
|
||||
releasedAt: timestamp('released_at', { withTimezone: true }),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(t) => [
|
||||
uniqueIndex('logical_agent_connector_lease_binding_idx').on(
|
||||
t.tenantId,
|
||||
t.logicalAgentId,
|
||||
t.bindingId,
|
||||
),
|
||||
index('logical_agent_connector_lease_expiry_idx').on(t.expiresAt),
|
||||
index('logical_agent_connector_lease_connector_idx').on(t.connectorId),
|
||||
],
|
||||
);
|
||||
|
||||
/** Append-only, credential-safe lease lifecycle and fencing denial metadata. */
|
||||
export const connectorLeaseAuditLog = pgTable(
|
||||
'connector_lease_audit_log',
|
||||
{
|
||||
id: uuid('id').primaryKey().defaultRandom(),
|
||||
tenantId: text('tenant_id').notNull(),
|
||||
logicalAgentId: text('logical_agent_id').notNull(),
|
||||
bindingId: text('binding_id').notNull(),
|
||||
connectorId: text('connector_id').notNull(),
|
||||
leaseId: uuid('lease_id'),
|
||||
leaseEpoch: bigint('lease_epoch', { mode: 'bigint' }),
|
||||
event: text('event', {
|
||||
enum: ['acquire', 'renew', 'takeover', 'reject', 'release', 'expiry'],
|
||||
}).notNull(),
|
||||
outcome: text('outcome', { enum: ['succeeded', 'denied'] }).notNull(),
|
||||
reason: text('reason'),
|
||||
correlationId: text('correlation_id').notNull(),
|
||||
occurredAt: timestamp('occurred_at', { withTimezone: true }).notNull(),
|
||||
},
|
||||
(t) => [
|
||||
index('connector_lease_audit_binding_occurred_idx').on(
|
||||
t.tenantId,
|
||||
t.logicalAgentId,
|
||||
t.bindingId,
|
||||
t.occurredAt.desc(),
|
||||
),
|
||||
index('connector_lease_audit_correlation_idx').on(t.correlationId),
|
||||
],
|
||||
);
|
||||
|
||||
// ─── Tess durable session state ─────────────────────────────────────────────
|
||||
// PostgreSQL is canonical for restart-safe Tess session recovery. The state
|
||||
// machine lives in @mosaicstack/agent; these records are its durable adapter.
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
attachInteractionSession,
|
||||
sendInteractionMessage,
|
||||
stopInteractionSession,
|
||||
} from './gateway-api.js';
|
||||
|
||||
const gateway = 'https://gateway.example.test';
|
||||
const cookie = 'session=test';
|
||||
const base = { agentName: 'Nova', correlationId: 'corr-1', sessionId: 'session-1' };
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
/** Unit coverage: the TUI preserves a non-success gateway response in its CLI error. */
|
||||
describe('interaction gateway error mapping', (): void => {
|
||||
it.each([
|
||||
{
|
||||
name: 'attach unauthorized',
|
||||
response: { status: 401, message: 'Invalid or expired session' },
|
||||
invoke: (): Promise<unknown> => attachInteractionSession(gateway, cookie, base),
|
||||
expected:
|
||||
'Failed to attach interaction session (401): {"message":"Invalid or expired session"}',
|
||||
},
|
||||
{
|
||||
name: 'send invalid request',
|
||||
response: { status: 403, message: 'Content and idempotency key are required' },
|
||||
invoke: (): Promise<unknown> =>
|
||||
sendInteractionMessage(gateway, cookie, {
|
||||
...base,
|
||||
content: 'hello',
|
||||
idempotencyKey: 'message-1',
|
||||
}),
|
||||
expected:
|
||||
'Failed to send interaction message (403): {"message":"Content and idempotency key are required"}',
|
||||
},
|
||||
{
|
||||
name: 'stop forbidden',
|
||||
response: { status: 403, message: 'Runtime termination approval denied' },
|
||||
invoke: (): Promise<unknown> =>
|
||||
stopInteractionSession(gateway, cookie, { ...base, approvalRef: 'approval-1' }),
|
||||
expected:
|
||||
'Failed to stop interaction session (403): {"message":"Runtime termination approval denied"}',
|
||||
},
|
||||
])(
|
||||
'$name preserves the typed gateway denial in the CLI error',
|
||||
async ({ response, invoke, expected }) => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response(JSON.stringify({ message: response.message }), {
|
||||
status: response.status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
await expect(invoke()).rejects.toThrow(expected);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,59 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
normalizeConnectorId,
|
||||
normalizeConnectorScopes,
|
||||
normalizeLeaseEpoch,
|
||||
normalizeLogicalAgentIdentity,
|
||||
normalizeLogicalBindingId,
|
||||
type ConnectorExecutionContext,
|
||||
type LogicalAgentIdentity,
|
||||
} from './connector-lease.dto.js';
|
||||
|
||||
describe('logical agent connector lease contract', (): void => {
|
||||
it('normalizes a runtime-neutral logical identity and binding vocabulary', (): void => {
|
||||
const identity = normalizeLogicalAgentIdentity({
|
||||
tenantId: ' tenant-01 ',
|
||||
logicalAgentId: ' MOS.Primary ',
|
||||
});
|
||||
|
||||
expect(identity).toEqual({ tenantId: 'tenant-01', logicalAgentId: 'mos.primary' });
|
||||
expect(normalizeLogicalBindingId(' Discord:Operations ')).toBe('discord:operations');
|
||||
expect(normalizeConnectorId(' PI.Worker-01 ')).toBe('pi.worker-01');
|
||||
expect(Object.isFrozen(identity)).toBe(true);
|
||||
expect(Object.keys(identity).sort()).toEqual(['logicalAgentId', 'tenantId']);
|
||||
});
|
||||
|
||||
it('canonicalizes scopes and decimal fencing epochs', (): void => {
|
||||
expect(normalizeConnectorScopes([' Runtime.Send ', 'tool.execute', 'runtime.send'])).toEqual([
|
||||
'runtime.send',
|
||||
'tool.execute',
|
||||
]);
|
||||
expect(normalizeLeaseEpoch('00042')).toBe('42');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['', 'mos'],
|
||||
['tenant', ''],
|
||||
['tenant', 'claude session/123'],
|
||||
])('rejects ambiguous identity values tenant=%j agent=%j', (tenantId, logicalAgentId): void => {
|
||||
expect(() => normalizeLogicalAgentIdentity({ tenantId, logicalAgentId })).toThrow();
|
||||
});
|
||||
|
||||
it('defines an adapter context without harness-native identity fields', (): void => {
|
||||
const identity: LogicalAgentIdentity = { tenantId: 'tenant-01', logicalAgentId: 'mos' };
|
||||
const context: ConnectorExecutionContext = {
|
||||
identity,
|
||||
bindingId: 'operator-chat',
|
||||
connectorId: 'connector-a',
|
||||
leaseId: '00000000-0000-4000-8000-000000000001',
|
||||
leaseEpoch: '7',
|
||||
scopes: ['runtime.send'],
|
||||
correlationId: 'correlation-1',
|
||||
grantExpiresAt: '2026-07-14T18:00:00.000Z',
|
||||
};
|
||||
|
||||
expect(context).not.toHaveProperty('sessionId');
|
||||
expect(context).not.toHaveProperty('tmuxSession');
|
||||
expect(context).not.toHaveProperty('providerSessionId');
|
||||
});
|
||||
});
|
||||
@@ -1,199 +0,0 @@
|
||||
const ID_PATTERN = /^[a-z0-9][a-z0-9._:@-]{0,127}$/;
|
||||
const TENANT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:@-]{0,127}$/;
|
||||
const SCOPE_PATTERN = /^[a-z][a-z0-9._:-]{0,127}$/;
|
||||
|
||||
/** Stable Mosaic identity. It intentionally contains no runtime/provider session identifier. */
|
||||
export interface LogicalAgentIdentity {
|
||||
readonly tenantId: string;
|
||||
readonly logicalAgentId: string;
|
||||
}
|
||||
|
||||
export interface LogicalAgentBinding {
|
||||
readonly identity: LogicalAgentIdentity;
|
||||
readonly bindingId: string;
|
||||
}
|
||||
|
||||
export interface ConnectorLease extends LogicalAgentBinding {
|
||||
readonly leaseId: string;
|
||||
readonly connectorId: string;
|
||||
readonly scopes: readonly string[];
|
||||
readonly leaseEpoch: string;
|
||||
readonly acquiredAt: string;
|
||||
readonly heartbeatAt: string;
|
||||
readonly expiresAt: string;
|
||||
readonly releasedAt?: string;
|
||||
}
|
||||
|
||||
export interface AcquireConnectorLeaseInput {
|
||||
readonly identity: LogicalAgentIdentity;
|
||||
readonly bindingId: string;
|
||||
readonly connectorId: string;
|
||||
readonly scopes: readonly string[];
|
||||
readonly ttlMs: number;
|
||||
readonly correlationId: string;
|
||||
}
|
||||
|
||||
export interface TakeoverConnectorLeaseInput extends AcquireConnectorLeaseInput {
|
||||
readonly expectedEpoch: string;
|
||||
}
|
||||
|
||||
export interface HeartbeatConnectorLeaseInput {
|
||||
readonly lease: ConnectorLease;
|
||||
readonly ttlMs: number;
|
||||
readonly correlationId: string;
|
||||
}
|
||||
|
||||
export interface ReleaseConnectorLeaseInput {
|
||||
readonly lease: ConnectorLease;
|
||||
readonly correlationId: string;
|
||||
}
|
||||
|
||||
export interface IssueConnectorExecutionGrantInput {
|
||||
readonly lease: ConnectorLease;
|
||||
readonly scopes: readonly string[];
|
||||
readonly ttlMs: number;
|
||||
readonly correlationId: string;
|
||||
}
|
||||
|
||||
/** Internal server grant. Object provenance is checked in addition to these fields. */
|
||||
export interface ConnectorExecutionGrant extends LogicalAgentBinding {
|
||||
readonly leaseId: string;
|
||||
readonly connectorId: string;
|
||||
readonly scopes: readonly string[];
|
||||
readonly leaseEpoch: string;
|
||||
readonly issuedAt: string;
|
||||
readonly expiresAt: string;
|
||||
readonly correlationId: string;
|
||||
}
|
||||
|
||||
/** Normalized context passed to an adapter only after current-lease validation. */
|
||||
export interface ConnectorExecutionContext extends LogicalAgentBinding {
|
||||
readonly leaseId: string;
|
||||
readonly connectorId: string;
|
||||
readonly scopes: readonly string[];
|
||||
readonly leaseEpoch: string;
|
||||
readonly correlationId: string;
|
||||
readonly grantExpiresAt: string;
|
||||
}
|
||||
|
||||
export interface FencedConnectorAdapter<TInput, TOutput> {
|
||||
execute(input: TInput, context: ConnectorExecutionContext): Promise<TOutput>;
|
||||
}
|
||||
|
||||
export type ConnectorLeaseAuditEventType =
|
||||
| 'acquire'
|
||||
| 'renew'
|
||||
| 'takeover'
|
||||
| 'reject'
|
||||
| 'release'
|
||||
| 'expiry';
|
||||
export type ConnectorLeaseAuditOutcome = 'succeeded' | 'denied';
|
||||
export type ConnectorLeaseRejectReason =
|
||||
| 'policy_denied'
|
||||
| 'lease_held'
|
||||
| 'takeover_required'
|
||||
| 'cas_mismatch'
|
||||
| 'lease_missing'
|
||||
| 'lease_released'
|
||||
| 'lease_expired'
|
||||
| 'stale_epoch'
|
||||
| 'connector_mismatch'
|
||||
| 'scope_denied'
|
||||
| 'forged_grant'
|
||||
| 'grant_expired';
|
||||
|
||||
/** Credential-safe metadata only: no grant object, scope set, payload, token, or approval ref. */
|
||||
export interface ConnectorLeaseAuditEvent extends LogicalAgentBinding {
|
||||
readonly event: ConnectorLeaseAuditEventType;
|
||||
readonly outcome: ConnectorLeaseAuditOutcome;
|
||||
readonly connectorId: string;
|
||||
readonly correlationId: string;
|
||||
readonly occurredAt: string;
|
||||
readonly leaseId?: string;
|
||||
readonly leaseEpoch?: string;
|
||||
readonly reason?: ConnectorLeaseRejectReason;
|
||||
}
|
||||
|
||||
export interface ConnectorLeaseAcquireMutation extends AcquireConnectorLeaseInput {
|
||||
readonly leaseId: string;
|
||||
readonly now: string;
|
||||
readonly expiresAt: string;
|
||||
}
|
||||
|
||||
export interface ConnectorLeaseTakeoverMutation extends TakeoverConnectorLeaseInput {
|
||||
readonly leaseId: string;
|
||||
readonly now: string;
|
||||
readonly expiresAt: string;
|
||||
}
|
||||
|
||||
export interface ConnectorLeaseHeartbeatMutation extends HeartbeatConnectorLeaseInput {
|
||||
readonly now: string;
|
||||
readonly expiresAt: string;
|
||||
}
|
||||
|
||||
export interface ConnectorLeaseReleaseMutation extends ReleaseConnectorLeaseInput {
|
||||
readonly now: string;
|
||||
}
|
||||
|
||||
export interface ConnectorLeaseStore {
|
||||
acquire(input: ConnectorLeaseAcquireMutation): Promise<ConnectorLease>;
|
||||
takeover(input: ConnectorLeaseTakeoverMutation): Promise<ConnectorLease>;
|
||||
heartbeat(input: ConnectorLeaseHeartbeatMutation): Promise<ConnectorLease>;
|
||||
release(input: ConnectorLeaseReleaseMutation): Promise<void>;
|
||||
findCurrent(binding: LogicalAgentBinding): Promise<ConnectorLease | null>;
|
||||
recordAudit(event: ConnectorLeaseAuditEvent): Promise<void>;
|
||||
}
|
||||
|
||||
export function normalizeLogicalAgentIdentity(input: LogicalAgentIdentity): LogicalAgentIdentity {
|
||||
const tenantId = requiredIdentifier(input.tenantId, 'tenant ID', TENANT_PATTERN, false);
|
||||
const logicalAgentId = requiredIdentifier(
|
||||
input.logicalAgentId,
|
||||
'logical agent ID',
|
||||
ID_PATTERN,
|
||||
true,
|
||||
);
|
||||
return Object.freeze({ tenantId, logicalAgentId });
|
||||
}
|
||||
|
||||
export function normalizeLogicalBindingId(value: string): string {
|
||||
return requiredIdentifier(value, 'logical binding ID', ID_PATTERN, true);
|
||||
}
|
||||
|
||||
export function normalizeConnectorId(value: string): string {
|
||||
return requiredIdentifier(value, 'connector ID', ID_PATTERN, true);
|
||||
}
|
||||
|
||||
export function normalizeCorrelationId(value: string): string {
|
||||
return requiredIdentifier(value, 'correlation ID', TENANT_PATTERN, false);
|
||||
}
|
||||
|
||||
export function normalizeConnectorScope(value: string): string {
|
||||
return requiredIdentifier(value, 'connector scope', SCOPE_PATTERN, true);
|
||||
}
|
||||
|
||||
export function normalizeConnectorScopes(values: readonly string[]): readonly string[] {
|
||||
if (values.length === 0) throw new Error('At least one connector scope is required');
|
||||
return Object.freeze(Array.from(new Set(values.map(normalizeConnectorScope))).sort());
|
||||
}
|
||||
|
||||
export function normalizeLeaseEpoch(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
if (!/^\d+$/.test(trimmed)) throw new Error('Lease epoch must be a positive decimal integer');
|
||||
const epoch = BigInt(trimmed);
|
||||
if (epoch < 1n) throw new Error('Lease epoch must be a positive decimal integer');
|
||||
return epoch.toString(10);
|
||||
}
|
||||
|
||||
function requiredIdentifier(
|
||||
value: string,
|
||||
label: string,
|
||||
pattern: RegExp,
|
||||
lowerCase: boolean,
|
||||
): string {
|
||||
const trimmed = value.trim();
|
||||
const normalized = lowerCase ? trimmed.toLowerCase() : trimmed;
|
||||
if (!pattern.test(normalized)) {
|
||||
throw new Error(`${label} has an invalid normalized format`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
@@ -4,4 +4,3 @@ export interface AgentSessionHandle {
|
||||
}
|
||||
|
||||
export * from './agent-runtime-provider.js';
|
||||
export * from './connector-lease.dto.js';
|
||||
|
||||
Reference in New Issue
Block a user