feat(mos): add logical identity connector fencing (#757)
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/push/publish Pipeline was successful

This commit was merged in pull request #757.
This commit is contained in:
2026-07-14 23:33:06 +00:00
parent 2e2280070a
commit eb4e14ae5c
22 changed files with 7031 additions and 0 deletions

View File

@@ -0,0 +1,261 @@
import { describe, expect, it, vi } from 'vitest';
import type {
ConnectorExecutionContext,
ConnectorExecutionGrant,
ConnectorLease,
ConnectorLeaseAuditEvent,
ConnectorLeaseStore,
FencedConnectorAdapter,
LogicalAgentBinding,
} from '@mosaicstack/types';
import {
ConnectorLeaseCoordinator,
MAX_CONNECTOR_GRANT_TTL_MS,
MAX_CONNECTOR_LEASE_TTL_MS,
} from './connector-lease.js';
import type { ConnectorLeaseError } from './connector-lease.js';
const identity = { tenantId: 'tenant-a', logicalAgentId: 'mos' } as const;
const binding: LogicalAgentBinding = { identity, bindingId: 'operator-chat' };
const activeLease: ConnectorLease = {
...binding,
leaseId: '00000000-0000-4000-8000-000000000001',
connectorId: 'connector-a',
scopes: ['runtime.send', 'tool.execute'],
leaseEpoch: '3',
acquiredAt: '2026-07-14T17:00:00.000Z',
heartbeatAt: '2026-07-14T17:00:00.000Z',
expiresAt: '2026-07-14T17:10:00.000Z',
};
class FakeLeaseStore implements ConnectorLeaseStore {
lease: ConnectorLease | null = activeLease;
readonly audits: ConnectorLeaseAuditEvent[] = [];
async acquire(): Promise<ConnectorLease> {
if (!this.lease) throw new Error('fixture has no lease');
return this.lease;
}
async takeover(): Promise<ConnectorLease> {
if (!this.lease) throw new Error('fixture has no lease');
return this.lease;
}
async heartbeat(): Promise<ConnectorLease> {
if (!this.lease) throw new Error('fixture has no lease');
return this.lease;
}
async release(): Promise<void> {}
async findCurrent(): Promise<ConnectorLease | null> {
return this.lease;
}
async recordAudit(event: ConnectorLeaseAuditEvent): Promise<void> {
this.audits.push(event);
}
}
describe('ConnectorLeaseCoordinator fencing', (): void => {
it('validates a server-minted grant immediately before invoking an adapter side effect', async (): Promise<void> => {
const store = new FakeLeaseStore();
const coordinator = new ConnectorLeaseCoordinator(store, {
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
});
const grant = await coordinator.issueGrant({
lease: activeLease,
scopes: ['runtime.send'],
ttlMs: 30_000,
correlationId: 'correlation-1',
});
const execute = vi.fn(async (_input: string, context: ConnectorExecutionContext) => context);
const adapter: FencedConnectorAdapter<string, ConnectorExecutionContext> = { execute };
const context = await coordinator.executeGrant(grant, 'runtime.send', 'hello', adapter);
expect(execute).toHaveBeenCalledOnce();
expect(context).toMatchObject({
identity,
bindingId: 'operator-chat',
connectorId: 'connector-a',
leaseEpoch: '3',
scopes: ['runtime.send'],
});
});
it('caps grant expiry to the durable current lease instead of submitted metadata', async (): Promise<void> => {
const store = new FakeLeaseStore();
store.lease = { ...activeLease, expiresAt: '2026-07-14T17:01:05.000Z' };
const coordinator = new ConnectorLeaseCoordinator(store, {
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
});
const grant = await coordinator.issueGrant({
lease: { ...activeLease, expiresAt: '2026-07-14T18:00:00.000Z' },
scopes: ['runtime.send'],
ttlMs: 30_000,
correlationId: 'correlation-durable-expiry',
});
expect(grant.expiresAt).toBe('2026-07-14T17:01:05.000Z');
});
it.each([
['forged clone', (grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({ ...grant })],
[
'cross-tenant clone',
(grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({
...grant,
identity: { ...grant.identity, tenantId: 'tenant-b' },
}),
],
[
'cross-agent clone',
(grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({
...grant,
identity: { ...grant.identity, logicalAgentId: 'other-agent' },
}),
],
[
'cross-binding clone',
(grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({
...grant,
bindingId: 'other-binding',
}),
],
[
'cross-connector clone',
(grant: ConnectorExecutionGrant): ConnectorExecutionGrant => ({
...grant,
connectorId: 'connector-b',
}),
],
])('denies and audits a %s before adapter invocation', async (_label, forge): Promise<void> => {
const store = new FakeLeaseStore();
const coordinator = new ConnectorLeaseCoordinator(store, {
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
});
const grant = await coordinator.issueGrant({
lease: activeLease,
scopes: ['runtime.send'],
ttlMs: 30_000,
correlationId: 'correlation-forged',
});
const adapter = { execute: vi.fn().mockResolvedValue(undefined) };
await expect(
coordinator.executeGrant(forge(grant), 'runtime.send', undefined, adapter),
).rejects.toMatchObject({ code: 'forged_grant' } satisfies Partial<ConnectorLeaseError>);
expect(adapter.execute).not.toHaveBeenCalled();
expect(store.audits.at(-1)).toMatchObject({
event: 'reject',
outcome: 'denied',
reason: 'forged_grant',
correlationId: 'correlation-forged',
});
});
it('denies malformed forged grants with sanitized audit metadata', async (): Promise<void> => {
const store = new FakeLeaseStore();
const coordinator = new ConnectorLeaseCoordinator(store, {
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
});
const adapter = { execute: vi.fn().mockResolvedValue(undefined) };
await expect(
coordinator.executeGrant(
// @ts-expect-error Deliberately exercise malformed runtime input at the trust boundary.
{},
'runtime.send',
undefined,
adapter,
),
).rejects.toMatchObject({ code: 'forged_grant' } satisfies Partial<ConnectorLeaseError>);
expect(adapter.execute).not.toHaveBeenCalled();
expect(store.audits).toContainEqual(
expect.objectContaining({
identity: { tenantId: 'untrusted', logicalAgentId: 'untrusted' },
bindingId: 'untrusted',
connectorId: 'untrusted',
correlationId: 'untrusted',
reason: 'forged_grant',
}),
);
});
it('rejects lease and grant TTLs above server-side safety caps', async (): Promise<void> => {
const store = new FakeLeaseStore();
const coordinator = new ConnectorLeaseCoordinator(store, {
now: (): Date => new Date('2026-07-14T17:01:00.000Z'),
});
expect(
() =>
new ConnectorLeaseCoordinator(store, {
maxLeaseTtlMs: MAX_CONNECTOR_LEASE_TTL_MS + 1,
}),
).toThrow(/no greater than/);
await expect(
coordinator.acquire({
identity,
bindingId: 'operator-chat',
connectorId: 'connector-a',
scopes: ['runtime.send'],
ttlMs: MAX_CONNECTOR_LEASE_TTL_MS + 1,
correlationId: 'correlation-ttl',
}),
).rejects.toThrow(/no greater than/);
await expect(
coordinator.issueGrant({
lease: activeLease,
scopes: ['runtime.send'],
ttlMs: MAX_CONNECTOR_GRANT_TTL_MS + 1,
correlationId: 'correlation-ttl',
}),
).rejects.toThrow(/no greater than/);
});
it('denies stale epoch, expired grant, and unauthorized scope before side effects', async (): Promise<void> => {
let now = new Date('2026-07-14T17:01:00.000Z');
const store = new FakeLeaseStore();
const coordinator = new ConnectorLeaseCoordinator(store, { now: (): Date => now });
const stale = await coordinator.issueGrant({
lease: activeLease,
scopes: ['runtime.send'],
ttlMs: 30_000,
correlationId: 'correlation-stale',
});
store.lease = { ...activeLease, leaseEpoch: '4', connectorId: 'connector-b' };
const adapter = { execute: vi.fn().mockResolvedValue(undefined) };
await expect(
coordinator.executeGrant(stale, 'runtime.send', undefined, adapter),
).rejects.toMatchObject({ code: 'stale_epoch' } satisfies Partial<ConnectorLeaseError>);
store.lease = activeLease;
const expiring = await coordinator.issueGrant({
lease: activeLease,
scopes: ['runtime.send'],
ttlMs: 1_000,
correlationId: 'correlation-expired',
});
now = new Date('2026-07-14T17:01:02.000Z');
await expect(
coordinator.executeGrant(expiring, 'runtime.send', undefined, adapter),
).rejects.toMatchObject({ code: 'grant_expired' } satisfies Partial<ConnectorLeaseError>);
now = new Date('2026-07-14T17:01:00.000Z');
const scoped = await coordinator.issueGrant({
lease: activeLease,
scopes: ['runtime.send'],
ttlMs: 30_000,
correlationId: 'correlation-scope',
});
await expect(
coordinator.executeGrant(scoped, 'tool.execute', undefined, adapter),
).rejects.toMatchObject({ code: 'scope_denied' } satisfies Partial<ConnectorLeaseError>);
expect(adapter.execute).not.toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,381 @@
import { randomUUID } from 'node:crypto';
import {
normalizeConnectorId,
normalizeConnectorScope,
normalizeConnectorScopes,
normalizeCorrelationId,
normalizeLeaseEpoch,
normalizeLogicalAgentIdentity,
normalizeLogicalBindingId,
type AcquireConnectorLeaseInput,
type ConnectorExecutionContext,
type ConnectorExecutionGrant,
type ConnectorLease,
type ConnectorLeaseAuditEvent,
type ConnectorLeaseRejectReason,
type ConnectorLeaseStore,
type FencedConnectorAdapter,
type HeartbeatConnectorLeaseInput,
type IssueConnectorExecutionGrantInput,
type LogicalAgentBinding,
type ReleaseConnectorLeaseInput,
type TakeoverConnectorLeaseInput,
} from '@mosaicstack/types';
export const MAX_CONNECTOR_LEASE_TTL_MS = 5 * 60 * 1000;
export const MAX_CONNECTOR_GRANT_TTL_MS = 30 * 1000;
export interface ConnectorLeaseCoordinatorOptions {
readonly now?: () => Date;
readonly maxLeaseTtlMs?: number;
readonly maxGrantTtlMs?: number;
}
export class ConnectorLeaseError extends Error {
constructor(
readonly code: ConnectorLeaseRejectReason,
message: string,
) {
super(message);
this.name = ConnectorLeaseError.name;
}
}
/**
* Runtime-neutral lease coordinator. Durable CAS lives in the store adapter;
* grant provenance remains process-local so a restart fails closed and mints
* fresh grants from the durable current lease.
*/
export class ConnectorLeaseCoordinator {
private readonly issuedGrants = new WeakSet<object>();
private readonly now: () => Date;
private readonly maxLeaseTtlMs: number;
private readonly maxGrantTtlMs: number;
constructor(
private readonly store: ConnectorLeaseStore,
options: ConnectorLeaseCoordinatorOptions = {},
) {
this.now = options.now ?? (() => new Date());
this.maxLeaseTtlMs = normalizeTtlLimit(
options.maxLeaseTtlMs ?? MAX_CONNECTOR_LEASE_TTL_MS,
MAX_CONNECTOR_LEASE_TTL_MS,
'lease',
);
this.maxGrantTtlMs = normalizeTtlLimit(
options.maxGrantTtlMs ?? MAX_CONNECTOR_GRANT_TTL_MS,
MAX_CONNECTOR_GRANT_TTL_MS,
'grant',
);
}
async acquire(input: AcquireConnectorLeaseInput): Promise<ConnectorLease> {
const command = normalizeAcquireInput(input, this.maxLeaseTtlMs);
const now = this.now();
return this.store.acquire({
...command,
leaseId: randomUUID(),
now: now.toISOString(),
expiresAt: expiresAt(now, command.ttlMs),
});
}
async takeover(input: TakeoverConnectorLeaseInput): Promise<ConnectorLease> {
const command = normalizeAcquireInput(input, this.maxLeaseTtlMs);
const now = this.now();
return this.store.takeover({
...command,
expectedEpoch: normalizeLeaseEpoch(input.expectedEpoch),
leaseId: randomUUID(),
now: now.toISOString(),
expiresAt: expiresAt(now, command.ttlMs),
});
}
async heartbeat(input: HeartbeatConnectorLeaseInput): Promise<ConnectorLease> {
const now = this.now();
const lease = normalizeConnectorLease(input.lease);
const ttlMs = normalizeTtl(input.ttlMs, this.maxLeaseTtlMs, 'lease');
return this.store.heartbeat({
lease,
ttlMs,
correlationId: normalizeCorrelationId(input.correlationId),
now: now.toISOString(),
expiresAt: expiresAt(now, ttlMs),
});
}
async release(input: ReleaseConnectorLeaseInput): Promise<void> {
await this.store.release({
lease: normalizeConnectorLease(input.lease),
correlationId: normalizeCorrelationId(input.correlationId),
now: this.now().toISOString(),
});
}
async current(binding: LogicalAgentBinding): Promise<ConnectorLease | null> {
return this.store.findCurrent(normalizeBinding(binding));
}
async issueGrant(input: IssueConnectorExecutionGrantInput): Promise<ConnectorExecutionGrant> {
const now = this.now();
const lease = normalizeConnectorLease(input.lease);
const scopes = normalizeConnectorScopes(input.scopes);
const correlationId = normalizeCorrelationId(input.correlationId);
const ttlMs = normalizeTtl(input.ttlMs, this.maxGrantTtlMs, 'grant');
const current = await this.store.findCurrent(lease);
await this.assertCurrentLease(current, lease, now, correlationId);
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
if (!isScopeSubset(scopes, lease.scopes) || !isScopeSubset(scopes, current.scopes)) {
await this.reject(lease, correlationId, now, 'scope_denied');
}
const requestedExpiry = new Date(now.getTime() + ttlMs);
const leaseExpiry = new Date(current.expiresAt);
const grantExpiry = requestedExpiry < leaseExpiry ? requestedExpiry : leaseExpiry;
const grant: ConnectorExecutionGrant = Object.freeze({
identity: current.identity,
bindingId: current.bindingId,
leaseId: current.leaseId,
connectorId: current.connectorId,
scopes,
leaseEpoch: current.leaseEpoch,
issuedAt: now.toISOString(),
expiresAt: grantExpiry.toISOString(),
correlationId,
});
this.issuedGrants.add(grant);
return grant;
}
async executeGrant<TInput, TOutput>(
grant: ConnectorExecutionGrant,
requiredScope: string,
input: TInput,
adapter: FencedConnectorAdapter<TInput, TOutput>,
): Promise<TOutput> {
const now = this.now();
const normalizedScope = normalizeConnectorScope(requiredScope);
if (!this.issuedGrants.has(grant)) {
await this.rejectForgedGrant(grant, now);
}
if (new Date(grant.expiresAt) <= now) {
await this.rejectGrant(grant, now, 'grant_expired');
}
const current = await this.store.findCurrent(normalizeBinding(grant));
await this.assertCurrentLease(current, grant, now, grant.correlationId);
if (!grant.scopes.includes(normalizedScope) || !current?.scopes.includes(normalizedScope)) {
await this.rejectGrant(grant, now, 'scope_denied');
}
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
const context: ConnectorExecutionContext = Object.freeze({
identity: current.identity,
bindingId: current.bindingId,
leaseId: current.leaseId,
connectorId: current.connectorId,
scopes: Object.freeze([...grant.scopes]),
leaseEpoch: current.leaseEpoch,
correlationId: grant.correlationId,
grantExpiresAt: grant.expiresAt,
});
return adapter.execute(input, context);
}
private async assertCurrentLease(
current: ConnectorLease | null,
authority: ConnectorLease | ConnectorExecutionGrant,
now: Date,
correlationId: string,
): Promise<void> {
if (!current) await this.reject(authority, correlationId, now, 'lease_missing');
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
if (current.releasedAt) await this.reject(authority, correlationId, now, 'lease_released');
if (new Date(current.expiresAt) <= now) {
await this.store.recordAudit(auditEvent(current, correlationId, now, 'expiry', 'succeeded'));
await this.reject(authority, correlationId, now, 'lease_expired');
}
if (current.leaseEpoch !== authority.leaseEpoch) {
await this.reject(authority, correlationId, now, 'stale_epoch');
}
if (current.leaseId !== authority.leaseId || current.connectorId !== authority.connectorId) {
await this.reject(authority, correlationId, now, 'connector_mismatch');
}
}
private async rejectGrant(
grant: ConnectorExecutionGrant,
now: Date,
reason: ConnectorLeaseRejectReason,
): Promise<never> {
return this.reject(grant, grant.correlationId, now, reason);
}
private async rejectForgedGrant(grant: unknown, now: Date): Promise<never> {
const event = safeForgedGrantAudit(grant, now);
await this.store.recordAudit(event);
throw new ConnectorLeaseError('forged_grant', safeReasonMessage('forged_grant'));
}
private async reject(
authority: LogicalAgentBinding & {
readonly connectorId: string;
readonly leaseId?: string;
readonly leaseEpoch?: string;
},
correlationId: string,
now: Date,
reason: ConnectorLeaseRejectReason,
): Promise<never> {
await this.store.recordAudit(
auditEvent(authority, correlationId, now, 'reject', 'denied', reason),
);
throw new ConnectorLeaseError(reason, safeReasonMessage(reason));
}
}
function normalizeAcquireInput(
input: AcquireConnectorLeaseInput,
maxLeaseTtlMs: number,
): AcquireConnectorLeaseInput {
return {
identity: normalizeLogicalAgentIdentity(input.identity),
bindingId: normalizeLogicalBindingId(input.bindingId),
connectorId: normalizeConnectorId(input.connectorId),
scopes: normalizeConnectorScopes(input.scopes),
ttlMs: normalizeTtl(input.ttlMs, maxLeaseTtlMs, 'lease'),
correlationId: normalizeCorrelationId(input.correlationId),
};
}
function normalizeBinding(input: LogicalAgentBinding): LogicalAgentBinding {
return {
identity: normalizeLogicalAgentIdentity(input.identity),
bindingId: normalizeLogicalBindingId(input.bindingId),
};
}
export function normalizeConnectorLease(lease: ConnectorLease): ConnectorLease {
const binding = normalizeBinding(lease);
return Object.freeze({
...binding,
leaseId: lease.leaseId,
connectorId: normalizeConnectorId(lease.connectorId),
scopes: normalizeConnectorScopes(lease.scopes),
leaseEpoch: normalizeLeaseEpoch(lease.leaseEpoch),
acquiredAt: normalizeTimestamp(lease.acquiredAt, 'lease acquisition'),
heartbeatAt: normalizeTimestamp(lease.heartbeatAt, 'lease heartbeat'),
expiresAt: normalizeTimestamp(lease.expiresAt, 'lease expiry'),
...(lease.releasedAt
? { releasedAt: normalizeTimestamp(lease.releasedAt, 'lease release') }
: {}),
});
}
function normalizeTtl(ttlMs: number, maximum: number, kind: 'lease' | 'grant'): number {
if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0 || ttlMs > maximum) {
throw new Error(
`Connector ${kind} TTL must be a positive safe integer no greater than ${maximum}ms`,
);
}
return ttlMs;
}
function normalizeTtlLimit(ttlMs: number, hardMaximum: number, kind: 'lease' | 'grant'): number {
if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0 || ttlMs > hardMaximum) {
throw new Error(
`Maximum connector ${kind} TTL must be a positive safe integer no greater than ${hardMaximum}ms`,
);
}
return ttlMs;
}
function normalizeTimestamp(value: string, label: string): string {
const date = new Date(value);
if (Number.isNaN(date.getTime())) throw new Error(`${label} timestamp is invalid`);
return date.toISOString();
}
function expiresAt(now: Date, ttlMs: number): string {
const expiry = new Date(now.getTime() + ttlMs);
if (Number.isNaN(expiry.getTime())) throw new Error('Connector lease TTL exceeds date range');
return expiry.toISOString();
}
function isScopeSubset(requested: readonly string[], allowed: readonly string[]): boolean {
return requested.every((scope) => allowed.includes(scope));
}
function auditEvent(
authority: LogicalAgentBinding & {
readonly connectorId: string;
readonly leaseId?: string;
readonly leaseEpoch?: string;
},
correlationId: string,
now: Date,
event: ConnectorLeaseAuditEvent['event'],
outcome: ConnectorLeaseAuditEvent['outcome'],
reason?: ConnectorLeaseRejectReason,
): ConnectorLeaseAuditEvent {
return {
identity: authority.identity,
bindingId: authority.bindingId,
connectorId: authority.connectorId,
correlationId,
occurredAt: now.toISOString(),
event,
outcome,
...(authority.leaseId ? { leaseId: authority.leaseId } : {}),
...(authority.leaseEpoch ? { leaseEpoch: authority.leaseEpoch } : {}),
...(reason ? { reason } : {}),
};
}
function safeForgedGrantAudit(grant: unknown, now: Date): ConnectorLeaseAuditEvent {
const fallback: ConnectorLeaseAuditEvent = {
identity: { tenantId: 'untrusted', logicalAgentId: 'untrusted' },
bindingId: 'untrusted',
connectorId: 'untrusted',
correlationId: 'untrusted',
occurredAt: now.toISOString(),
event: 'reject',
outcome: 'denied',
reason: 'forged_grant',
};
if (typeof grant !== 'object' || grant === null || !('identity' in grant)) return fallback;
const identity = grant.identity;
if (typeof identity !== 'object' || identity === null) return fallback;
if (!('tenantId' in identity) || !('logicalAgentId' in identity)) return fallback;
if (!('bindingId' in grant) || !('connectorId' in grant) || !('correlationId' in grant)) {
return fallback;
}
if (
typeof identity.tenantId !== 'string' ||
typeof identity.logicalAgentId !== 'string' ||
typeof grant.bindingId !== 'string' ||
typeof grant.connectorId !== 'string' ||
typeof grant.correlationId !== 'string'
) {
return fallback;
}
try {
return {
identity: normalizeLogicalAgentIdentity({
tenantId: identity.tenantId,
logicalAgentId: identity.logicalAgentId,
}),
bindingId: normalizeLogicalBindingId(grant.bindingId),
connectorId: normalizeConnectorId(grant.connectorId),
correlationId: normalizeCorrelationId(grant.correlationId),
occurredAt: now.toISOString(),
event: 'reject',
outcome: 'denied',
reason: 'forged_grant',
};
} catch {
return fallback;
}
}
function safeReasonMessage(reason: ConnectorLeaseRejectReason): string {
return `Connector authority denied: ${reason}`;
}

View File

@@ -5,3 +5,4 @@ export * from './tmux-fleet-runtime-provider.js';
export * from './hermes-runtime-provider.js';
export * from './matrix-native-runtime-provider.js';
export * from './durable-session.js';
export * from './connector-lease.js';