feat(mos): add logical identity connector fencing (#757)

2026-07-14 23:33:06 +00:00
parent 2e2280070a
commit eb4e14ae5c
22 changed files with 7031 additions and 0 deletions
--- a/packages/agent/src/connector-lease.ts
+++ b/packages/agent/src/connector-lease.ts
@@ -0,0 +1,381 @@
+import { randomUUID } from 'node:crypto';
+import {
+  normalizeConnectorId,
+  normalizeConnectorScope,
+  normalizeConnectorScopes,
+  normalizeCorrelationId,
+  normalizeLeaseEpoch,
+  normalizeLogicalAgentIdentity,
+  normalizeLogicalBindingId,
+  type AcquireConnectorLeaseInput,
+  type ConnectorExecutionContext,
+  type ConnectorExecutionGrant,
+  type ConnectorLease,
+  type ConnectorLeaseAuditEvent,
+  type ConnectorLeaseRejectReason,
+  type ConnectorLeaseStore,
+  type FencedConnectorAdapter,
+  type HeartbeatConnectorLeaseInput,
+  type IssueConnectorExecutionGrantInput,
+  type LogicalAgentBinding,
+  type ReleaseConnectorLeaseInput,
+  type TakeoverConnectorLeaseInput,
+} from '@mosaicstack/types';
+
+export const MAX_CONNECTOR_LEASE_TTL_MS = 5 * 60 * 1000;
+export const MAX_CONNECTOR_GRANT_TTL_MS = 30 * 1000;
+
+export interface ConnectorLeaseCoordinatorOptions {
+  readonly now?: () => Date;
+  readonly maxLeaseTtlMs?: number;
+  readonly maxGrantTtlMs?: number;
+}
+
+export class ConnectorLeaseError extends Error {
+  constructor(
+    readonly code: ConnectorLeaseRejectReason,
+    message: string,
+  ) {
+    super(message);
+    this.name = ConnectorLeaseError.name;
+  }
+}
+
+/**
+ * Runtime-neutral lease coordinator. Durable CAS lives in the store adapter;
+ * grant provenance remains process-local so a restart fails closed and mints
+ * fresh grants from the durable current lease.
+ */
+export class ConnectorLeaseCoordinator {
+  private readonly issuedGrants = new WeakSet<object>();
+  private readonly now: () => Date;
+  private readonly maxLeaseTtlMs: number;
+  private readonly maxGrantTtlMs: number;
+
+  constructor(
+    private readonly store: ConnectorLeaseStore,
+    options: ConnectorLeaseCoordinatorOptions = {},
+  ) {
+    this.now = options.now ?? (() => new Date());
+    this.maxLeaseTtlMs = normalizeTtlLimit(
+      options.maxLeaseTtlMs ?? MAX_CONNECTOR_LEASE_TTL_MS,
+      MAX_CONNECTOR_LEASE_TTL_MS,
+      'lease',
+    );
+    this.maxGrantTtlMs = normalizeTtlLimit(
+      options.maxGrantTtlMs ?? MAX_CONNECTOR_GRANT_TTL_MS,
+      MAX_CONNECTOR_GRANT_TTL_MS,
+      'grant',
+    );
+  }
+
+  async acquire(input: AcquireConnectorLeaseInput): Promise<ConnectorLease> {
+    const command = normalizeAcquireInput(input, this.maxLeaseTtlMs);
+    const now = this.now();
+    return this.store.acquire({
+      ...command,
+      leaseId: randomUUID(),
+      now: now.toISOString(),
+      expiresAt: expiresAt(now, command.ttlMs),
+    });
+  }
+
+  async takeover(input: TakeoverConnectorLeaseInput): Promise<ConnectorLease> {
+    const command = normalizeAcquireInput(input, this.maxLeaseTtlMs);
+    const now = this.now();
+    return this.store.takeover({
+      ...command,
+      expectedEpoch: normalizeLeaseEpoch(input.expectedEpoch),
+      leaseId: randomUUID(),
+      now: now.toISOString(),
+      expiresAt: expiresAt(now, command.ttlMs),
+    });
+  }
+
+  async heartbeat(input: HeartbeatConnectorLeaseInput): Promise<ConnectorLease> {
+    const now = this.now();
+    const lease = normalizeConnectorLease(input.lease);
+    const ttlMs = normalizeTtl(input.ttlMs, this.maxLeaseTtlMs, 'lease');
+    return this.store.heartbeat({
+      lease,
+      ttlMs,
+      correlationId: normalizeCorrelationId(input.correlationId),
+      now: now.toISOString(),
+      expiresAt: expiresAt(now, ttlMs),
+    });
+  }
+
+  async release(input: ReleaseConnectorLeaseInput): Promise<void> {
+    await this.store.release({
+      lease: normalizeConnectorLease(input.lease),
+      correlationId: normalizeCorrelationId(input.correlationId),
+      now: this.now().toISOString(),
+    });
+  }
+
+  async current(binding: LogicalAgentBinding): Promise<ConnectorLease | null> {
+    return this.store.findCurrent(normalizeBinding(binding));
+  }
+
+  async issueGrant(input: IssueConnectorExecutionGrantInput): Promise<ConnectorExecutionGrant> {
+    const now = this.now();
+    const lease = normalizeConnectorLease(input.lease);
+    const scopes = normalizeConnectorScopes(input.scopes);
+    const correlationId = normalizeCorrelationId(input.correlationId);
+    const ttlMs = normalizeTtl(input.ttlMs, this.maxGrantTtlMs, 'grant');
+    const current = await this.store.findCurrent(lease);
+    await this.assertCurrentLease(current, lease, now, correlationId);
+    if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
+    if (!isScopeSubset(scopes, lease.scopes) || !isScopeSubset(scopes, current.scopes)) {
+      await this.reject(lease, correlationId, now, 'scope_denied');
+    }
+    const requestedExpiry = new Date(now.getTime() + ttlMs);
+    const leaseExpiry = new Date(current.expiresAt);
+    const grantExpiry = requestedExpiry < leaseExpiry ? requestedExpiry : leaseExpiry;
+    const grant: ConnectorExecutionGrant = Object.freeze({
+      identity: current.identity,
+      bindingId: current.bindingId,
+      leaseId: current.leaseId,
+      connectorId: current.connectorId,
+      scopes,
+      leaseEpoch: current.leaseEpoch,
+      issuedAt: now.toISOString(),
+      expiresAt: grantExpiry.toISOString(),
+      correlationId,
+    });
+    this.issuedGrants.add(grant);
+    return grant;
+  }
+
+  async executeGrant<TInput, TOutput>(
+    grant: ConnectorExecutionGrant,
+    requiredScope: string,
+    input: TInput,
+    adapter: FencedConnectorAdapter<TInput, TOutput>,
+  ): Promise<TOutput> {
+    const now = this.now();
+    const normalizedScope = normalizeConnectorScope(requiredScope);
+    if (!this.issuedGrants.has(grant)) {
+      await this.rejectForgedGrant(grant, now);
+    }
+    if (new Date(grant.expiresAt) <= now) {
+      await this.rejectGrant(grant, now, 'grant_expired');
+    }
+    const current = await this.store.findCurrent(normalizeBinding(grant));
+    await this.assertCurrentLease(current, grant, now, grant.correlationId);
+    if (!grant.scopes.includes(normalizedScope) || !current?.scopes.includes(normalizedScope)) {
+      await this.rejectGrant(grant, now, 'scope_denied');
+    }
+    if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
+    const context: ConnectorExecutionContext = Object.freeze({
+      identity: current.identity,
+      bindingId: current.bindingId,
+      leaseId: current.leaseId,
+      connectorId: current.connectorId,
+      scopes: Object.freeze([...grant.scopes]),
+      leaseEpoch: current.leaseEpoch,
+      correlationId: grant.correlationId,
+      grantExpiresAt: grant.expiresAt,
+    });
+    return adapter.execute(input, context);
+  }
+
+  private async assertCurrentLease(
+    current: ConnectorLease | null,
+    authority: ConnectorLease | ConnectorExecutionGrant,
+    now: Date,
+    correlationId: string,
+  ): Promise<void> {
+    if (!current) await this.reject(authority, correlationId, now, 'lease_missing');
+    if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
+    if (current.releasedAt) await this.reject(authority, correlationId, now, 'lease_released');
+    if (new Date(current.expiresAt) <= now) {
+      await this.store.recordAudit(auditEvent(current, correlationId, now, 'expiry', 'succeeded'));
+      await this.reject(authority, correlationId, now, 'lease_expired');
+    }
+    if (current.leaseEpoch !== authority.leaseEpoch) {
+      await this.reject(authority, correlationId, now, 'stale_epoch');
+    }
+    if (current.leaseId !== authority.leaseId || current.connectorId !== authority.connectorId) {
+      await this.reject(authority, correlationId, now, 'connector_mismatch');
+    }
+  }
+
+  private async rejectGrant(
+    grant: ConnectorExecutionGrant,
+    now: Date,
+    reason: ConnectorLeaseRejectReason,
+  ): Promise<never> {
+    return this.reject(grant, grant.correlationId, now, reason);
+  }
+
+  private async rejectForgedGrant(grant: unknown, now: Date): Promise<never> {
+    const event = safeForgedGrantAudit(grant, now);
+    await this.store.recordAudit(event);
+    throw new ConnectorLeaseError('forged_grant', safeReasonMessage('forged_grant'));
+  }
+
+  private async reject(
+    authority: LogicalAgentBinding & {
+      readonly connectorId: string;
+      readonly leaseId?: string;
+      readonly leaseEpoch?: string;
+    },
+    correlationId: string,
+    now: Date,
+    reason: ConnectorLeaseRejectReason,
+  ): Promise<never> {
+    await this.store.recordAudit(
+      auditEvent(authority, correlationId, now, 'reject', 'denied', reason),
+    );
+    throw new ConnectorLeaseError(reason, safeReasonMessage(reason));
+  }
+}
+
+function normalizeAcquireInput(
+  input: AcquireConnectorLeaseInput,
+  maxLeaseTtlMs: number,
+): AcquireConnectorLeaseInput {
+  return {
+    identity: normalizeLogicalAgentIdentity(input.identity),
+    bindingId: normalizeLogicalBindingId(input.bindingId),
+    connectorId: normalizeConnectorId(input.connectorId),
+    scopes: normalizeConnectorScopes(input.scopes),
+    ttlMs: normalizeTtl(input.ttlMs, maxLeaseTtlMs, 'lease'),
+    correlationId: normalizeCorrelationId(input.correlationId),
+  };
+}
+
+function normalizeBinding(input: LogicalAgentBinding): LogicalAgentBinding {
+  return {
+    identity: normalizeLogicalAgentIdentity(input.identity),
+    bindingId: normalizeLogicalBindingId(input.bindingId),
+  };
+}
+
+export function normalizeConnectorLease(lease: ConnectorLease): ConnectorLease {
+  const binding = normalizeBinding(lease);
+  return Object.freeze({
+    ...binding,
+    leaseId: lease.leaseId,
+    connectorId: normalizeConnectorId(lease.connectorId),
+    scopes: normalizeConnectorScopes(lease.scopes),
+    leaseEpoch: normalizeLeaseEpoch(lease.leaseEpoch),
+    acquiredAt: normalizeTimestamp(lease.acquiredAt, 'lease acquisition'),
+    heartbeatAt: normalizeTimestamp(lease.heartbeatAt, 'lease heartbeat'),
+    expiresAt: normalizeTimestamp(lease.expiresAt, 'lease expiry'),
+    ...(lease.releasedAt
+      ? { releasedAt: normalizeTimestamp(lease.releasedAt, 'lease release') }
+      : {}),
+  });
+}
+
+function normalizeTtl(ttlMs: number, maximum: number, kind: 'lease' | 'grant'): number {
+  if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0 || ttlMs > maximum) {
+    throw new Error(
+      `Connector ${kind} TTL must be a positive safe integer no greater than ${maximum}ms`,
+    );
+  }
+  return ttlMs;
+}
+
+function normalizeTtlLimit(ttlMs: number, hardMaximum: number, kind: 'lease' | 'grant'): number {
+  if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0 || ttlMs > hardMaximum) {
+    throw new Error(
+      `Maximum connector ${kind} TTL must be a positive safe integer no greater than ${hardMaximum}ms`,
+    );
+  }
+  return ttlMs;
+}
+
+function normalizeTimestamp(value: string, label: string): string {
+  const date = new Date(value);
+  if (Number.isNaN(date.getTime())) throw new Error(`${label} timestamp is invalid`);
+  return date.toISOString();
+}
+
+function expiresAt(now: Date, ttlMs: number): string {
+  const expiry = new Date(now.getTime() + ttlMs);
+  if (Number.isNaN(expiry.getTime())) throw new Error('Connector lease TTL exceeds date range');
+  return expiry.toISOString();
+}
+
+function isScopeSubset(requested: readonly string[], allowed: readonly string[]): boolean {
+  return requested.every((scope) => allowed.includes(scope));
+}
+
+function auditEvent(
+  authority: LogicalAgentBinding & {
+    readonly connectorId: string;
+    readonly leaseId?: string;
+    readonly leaseEpoch?: string;
+  },
+  correlationId: string,
+  now: Date,
+  event: ConnectorLeaseAuditEvent['event'],
+  outcome: ConnectorLeaseAuditEvent['outcome'],
+  reason?: ConnectorLeaseRejectReason,
+): ConnectorLeaseAuditEvent {
+  return {
+    identity: authority.identity,
+    bindingId: authority.bindingId,
+    connectorId: authority.connectorId,
+    correlationId,
+    occurredAt: now.toISOString(),
+    event,
+    outcome,
+    ...(authority.leaseId ? { leaseId: authority.leaseId } : {}),
+    ...(authority.leaseEpoch ? { leaseEpoch: authority.leaseEpoch } : {}),
+    ...(reason ? { reason } : {}),
+  };
+}
+
+function safeForgedGrantAudit(grant: unknown, now: Date): ConnectorLeaseAuditEvent {
+  const fallback: ConnectorLeaseAuditEvent = {
+    identity: { tenantId: 'untrusted', logicalAgentId: 'untrusted' },
+    bindingId: 'untrusted',
+    connectorId: 'untrusted',
+    correlationId: 'untrusted',
+    occurredAt: now.toISOString(),
+    event: 'reject',
+    outcome: 'denied',
+    reason: 'forged_grant',
+  };
+  if (typeof grant !== 'object' || grant === null || !('identity' in grant)) return fallback;
+  const identity = grant.identity;
+  if (typeof identity !== 'object' || identity === null) return fallback;
+  if (!('tenantId' in identity) || !('logicalAgentId' in identity)) return fallback;
+  if (!('bindingId' in grant) || !('connectorId' in grant) || !('correlationId' in grant)) {
+    return fallback;
+  }
+  if (
+    typeof identity.tenantId !== 'string' ||
+    typeof identity.logicalAgentId !== 'string' ||
+    typeof grant.bindingId !== 'string' ||
+    typeof grant.connectorId !== 'string' ||
+    typeof grant.correlationId !== 'string'
+  ) {
+    return fallback;
+  }
+  try {
+    return {
+      identity: normalizeLogicalAgentIdentity({
+        tenantId: identity.tenantId,
+        logicalAgentId: identity.logicalAgentId,
+      }),
+      bindingId: normalizeLogicalBindingId(grant.bindingId),
+      connectorId: normalizeConnectorId(grant.connectorId),
+      correlationId: normalizeCorrelationId(grant.correlationId),
+      occurredAt: now.toISOString(),
+      event: 'reject',
+      outcome: 'denied',
+      reason: 'forged_grant',
+    };
+  } catch {
+    return fallback;
+  }
+}
+
+function safeReasonMessage(reason: ConnectorLeaseRejectReason): string {
+  return `Connector authority denied: ${reason}`;
+}