382 lines
13 KiB
TypeScript
382 lines
13 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import {
|
|
normalizeConnectorId,
|
|
normalizeConnectorScope,
|
|
normalizeConnectorScopes,
|
|
normalizeCorrelationId,
|
|
normalizeLeaseEpoch,
|
|
normalizeLogicalAgentIdentity,
|
|
normalizeLogicalBindingId,
|
|
type AcquireConnectorLeaseInput,
|
|
type ConnectorExecutionContext,
|
|
type ConnectorExecutionGrant,
|
|
type ConnectorLease,
|
|
type ConnectorLeaseAuditEvent,
|
|
type ConnectorLeaseRejectReason,
|
|
type ConnectorLeaseStore,
|
|
type FencedConnectorAdapter,
|
|
type HeartbeatConnectorLeaseInput,
|
|
type IssueConnectorExecutionGrantInput,
|
|
type LogicalAgentBinding,
|
|
type ReleaseConnectorLeaseInput,
|
|
type TakeoverConnectorLeaseInput,
|
|
} from '@mosaicstack/types';
|
|
|
|
export const MAX_CONNECTOR_LEASE_TTL_MS = 5 * 60 * 1000;
|
|
export const MAX_CONNECTOR_GRANT_TTL_MS = 30 * 1000;
|
|
|
|
export interface ConnectorLeaseCoordinatorOptions {
|
|
readonly now?: () => Date;
|
|
readonly maxLeaseTtlMs?: number;
|
|
readonly maxGrantTtlMs?: number;
|
|
}
|
|
|
|
export class ConnectorLeaseError extends Error {
|
|
constructor(
|
|
readonly code: ConnectorLeaseRejectReason,
|
|
message: string,
|
|
) {
|
|
super(message);
|
|
this.name = ConnectorLeaseError.name;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Runtime-neutral lease coordinator. Durable CAS lives in the store adapter;
|
|
* grant provenance remains process-local so a restart fails closed and mints
|
|
* fresh grants from the durable current lease.
|
|
*/
|
|
export class ConnectorLeaseCoordinator {
|
|
private readonly issuedGrants = new WeakSet<object>();
|
|
private readonly now: () => Date;
|
|
private readonly maxLeaseTtlMs: number;
|
|
private readonly maxGrantTtlMs: number;
|
|
|
|
constructor(
|
|
private readonly store: ConnectorLeaseStore,
|
|
options: ConnectorLeaseCoordinatorOptions = {},
|
|
) {
|
|
this.now = options.now ?? (() => new Date());
|
|
this.maxLeaseTtlMs = normalizeTtlLimit(
|
|
options.maxLeaseTtlMs ?? MAX_CONNECTOR_LEASE_TTL_MS,
|
|
MAX_CONNECTOR_LEASE_TTL_MS,
|
|
'lease',
|
|
);
|
|
this.maxGrantTtlMs = normalizeTtlLimit(
|
|
options.maxGrantTtlMs ?? MAX_CONNECTOR_GRANT_TTL_MS,
|
|
MAX_CONNECTOR_GRANT_TTL_MS,
|
|
'grant',
|
|
);
|
|
}
|
|
|
|
async acquire(input: AcquireConnectorLeaseInput): Promise<ConnectorLease> {
|
|
const command = normalizeAcquireInput(input, this.maxLeaseTtlMs);
|
|
const now = this.now();
|
|
return this.store.acquire({
|
|
...command,
|
|
leaseId: randomUUID(),
|
|
now: now.toISOString(),
|
|
expiresAt: expiresAt(now, command.ttlMs),
|
|
});
|
|
}
|
|
|
|
async takeover(input: TakeoverConnectorLeaseInput): Promise<ConnectorLease> {
|
|
const command = normalizeAcquireInput(input, this.maxLeaseTtlMs);
|
|
const now = this.now();
|
|
return this.store.takeover({
|
|
...command,
|
|
expectedEpoch: normalizeLeaseEpoch(input.expectedEpoch),
|
|
leaseId: randomUUID(),
|
|
now: now.toISOString(),
|
|
expiresAt: expiresAt(now, command.ttlMs),
|
|
});
|
|
}
|
|
|
|
async heartbeat(input: HeartbeatConnectorLeaseInput): Promise<ConnectorLease> {
|
|
const now = this.now();
|
|
const lease = normalizeConnectorLease(input.lease);
|
|
const ttlMs = normalizeTtl(input.ttlMs, this.maxLeaseTtlMs, 'lease');
|
|
return this.store.heartbeat({
|
|
lease,
|
|
ttlMs,
|
|
correlationId: normalizeCorrelationId(input.correlationId),
|
|
now: now.toISOString(),
|
|
expiresAt: expiresAt(now, ttlMs),
|
|
});
|
|
}
|
|
|
|
async release(input: ReleaseConnectorLeaseInput): Promise<void> {
|
|
await this.store.release({
|
|
lease: normalizeConnectorLease(input.lease),
|
|
correlationId: normalizeCorrelationId(input.correlationId),
|
|
now: this.now().toISOString(),
|
|
});
|
|
}
|
|
|
|
async current(binding: LogicalAgentBinding): Promise<ConnectorLease | null> {
|
|
return this.store.findCurrent(normalizeBinding(binding));
|
|
}
|
|
|
|
async issueGrant(input: IssueConnectorExecutionGrantInput): Promise<ConnectorExecutionGrant> {
|
|
const now = this.now();
|
|
const lease = normalizeConnectorLease(input.lease);
|
|
const scopes = normalizeConnectorScopes(input.scopes);
|
|
const correlationId = normalizeCorrelationId(input.correlationId);
|
|
const ttlMs = normalizeTtl(input.ttlMs, this.maxGrantTtlMs, 'grant');
|
|
const current = await this.store.findCurrent(lease);
|
|
await this.assertCurrentLease(current, lease, now, correlationId);
|
|
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
|
|
if (!isScopeSubset(scopes, lease.scopes) || !isScopeSubset(scopes, current.scopes)) {
|
|
await this.reject(lease, correlationId, now, 'scope_denied');
|
|
}
|
|
const requestedExpiry = new Date(now.getTime() + ttlMs);
|
|
const leaseExpiry = new Date(current.expiresAt);
|
|
const grantExpiry = requestedExpiry < leaseExpiry ? requestedExpiry : leaseExpiry;
|
|
const grant: ConnectorExecutionGrant = Object.freeze({
|
|
identity: current.identity,
|
|
bindingId: current.bindingId,
|
|
leaseId: current.leaseId,
|
|
connectorId: current.connectorId,
|
|
scopes,
|
|
leaseEpoch: current.leaseEpoch,
|
|
issuedAt: now.toISOString(),
|
|
expiresAt: grantExpiry.toISOString(),
|
|
correlationId,
|
|
});
|
|
this.issuedGrants.add(grant);
|
|
return grant;
|
|
}
|
|
|
|
async executeGrant<TInput, TOutput>(
|
|
grant: ConnectorExecutionGrant,
|
|
requiredScope: string,
|
|
input: TInput,
|
|
adapter: FencedConnectorAdapter<TInput, TOutput>,
|
|
): Promise<TOutput> {
|
|
const now = this.now();
|
|
const normalizedScope = normalizeConnectorScope(requiredScope);
|
|
if (!this.issuedGrants.has(grant)) {
|
|
await this.rejectForgedGrant(grant, now);
|
|
}
|
|
if (new Date(grant.expiresAt) <= now) {
|
|
await this.rejectGrant(grant, now, 'grant_expired');
|
|
}
|
|
const current = await this.store.findCurrent(normalizeBinding(grant));
|
|
await this.assertCurrentLease(current, grant, now, grant.correlationId);
|
|
if (!grant.scopes.includes(normalizedScope) || !current?.scopes.includes(normalizedScope)) {
|
|
await this.rejectGrant(grant, now, 'scope_denied');
|
|
}
|
|
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
|
|
const context: ConnectorExecutionContext = Object.freeze({
|
|
identity: current.identity,
|
|
bindingId: current.bindingId,
|
|
leaseId: current.leaseId,
|
|
connectorId: current.connectorId,
|
|
scopes: Object.freeze([...grant.scopes]),
|
|
leaseEpoch: current.leaseEpoch,
|
|
correlationId: grant.correlationId,
|
|
grantExpiresAt: grant.expiresAt,
|
|
});
|
|
return adapter.execute(input, context);
|
|
}
|
|
|
|
private async assertCurrentLease(
|
|
current: ConnectorLease | null,
|
|
authority: ConnectorLease | ConnectorExecutionGrant,
|
|
now: Date,
|
|
correlationId: string,
|
|
): Promise<void> {
|
|
if (!current) await this.reject(authority, correlationId, now, 'lease_missing');
|
|
if (!current) throw new ConnectorLeaseError('lease_missing', 'Connector lease is unavailable');
|
|
if (current.releasedAt) await this.reject(authority, correlationId, now, 'lease_released');
|
|
if (new Date(current.expiresAt) <= now) {
|
|
await this.store.recordAudit(auditEvent(current, correlationId, now, 'expiry', 'succeeded'));
|
|
await this.reject(authority, correlationId, now, 'lease_expired');
|
|
}
|
|
if (current.leaseEpoch !== authority.leaseEpoch) {
|
|
await this.reject(authority, correlationId, now, 'stale_epoch');
|
|
}
|
|
if (current.leaseId !== authority.leaseId || current.connectorId !== authority.connectorId) {
|
|
await this.reject(authority, correlationId, now, 'connector_mismatch');
|
|
}
|
|
}
|
|
|
|
private async rejectGrant(
|
|
grant: ConnectorExecutionGrant,
|
|
now: Date,
|
|
reason: ConnectorLeaseRejectReason,
|
|
): Promise<never> {
|
|
return this.reject(grant, grant.correlationId, now, reason);
|
|
}
|
|
|
|
private async rejectForgedGrant(grant: unknown, now: Date): Promise<never> {
|
|
const event = safeForgedGrantAudit(grant, now);
|
|
await this.store.recordAudit(event);
|
|
throw new ConnectorLeaseError('forged_grant', safeReasonMessage('forged_grant'));
|
|
}
|
|
|
|
private async reject(
|
|
authority: LogicalAgentBinding & {
|
|
readonly connectorId: string;
|
|
readonly leaseId?: string;
|
|
readonly leaseEpoch?: string;
|
|
},
|
|
correlationId: string,
|
|
now: Date,
|
|
reason: ConnectorLeaseRejectReason,
|
|
): Promise<never> {
|
|
await this.store.recordAudit(
|
|
auditEvent(authority, correlationId, now, 'reject', 'denied', reason),
|
|
);
|
|
throw new ConnectorLeaseError(reason, safeReasonMessage(reason));
|
|
}
|
|
}
|
|
|
|
function normalizeAcquireInput(
|
|
input: AcquireConnectorLeaseInput,
|
|
maxLeaseTtlMs: number,
|
|
): AcquireConnectorLeaseInput {
|
|
return {
|
|
identity: normalizeLogicalAgentIdentity(input.identity),
|
|
bindingId: normalizeLogicalBindingId(input.bindingId),
|
|
connectorId: normalizeConnectorId(input.connectorId),
|
|
scopes: normalizeConnectorScopes(input.scopes),
|
|
ttlMs: normalizeTtl(input.ttlMs, maxLeaseTtlMs, 'lease'),
|
|
correlationId: normalizeCorrelationId(input.correlationId),
|
|
};
|
|
}
|
|
|
|
function normalizeBinding(input: LogicalAgentBinding): LogicalAgentBinding {
|
|
return {
|
|
identity: normalizeLogicalAgentIdentity(input.identity),
|
|
bindingId: normalizeLogicalBindingId(input.bindingId),
|
|
};
|
|
}
|
|
|
|
export function normalizeConnectorLease(lease: ConnectorLease): ConnectorLease {
|
|
const binding = normalizeBinding(lease);
|
|
return Object.freeze({
|
|
...binding,
|
|
leaseId: lease.leaseId,
|
|
connectorId: normalizeConnectorId(lease.connectorId),
|
|
scopes: normalizeConnectorScopes(lease.scopes),
|
|
leaseEpoch: normalizeLeaseEpoch(lease.leaseEpoch),
|
|
acquiredAt: normalizeTimestamp(lease.acquiredAt, 'lease acquisition'),
|
|
heartbeatAt: normalizeTimestamp(lease.heartbeatAt, 'lease heartbeat'),
|
|
expiresAt: normalizeTimestamp(lease.expiresAt, 'lease expiry'),
|
|
...(lease.releasedAt
|
|
? { releasedAt: normalizeTimestamp(lease.releasedAt, 'lease release') }
|
|
: {}),
|
|
});
|
|
}
|
|
|
|
function normalizeTtl(ttlMs: number, maximum: number, kind: 'lease' | 'grant'): number {
|
|
if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0 || ttlMs > maximum) {
|
|
throw new Error(
|
|
`Connector ${kind} TTL must be a positive safe integer no greater than ${maximum}ms`,
|
|
);
|
|
}
|
|
return ttlMs;
|
|
}
|
|
|
|
function normalizeTtlLimit(ttlMs: number, hardMaximum: number, kind: 'lease' | 'grant'): number {
|
|
if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0 || ttlMs > hardMaximum) {
|
|
throw new Error(
|
|
`Maximum connector ${kind} TTL must be a positive safe integer no greater than ${hardMaximum}ms`,
|
|
);
|
|
}
|
|
return ttlMs;
|
|
}
|
|
|
|
function normalizeTimestamp(value: string, label: string): string {
|
|
const date = new Date(value);
|
|
if (Number.isNaN(date.getTime())) throw new Error(`${label} timestamp is invalid`);
|
|
return date.toISOString();
|
|
}
|
|
|
|
function expiresAt(now: Date, ttlMs: number): string {
|
|
const expiry = new Date(now.getTime() + ttlMs);
|
|
if (Number.isNaN(expiry.getTime())) throw new Error('Connector lease TTL exceeds date range');
|
|
return expiry.toISOString();
|
|
}
|
|
|
|
function isScopeSubset(requested: readonly string[], allowed: readonly string[]): boolean {
|
|
return requested.every((scope) => allowed.includes(scope));
|
|
}
|
|
|
|
function auditEvent(
|
|
authority: LogicalAgentBinding & {
|
|
readonly connectorId: string;
|
|
readonly leaseId?: string;
|
|
readonly leaseEpoch?: string;
|
|
},
|
|
correlationId: string,
|
|
now: Date,
|
|
event: ConnectorLeaseAuditEvent['event'],
|
|
outcome: ConnectorLeaseAuditEvent['outcome'],
|
|
reason?: ConnectorLeaseRejectReason,
|
|
): ConnectorLeaseAuditEvent {
|
|
return {
|
|
identity: authority.identity,
|
|
bindingId: authority.bindingId,
|
|
connectorId: authority.connectorId,
|
|
correlationId,
|
|
occurredAt: now.toISOString(),
|
|
event,
|
|
outcome,
|
|
...(authority.leaseId ? { leaseId: authority.leaseId } : {}),
|
|
...(authority.leaseEpoch ? { leaseEpoch: authority.leaseEpoch } : {}),
|
|
...(reason ? { reason } : {}),
|
|
};
|
|
}
|
|
|
|
function safeForgedGrantAudit(grant: unknown, now: Date): ConnectorLeaseAuditEvent {
|
|
const fallback: ConnectorLeaseAuditEvent = {
|
|
identity: { tenantId: 'untrusted', logicalAgentId: 'untrusted' },
|
|
bindingId: 'untrusted',
|
|
connectorId: 'untrusted',
|
|
correlationId: 'untrusted',
|
|
occurredAt: now.toISOString(),
|
|
event: 'reject',
|
|
outcome: 'denied',
|
|
reason: 'forged_grant',
|
|
};
|
|
if (typeof grant !== 'object' || grant === null || !('identity' in grant)) return fallback;
|
|
const identity = grant.identity;
|
|
if (typeof identity !== 'object' || identity === null) return fallback;
|
|
if (!('tenantId' in identity) || !('logicalAgentId' in identity)) return fallback;
|
|
if (!('bindingId' in grant) || !('connectorId' in grant) || !('correlationId' in grant)) {
|
|
return fallback;
|
|
}
|
|
if (
|
|
typeof identity.tenantId !== 'string' ||
|
|
typeof identity.logicalAgentId !== 'string' ||
|
|
typeof grant.bindingId !== 'string' ||
|
|
typeof grant.connectorId !== 'string' ||
|
|
typeof grant.correlationId !== 'string'
|
|
) {
|
|
return fallback;
|
|
}
|
|
try {
|
|
return {
|
|
identity: normalizeLogicalAgentIdentity({
|
|
tenantId: identity.tenantId,
|
|
logicalAgentId: identity.logicalAgentId,
|
|
}),
|
|
bindingId: normalizeLogicalBindingId(grant.bindingId),
|
|
connectorId: normalizeConnectorId(grant.connectorId),
|
|
correlationId: normalizeCorrelationId(grant.correlationId),
|
|
occurredAt: now.toISOString(),
|
|
event: 'reject',
|
|
outcome: 'denied',
|
|
reason: 'forged_grant',
|
|
};
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
function safeReasonMessage(reason: ConnectorLeaseRejectReason): string {
|
|
return `Connector authority denied: ${reason}`;
|
|
}
|