fix(gateway): fence lifecycle authority against durable lease
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
This commit is contained in:
@@ -17,6 +17,7 @@ import {
|
|||||||
CONNECTOR_LEASE_POLICY,
|
CONNECTOR_LEASE_POLICY,
|
||||||
ConnectorLeaseService,
|
ConnectorLeaseService,
|
||||||
type ConnectorLeasePolicy,
|
type ConnectorLeasePolicy,
|
||||||
|
type ConnectorLeasePolicySubject,
|
||||||
} from './connector-lease.service.js';
|
} from './connector-lease.service.js';
|
||||||
|
|
||||||
const authorize = vi.fn().mockResolvedValue(true);
|
const authorize = vi.fn().mockResolvedValue(true);
|
||||||
@@ -31,6 +32,7 @@ describe('gateway connector lease fencing integration', (): void => {
|
|||||||
let handle: DbHandle;
|
let handle: DbHandle;
|
||||||
let moduleRef: TestingModule;
|
let moduleRef: TestingModule;
|
||||||
let service: ConnectorLeaseService;
|
let service: ConnectorLeaseService;
|
||||||
|
let repository: ConnectorLeaseRepository;
|
||||||
|
|
||||||
beforeAll(async (): Promise<void> => {
|
beforeAll(async (): Promise<void> => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
@@ -47,6 +49,7 @@ describe('gateway connector lease fencing integration', (): void => {
|
|||||||
],
|
],
|
||||||
}).compile();
|
}).compile();
|
||||||
service = moduleRef.get(ConnectorLeaseService);
|
service = moduleRef.get(ConnectorLeaseService);
|
||||||
|
repository = moduleRef.get(ConnectorLeaseRepository);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterAll(async (): Promise<void> => {
|
afterAll(async (): Promise<void> => {
|
||||||
@@ -229,4 +232,110 @@ describe('gateway connector lease fencing integration', (): void => {
|
|||||||
await expect(service.executeGrant(grant, 'runtime.send', undefined, adapter)).rejects.toThrow();
|
await expect(service.executeGrant(grant, 'runtime.send', undefined, adapter)).rejects.toThrow();
|
||||||
expect(adapter.execute).not.toHaveBeenCalled();
|
expect(adapter.execute).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('rejects submitted lifecycle scopes that differ from durable authority before policy or mutation', async (): Promise<void> => {
|
||||||
|
authorize.mockResolvedValue(true);
|
||||||
|
const heartbeatLease = await service.acquire(
|
||||||
|
{
|
||||||
|
logicalAgentId: 'mos',
|
||||||
|
bindingId: 'operator-chat-heartbeat-scope',
|
||||||
|
connectorId: 'pi-worker-a',
|
||||||
|
scopes: ['runtime.send'],
|
||||||
|
ttlMs: 60_000,
|
||||||
|
},
|
||||||
|
{ ...context, correlationId: 'correlation-heartbeat-scope-setup' },
|
||||||
|
);
|
||||||
|
const releaseLease = await service.acquire(
|
||||||
|
{
|
||||||
|
logicalAgentId: 'mos',
|
||||||
|
bindingId: 'operator-chat-release-scope',
|
||||||
|
connectorId: 'pi-worker-a',
|
||||||
|
scopes: ['runtime.send'],
|
||||||
|
ttlMs: 60_000,
|
||||||
|
},
|
||||||
|
{ ...context, correlationId: 'correlation-release-scope-setup' },
|
||||||
|
);
|
||||||
|
const forgedHeartbeat = { ...heartbeatLease, scopes: ['tool.execute'] };
|
||||||
|
const forgedRelease = { ...releaseLease, scopes: ['tool.execute'] };
|
||||||
|
|
||||||
|
authorize.mockImplementation(async (subject: ConnectorLeasePolicySubject) => {
|
||||||
|
return subject.requestedScopes.length === 1 && subject.requestedScopes[0] === 'tool.execute';
|
||||||
|
});
|
||||||
|
authorize.mockClear();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.heartbeat(forgedHeartbeat, 30_000, {
|
||||||
|
...context,
|
||||||
|
correlationId: 'correlation-heartbeat-scope-forgery',
|
||||||
|
}),
|
||||||
|
).rejects.toThrow('Connector authority policy denied');
|
||||||
|
await expect(
|
||||||
|
service.release(forgedRelease, {
|
||||||
|
...context,
|
||||||
|
correlationId: 'correlation-release-scope-forgery',
|
||||||
|
}),
|
||||||
|
).rejects.toThrow('Connector authority policy denied');
|
||||||
|
expect(authorize).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
const currentHeartbeat = await repository.findCurrent({
|
||||||
|
identity: heartbeatLease.identity,
|
||||||
|
bindingId: heartbeatLease.bindingId,
|
||||||
|
});
|
||||||
|
const currentRelease = await repository.findCurrent({
|
||||||
|
identity: releaseLease.identity,
|
||||||
|
bindingId: releaseLease.bindingId,
|
||||||
|
});
|
||||||
|
expect(currentHeartbeat).toMatchObject({
|
||||||
|
leaseId: heartbeatLease.leaseId,
|
||||||
|
scopes: ['runtime.send'],
|
||||||
|
heartbeatAt: heartbeatLease.heartbeatAt,
|
||||||
|
expiresAt: heartbeatLease.expiresAt,
|
||||||
|
});
|
||||||
|
expect(currentRelease).toMatchObject({
|
||||||
|
leaseId: releaseLease.leaseId,
|
||||||
|
scopes: ['runtime.send'],
|
||||||
|
});
|
||||||
|
expect(currentRelease?.releasedAt).toBeUndefined();
|
||||||
|
|
||||||
|
const forgedAudits = await handle.db
|
||||||
|
.select()
|
||||||
|
.from(connectorLeaseAuditLog)
|
||||||
|
.where(eq(connectorLeaseAuditLog.correlationId, 'correlation-heartbeat-scope-forgery'));
|
||||||
|
expect(forgedAudits).toHaveLength(1);
|
||||||
|
expect(forgedAudits[0]).toMatchObject({
|
||||||
|
bindingId: heartbeatLease.bindingId,
|
||||||
|
connectorId: heartbeatLease.connectorId,
|
||||||
|
event: 'reject',
|
||||||
|
outcome: 'denied',
|
||||||
|
reason: 'policy_denied',
|
||||||
|
});
|
||||||
|
const forgedReleaseAudits = await handle.db
|
||||||
|
.select()
|
||||||
|
.from(connectorLeaseAuditLog)
|
||||||
|
.where(eq(connectorLeaseAuditLog.correlationId, 'correlation-release-scope-forgery'));
|
||||||
|
expect(forgedReleaseAudits).toHaveLength(1);
|
||||||
|
expect(forgedReleaseAudits[0]).toMatchObject({
|
||||||
|
bindingId: releaseLease.bindingId,
|
||||||
|
connectorId: releaseLease.connectorId,
|
||||||
|
event: 'reject',
|
||||||
|
outcome: 'denied',
|
||||||
|
reason: 'policy_denied',
|
||||||
|
});
|
||||||
|
|
||||||
|
authorize.mockImplementation(async (subject: ConnectorLeasePolicySubject) => {
|
||||||
|
return subject.requestedScopes.length === 1 && subject.requestedScopes[0] === 'runtime.send';
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
service.heartbeat(heartbeatLease, 30_000, {
|
||||||
|
...context,
|
||||||
|
correlationId: 'correlation-heartbeat-scope-canonical',
|
||||||
|
}),
|
||||||
|
).resolves.toMatchObject({ scopes: ['runtime.send'] });
|
||||||
|
await expect(
|
||||||
|
service.release(releaseLease, {
|
||||||
|
...context,
|
||||||
|
correlationId: 'correlation-release-scope-canonical',
|
||||||
|
}),
|
||||||
|
).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -111,16 +111,10 @@ export class ConnectorLeaseService {
|
|||||||
context: ConnectorLeaseRequestContext,
|
context: ConnectorLeaseRequestContext,
|
||||||
): Promise<ConnectorLease> {
|
): Promise<ConnectorLease> {
|
||||||
const normalizedLease = normalizeConnectorLease(lease);
|
const normalizedLease = normalizeConnectorLease(lease);
|
||||||
await this.assertTenant(normalizedLease, context);
|
const durableLease = await this.durableLifecycleLease(normalizedLease, context);
|
||||||
await this.assertPolicy(
|
await this.assertPolicy('lease.heartbeat', durableLease, context, durableLease.scopes, ttlMs);
|
||||||
'lease.heartbeat',
|
|
||||||
normalizedLease,
|
|
||||||
context,
|
|
||||||
normalizedLease.scopes,
|
|
||||||
ttlMs,
|
|
||||||
);
|
|
||||||
return this.coordinator.heartbeat({
|
return this.coordinator.heartbeat({
|
||||||
lease: normalizedLease,
|
lease: durableLease,
|
||||||
ttlMs,
|
ttlMs,
|
||||||
correlationId: this.correlation(context),
|
correlationId: this.correlation(context),
|
||||||
});
|
});
|
||||||
@@ -128,16 +122,10 @@ export class ConnectorLeaseService {
|
|||||||
|
|
||||||
async release(lease: ConnectorLease, context: ConnectorLeaseRequestContext): Promise<void> {
|
async release(lease: ConnectorLease, context: ConnectorLeaseRequestContext): Promise<void> {
|
||||||
const normalizedLease = normalizeConnectorLease(lease);
|
const normalizedLease = normalizeConnectorLease(lease);
|
||||||
await this.assertTenant(normalizedLease, context);
|
const durableLease = await this.durableLifecycleLease(normalizedLease, context);
|
||||||
await this.assertPolicy(
|
await this.assertPolicy('lease.release', durableLease, context, durableLease.scopes, null);
|
||||||
'lease.release',
|
|
||||||
normalizedLease,
|
|
||||||
context,
|
|
||||||
normalizedLease.scopes,
|
|
||||||
null,
|
|
||||||
);
|
|
||||||
await this.coordinator.release({
|
await this.coordinator.release({
|
||||||
lease: normalizedLease,
|
lease: durableLease,
|
||||||
correlationId: this.correlation(context),
|
correlationId: this.correlation(context),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -220,6 +208,19 @@ export class ConnectorLeaseService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async durableLifecycleLease(
|
||||||
|
submittedLease: ConnectorLease,
|
||||||
|
context: ConnectorLeaseRequestContext,
|
||||||
|
): Promise<ConnectorLease> {
|
||||||
|
await this.assertTenant(submittedLease, context);
|
||||||
|
const durableLease = await this.coordinator.current(submittedLease);
|
||||||
|
if (!durableLease || !hasSameLifecycleAuthority(submittedLease, durableLease)) {
|
||||||
|
await this.recordPolicyDenial(durableLease ?? submittedLease, context);
|
||||||
|
throw new ForbiddenException('Connector authority policy denied');
|
||||||
|
}
|
||||||
|
return durableLease;
|
||||||
|
}
|
||||||
|
|
||||||
private async assertPolicy(
|
private async assertPolicy(
|
||||||
action: ConnectorLeasePolicyAction,
|
action: ConnectorLeasePolicyAction,
|
||||||
subject: Pick<ConnectorLease, 'identity' | 'bindingId' | 'connectorId'>,
|
subject: Pick<ConnectorLease, 'identity' | 'bindingId' | 'connectorId'>,
|
||||||
@@ -264,3 +265,21 @@ export class ConnectorLeaseService {
|
|||||||
return normalizeCorrelationId(context.correlationId);
|
return normalizeCorrelationId(context.correlationId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hasSameLifecycleAuthority(
|
||||||
|
submittedLease: ConnectorLease,
|
||||||
|
durableLease: ConnectorLease,
|
||||||
|
): boolean {
|
||||||
|
return (
|
||||||
|
submittedLease.identity.tenantId === durableLease.identity.tenantId &&
|
||||||
|
submittedLease.identity.logicalAgentId === durableLease.identity.logicalAgentId &&
|
||||||
|
submittedLease.bindingId === durableLease.bindingId &&
|
||||||
|
submittedLease.leaseId === durableLease.leaseId &&
|
||||||
|
submittedLease.connectorId === durableLease.connectorId &&
|
||||||
|
submittedLease.leaseEpoch === durableLease.leaseEpoch &&
|
||||||
|
submittedLease.scopes.length === durableLease.scopes.length &&
|
||||||
|
submittedLease.scopes.every((scope: string, index: number): boolean => {
|
||||||
|
return scope === durableLease.scopes[index];
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -138,3 +138,11 @@ Latest independent security review: no critical/high/medium/low findings. Final
|
|||||||
- Generated schema check: `pnpm --filter @mosaicstack/db db:generate` reported no schema changes; migration `0016_salty_morlocks`, snapshot, and journal were unchanged by generation.
|
- Generated schema check: `pnpm --filter @mosaicstack/db db:generate` reported no schema changes; migration `0016_salty_morlocks`, snapshot, and journal were unchanged by generation.
|
||||||
- Full verification: `pnpm typecheck` 42/42; `pnpm lint` 23/23; `pnpm format:check` passed; `pnpm test` 42/42 (gateway 627 passed / 12 environment-gated skipped). Scoped diff check and local documentation-link scan passed.
|
- Full verification: `pnpm typecheck` 42/42; `pnpm lint` 23/23; `pnpm format:check` passed; `pnpm test` 42/42 (gateway 627 passed / 12 environment-gated skipped). Scoped diff check and local documentation-link scan passed.
|
||||||
- Pending only: commit this reconciliation record, queue guard, force-with-lease push of the rebased existing branch, then fresh independent DB/code/security review and Ultron. Do not claim merge or issue closure.
|
- Pending only: commit this reconciliation record, queue guard, force-with-lease push of the rebased existing branch, then fresh independent DB/code/security review and Ultron. Do not claim merge or issue closure.
|
||||||
|
|
||||||
|
## Durable lifecycle-authority remediation (2026-07-14)
|
||||||
|
|
||||||
|
- Independent review found that heartbeat and release authorized the submitted lease, allowing forged lifecycle scope data to influence policy before the durable row was consulted.
|
||||||
|
- Remediation: lifecycle operations tenant-check the submitted lease, load the durable current lease, compare tenant, logical agent, binding, lease UUID, connector, epoch, and canonical ordered scopes, audit and deny any absent/mismatched authority, then run policy and coordinator lifecycle calls with the durable lease. Coordinator CAS/fencing checks remain unchanged.
|
||||||
|
- Adversarial gateway integration coverage proves forged `tool.execute` scopes on a durable `runtime.send` lease deny before policy or mutation, preserve heartbeat/release state, and write a denial audit for both lifecycle actions; canonical heartbeat and release remain accepted.
|
||||||
|
- Verification: focused types 6/6; agent 10/10; gateway PGlite integration/repository 9/9 with one `DATABASE_URL`-gated PostgreSQL test skipped; Drizzle check passed; root typecheck 42/42; lint 23/23; format check passed; root test 42/42 (gateway 628 passed / 12 environment-gated skipped).
|
||||||
|
- Pending: commit, queue-guard, force-with-lease push, then independent DB/code/security rereview and CI. Do not merge or close #755.
|
||||||
|
|||||||
Reference in New Issue
Block a user