fix(tess): harden redacted stream boundaries
This commit is contained in:
@@ -69,6 +69,51 @@ describe('ChatGateway redaction boundary', (): void => {
|
|||||||
expect(JSON.stringify(client.emit.mock.calls)).not.toContain(CANARY);
|
expect(JSON.stringify(client.emit.mock.calls)).not.toContain(CANARY);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('retains a split secret label until its value can be redacted', (): void => {
|
||||||
|
const { gateway } = buildGateway();
|
||||||
|
const client = {
|
||||||
|
connected: true,
|
||||||
|
id: 'client-1',
|
||||||
|
data: { user: { id: 'user-1' } },
|
||||||
|
emit: vi.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
gateway.relayEvent(client, CONVERSATION_ID, {
|
||||||
|
type: 'message_update',
|
||||||
|
assistantMessageEvent: { type: 'text_delta', delta: 'token ' },
|
||||||
|
});
|
||||||
|
gateway.relayEvent(client, CONVERSATION_ID, {
|
||||||
|
type: 'message_update',
|
||||||
|
assistantMessageEvent: { type: 'text_delta', delta: '=canaryvalue123 ' },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(client.emit).toHaveBeenCalledWith('agent:text', {
|
||||||
|
conversationId: CONVERSATION_ID,
|
||||||
|
text: '[REDACTED_SECRET] ',
|
||||||
|
});
|
||||||
|
expect(JSON.stringify(client.emit.mock.calls)).not.toContain('canaryvalue123');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('drops an oversized unterminated stream fragment rather than retaining it', (): void => {
|
||||||
|
const { gateway } = buildGateway();
|
||||||
|
const client = {
|
||||||
|
connected: true,
|
||||||
|
id: 'client-1',
|
||||||
|
data: { user: { id: 'user-1' } },
|
||||||
|
emit: vi.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
gateway.relayEvent(client, CONVERSATION_ID, {
|
||||||
|
type: 'message_update',
|
||||||
|
assistantMessageEvent: { type: 'text_delta', delta: 'x'.repeat(8_193) },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(client.emit).toHaveBeenCalledWith('agent:text', {
|
||||||
|
conversationId: CONVERSATION_ID,
|
||||||
|
text: '[REDACTED_STREAM_OVERFLOW]',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('persists only redacted assistant content with classifications', (): void => {
|
it('persists only redacted assistant content with classifications', (): void => {
|
||||||
const { gateway, brain } = buildGateway();
|
const { gateway, brain } = buildGateway();
|
||||||
const client = {
|
const client = {
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ interface ClientSession {
|
|||||||
* Keyed by conversationId, value is the model name to use.
|
* Keyed by conversationId, value is the model name to use.
|
||||||
*/
|
*/
|
||||||
const modelOverrides = new Map<string, string>();
|
const modelOverrides = new Map<string, string>();
|
||||||
|
const MAX_REDACTION_BUFFER_LENGTH = 8_192;
|
||||||
|
|
||||||
function isDiscordIngressEnvelope(value: unknown): value is DiscordIngressEnvelope {
|
function isDiscordIngressEnvelope(value: unknown): value is DiscordIngressEnvelope {
|
||||||
if (typeof value !== 'object' || value === null) return false;
|
if (typeof value !== 'object' || value === null) return false;
|
||||||
@@ -111,6 +112,7 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
|
|||||||
/** Raw stream fragments are kept in memory only until they are safe to redact and emit. */
|
/** Raw stream fragments are kept in memory only until they are safe to redact and emit. */
|
||||||
private readonly textEgressBuffers = new Map<string, string>();
|
private readonly textEgressBuffers = new Map<string, string>();
|
||||||
private readonly thinkingEgressBuffers = new Map<string, string>();
|
private readonly thinkingEgressBuffers = new Map<string, string>();
|
||||||
|
private readonly overflowedEgress = new Set<string>();
|
||||||
private readonly discordReplayProtector = new DiscordReplayProtector();
|
private readonly discordReplayProtector = new DiscordReplayProtector();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@@ -161,6 +163,8 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
|
|||||||
}
|
}
|
||||||
this.textEgressBuffers.delete(client.id);
|
this.textEgressBuffers.delete(client.id);
|
||||||
this.thinkingEgressBuffers.delete(client.id);
|
this.thinkingEgressBuffers.delete(client.id);
|
||||||
|
this.overflowedEgress.delete(this.egressKey(client, 'agent:text'));
|
||||||
|
this.overflowedEgress.delete(this.egressKey(client, 'agent:thinking'));
|
||||||
}
|
}
|
||||||
|
|
||||||
private getClientScope(client: Socket): ActorTenantScope | null {
|
private getClientScope(client: Socket): ActorTenantScope | null {
|
||||||
@@ -758,7 +762,18 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
|
|||||||
buffers: Map<string, string>,
|
buffers: Map<string, string>,
|
||||||
delta: string,
|
delta: string,
|
||||||
): void {
|
): void {
|
||||||
buffers.set(client.id, `${buffers.get(client.id) ?? ''}${delta}`);
|
const key = this.egressKey(client, eventName);
|
||||||
|
if (this.overflowedEgress.has(key)) return;
|
||||||
|
|
||||||
|
const buffered = `${buffers.get(client.id) ?? ''}${delta}`;
|
||||||
|
if (buffered.length > MAX_REDACTION_BUFFER_LENGTH) {
|
||||||
|
buffers.delete(client.id);
|
||||||
|
this.overflowedEgress.add(key);
|
||||||
|
client.emit(eventName, { conversationId, text: '[REDACTED_STREAM_OVERFLOW]' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
buffers.set(client.id, buffered);
|
||||||
this.flushRedactedEgress(client, conversationId, eventName, buffers, false);
|
this.flushRedactedEgress(client, conversationId, eventName, buffers, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -774,6 +789,12 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
|
|||||||
buffers: Map<string, string>,
|
buffers: Map<string, string>,
|
||||||
final: boolean,
|
final: boolean,
|
||||||
): void {
|
): void {
|
||||||
|
const key = this.egressKey(client, eventName);
|
||||||
|
if (this.overflowedEgress.has(key)) {
|
||||||
|
if (final) this.overflowedEgress.delete(key);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const buffered = buffers.get(client.id) ?? '';
|
const buffered = buffers.get(client.id) ?? '';
|
||||||
const releaseLength = final ? buffered.length : this.safeRedactionPrefixLength(buffered);
|
const releaseLength = final ? buffered.length : this.safeRedactionPrefixLength(buffered);
|
||||||
const released = buffered.slice(0, releaseLength);
|
const released = buffered.slice(0, releaseLength);
|
||||||
@@ -806,7 +827,21 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
|
|||||||
|
|
||||||
// The secret classifier accepts whitespace around ':' and '=', so preserve
|
// The secret classifier accepts whitespace around ':' and '=', so preserve
|
||||||
// a pending label until its value and delimiter are both complete.
|
// a pending label until its value and delimiter are both complete.
|
||||||
const secretLabel = /(?:api[_-]?key|token|password|secret)\s*[:=]\s*$/i.exec(content);
|
const pendingSecretLabel =
|
||||||
|
/(?:^|[^A-Za-z0-9_])((?:api[_-]?key|token|password|secret|bearer|authorization)\s*)$/i.exec(
|
||||||
|
content,
|
||||||
|
);
|
||||||
|
if (pendingSecretLabel) {
|
||||||
|
const label = pendingSecretLabel[1] ?? '';
|
||||||
|
retainedFrom = Math.min(
|
||||||
|
retainedFrom,
|
||||||
|
pendingSecretLabel.index + pendingSecretLabel[0].length - label.length,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const secretLabel = /(?:api[_-]?key|token|password|secret|authorization)\s*[:=]\s*$/i.exec(
|
||||||
|
content,
|
||||||
|
);
|
||||||
if (secretLabel) {
|
if (secretLabel) {
|
||||||
retainedFrom = Math.min(retainedFrom, secretLabel.index);
|
retainedFrom = Math.min(retainedFrom, secretLabel.index);
|
||||||
}
|
}
|
||||||
@@ -826,6 +861,10 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
|
|||||||
return retainedFrom;
|
return retainedFrom;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private egressKey(client: Socket, eventName: 'agent:text' | 'agent:thinking'): string {
|
||||||
|
return `${client.id}:${eventName}`;
|
||||||
|
}
|
||||||
|
|
||||||
private relayEvent(client: Socket, conversationId: string, event: AgentSessionEvent): void {
|
private relayEvent(client: Socket, conversationId: string, event: AgentSessionEvent): void {
|
||||||
if (!client.connected) {
|
if (!client.connected) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
@@ -845,6 +884,8 @@ export class ChatGateway implements OnGatewayInit, OnGatewayConnection, OnGatewa
|
|||||||
}
|
}
|
||||||
this.textEgressBuffers.set(client.id, '');
|
this.textEgressBuffers.set(client.id, '');
|
||||||
this.thinkingEgressBuffers.set(client.id, '');
|
this.thinkingEgressBuffers.set(client.id, '');
|
||||||
|
this.overflowedEgress.delete(this.egressKey(client, 'agent:text'));
|
||||||
|
this.overflowedEgress.delete(this.egressKey(client, 'agent:thinking'));
|
||||||
client.emit('agent:start', { conversationId });
|
client.emit('agent:start', { conversationId });
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,4 +11,15 @@ describe('redactSensitiveContent', (): void => {
|
|||||||
expect(result.content).not.toContain('+1 555 555 1212');
|
expect(result.content).not.toContain('+1 555 555 1212');
|
||||||
expect(result.classifications).toEqual(['secret', 'pii']);
|
expect(result.classifications).toEqual(['secret', 'pii']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('redacts common provider credential formats', (): void => {
|
||||||
|
const result = redactSensitiveContent(
|
||||||
|
'Authorization: Bearer canary.bearer.token jwt eyJcanary.eyJpayload.eyJsignature aws AKIACANARY1234567890',
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.content).not.toContain('canary.bearer.token');
|
||||||
|
expect(result.content).not.toContain('eyJcanary.eyJpayload.eyJsignature');
|
||||||
|
expect(result.content).not.toContain('AKIACANARY1234567890');
|
||||||
|
expect(result.classifications).toEqual(['secret']);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,6 +8,11 @@ export interface RedactionResult {
|
|||||||
const SECRET_PATTERNS: RegExp[] = [
|
const SECRET_PATTERNS: RegExp[] = [
|
||||||
/\b(?:sk|ghp|gitea)_[A-Za-z0-9_-]{8,}\b/g,
|
/\b(?:sk|ghp|gitea)_[A-Za-z0-9_-]{8,}\b/g,
|
||||||
/\b(?:api[_-]?key|token|password|secret)\s*[:=]\s*[^\s,;]+/gi,
|
/\b(?:api[_-]?key|token|password|secret)\s*[:=]\s*[^\s,;]+/gi,
|
||||||
|
/\b(?:authorization\s*:\s*)?bearer\s+[A-Za-z0-9._~+/-]+=*/gi,
|
||||||
|
/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g,
|
||||||
|
/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
|
||||||
|
/-----BEGIN(?: [A-Z]+)* PRIVATE KEY-----[\s\S]*?-----END(?: [A-Z]+)* PRIVATE KEY-----/g,
|
||||||
|
/https?:\/\/[^\s?#]+[^\s]*[?&](?:token|key|secret|signature|sig)=[^\s&#]+/gi,
|
||||||
];
|
];
|
||||||
const PII_PATTERNS: RegExp[] = [
|
const PII_PATTERNS: RegExp[] = [
|
||||||
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
|
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
|
||||||
|
|||||||
Reference in New Issue
Block a user