fix(chat-fail): review round 3: the link, email and mixed-run redaction patterns start only where a run without spaces starts, so redacting the whole kept stdout stays linear (a 128000-character run took 10 to 15 s, now a few ms) and matches the same text; a long text that redacts below the read limit is not cut and keeps its first word, and a cause that is only [redacted] is dropped; the proxy page reads relogin, plural and past forms of an expired sign-in as a sign-in (#1873, lane chat-fail)
This commit is contained in:
1 parent
fc9e4dfca8
commit
07eca4a670
5 files changed
+35
-7
No files matched your search
@@ -28,7 +28,7 @@ const SIGN_IN = 'The account needs to sign in again. Sign in again, or add anoth
|
||||
const NO_REASON = 'The model service gave no reason. Wait and try again later, sign in again, or add another account.';
|
||||
const USAGE_TEXT = /usage limit|quota|rate.?limit|limit reached|too many requests|\b429\b|insufficient.?balance|credit balance/i;
|
||||
// A bare "expired" is not enough: a usage window expires too.
|
||||
const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|\b(?:\w*[_-])?(?:token|session|credentials?|login|auth\w*|grant)\s+(?:(?:has|is)\s+)?(?:been\s+)?expired|\brevoked\b|\bsign.?in\b|\blog.?in\b|\bre-?auth/i;
|
||||
const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|\b(?:\w*[_-])?(?:tokens?|sessions?|credentials?|logins?|auth\w*|grants?)\s+(?:(?:has|have|is|are|was|were)\s+)?(?:been\s+)?expired|\brevoked\b|\bsign.?in\b|\b(?:re-?)?log.?in\b|\bre-?auth/i;
|
||||
const RFC3339 = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/;
|
||||
// A reset time the read gives, as "YYYY-MM-DD HH:MM UTC" made from the parsed date, only when
|
||||
// it is a valid RFC 3339 time still ahead of now. Anything else gives no time.
|
||||
|
||||
@@ -75,3 +75,8 @@ test('a message word inside another word is not a sign-in; a prefixed or "is exp
|
||||
for (const status_message of ['access_token expired', 'token is expired', 'refresh-token has been expired', 'please sign in'])
|
||||
assert.equal((await reasonOf(t, { status: 'error', status_message })).reason, SIGN_IN, status_message);
|
||||
});
|
||||
// Review round 3: "relogin", plurals and past forms of "expired" read as a sign-in too.
|
||||
test('relogin, plural and past forms of an expired sign-in say sign in again', async t => {
|
||||
for (const status_message of ['relogin required', 'tokens have expired', 'session was expired', 'credentials were expired'])
|
||||
assert.equal((await reasonOf(t, { status: 'error', status_message })).reason, SIGN_IN, status_message);
|
||||
});
|
||||
@@ -89,10 +89,13 @@ export function secretValues(env) {
|
||||
|
||||
// A key word, with any prefix joined by _ or - (access_token, client_secret, x-api-key).
|
||||
const KEY_WORD = "(?:\\w*[_-])?(?:api[_-]?key|key|token|secret|password|passwd|authorization|cookie|session)";
|
||||
// Redaction runs over the whole text, up to the runner's 128000 characters, so a pattern that
|
||||
// could start anywhere inside a long run without spaces starts only where the run starts: it
|
||||
// matches the same text in linear time, not quadratic.
|
||||
const REDACT = [
|
||||
/\b[a-z][a-z0-9+.-]*:\/\/\S*/gi, // any link, its userinfo and query included
|
||||
/(?<![a-z0-9+.-])[a-z0-9+.-]+:\/\/\S*/gi, // any link, its userinfo and query included, from the run's start
|
||||
/\bwww\.\S+/gi,
|
||||
/\S+@\S+/g, // email addresses and user@host
|
||||
/(?<!\S)(?=\S*\S@\S)\S+/g, // email addresses and user@host: the whole run holding the @
|
||||
/(?:^|(?<=[\s("'=:]))(?:~|\.{1,2})?\/\S+/g, // absolute and home paths
|
||||
/\b[A-Za-z]:\\\S*/g, // Windows paths
|
||||
/\\\\\S+/g, // UNC paths
|
||||
@@ -105,7 +108,7 @@ const REDACT = [
|
||||
/\beyJ\S+/g, // JWT
|
||||
// A long run that mixes letters and digits, or any very long run, reads as a key.
|
||||
/[A-Za-z0-9_\-+/=.]{32,}/g,
|
||||
/(?=[A-Za-z0-9_\-+/=.]*\d)(?=[A-Za-z0-9_\-+/=.]*[A-Za-z])[A-Za-z0-9_\-+/=.]{16,}/g,
|
||||
/(?<![A-Za-z0-9_\-+/=.])(?=[A-Za-z0-9_\-+/=.]*\d)(?=[A-Za-z0-9_\-+/=.]*[A-Za-z])[A-Za-z0-9_\-+/=.]{16,}/g,
|
||||
];
|
||||
|
||||
export function cleanText(text) {
|
||||
@@ -139,8 +142,10 @@ export function turnCause(stdout, secrets = []) {
|
||||
let shown = clean;
|
||||
for (const re of REDACT) shown = shown.replace(re, "[redacted]");
|
||||
shown = shown.replace(/(?:\[redacted\]\s*){2,}/g, "[redacted] ").trim();
|
||||
if (tail) shown = shown.slice(-READ_MAX).replace(/^\S+\s*/, "");
|
||||
return shown ? cap(shown, tail) : null;
|
||||
// The cut drops the partial word it lands in; a redacted text short enough needs no cut.
|
||||
if (shown.length > READ_MAX) shown = shown.slice(-READ_MAX).replace(/^\S+\s*/, "");
|
||||
// Text that redacts to nothing but [redacted] says nothing; the technical reason stands alone.
|
||||
return shown && shown !== "[redacted]" ? cap(shown, tail) : null;
|
||||
}
|
||||
|
||||
// The chat body for a failed turn: the plain cause first, the technical reason after.
|
||||
|
||||
@@ -367,7 +367,8 @@ test("chat-fail F1 error text is cleaned, redacted and capped; a held secret wit
|
||||
const [hostile, leak, quiet] = ctx.inbox.map(m => m.body);
|
||||
const [cause, details] = hostile.slice("The turn failed: ".length).split("\nDetails: ");
|
||||
assert.equal(details, "adapter exited 1");
|
||||
assert.match(cause, /^API Error: 500 boom evil /);
|
||||
// The long token before the error is redacted as a whole: it shows, if at all, as one [redacted].
|
||||
assert.match(cause, /^(?:\[redacted\] )?API Error: 500 boom evil /);
|
||||
assert.ok(Array.from(cause).length <= 300, cause.length);
|
||||
assert.equal(cause.includes("www"), false, "the long text before the error is not shown");
|
||||
assert.doesNotMatch(hostile, /[\u0000-\u0009\u000b-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069\ufeff]/u, "no control or format character but the one separating newline");
|
||||
|
||||
@@ -95,3 +95,20 @@ test("review round 2: a tail cut between a key word and its value never shows th
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Review round 3: redaction runs over the whole kept stdout (up to 128000 characters), so a long
|
||||
// run with no spaces (base64, a data URI, minified JSON) must not make it slow.
|
||||
test("review round 3: a 128000-character run with no spaces is redacted in well under a second", () => {
|
||||
const N = 128000;
|
||||
for (const run of ["a".repeat(N), "a.".repeat(N / 2), "a-".repeat(N / 2), "a/".repeat(N / 2), "a:".repeat(N / 2), 'a"'.repeat(N / 2)]) {
|
||||
const started = performance.now();
|
||||
const cause = turnCause(`${run} then the adapter crashed`);
|
||||
const took = performance.now() - started;
|
||||
assert.ok(took < 1000, `${run.slice(0, 4)}... took ${Math.round(took)} ms`);
|
||||
assert.match(cause, /then the adapter crashed$/);
|
||||
}
|
||||
});
|
||||
// Review round 3: a long text that redacts down below the read limit isn't cut, so its first word stays.
|
||||
test("review round 3: a long text that redaction makes short keeps its first word", () => {
|
||||
assert.equal(turnCause(`Crash: adapter died at https://e.example/${"a".repeat(2500)}`), "Crash: adapter died at [redacted]");
|
||||
});
|
||||
Reference in new issue
Block a user