From 6053e1ee4c531ea8906b9b8bb291300295ebf202 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Wed, 22 Jul 2026 00:05:39 -0500 Subject: [PATCH] fix(git): fail closed on tabs the YAML fallback recognizer would swallow (#865) PyYAML raises a ScannerError on a tab used anywhere outside a quoted scalar (leading/trailing/embedded in a plain value, immediately before or after a key colon, or as indentation) and yields no token, accepting tabs ONLY inside single/double-quoted scalars. The conservative block-YAML fallback in get_gitea_token_for_login normalized those tabs away -- via _scalar's top-level .strip(), _KEY_RE's [ \t] separators, _Parser.__init__'s body.rstrip(), parse_seq's body[1:].strip(), and the inline-map emptiness checks -- and still emitted the login token: a credential fail-open in the dangerous direction (less conservative than PyYAML). Extend the whole-document "only ever more conservative than PyYAML, never less" invariant to tab/scanner parity: - _KEY_RE now uses SPACE-only separators (` *:` / `[ ](.*)`), so a tab in a key/value separator makes the line fail to match and the caller fails closed. - Every whitespace-normalization site strips SPACES only (strip(" ")/rstrip(" ")) so a tab survives to a fail-closed guard instead of being silently removed: _scalar top-level strip, quoted-trailing strip, post-comment strip, _Parser.__init__ body rstrip, parse_seq item strip, and the three inline emptiness checks. - _scalar fails closed on any tab remaining in a plain scalar. Tabs strictly inside quoted scalars are preserved verbatim (unchanged parity), matching exactly what PyYAML accepts. Verified empirically against PyYAML 6.0.3: ScannerError for each rejected tab position; string-preserved for quoted inner tabs. Differential fuzz with tab re-included: 0 fail-opens over ~6000 inputs; 0 over-rejection across 220 PyYAML-accepted quoted-tab cases. Adds section 17 to the regression harness (fail-close fixtures for trailing/leading/embedded/after-colon/indentation tabs; parity fixtures for double- and single-quoted inner tabs). Co-Authored-By: Claude Opus 4.8 --- .../framework/tools/git/detect-platform.sh | 37 ++++++++---- .../tools/git/test-gitea-login-resolution.sh | 57 +++++++++++++++++++ 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/packages/mosaic/framework/tools/git/detect-platform.sh b/packages/mosaic/framework/tools/git/detect-platform.sh index 0281b5ca..e552743f 100755 --- a/packages/mosaic/framework/tools/git/detect-platform.sh +++ b/packages/mosaic/framework/tools/git/detect-platform.sh @@ -807,7 +807,12 @@ def _constructible(text): # Sentinel: "outside the supported subset -> fail closed". Distinct from a # genuine null (None), which is a valid resolved value. _FAIL = object() -_KEY_RE = re.compile(r"^([A-Za-z0-9_][A-Za-z0-9_.\-]*)[ \t]*:(?:[ \t](.*)|)$") +# Separators are SPACE-only: PyYAML rejects a tab used as key/value whitespace +# (before or after the ':') with a scanner error, so a tab there must NOT be +# treated as a benign separator. Space before the colon and one space after it +# stay valid (PyYAML strips a plain key's trailing spaces); a tab in either +# position makes the whole line fail to match -> the caller fails closed. +_KEY_RE = re.compile(r"^([A-Za-z0-9_][A-Za-z0-9_.\-]*) *:(?:[ ](.*)|)$") _FLOW = set("[]{}*&!") @@ -830,7 +835,10 @@ def _scalar(raw): # subject to PyYAML's implicit typing: forms like 12345 / null / ~ / yes / # 3.14 / a timestamp resolve to a NON-string (int/None/bool/float/date), so # they return None here (PyYAML's path rejects a non-str token via _accept). - value = raw.strip() + # Strip SPACES only, never tabs: PyYAML raises a scanner error on a tab in a + # plain/leading/trailing scalar position, so a tab must be PRESERVED here to + # trip the fail-closed guard below rather than be silently normalized away. + value = raw.strip(" ") if not value: return None # empty plain scalar -> null if value[0] in ("'", '"'): @@ -838,7 +846,7 @@ def _scalar(raw): end = value.find(quote, 1) if end == -1: return _FAIL # unterminated quote: PyYAML would error / continue - rest = value[end + 1:].strip() + rest = value[end + 1:].strip(" ") if rest and not rest.startswith("#"): return _FAIL # trailing junk after a quoted scalar inner = value[1:end] @@ -854,7 +862,7 @@ def _scalar(raw): if ch == "#" and (i == 0 or value[i - 1] in (" ", "\t")): value = value[:i] break - value = value.strip() + value = value.strip(" ") # spaces only; a tab must survive to fail closed if not value: return None if value[0] in _FLOW or value[0] in ("|", ">", "?", "@", "`", '"', "'", "%", ","): @@ -869,7 +877,13 @@ def _scalar(raw): return _FAIL if any(ch in _FLOW for ch in value): return _FAIL - if ": " in value or value.endswith(":") or "\t" in value: + if "\t" in value: + # A tab anywhere in a plain scalar (leading, trailing, or embedded) is a + # PyYAML scanner error on the whole document -- it accepts tabs ONLY + # inside quoted scalars (handled above, returned verbatim). Fail closed + # so the fallback never emits a token PyYAML would refuse over a tab. + return _FAIL + if ": " in value or value.endswith(":"): return _FAIL # nested-mapping-in-scalar / ambiguous if _implicit_nonstring(value): # A plain scalar PyYAML would tag as a non-string (null/bool/int/float/ @@ -913,7 +927,10 @@ class _Parser: continue if re.match(r"^(---|\.\.\.)(\s|$)", body) or body in ("---", "..."): raise _Bail() # document / end marker -> multi-doc -> fail closed - self.toks.append((indent, body.rstrip())) + # rstrip SPACES only: a trailing tab is a PyYAML scanner error, so it + # must be kept on the token body to reach the fail-closed guards + # (rstrip() would swallow it and let a bad line resolve a token). + self.toks.append((indent, body.rstrip(" "))) self.i = 0 def peek(self): @@ -959,7 +976,7 @@ class _Parser: self.i += 1 if key in result: raise _Bail() # duplicate mapping key (PyYAML last-wins; we bail) - if inline is not None and inline.strip() != "": + if inline is not None and inline.strip(" ") != "": val = _scalar(inline) if val is _FAIL: raise _Bail() @@ -997,7 +1014,7 @@ class _Parser: raise _Bail() if not (body.startswith("-") and (len(body) == 1 or body[1] in (" ", "\t"))): break # a mapping entry at this indent ends the sequence - rest = body[1:].strip() + rest = body[1:].strip(" ") # spaces only; a tab must survive to bail self.i += 1 if rest == "": nxt = self.peek() @@ -1023,7 +1040,7 @@ class _Parser: result = {} key = first_match.group(1) inline = first_match.group(2) - if inline is not None and inline.strip() != "": + if inline is not None and inline.strip(" ") != "": val = _scalar(inline) if val is _FAIL: raise _Bail() @@ -1049,7 +1066,7 @@ class _Parser: self.i += 1 if k in result: raise _Bail() - if iv is not None and iv.strip() != "": + if iv is not None and iv.strip(" ") != "": v = _scalar(iv) if v is _FAIL: raise _Bail() diff --git a/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh b/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh index ad571b57..c6d4bdb6 100755 --- a/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh +++ b/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh @@ -719,4 +719,61 @@ assert_token "4.e8 float look-alike still resolves token" "TOK_PLAIN" primary gi write_bad_key_fixture '0x1f' assert_token "valid hex int unrelated key still resolves token" "TOK_PLAIN" primary git.example +# 17. TAB / SCANNER PARITY: PyYAML raises a ScannerError on a tab used anywhere +# outside a quoted scalar -- leading, trailing, or embedded in a plain value, +# immediately after a key colon, before a key colon, or as indentation -- and +# yields NO token, accepting tabs ONLY inside single/double-quoted scalars +# (where the tab is preserved as string content). A prior fallback swallowed +# those tabs (via .strip()/.rstrip() normalization and [ \t] key separators) +# and still emitted the login token -- a fail-open in the dangerous direction. +# The recognizer now fails CLOSED for the whole document on any tab PyYAML +# rejects, while preserving the tabs PyYAML keeps (inside quotes). All tab +# positions were verified empirically against PyYAML 6.0.3 (ScannerError for +# each rejected position; string-preserved for quoted inner tabs). +TAB=$'\t' +# Fail-close: a tab in a plain value position (trailing / leading / embedded). +write_bad_key_fixture "l4o${TAB}" +assert_both_fail_closed "trailing tab in plain value fails closed" primary git.example +write_bad_key_fixture "${TAB}9" +assert_both_fail_closed "leading tab in plain value fails closed" primary git.example +write_bad_key_fixture "a${TAB}b" +assert_both_fail_closed "embedded tab in plain value fails closed" primary git.example +# Fail-close: a tab immediately after the key colon (no separating space). +write_fixture "bad:${TAB}9 +logins: + - name: primary + url: https://git.example + token: TOK_PLAIN +" +assert_both_fail_closed "tab immediately after key colon fails closed" primary git.example +# Fail-close: a tab used as indentation (before a sequence dash). +write_fixture "logins: +${TAB}- name: primary + url: https://git.example + token: TOK_PLAIN +" +assert_both_fail_closed "tab used as indentation fails closed" primary git.example +# Fail-close: a tab trailing a sequence-mapping field value. +write_fixture "logins: + - name: primary + url: https://git.example + token: TOK_PLAIN${TAB} +" +assert_both_fail_closed "tab trailing a seq field value fails closed" primary git.example +# NOT over-broad: a tab strictly INSIDE a quoted scalar is valid YAML (PyYAML +# keeps it as string content), so the document parses and the login token still +# resolves in BOTH paths -- double-quoted and single-quoted. +write_bad_key_fixture "\"a${TAB}b\"" +assert_token "tab inside a double-quoted value still resolves token" "TOK_PLAIN" primary git.example +write_bad_key_fixture "'a${TAB}b'" +assert_token "tab inside a single-quoted value still resolves token" "TOK_PLAIN" primary git.example +# ...and a quoted token value carrying an inner tab resolves to the exact string +# (tab preserved), identical to PyYAML's construction. +write_fixture "logins: + - name: primary + url: https://git.example + token: \"T${TAB}OK\" +" +assert_token "quoted token with inner tab resolves verbatim" "T${TAB}OK" primary git.example + echo "Gitea login resolution regression harness passed"