fix(git): fail closed on tabs the YAML fallback recognizer would swallow (#865)
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful

PyYAML raises a ScannerError on a tab used anywhere outside a quoted
scalar (leading/trailing/embedded in a plain value, immediately before or
after a key colon, or as indentation) and yields no token, accepting tabs
ONLY inside single/double-quoted scalars. The conservative block-YAML
fallback in get_gitea_token_for_login normalized those tabs away -- via
_scalar's top-level .strip(), _KEY_RE's [ \t] separators, _Parser.__init__'s
body.rstrip(), parse_seq's body[1:].strip(), and the inline-map emptiness
checks -- and still emitted the login token: a credential fail-open in the
dangerous direction (less conservative than PyYAML).

Extend the whole-document "only ever more conservative than PyYAML, never
less" invariant to tab/scanner parity:
- _KEY_RE now uses SPACE-only separators (` *:` / `[ ](.*)`), so a tab in a
  key/value separator makes the line fail to match and the caller fails closed.
- Every whitespace-normalization site strips SPACES only (strip(" ")/rstrip(" "))
  so a tab survives to a fail-closed guard instead of being silently removed:
  _scalar top-level strip, quoted-trailing strip, post-comment strip,
  _Parser.__init__ body rstrip, parse_seq item strip, and the three inline
  emptiness checks.
- _scalar fails closed on any tab remaining in a plain scalar.
Tabs strictly inside quoted scalars are preserved verbatim (unchanged parity),
matching exactly what PyYAML accepts.

Verified empirically against PyYAML 6.0.3: ScannerError for each rejected
tab position; string-preserved for quoted inner tabs. Differential fuzz with
tab re-included: 0 fail-opens over ~6000 inputs; 0 over-rejection across 220
PyYAML-accepted quoted-tab cases. Adds section 17 to the regression harness
(fail-close fixtures for trailing/leading/embedded/after-colon/indentation
tabs; parity fixtures for double- and single-quoted inner tabs).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hermes Agent
2026-07-22 00:05:39 -05:00
parent 044744339b
commit 6053e1ee4c
2 changed files with 84 additions and 10 deletions

View File

@@ -807,7 +807,12 @@ def _constructible(text):
# Sentinel: "outside the supported subset -> fail closed". Distinct from a
# genuine null (None), which is a valid resolved value.
_FAIL = object()
_KEY_RE = re.compile(r"^([A-Za-z0-9_][A-Za-z0-9_.\-]*)[ \t]*:(?:[ \t](.*)|)$")
# Separators are SPACE-only: PyYAML rejects a tab used as key/value whitespace
# (before or after the ':') with a scanner error, so a tab there must NOT be
# treated as a benign separator. Space before the colon and one space after it
# stay valid (PyYAML strips a plain key's trailing spaces); a tab in either
# position makes the whole line fail to match -> the caller fails closed.
_KEY_RE = re.compile(r"^([A-Za-z0-9_][A-Za-z0-9_.\-]*) *:(?:[ ](.*)|)$")
_FLOW = set("[]{}*&!")
@@ -830,7 +835,10 @@ def _scalar(raw):
# subject to PyYAML's implicit typing: forms like 12345 / null / ~ / yes /
# 3.14 / a timestamp resolve to a NON-string (int/None/bool/float/date), so
# they return None here (PyYAML's path rejects a non-str token via _accept).
value = raw.strip()
# Strip SPACES only, never tabs: PyYAML raises a scanner error on a tab in a
# plain/leading/trailing scalar position, so a tab must be PRESERVED here to
# trip the fail-closed guard below rather than be silently normalized away.
value = raw.strip(" ")
if not value:
return None # empty plain scalar -> null
if value[0] in ("'", '"'):
@@ -838,7 +846,7 @@ def _scalar(raw):
end = value.find(quote, 1)
if end == -1:
return _FAIL # unterminated quote: PyYAML would error / continue
rest = value[end + 1:].strip()
rest = value[end + 1:].strip(" ")
if rest and not rest.startswith("#"):
return _FAIL # trailing junk after a quoted scalar
inner = value[1:end]
@@ -854,7 +862,7 @@ def _scalar(raw):
if ch == "#" and (i == 0 or value[i - 1] in (" ", "\t")):
value = value[:i]
break
value = value.strip()
value = value.strip(" ") # spaces only; a tab must survive to fail closed
if not value:
return None
if value[0] in _FLOW or value[0] in ("|", ">", "?", "@", "`", '"', "'", "%", ","):
@@ -869,7 +877,13 @@ def _scalar(raw):
return _FAIL
if any(ch in _FLOW for ch in value):
return _FAIL
if ": " in value or value.endswith(":") or "\t" in value:
if "\t" in value:
# A tab anywhere in a plain scalar (leading, trailing, or embedded) is a
# PyYAML scanner error on the whole document -- it accepts tabs ONLY
# inside quoted scalars (handled above, returned verbatim). Fail closed
# so the fallback never emits a token PyYAML would refuse over a tab.
return _FAIL
if ": " in value or value.endswith(":"):
return _FAIL # nested-mapping-in-scalar / ambiguous
if _implicit_nonstring(value):
# A plain scalar PyYAML would tag as a non-string (null/bool/int/float/
@@ -913,7 +927,10 @@ class _Parser:
continue
if re.match(r"^(---|\.\.\.)(\s|$)", body) or body in ("---", "..."):
raise _Bail() # document / end marker -> multi-doc -> fail closed
self.toks.append((indent, body.rstrip()))
# rstrip SPACES only: a trailing tab is a PyYAML scanner error, so it
# must be kept on the token body to reach the fail-closed guards
# (rstrip() would swallow it and let a bad line resolve a token).
self.toks.append((indent, body.rstrip(" ")))
self.i = 0
def peek(self):
@@ -959,7 +976,7 @@ class _Parser:
self.i += 1
if key in result:
raise _Bail() # duplicate mapping key (PyYAML last-wins; we bail)
if inline is not None and inline.strip() != "":
if inline is not None and inline.strip(" ") != "":
val = _scalar(inline)
if val is _FAIL:
raise _Bail()
@@ -997,7 +1014,7 @@ class _Parser:
raise _Bail()
if not (body.startswith("-") and (len(body) == 1 or body[1] in (" ", "\t"))):
break # a mapping entry at this indent ends the sequence
rest = body[1:].strip()
rest = body[1:].strip(" ") # spaces only; a tab must survive to bail
self.i += 1
if rest == "":
nxt = self.peek()
@@ -1023,7 +1040,7 @@ class _Parser:
result = {}
key = first_match.group(1)
inline = first_match.group(2)
if inline is not None and inline.strip() != "":
if inline is not None and inline.strip(" ") != "":
val = _scalar(inline)
if val is _FAIL:
raise _Bail()
@@ -1049,7 +1066,7 @@ class _Parser:
self.i += 1
if k in result:
raise _Bail()
if iv is not None and iv.strip() != "":
if iv is not None and iv.strip(" ") != "":
v = _scalar(iv)
if v is _FAIL:
raise _Bail()