fix(git): fail closed on tabs the YAML fallback recognizer would swallow (#865)
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
PyYAML raises a ScannerError on a tab used anywhere outside a quoted
scalar (leading/trailing/embedded in a plain value, immediately before or
after a key colon, or as indentation) and yields no token, accepting tabs
ONLY inside single/double-quoted scalars. The conservative block-YAML
fallback in get_gitea_token_for_login normalized those tabs away -- via
_scalar's top-level .strip(), _KEY_RE's [ \t] separators, _Parser.__init__'s
body.rstrip(), parse_seq's body[1:].strip(), and the inline-map emptiness
checks -- and still emitted the login token: a credential fail-open in the
dangerous direction (less conservative than PyYAML).
Extend the whole-document "only ever more conservative than PyYAML, never
less" invariant to tab/scanner parity:
- _KEY_RE now uses SPACE-only separators (` *:` / `[ ](.*)`), so a tab in a
key/value separator makes the line fail to match and the caller fails closed.
- Every whitespace-normalization site strips SPACES only (strip(" ")/rstrip(" "))
so a tab survives to a fail-closed guard instead of being silently removed:
_scalar top-level strip, quoted-trailing strip, post-comment strip,
_Parser.__init__ body rstrip, parse_seq item strip, and the three inline
emptiness checks.
- _scalar fails closed on any tab remaining in a plain scalar.
Tabs strictly inside quoted scalars are preserved verbatim (unchanged parity),
matching exactly what PyYAML accepts.
Verified empirically against PyYAML 6.0.3: ScannerError for each rejected
tab position; string-preserved for quoted inner tabs. Differential fuzz with
tab re-included: 0 fail-opens over ~6000 inputs; 0 over-rejection across 220
PyYAML-accepted quoted-tab cases. Adds section 17 to the regression harness
(fail-close fixtures for trailing/leading/embedded/after-colon/indentation
tabs; parity fixtures for double- and single-quoted inner tabs).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -807,7 +807,12 @@ def _constructible(text):
|
|||||||
# Sentinel: "outside the supported subset -> fail closed". Distinct from a
|
# Sentinel: "outside the supported subset -> fail closed". Distinct from a
|
||||||
# genuine null (None), which is a valid resolved value.
|
# genuine null (None), which is a valid resolved value.
|
||||||
_FAIL = object()
|
_FAIL = object()
|
||||||
_KEY_RE = re.compile(r"^([A-Za-z0-9_][A-Za-z0-9_.\-]*)[ \t]*:(?:[ \t](.*)|)$")
|
# Separators are SPACE-only: PyYAML rejects a tab used as key/value whitespace
|
||||||
|
# (before or after the ':') with a scanner error, so a tab there must NOT be
|
||||||
|
# treated as a benign separator. Space before the colon and one space after it
|
||||||
|
# stay valid (PyYAML strips a plain key's trailing spaces); a tab in either
|
||||||
|
# position makes the whole line fail to match -> the caller fails closed.
|
||||||
|
_KEY_RE = re.compile(r"^([A-Za-z0-9_][A-Za-z0-9_.\-]*) *:(?:[ ](.*)|)$")
|
||||||
_FLOW = set("[]{}*&!")
|
_FLOW = set("[]{}*&!")
|
||||||
|
|
||||||
|
|
||||||
@@ -830,7 +835,10 @@ def _scalar(raw):
|
|||||||
# subject to PyYAML's implicit typing: forms like 12345 / null / ~ / yes /
|
# subject to PyYAML's implicit typing: forms like 12345 / null / ~ / yes /
|
||||||
# 3.14 / a timestamp resolve to a NON-string (int/None/bool/float/date), so
|
# 3.14 / a timestamp resolve to a NON-string (int/None/bool/float/date), so
|
||||||
# they return None here (PyYAML's path rejects a non-str token via _accept).
|
# they return None here (PyYAML's path rejects a non-str token via _accept).
|
||||||
value = raw.strip()
|
# Strip SPACES only, never tabs: PyYAML raises a scanner error on a tab in a
|
||||||
|
# plain/leading/trailing scalar position, so a tab must be PRESERVED here to
|
||||||
|
# trip the fail-closed guard below rather than be silently normalized away.
|
||||||
|
value = raw.strip(" ")
|
||||||
if not value:
|
if not value:
|
||||||
return None # empty plain scalar -> null
|
return None # empty plain scalar -> null
|
||||||
if value[0] in ("'", '"'):
|
if value[0] in ("'", '"'):
|
||||||
@@ -838,7 +846,7 @@ def _scalar(raw):
|
|||||||
end = value.find(quote, 1)
|
end = value.find(quote, 1)
|
||||||
if end == -1:
|
if end == -1:
|
||||||
return _FAIL # unterminated quote: PyYAML would error / continue
|
return _FAIL # unterminated quote: PyYAML would error / continue
|
||||||
rest = value[end + 1:].strip()
|
rest = value[end + 1:].strip(" ")
|
||||||
if rest and not rest.startswith("#"):
|
if rest and not rest.startswith("#"):
|
||||||
return _FAIL # trailing junk after a quoted scalar
|
return _FAIL # trailing junk after a quoted scalar
|
||||||
inner = value[1:end]
|
inner = value[1:end]
|
||||||
@@ -854,7 +862,7 @@ def _scalar(raw):
|
|||||||
if ch == "#" and (i == 0 or value[i - 1] in (" ", "\t")):
|
if ch == "#" and (i == 0 or value[i - 1] in (" ", "\t")):
|
||||||
value = value[:i]
|
value = value[:i]
|
||||||
break
|
break
|
||||||
value = value.strip()
|
value = value.strip(" ") # spaces only; a tab must survive to fail closed
|
||||||
if not value:
|
if not value:
|
||||||
return None
|
return None
|
||||||
if value[0] in _FLOW or value[0] in ("|", ">", "?", "@", "`", '"', "'", "%", ","):
|
if value[0] in _FLOW or value[0] in ("|", ">", "?", "@", "`", '"', "'", "%", ","):
|
||||||
@@ -869,7 +877,13 @@ def _scalar(raw):
|
|||||||
return _FAIL
|
return _FAIL
|
||||||
if any(ch in _FLOW for ch in value):
|
if any(ch in _FLOW for ch in value):
|
||||||
return _FAIL
|
return _FAIL
|
||||||
if ": " in value or value.endswith(":") or "\t" in value:
|
if "\t" in value:
|
||||||
|
# A tab anywhere in a plain scalar (leading, trailing, or embedded) is a
|
||||||
|
# PyYAML scanner error on the whole document -- it accepts tabs ONLY
|
||||||
|
# inside quoted scalars (handled above, returned verbatim). Fail closed
|
||||||
|
# so the fallback never emits a token PyYAML would refuse over a tab.
|
||||||
|
return _FAIL
|
||||||
|
if ": " in value or value.endswith(":"):
|
||||||
return _FAIL # nested-mapping-in-scalar / ambiguous
|
return _FAIL # nested-mapping-in-scalar / ambiguous
|
||||||
if _implicit_nonstring(value):
|
if _implicit_nonstring(value):
|
||||||
# A plain scalar PyYAML would tag as a non-string (null/bool/int/float/
|
# A plain scalar PyYAML would tag as a non-string (null/bool/int/float/
|
||||||
@@ -913,7 +927,10 @@ class _Parser:
|
|||||||
continue
|
continue
|
||||||
if re.match(r"^(---|\.\.\.)(\s|$)", body) or body in ("---", "..."):
|
if re.match(r"^(---|\.\.\.)(\s|$)", body) or body in ("---", "..."):
|
||||||
raise _Bail() # document / end marker -> multi-doc -> fail closed
|
raise _Bail() # document / end marker -> multi-doc -> fail closed
|
||||||
self.toks.append((indent, body.rstrip()))
|
# rstrip SPACES only: a trailing tab is a PyYAML scanner error, so it
|
||||||
|
# must be kept on the token body to reach the fail-closed guards
|
||||||
|
# (rstrip() would swallow it and let a bad line resolve a token).
|
||||||
|
self.toks.append((indent, body.rstrip(" ")))
|
||||||
self.i = 0
|
self.i = 0
|
||||||
|
|
||||||
def peek(self):
|
def peek(self):
|
||||||
@@ -959,7 +976,7 @@ class _Parser:
|
|||||||
self.i += 1
|
self.i += 1
|
||||||
if key in result:
|
if key in result:
|
||||||
raise _Bail() # duplicate mapping key (PyYAML last-wins; we bail)
|
raise _Bail() # duplicate mapping key (PyYAML last-wins; we bail)
|
||||||
if inline is not None and inline.strip() != "":
|
if inline is not None and inline.strip(" ") != "":
|
||||||
val = _scalar(inline)
|
val = _scalar(inline)
|
||||||
if val is _FAIL:
|
if val is _FAIL:
|
||||||
raise _Bail()
|
raise _Bail()
|
||||||
@@ -997,7 +1014,7 @@ class _Parser:
|
|||||||
raise _Bail()
|
raise _Bail()
|
||||||
if not (body.startswith("-") and (len(body) == 1 or body[1] in (" ", "\t"))):
|
if not (body.startswith("-") and (len(body) == 1 or body[1] in (" ", "\t"))):
|
||||||
break # a mapping entry at this indent ends the sequence
|
break # a mapping entry at this indent ends the sequence
|
||||||
rest = body[1:].strip()
|
rest = body[1:].strip(" ") # spaces only; a tab must survive to bail
|
||||||
self.i += 1
|
self.i += 1
|
||||||
if rest == "":
|
if rest == "":
|
||||||
nxt = self.peek()
|
nxt = self.peek()
|
||||||
@@ -1023,7 +1040,7 @@ class _Parser:
|
|||||||
result = {}
|
result = {}
|
||||||
key = first_match.group(1)
|
key = first_match.group(1)
|
||||||
inline = first_match.group(2)
|
inline = first_match.group(2)
|
||||||
if inline is not None and inline.strip() != "":
|
if inline is not None and inline.strip(" ") != "":
|
||||||
val = _scalar(inline)
|
val = _scalar(inline)
|
||||||
if val is _FAIL:
|
if val is _FAIL:
|
||||||
raise _Bail()
|
raise _Bail()
|
||||||
@@ -1049,7 +1066,7 @@ class _Parser:
|
|||||||
self.i += 1
|
self.i += 1
|
||||||
if k in result:
|
if k in result:
|
||||||
raise _Bail()
|
raise _Bail()
|
||||||
if iv is not None and iv.strip() != "":
|
if iv is not None and iv.strip(" ") != "":
|
||||||
v = _scalar(iv)
|
v = _scalar(iv)
|
||||||
if v is _FAIL:
|
if v is _FAIL:
|
||||||
raise _Bail()
|
raise _Bail()
|
||||||
|
|||||||
@@ -719,4 +719,61 @@ assert_token "4.e8 float look-alike still resolves token" "TOK_PLAIN" primary gi
|
|||||||
write_bad_key_fixture '0x1f'
|
write_bad_key_fixture '0x1f'
|
||||||
assert_token "valid hex int unrelated key still resolves token" "TOK_PLAIN" primary git.example
|
assert_token "valid hex int unrelated key still resolves token" "TOK_PLAIN" primary git.example
|
||||||
|
|
||||||
|
# 17. TAB / SCANNER PARITY: PyYAML raises a ScannerError on a tab used anywhere
|
||||||
|
# outside a quoted scalar -- leading, trailing, or embedded in a plain value,
|
||||||
|
# immediately after a key colon, before a key colon, or as indentation -- and
|
||||||
|
# yields NO token, accepting tabs ONLY inside single/double-quoted scalars
|
||||||
|
# (where the tab is preserved as string content). A prior fallback swallowed
|
||||||
|
# those tabs (via .strip()/.rstrip() normalization and [ \t] key separators)
|
||||||
|
# and still emitted the login token -- a fail-open in the dangerous direction.
|
||||||
|
# The recognizer now fails CLOSED for the whole document on any tab PyYAML
|
||||||
|
# rejects, while preserving the tabs PyYAML keeps (inside quotes). All tab
|
||||||
|
# positions were verified empirically against PyYAML 6.0.3 (ScannerError for
|
||||||
|
# each rejected position; string-preserved for quoted inner tabs).
|
||||||
|
TAB=$'\t'
|
||||||
|
# Fail-close: a tab in a plain value position (trailing / leading / embedded).
|
||||||
|
write_bad_key_fixture "l4o${TAB}"
|
||||||
|
assert_both_fail_closed "trailing tab in plain value fails closed" primary git.example
|
||||||
|
write_bad_key_fixture "${TAB}9"
|
||||||
|
assert_both_fail_closed "leading tab in plain value fails closed" primary git.example
|
||||||
|
write_bad_key_fixture "a${TAB}b"
|
||||||
|
assert_both_fail_closed "embedded tab in plain value fails closed" primary git.example
|
||||||
|
# Fail-close: a tab immediately after the key colon (no separating space).
|
||||||
|
write_fixture "bad:${TAB}9
|
||||||
|
logins:
|
||||||
|
- name: primary
|
||||||
|
url: https://git.example
|
||||||
|
token: TOK_PLAIN
|
||||||
|
"
|
||||||
|
assert_both_fail_closed "tab immediately after key colon fails closed" primary git.example
|
||||||
|
# Fail-close: a tab used as indentation (before a sequence dash).
|
||||||
|
write_fixture "logins:
|
||||||
|
${TAB}- name: primary
|
||||||
|
url: https://git.example
|
||||||
|
token: TOK_PLAIN
|
||||||
|
"
|
||||||
|
assert_both_fail_closed "tab used as indentation fails closed" primary git.example
|
||||||
|
# Fail-close: a tab trailing a sequence-mapping field value.
|
||||||
|
write_fixture "logins:
|
||||||
|
- name: primary
|
||||||
|
url: https://git.example
|
||||||
|
token: TOK_PLAIN${TAB}
|
||||||
|
"
|
||||||
|
assert_both_fail_closed "tab trailing a seq field value fails closed" primary git.example
|
||||||
|
# NOT over-broad: a tab strictly INSIDE a quoted scalar is valid YAML (PyYAML
|
||||||
|
# keeps it as string content), so the document parses and the login token still
|
||||||
|
# resolves in BOTH paths -- double-quoted and single-quoted.
|
||||||
|
write_bad_key_fixture "\"a${TAB}b\""
|
||||||
|
assert_token "tab inside a double-quoted value still resolves token" "TOK_PLAIN" primary git.example
|
||||||
|
write_bad_key_fixture "'a${TAB}b'"
|
||||||
|
assert_token "tab inside a single-quoted value still resolves token" "TOK_PLAIN" primary git.example
|
||||||
|
# ...and a quoted token value carrying an inner tab resolves to the exact string
|
||||||
|
# (tab preserved), identical to PyYAML's construction.
|
||||||
|
write_fixture "logins:
|
||||||
|
- name: primary
|
||||||
|
url: https://git.example
|
||||||
|
token: \"T${TAB}OK\"
|
||||||
|
"
|
||||||
|
assert_token "quoted token with inner tab resolves verbatim" "T${TAB}OK" primary git.example
|
||||||
|
|
||||||
echo "Gitea login resolution regression harness passed"
|
echo "Gitea login resolution regression harness passed"
|
||||||
|
|||||||
Reference in New Issue
Block a user