From 2d821324fe5b63a2e0bd9a874e361b6d19ed8c7e Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Wed, 22 Jul 2026 02:24:33 -0500 Subject: [PATCH] fix(git): PyYAML-absent fallback parity for quoted-scalar breaks and tag/anchor properties (#865 round 11) Two residual over-rejections in the PyYAML-absent fallback of get_gitea_token_for_login, both fail-closed today but diverging from real PyYAML 6.0.3 for valid inputs. Neither is a fail-open; the fix loosens ONLY to exact PyYAML parity and keeps failing closed elsewhere. Blocker A - embedded printable line break inside a quoted scalar. The fallback split the raw document with str.splitlines(), which breaks at NEL(U+0085), LS(U+2028) and PS(U+2029) -- all PRINTABLE to PyYAML's Reader. Inside a flow (quoted) scalar PyYAML does not break at these: it line-folds a double/single-quoted scalar (NEL/LF/CR -> one space; LS/PS verbatim), resolving one token, while splitlines() cut mid-quote and failed the whole document closed. Replace splitlines() with _split_logical_lines, a quote-aware splitter that reproduces PyYAML's flow folding (scan_flow_scalar_spaces/breaks) and raises on a col-0 doc marker (---/...) in a folded continuation. The same code points UNQUOTED or in a comment still make PyYAML raise, so those stay closed. Blocker B - leading non-specific tag / anchor property. The recognizer blanket-rejected any scalar starting with '!' or '&'. But '! ' (bang + space) and '&name ' are transparent node properties: PyYAML strips them and applies normal implicit typing, so '! x'/'&a x' resolve the string 'x'. Add _strip_properties, consuming <=1 non-specific tag and <=1 anchor (either order) and recursing on the remainder. '!x' (tag handle), '!foo x', non-string nodes ('! 123'), duplicate properties and explicit tags ('!!str x') still fail closed. Fallback now matches real PyYAML 6.0.3 (oracle) on a two-direction differential fuzz of the full Unicode line-break set x {double, single, plain, comment, token-position} and the full leading tag/anchor space: 833 cases, 0 fail-opens, 0 present-mismatches, 52 endorsed fail-closed over-rejects (blank-line-in-quote \n folds; explicit !!str/verbose tags; property+quoted; anchor-without-space). Wrapper scripts pr-review.sh and issue-comment.sh unchanged (0-line diff). Tests: additions-only sections 23 (quoted-scalar breaks) and 24 (tag/anchor properties). Co-Authored-By: Claude Opus 4.8 --- .../framework/tools/git/detect-platform.sh | 204 +++++++++++++++++- .../tools/git/test-gitea-login-resolution.sh | 116 +++++++++- 2 files changed, 310 insertions(+), 10 deletions(-) diff --git a/packages/mosaic/framework/tools/git/detect-platform.sh b/packages/mosaic/framework/tools/git/detect-platform.sh index 781977c0..0d41de6e 100755 --- a/packages/mosaic/framework/tools/git/detect-platform.sh +++ b/packages/mosaic/framework/tools/git/detect-platform.sh @@ -843,6 +843,9 @@ _FAIL = object() # position makes the whole line fail to match -> the caller fails closed. _KEY_RE = re.compile(r"^([A-Za-z0-9_][A-Za-z0-9_.\-]*) *:(?:[ ](.*)|)$") _FLOW = set("[]{}*&!") +# A plain anchor property: '&' then PyYAML's exact anchor charset [0-9A-Za-z-_]+, +# then whatever follows (captured for the caller to require a space or EOL). +_ANCHOR_RE = re.compile(r"^&[0-9A-Za-z\-_]+(.*)$", re.S) class _Bail(Exception): @@ -852,6 +855,46 @@ class _Bail(Exception): pass +def _strip_properties(value): + # Consume leading YAML node properties -- at most ONE non-specific tag + # ('!' + space) and at most ONE plain anchor ('&name' + space), in either + # order -- and return the remaining node text for normal resolution. PyYAML's + # SafeLoader applies these TRANSPARENT properties and then resolves the + # following node with its ordinary implicit typing, so (verified against real + # PyYAML 6.0.3) '! x' -> 'x', '&a 123' -> int 123, '! ' / '&a' -> null, and + # '! &a x' / '&a ! x' -> 'x', exactly as the bare node would resolve. Returns + # _FAIL for every form we do NOT reproduce identically, so the caller fails + # closed: a tag shorthand ('!x' -> ConstructorError), an explicit or verbatim + # tag ('!!str' / '!!int' / '!<...>' / '!foo') whose type coercion we do not + # emulate, a tab separator (PyYAML scanner error), a duplicate tag or anchor + # ('! ! x' / '&a &b x' -> ParserError), or an anchor glued to a non-space. + # The returned remainder (possibly '') is re-resolved by the caller; '' is a + # null node -> fail closed for a token field, matching PyYAML. + seen_tag = False + seen_anchor = False + while value[:1] in ("!", "&"): + if value[0] == "!": + # Non-specific tag: '!' then AT LEAST ONE SPACE. A tab is a PyYAML + # scanner error; '!x' / '!!type' / '!' are typed/short tags. + if seen_tag or len(value) < 2 or value[1] != " ": + return _FAIL + seen_tag = True + value = value[1:].lstrip(" ") + else: # anchor '&name' + m = _ANCHOR_RE.match(value) + if seen_anchor or m is None: + return _FAIL + rest = m.group(1) + if rest == "": + value = "" # bare '&name' with no following node -> null node + elif rest[0] == " ": + value = rest.lstrip(" ") + else: + return _FAIL # '&name' glued to a non-space (e.g. '&a:') -> error + seen_anchor = True + return value + + def _scalar(raw): # Resolve a single flow scalar (quoted or plain) the way PyYAML would for # tea's simple values, or _FAIL when it is outside the supported subset so @@ -894,8 +937,18 @@ def _scalar(raw): value = value.strip(" ") # spaces only; a tab must survive to fail closed if not value: return None + if value[0] in ("!", "&"): + # Leading node property (non-specific tag '! ' / plain anchor '&name '): + # PyYAML applies it to the FOLLOWING node and resolves THAT, so strip the + # transparent forms and re-resolve the remainder EXACTLY (implicit typing + # and all). _strip_properties returns _FAIL for the tag/anchor forms + # PyYAML rejects or type-coerces, which fail closed here. + remainder = _strip_properties(value) + if remainder is _FAIL: + return _FAIL + return _scalar(remainder) if value[0] in _FLOW or value[0] in ("|", ">", "@", "`", '"', "'", "%", ","): - return _FAIL # flow / block-scalar / reserved / directive / anchor / quote + return _FAIL # flow / block-scalar / reserved / directive / quote # NOTE: "?" is deliberately NOT in the blanket-reject set above. Empirically # verified against real PyYAML 6.0.3: "?x" (indicator immediately followed by # a non-space) is a plain scalar string "?x" like "-x" and ":x" are -- only @@ -1189,6 +1242,138 @@ def _token_via_pyyaml(): return None +_BREAKS = "\r\n\x85

" +_VALUE_OPEN_RE = re.compile(r"^\s*(?:-\s+)*[A-Za-z0-9_][A-Za-z0-9_.\-]*:$") +_SEQ_OPEN_RE = re.compile(r"^\s*(?:-\s+)*-$") + + +def _value_open_before(prefix): + # True when a quote appearing at the END of `prefix` (the current logical + # line so far) begins a flow SCALAR value -- the only position where a quote + # opens a quoted scalar whose embedded line breaks PyYAML folds instead of + # splitting. That is: the prefix is empty/all-indent (root or block scalar + # start), or ends with a mapping ': ' or a sequence '- ' indicator that is + # SPACE-separated from the quote. A quote glued to the previous char (e.g. + # 'key:"x') is NOT a value opener (PyYAML needs the space), and a quote mid + # -content is literal -- both fail this test, so their breaks split the line + # exactly as before (fail closed / plain-scalar behavior preserved). + stripped = prefix.rstrip(" ") + if stripped == "": + return True # start of line (after any indentation) -> node value start + if prefix == stripped: + return False # no space before the quote -> not a value opener + return bool(_VALUE_OPEN_RE.match(stripped) or _SEQ_OPEN_RE.match(stripped)) + + +def _fold_quoted_break_run(run): + # Reproduce PyYAML's flow-scalar folding (scan_flow_scalar_spaces + + # scan_flow_scalar_breaks) for the maximal ' \t' + line-break run inside a + # quoted scalar. A run with NO break is literal whitespace (verbatim). With a + # break: surrounding spaces/tabs are dropped; a single \n-class break + # (\n / \r / \r\n / \x85 NEL) folds to ONE space; 
 (LS) / 
 (PS) + # are kept verbatim; each ADDITIONAL break (a blank line) contributes its own + # break char (\n for the \n-class). Verified against real PyYAML 6.0.3 for + # both double- and single-quoted scalars (they fold identically here). + def _lb(s, j): + if s[j] == "\r" and j + 1 < len(s) and s[j + 1] == "\n": + return "\n", 2 + if s[j] in "\r\n\x85": + return "\n", 1 + return s[j], 1 # 
 / 
 preserved verbatim + n = len(run) + j = 0 + while j < n and run[j] in " \t": + j += 1 + if j >= n: + return run # pure whitespace, no break -> literal content + line_break, adv = _lb(run, j) + j += adv + breaks = [] + while True: + while j < n and run[j] in " \t": + j += 1 + if j < n and run[j] in _BREAKS: + b, adv = _lb(run, j) + j += adv + breaks.append(b) + else: + break + out = "" + if line_break != "\n": + out += line_break + elif not breaks: + out += " " + return out + "".join(breaks) + + +def _split_logical_lines(raw_text): + # Split like str.splitlines() EXCEPT a line break INSIDE a value-opening + # quoted scalar does NOT end the line: PyYAML keeps the quoted scalar together + # and folds the break (flow folding above), so we fold it and continue the + # same logical line. Only the break chars that survive the _FORBIDDEN_CONTROL + # gate reach here: \n, \r, \x85 (NEL), 
 (LS), 
 (PS). OUTSIDE + # quotes every one ends the line (matching PyYAML's block-level line breaks + # and str.splitlines), so a NEL/LS/PS used as the document's line-terminator + # style, and an UNQUOTED plain scalar carrying an embedded break, split and + # fail closed exactly as before. A quoted scalar left unterminated at EOF + # stays on the final line and fails closed in _scalar (PyYAML errors too). + out = [] + cur = [] + i = 0 + n = len(raw_text) + quote = None + while i < n: + ch = raw_text[i] + if quote is None: + if ch in _BREAKS: + out.append("".join(cur)) + cur = [] + i += 2 if (ch == "\r" and i + 1 < n and raw_text[i + 1] == "\n") else 1 + continue + if ch in ("'", '"') and _value_open_before("".join(cur)): + quote = ch + cur.append(ch) + i += 1 + continue + cur.append(ch) + i += 1 + continue + # inside a value-opening quoted scalar + if ch == quote: + cur.append(ch) + quote = None + i += 1 + continue + if ch in " \t" or ch in _BREAKS: + j = i + has_break = False + while j < n and (raw_text[j] in " \t" or raw_text[j] in _BREAKS): + if raw_text[j] in _BREAKS: + has_break = True + j += 1 + run = raw_text[i:j] + if has_break: + # A continuation line beginning (at column 0, no indent) with a + # document marker is a PyYAML scanner error even inside a quoted + # scalar; when the run ends on a break the continuation content at + # j is at column 0, so guard it and fail closed. (An INDENTED + # '---' is literal content to PyYAML and folds normally.) + if run[-1] in _BREAKS and raw_text[j:j + 3] in ("---", "...") and ( + j + 3 >= n or raw_text[j + 3] in "\0 \t" + _BREAKS + ): + raise _Bail() + cur.append(_fold_quoted_break_run(run)) + else: + cur.append(run) + i = j + continue + cur.append(ch) + i += 1 + if cur or not out: + out.append("".join(cur)) + return out + + def _token_via_lines(): # Conservative fallback for hosts without PyYAML. It parses config.yml with a # strict recognizer (_safe_parse) of the narrow block-style subset tea writes, @@ -1208,17 +1393,18 @@ def _token_via_lines(): # block, surrogates, and 0xFFFE/0xFFFF) appears ANYWHERE in the raw stream -- # in a plain scalar, inside single- or double-quoted scalars, in a comment, # or in a field this recognizer never even looks at. Checking the raw text - # (before splitlines(), which itself splits on some of these same code points - # -- e.g. \x0b, \x0c, \x1c-\x1e, and NEL \x85 -- and would otherwise obscure - # them) mirrors that exactly: fail closed for the whole document rather than - # risk emitting a token PyYAML would have refused. (NEL 0x85 is PRINTABLE to - # PyYAML's Reader, so it is NOT in _FORBIDDEN_CONTROL; where PyYAML treats it - # as a line break the fallback's splitlines() agrees, and where PyYAML then - # errors the recognizer's structural guards fail closed identically.) + # (before line splitting, which would otherwise split on some of these same + # code points -- e.g. \x0b, \x0c, \x1c-\x1e -- and obscure them) mirrors that + # exactly: fail closed for the whole document. (NEL 0x85, LS 0x2028 and PS + # 0x2029 are PRINTABLE to PyYAML's Reader, so they are NOT in + # _FORBIDDEN_CONTROL; _split_logical_lines below reproduces PyYAML's line-break + # semantics for them -- a structural break outside quotes, a FOLDED break + # inside a quoted scalar -- rather than str.splitlines(), which would wrongly + # split them mid-quote and fail-close a token PyYAML resolves.) if _FORBIDDEN_CONTROL.search(raw_text): return None - lines = raw_text.splitlines() + lines = _split_logical_lines(raw_text) config = _safe_parse(lines) if config is _FAIL: diff --git a/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh b/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh index 2c265dfb..c17338fe 100755 --- a/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh +++ b/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh @@ -922,7 +922,9 @@ elif pos == "token": elif pos == "nelterm": # NEL (U+0085) used as the line terminator throughout: PyYAML treats it as a # line break (printable, NOT a ReaderError) and resolves the token; the - # fallback's splitlines() splits on NEL identically -> parity, token resolves. + # fallback's _split_logical_lines splits on NEL identically OUTSIDE a quote + # -> parity, token resolves. (Round 23 covers NEL/LS/PS INSIDE a quote, where + # PyYAML folds rather than breaks and a naive splitlines() would over-split.) doc = ("logins:" + ch + " - name: primary" + ch + " url: https://git.example" + ch + " token: TOK_NEL" + ch) else: @@ -1007,4 +1009,116 @@ write_fixture 'logins: ' assert_both_fail_closed "trailing colon (map indicator) token fails closed" primary git.example +# 23. EMBEDDED LINE-BREAK INSIDE A QUOTED SCALAR (#865 round-11 blocker A): the +# round-10 fallback split the raw document with str.splitlines(), which breaks +# at NEL(U+0085), LS(U+2028) and PS(U+2029) -- code points that are PRINTABLE +# to PyYAML's Reader. Inside a flow (quoted) scalar PyYAML does NOT break at +# these: it LINE-FOLDS a double/single-quoted scalar (NEL/LF/CR -> a single +# space; LS/PS -> the char verbatim), so it resolves ONE token, while +# splitlines() cut the value mid-quote and failed the whole document closed +# (over-rejection). The fallback now uses _split_logical_lines, which +# reproduces PyYAML's flow-folding. Verified empirically vs real PyYAML 6.0.3. +# write_break_fixture emits a chosen break code point in a selectable context. +write_break_fixture() { + # $1 = hex code point of the break; $2 = context: dq|sq|plain|comment|dq2 + CP_HEX="$1" Q_STYLE="$2" python3 - "$FIXTURE_XDG/tea/config.yml" <<'PY' +import sys, os +cp = int(os.environ["CP_HEX"], 16) +q = os.environ["Q_STYLE"] +ch = chr(cp) +head = "logins:\n - name: primary\n url: https://git.example\n token: " +if q == "dq": + doc = head + '"tok' + ch + 'en"' +elif q == "sq": + doc = head + "'tok" + ch + "en'" +elif q == "plain": + doc = head + "tok" + ch + "en" +elif q == "dq2": + # blank line inside a quoted scalar: PyYAML folds a two-break run to a literal + # newline, which the recognizer's key regex cannot carry -> endorsed + # fail-closed over-reject (see assert_fallback_fails_closed below). + doc = head + '"tok' + ch + ch + 'en"' +elif q == "comment": + doc = ("logins:\n - name: primary\n url: https://git.example\n" + " token: TOK_PLAIN\n# c" + ch + "x") +else: + raise SystemExit("bad q") +with open(sys.argv[1], "w", encoding="utf-8") as f: + f.write(doc + "\n") +PY +} + +# NEL folds to a single space inside double- AND single-quoted scalars: the token +# resolves identically in both paths (fallback no longer over-splits). +write_break_fixture 0x85 dq +assert_token "NEL inside double-quote folds to space, token resolves" "tok en" primary git.example +write_break_fixture 0x85 sq +assert_token "NEL inside single-quote folds to space, token resolves" "tok en" primary git.example +# LS(U+2028)/PS(U+2029) are preserved VERBATIM by PyYAML's flow fold (they are +# not \n-class breaks); the fallback must surface them byte-for-byte. +write_break_fixture 0x2028 dq +assert_token "LS inside double-quote is verbatim" "$(printf 'tok\342\200\250en')" primary git.example +write_break_fixture 0x2028 sq +assert_token "LS inside single-quote is verbatim" "$(printf 'tok\342\200\250en')" primary git.example +write_break_fixture 0x2029 dq +assert_token "PS inside double-quote is verbatim" "$(printf 'tok\342\200\251en')" primary git.example + +# Direction-sensitivity: the SAME code points UNQUOTED (a plain scalar) or in a +# COMMENT make PyYAML raise a scanner error, so both paths must fail closed. The +# fold rule applies ONLY inside a quoted scalar. +write_break_fixture 0x85 plain +assert_token "NEL in an unquoted plain scalar fails closed" "" primary git.example +write_break_fixture 0x2028 plain +assert_token "LS in an unquoted plain scalar fails closed" "" primary git.example +write_break_fixture 0x85 comment +assert_token "NEL in a comment fails closed" "" primary git.example + +# Endorsed fail-closed over-reject: a blank line inside a quoted scalar folds to a +# literal newline that the recognizer cannot carry -- PyYAML resolves a +# (newline-bearing) token, the fallback fails closed (strictly safer). +write_break_fixture 0x85 dq2 +assert_fallback_fails_closed "blank-line-in-quote (NEL run) fails closed" primary git.example + +# 24. LEADING NON-SPECIFIC TAG / ANCHOR PROPERTY (#865 round-11 blocker B): the +# round-10 recognizer blanket-rejected any scalar beginning with '!' or '&'. +# But a NON-SPECIFIC tag '! ' (bang + SPACE) and an anchor '&name ' are +# transparent node properties: PyYAML strips them and applies normal implicit +# typing to the node, so '! x'/'&a x' resolve the STRING 'x'. The fallback now +# consumes <=1 non-specific tag and <=1 anchor (either order) and recurses on +# the remainder. Verified empirically vs real PyYAML 6.0.3. +for pair in '! x=x' '&a x=x' '! x y=x y' '! "q"=q' "! 'q'=q" '! &b x=x' '&b ! x=x'; do + tv="${pair%%=*}"; want="${pair#*=}" + write_fixture "logins: + - name: primary + url: https://git.example + token: ${tv} +" + assert_token "property token '$tv' resolves node string" "$want" primary git.example +done + +# Fail-OPEN direction stays shut. '!x' (bang + NON-space) is a tag HANDLE -> +# ConstructorError; '!foo x'/'* a'/'! !x' raise; a property over a NON-string node +# ('! 123'/'! true'/'! null') types non-str -> no usable token. All fail closed in +# BOTH paths. +for tv in '!x' '!foo x' '* a' '! !x' '! 123' '! true' '! null' '&a &b x'; do + write_fixture "logins: + - name: primary + url: https://git.example + token: ${tv} +" + assert_token "non-resolving property token '$tv' fails closed" "" primary git.example +done + +# Endorsed fail-closed over-reject: an EXPLICIT tag ('!!str x', verbose +# '! x') forces a string PyYAML resolves, but the fallback +# recognizes only the transparent non-specific tag and fails closed (safer). +for tv in '!!str x' '! x'; do + write_fixture "logins: + - name: primary + url: https://git.example + token: ${tv} +" + assert_fallback_fails_closed "explicit-tag token '$tv' fails closed" primary git.example +done + echo "Gitea login resolution regression harness passed"