diff --git a/packages/mosaic/framework/tools/git/detect-platform.sh b/packages/mosaic/framework/tools/git/detect-platform.sh index 7d55ad10..781977c0 100755 --- a/packages/mosaic/framework/tools/git/detect-platform.sh +++ b/packages/mosaic/framework/tools/git/detect-platform.sh @@ -810,19 +810,27 @@ def _constructible(text): # PyYAML's Reader.check_printable scans the ENTIRE raw input stream (not just -# scalar contents, and NOT scoped by quoting) for bytes outside its printable -# set and raises ReaderError -- a whole-document reject -- the instant one is -# found, no matter where it sits (an unrelated field, a comment, inside or -# outside quotes). Empirically verified against real PyYAML 6.0.3: EVERY C0 -# control byte below plus DEL (0x7F) rejects in plain scalars, inside -# double-quoted scalars, and inside single-quoted scalars alike; only -# TAB(0x09), LF(0x0A), and CR(0x0D) are accepted among the C0 range (LF/CR are -# line separators, handled elsewhere; TAB has its own narrower, position-aware -# guard above since PyYAML accepts it in some contexts). This set therefore -# excludes 0x09/0x0A/0x0D and matches the forbidden C0 set {0x00-0x08, 0x0B, -# 0x0C, 0x0E-0x1F} plus DEL (0x7F). +# scalar contents, and NOT scoped by quoting) for code points outside its +# printable set and raises ReaderError -- a whole-document reject -- the instant +# one is found, no matter where it sits (an unrelated field, a comment, inside or +# outside quotes). To guarantee parity WITHOUT a hand-rolled subset (which missed +# the C1 block and BMP noncharacters), this is PyYAML 6.0.3's EXACT Reader +# printable definition, negated verbatim: +# Reader.NON_PRINTABLE = +# re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-퟿-�' +# '\U00010000-\U0010FFFF]') +# i.e. PRINTABLE = {TAB(0x09), LF(0x0A), CR(0x0D), 0x20-0x7E, NEL(0x85), +# 0xA0-0xD7FF, 0xE000-0xFFFD, 0x10000-0x10FFFF}; EVERYTHING else (all other C0 +# controls, DEL 0x7F, the entire C1 block 0x80-0x84/0x86-0x9F, the surrogate +# range 0xD800-0xDFFF, and the BMP noncharacters 0xFFFE/0xFFFF) is non-printable +# -> whole-document fail closed. Verified empirically against real PyYAML 6.0.3 +# in both directions: the C1 bytes and 0xFFFE/0xFFFF reject; NEL(0x85), 0xA0, and +# astral code points (e.g. U+1F600) are accepted and resolve the token. The prior +# C0/DEL/tab behavior is a strict subset of this. (A surrogate code point cannot +# occur in valid UTF-8, so it also trips the UTF-8 decode on read and fails +# closed; it is included here for exact definitional parity.) _FORBIDDEN_CONTROL = re.compile( - "[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]" + "[^\x09\x0a\x0d\x20-\x7e\x85\xa0-퟿-�\U00010000-\U0010ffff]" ) # Sentinel: "outside the supported subset -> fail closed". Distinct from a @@ -903,8 +911,21 @@ def _scalar(raw): # remain valid plain scalars and fall through. Fail closed on the bare # indicator so the fallback never emits a token PyYAML would refuse. return _FAIL - if any(ch in _FLOW for ch in value): - return _FAIL + # NO blanket internal-indicator reject here. In BLOCK context (where a tea + # config value always sits) PyYAML treats ',[]{}' as ordinary plain-scalar + # content -- they are flow indicators ONLY inside a flow collection -- and + # treats '!&*|>#%@`' plus quotes as significant ONLY at the FIRST non-space + # character of a node (a leading one starts a tag/anchor/alias/block-scalar/ + # comment/directive/reserved/quote), which the leading-char guard above + # (value[0] ...) already rejects. Verified empirically against real PyYAML + # 6.0.3: an INTERNAL '!' ',' '[' ']' '{' '}' '&' '*' '#'(not space-preceded) + # ':'(not space-followed) all resolve as a plain-string token. The only + # internal positions PyYAML treats as significant in block context are ' #' + # (whitespace-preceded '#' -> comment; stripped by the loop above) and ': ' + # or a trailing ':' (-> mapping; rejected by the ": "/endswith(":") guard + # below). A blanket any(ch in _FLOW ...) scan over-rejected those internal + # indicators, dropping a token PyYAML emits verbatim; removing it restores + # parity while the position-specific guards keep the fail-open direction shut. if "\t" in value: # A tab anywhere in a plain scalar (leading, trailing, or embedded) is a # PyYAML scanner error on the whole document -- it accepts tabs ONLY @@ -1182,14 +1203,18 @@ def _token_via_lines(): raw_text = handle.read() # Whole-document, position-independent reject: PyYAML's Reader rejects the - # ENTIRE document (ReaderError) if a forbidden C0/DEL control byte appears - # ANYWHERE in the raw stream -- in a plain scalar, inside single- or - # double-quoted scalars, in a comment, or in a field this recognizer never - # even looks at. Checking the raw text (before splitlines(), which itself - # splits on some of these same control bytes -- e.g. \x0b, \x0c, \x1c-\x1e - # -- and would otherwise obscure them) mirrors that exactly: fail closed - # for the whole document rather than risk emitting a token PyYAML would - # have refused. + # ENTIRE document (ReaderError) if a NON-printable code point (per its exact + # printable definition -- see _FORBIDDEN_CONTROL: C0 controls, DEL, the C1 + # block, surrogates, and 0xFFFE/0xFFFF) appears ANYWHERE in the raw stream -- + # in a plain scalar, inside single- or double-quoted scalars, in a comment, + # or in a field this recognizer never even looks at. Checking the raw text + # (before splitlines(), which itself splits on some of these same code points + # -- e.g. \x0b, \x0c, \x1c-\x1e, and NEL \x85 -- and would otherwise obscure + # them) mirrors that exactly: fail closed for the whole document rather than + # risk emitting a token PyYAML would have refused. (NEL 0x85 is PRINTABLE to + # PyYAML's Reader, so it is NOT in _FORBIDDEN_CONTROL; where PyYAML treats it + # as a line break the fallback's splitlines() agrees, and where PyYAML then + # errors the recognizer's structural guards fail closed identically.) if _FORBIDDEN_CONTROL.search(raw_text): return None diff --git a/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh b/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh index 1445f924..2c265dfb 100755 --- a/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh +++ b/packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh @@ -893,4 +893,118 @@ write_fixture 'logins: ' assert_token "question-mark-not-space is a plain string, token resolves" "?x" primary git.example +# 21. PRINTABLE-BOUNDARY FAIL-CLOSE (#865 round-10 blocker 1): the round-9 guard +# used a hand-rolled C0/DEL subset that MISSED code points PyYAML's Reader +# also rejects -- the C1 block (U+0080-0084, U+0086-009F) and the BMP +# noncharacters U+FFFE/U+FFFF -- so the fallback still emitted the token from +# documents PyYAML rejects whole (fail-open). The guard now uses PyYAML +# 6.0.3's EXACT Reader.NON_PRINTABLE character class (see detect-platform.sh +# _FORBIDDEN_CONTROL). Verified empirically against real PyYAML 6.0.3: +# PRINTABLE = {0x09,0x0A,0x0D, 0x20-0x7E, 0x85(NEL), 0xA0-0xD7FF, +# 0xE000-0xFFFD, 0x10000-0x10FFFF}; everything else fails the whole document +# closed. write_codepoint_fixture emits a chosen Unicode code point's real +# UTF-8 bytes (via python3, since bash strings cannot faithfully carry many +# of these) into a selectable position, then the login block follows. +write_codepoint_fixture() { + # $1 = hex code point (e.g. 0x80); $2 = position: field|comment|token|nelterm + CP_HEX="$1" CP_POS="$2" python3 - "$FIXTURE_XDG/tea/config.yml" <<'PY' +import sys +cp = int(__import__("os").environ["CP_HEX"], 16) +pos = __import__("os").environ["CP_POS"] +ch = chr(cp) +head = "logins:\n - name: primary\n url: https://git.example\n token: TOK_PLAIN\n" +if pos == "field": + doc = head + "other: x" + ch + "y\n" +elif pos == "comment": + doc = head + "# note x" + ch + "y here\n" +elif pos == "token": + doc = "logins:\n - name: primary\n url: https://git.example\n token: T" + ch + "K\n" +elif pos == "nelterm": + # NEL (U+0085) used as the line terminator throughout: PyYAML treats it as a + # line break (printable, NOT a ReaderError) and resolves the token; the + # fallback's splitlines() splits on NEL identically -> parity, token resolves. + doc = ("logins:" + ch + " - name: primary" + ch + + " url: https://git.example" + ch + " token: TOK_NEL" + ch) +else: + raise SystemExit("bad pos") +with open(sys.argv[1], "w", encoding="utf-8") as f: + f.write(doc) +PY +} + +# C1-block + BMP-noncharacter code points fail the WHOLE document closed in an +# unrelated field and in a comment, exactly as PyYAML's ReaderError does. +for cphex in 0x80 0x81 0x84 0x86 0x9f 0xfffe 0xffff; do + write_codepoint_fixture "$cphex" field + assert_both_fail_closed "code point $cphex in unrelated field fails closed" primary git.example + write_codepoint_fixture "$cphex" comment + assert_both_fail_closed "code point $cphex in a comment fails closed" primary git.example +done + +# NOT over-broad: printable code points PyYAML ACCEPTS must still resolve the +# token in BOTH paths -- NEL(0x85) as a line separator, U+00A0 (NBSP) inside a +# value, and an astral code point (U+1F600) inside the token value. +write_codepoint_fixture 0x85 nelterm +assert_token "NEL (U+0085) line-terminator resolves token" "TOK_NEL" primary git.example +write_codepoint_fixture 0xa0 field +assert_token "U+00A0 in unrelated value still resolves token" "TOK_PLAIN" primary git.example +write_codepoint_fixture 0x1f600 token +assert_token "astral U+1F600 inside token resolves verbatim" "$(printf 'T\360\237\230\200K')" primary git.example + +# 22. INTERNAL-INDICATOR OVER-REJECTION (#865 round-10 blocker 2): the round-9 +# recognizer blanket-rejected any plain scalar CONTAINING a flow indicator +# ([]{}*&!), but in BLOCK context PyYAML treats ',[]{}' as ordinary content +# and treats '!&*#...' as significant ONLY at the FIRST non-space char. So an +# INTERNAL indicator is legal plain-string content and PyYAML emits the token +# verbatim; the fallback dropped it (over-rejection). Verified empirically +# against real PyYAML 6.0.3. The fix removes the blanket internal scan while +# the leading-char guard and the ' #'/': '/trailing-':' guards keep the +# fail-OPEN direction shut. +for tv in 'a!b' 'a,b' 'a[b' 'a]b' 'a{b' 'a}b' 'a&b' 'a*b' 'a[b]c' 'a{b}c' 'a,b,c' 'a#b' 'a:b'; do + write_fixture "logins: + - name: primary + url: https://git.example + token: ${tv} +" + assert_token "internal-indicator token '$tv' resolves verbatim" "$tv" primary git.example +done + +# Fail-OPEN direction stays shut: a LEADING indicator, a ' #' comment tail, an +# internal ': ' (colon-space) inline map, and a trailing ':' each make PyYAML +# resolve NO usable string token (tag/flow/anchor reject or None, comment strip, +# or a mapping), so BOTH must fail closed. (Leading tag/anchor/flow are the +# documented, round-9-approved structural fail-closed class; kept intact here.) +write_fixture 'logins: + - name: primary + url: https://git.example + token: !x +' +assert_both_fail_closed "leading '!' tag token fails closed" primary git.example +write_fixture 'logins: + - name: primary + url: https://git.example + token: [a] +' +assert_both_fail_closed "leading '[' flow-seq token fails closed" primary git.example +write_fixture 'logins: + - name: primary + url: https://git.example + token: a # trailing comment +' +# ' #' comment tail: PyYAML strips the comment -> token is the string 'a', which +# still resolves. This is the NOT-over-broad boundary partner of the guard. +assert_token "space-hash comment tail strips to plain token" "a" primary git.example +write_fixture 'logins: + - name: primary + url: https://git.example + token: a: b +' +assert_both_fail_closed "internal colon-space (inline map) token fails closed" primary git.example +write_fixture 'logins: + - name: primary + url: https://git.example + token: ab: +' +assert_both_fail_closed "trailing colon (map indicator) token fails closed" primary git.example + echo "Gitea login resolution regression harness passed"