packages/mosaic/framework/tools/git/detect-platform.sh

@@ -810,19 +810,27 @@ def _constructible(text):
 
 
 # PyYAML's Reader.check_printable scans the ENTIRE raw input stream (not just
-# scalar contents, and NOT scoped by quoting) for bytes outside its printable
-# set and raises ReaderError -- a whole-document reject -- the instant one is
-# found, no matter where it sits (an unrelated field, a comment, inside or
-# outside quotes). Empirically verified against real PyYAML 6.0.3: EVERY C0
-# control byte below plus DEL (0x7F) rejects in plain scalars, inside
-# double-quoted scalars, and inside single-quoted scalars alike; only
-# TAB(0x09), LF(0x0A), and CR(0x0D) are accepted among the C0 range (LF/CR are
-# line separators, handled elsewhere; TAB has its own narrower, position-aware
-# guard above since PyYAML accepts it in some contexts). This set therefore
-# excludes 0x09/0x0A/0x0D and matches the forbidden C0 set {0x00-0x08, 0x0B,
-# 0x0C, 0x0E-0x1F} plus DEL (0x7F).
+# scalar contents, and NOT scoped by quoting) for code points outside its
+# printable set and raises ReaderError -- a whole-document reject -- the instant
+# one is found, no matter where it sits (an unrelated field, a comment, inside or
+# outside quotes). To guarantee parity WITHOUT a hand-rolled subset (which missed
+# the C1 block and BMP noncharacters), this is PyYAML 6.0.3's EXACT Reader
+# printable definition, negated verbatim:
+#   Reader.NON_PRINTABLE =
+#     re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-퟿-<2D>'
+#                '\U00010000-\U0010FFFF]')
+# i.e. PRINTABLE = {TAB(0x09), LF(0x0A), CR(0x0D), 0x20-0x7E, NEL(0x85),
+# 0xA0-0xD7FF, 0xE000-0xFFFD, 0x10000-0x10FFFF}; EVERYTHING else (all other C0
+# controls, DEL 0x7F, the entire C1 block 0x80-0x84/0x86-0x9F, the surrogate
+# range 0xD800-0xDFFF, and the BMP noncharacters 0xFFFE/0xFFFF) is non-printable
+# -> whole-document fail closed. Verified empirically against real PyYAML 6.0.3
+# in both directions: the C1 bytes and 0xFFFE/0xFFFF reject; NEL(0x85), 0xA0, and
+# astral code points (e.g. U+1F600) are accepted and resolve the token. The prior
+# C0/DEL/tab behavior is a strict subset of this. (A surrogate code point cannot
+# occur in valid UTF-8, so it also trips the UTF-8 decode on read and fails
+# closed; it is included here for exact definitional parity.)
 _FORBIDDEN_CONTROL = re.compile(
-    "[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]"
+    "[^\x09\x0a\x0d\x20-\x7e\x85\xa0-퟿-<2D>\U00010000-\U0010ffff]"
 )
 
 # Sentinel: "outside the supported subset -> fail closed". Distinct from a
@@ -903,8 +911,21 @@ def _scalar(raw):
         # remain valid plain scalars and fall through. Fail closed on the bare
         # indicator so the fallback never emits a token PyYAML would refuse.
         return _FAIL
-    if any(ch in _FLOW for ch in value):
-        return _FAIL
+    # NO blanket internal-indicator reject here. In BLOCK context (where a tea
+    # config value always sits) PyYAML treats ',[]{}' as ordinary plain-scalar
+    # content -- they are flow indicators ONLY inside a flow collection -- and
+    # treats '!&*|>#%@`' plus quotes as significant ONLY at the FIRST non-space
+    # character of a node (a leading one starts a tag/anchor/alias/block-scalar/
+    # comment/directive/reserved/quote), which the leading-char guard above
+    # (value[0] ...) already rejects. Verified empirically against real PyYAML
+    # 6.0.3: an INTERNAL '!' ',' '[' ']' '{' '}' '&' '*' '#'(not space-preceded)
+    # ':'(not space-followed) all resolve as a plain-string token. The only
+    # internal positions PyYAML treats as significant in block context are ' #'
+    # (whitespace-preceded '#' -> comment; stripped by the loop above) and ': '
+    # or a trailing ':' (-> mapping; rejected by the ": "/endswith(":") guard
+    # below). A blanket any(ch in _FLOW ...) scan over-rejected those internal
+    # indicators, dropping a token PyYAML emits verbatim; removing it restores
+    # parity while the position-specific guards keep the fail-open direction shut.
     if "\t" in value:
         # A tab anywhere in a plain scalar (leading, trailing, or embedded) is a
         # PyYAML scanner error on the whole document -- it accepts tabs ONLY
@@ -1182,14 +1203,18 @@ def _token_via_lines():
         raw_text = handle.read()
 
     # Whole-document, position-independent reject: PyYAML's Reader rejects the
-    # ENTIRE document (ReaderError) if a forbidden C0/DEL control byte appears
-    # ANYWHERE in the raw stream -- in a plain scalar, inside single- or
-    # double-quoted scalars, in a comment, or in a field this recognizer never
-    # even looks at. Checking the raw text (before splitlines(), which itself
-    # splits on some of these same control bytes -- e.g. \x0b, \x0c, \x1c-\x1e
-    # -- and would otherwise obscure them) mirrors that exactly: fail closed
-    # for the whole document rather than risk emitting a token PyYAML would
-    # have refused.
+    # ENTIRE document (ReaderError) if a NON-printable code point (per its exact
+    # printable definition -- see _FORBIDDEN_CONTROL: C0 controls, DEL, the C1
+    # block, surrogates, and 0xFFFE/0xFFFF) appears ANYWHERE in the raw stream --
+    # in a plain scalar, inside single- or double-quoted scalars, in a comment,
+    # or in a field this recognizer never even looks at. Checking the raw text
+    # (before splitlines(), which itself splits on some of these same code points
+    # -- e.g. \x0b, \x0c, \x1c-\x1e, and NEL \x85 -- and would otherwise obscure
+    # them) mirrors that exactly: fail closed for the whole document rather than
+    # risk emitting a token PyYAML would have refused. (NEL 0x85 is PRINTABLE to
+    # PyYAML's Reader, so it is NOT in _FORBIDDEN_CONTROL; where PyYAML treats it
+    # as a line break the fallback's splitlines() agrees, and where PyYAML then
+    # errors the recognizer's structural guards fail closed identically.)
     if _FORBIDDEN_CONTROL.search(raw_text):
         return None
 

packages/mosaic/framework/tools/git/test-gitea-login-resolution.sh

@@ -893,4 +893,118 @@ write_fixture 'logins:
 '
 assert_token "question-mark-not-space is a plain string, token resolves" "?x" primary git.example
 
+# 21. PRINTABLE-BOUNDARY FAIL-CLOSE (#865 round-10 blocker 1): the round-9 guard
+#     used a hand-rolled C0/DEL subset that MISSED code points PyYAML's Reader
+#     also rejects -- the C1 block (U+0080-0084, U+0086-009F) and the BMP
+#     noncharacters U+FFFE/U+FFFF -- so the fallback still emitted the token from
+#     documents PyYAML rejects whole (fail-open). The guard now uses PyYAML
+#     6.0.3's EXACT Reader.NON_PRINTABLE character class (see detect-platform.sh
+#     _FORBIDDEN_CONTROL). Verified empirically against real PyYAML 6.0.3:
+#     PRINTABLE = {0x09,0x0A,0x0D, 0x20-0x7E, 0x85(NEL), 0xA0-0xD7FF,
+#     0xE000-0xFFFD, 0x10000-0x10FFFF}; everything else fails the whole document
+#     closed. write_codepoint_fixture emits a chosen Unicode code point's real
+#     UTF-8 bytes (via python3, since bash strings cannot faithfully carry many
+#     of these) into a selectable position, then the login block follows.
+write_codepoint_fixture() {
+    # $1 = hex code point (e.g. 0x80); $2 = position: field|comment|token|nelterm
+    CP_HEX="$1" CP_POS="$2" python3 - "$FIXTURE_XDG/tea/config.yml" <<'PY'
+import sys
+cp = int(__import__("os").environ["CP_HEX"], 16)
+pos = __import__("os").environ["CP_POS"]
+ch = chr(cp)
+head = "logins:\n  - name: primary\n    url: https://git.example\n    token: TOK_PLAIN\n"
+if pos == "field":
+    doc = head + "other: x" + ch + "y\n"
+elif pos == "comment":
+    doc = head + "# note x" + ch + "y here\n"
+elif pos == "token":
+    doc = "logins:\n  - name: primary\n    url: https://git.example\n    token: T" + ch + "K\n"
+elif pos == "nelterm":
+    # NEL (U+0085) used as the line terminator throughout: PyYAML treats it as a
+    # line break (printable, NOT a ReaderError) and resolves the token; the
+    # fallback's splitlines() splits on NEL identically -> parity, token resolves.
+    doc = ("logins:" + ch + "  - name: primary" + ch
+           + "    url: https://git.example" + ch + "    token: TOK_NEL" + ch)
+else:
+    raise SystemExit("bad pos")
+with open(sys.argv[1], "w", encoding="utf-8") as f:
+    f.write(doc)
+PY
+}
+
+# C1-block + BMP-noncharacter code points fail the WHOLE document closed in an
+# unrelated field and in a comment, exactly as PyYAML's ReaderError does.
+for cphex in 0x80 0x81 0x84 0x86 0x9f 0xfffe 0xffff; do
+    write_codepoint_fixture "$cphex" field
+    assert_both_fail_closed "code point $cphex in unrelated field fails closed" primary git.example
+    write_codepoint_fixture "$cphex" comment
+    assert_both_fail_closed "code point $cphex in a comment fails closed" primary git.example
+done
+
+# NOT over-broad: printable code points PyYAML ACCEPTS must still resolve the
+# token in BOTH paths -- NEL(0x85) as a line separator, U+00A0 (NBSP) inside a
+# value, and an astral code point (U+1F600) inside the token value.
+write_codepoint_fixture 0x85 nelterm
+assert_token "NEL (U+0085) line-terminator resolves token" "TOK_NEL" primary git.example
+write_codepoint_fixture 0xa0 field
+assert_token "U+00A0 in unrelated value still resolves token" "TOK_PLAIN" primary git.example
+write_codepoint_fixture 0x1f600 token
+assert_token "astral U+1F600 inside token resolves verbatim" "$(printf 'T\360\237\230\200K')" primary git.example
+
+# 22. INTERNAL-INDICATOR OVER-REJECTION (#865 round-10 blocker 2): the round-9
+#     recognizer blanket-rejected any plain scalar CONTAINING a flow indicator
+#     ([]{}*&!), but in BLOCK context PyYAML treats ',[]{}' as ordinary content
+#     and treats '!&*#...' as significant ONLY at the FIRST non-space char. So an
+#     INTERNAL indicator is legal plain-string content and PyYAML emits the token
+#     verbatim; the fallback dropped it (over-rejection). Verified empirically
+#     against real PyYAML 6.0.3. The fix removes the blanket internal scan while
+#     the leading-char guard and the ' #'/': '/trailing-':' guards keep the
+#     fail-OPEN direction shut.
+for tv in 'a!b' 'a,b' 'a[b' 'a]b' 'a{b' 'a}b' 'a&b' 'a*b' 'a[b]c' 'a{b}c' 'a,b,c' 'a#b' 'a:b'; do
+    write_fixture "logins:
+  - name: primary
+    url: https://git.example
+    token: ${tv}
+"
+    assert_token "internal-indicator token '$tv' resolves verbatim" "$tv" primary git.example
+done
+
+# Fail-OPEN direction stays shut: a LEADING indicator, a ' #' comment tail, an
+# internal ': ' (colon-space) inline map, and a trailing ':' each make PyYAML
+# resolve NO usable string token (tag/flow/anchor reject or None, comment strip,
+# or a mapping), so BOTH must fail closed. (Leading tag/anchor/flow are the
+# documented, round-9-approved structural fail-closed class; kept intact here.)
+write_fixture 'logins:
+  - name: primary
+    url: https://git.example
+    token: !x
+'
+assert_both_fail_closed "leading '!' tag token fails closed" primary git.example
+write_fixture 'logins:
+  - name: primary
+    url: https://git.example
+    token: [a]
+'
+assert_both_fail_closed "leading '[' flow-seq token fails closed" primary git.example
+write_fixture 'logins:
+  - name: primary
+    url: https://git.example
+    token: a # trailing comment
+'
+# ' #' comment tail: PyYAML strips the comment -> token is the string 'a', which
+# still resolves. This is the NOT-over-broad boundary partner of the guard.
+assert_token "space-hash comment tail strips to plain token" "a" primary git.example
+write_fixture 'logins:
+  - name: primary
+    url: https://git.example
+    token: a: b
+'
+assert_both_fail_closed "internal colon-space (inline map) token fails closed" primary git.example
+write_fixture 'logins:
+  - name: primary
+    url: https://git.example
+    token: ab:
+'
+assert_both_fail_closed "trailing colon (map indicator) token fails closed" primary git.example
+
 echo "Gitea login resolution regression harness passed"