{"uuid": "1071e230-c3c9-4ab3-a5eb-dcf60d959c17", "vulnerability_lookup_origin": "1a89b78e-f703-45f3-bb86-59eb712668bd", "author": "9f56dd64-161d-43a6-b9c3-555944290a09", "vulnerability": "CVE-2021-42574", "type": "seen", "source": "https://gist.github.com/raeq/e78d5a79f40aaf81c6c6324f6b643c86", "content": "#!/usr/bin/env python3\n\"\"\"Evidence for raeq/disarm \u2014 literal invisible characters in the repository.\n\n`tests/test_readme_invisible_characters.py` enforces \"escapes, never literals\" for\nREADME.md and nothing else. This walks the whole tree with that file's own predicate\n(`Cf` or `Cc`, excluding newline and tab) and splits the result by risk class, because\nthe classes want different answers:\n\n  bidi-control     reorders the surrounding source in an editor and in a diff \u2014 the\n                   CVE-2021-42574 mechanism itself\n  other invisible  ZWSP, U+FEFF, word joiner, soft hyphen: render as nothing, so a\n                   reviewer cannot see them at all\n  ZWJ              usually legitimate emoji sequencing, where the literal renders as one\n                   glyph and is more readable than escapes\n  C0/C1 control    BEL, NEL\n\nRun from a disarm checkout:  python3 scan_literal_invisibles.py\n\"\"\"\n\nfrom __future__ import annotations\n\nimport collections\nimport pathlib\nimport unicodedata\n\nSKIP_DIRS = {\".git\", \"target\", \"node_modules\", \"site\", \".venv\", \"data\"}\nEXTENSIONS = {\n    \".py\", \".rs\", \".md\", \".rb\", \".mjs\", \".ts\", \".java\", \".kt\",\n    \".toml\", \".yml\", \".yaml\", \".sh\", \".c\", \".h\",\n}\n#: Newline and tab are `Cc`, and are simply how a text file is written.\nALLOWED = frozenset(\"\\n\\r\\t\")\n\nBIDI_CONTROLS = set(range(0x202A, 0x202F)) | set(range(0x2066, 0x206A)) | {0x200E, 0x200F, 0x061C}\n\n\ndef risk_class(ch: str) -> str:\n    cp, cat = ord(ch), unicodedata.category(ch)\n    if cp in BIDI_CONTROLS:\n        return \"bidi-control (Trojan Source)\"\n    if cp == 0x200D:\n        return \"ZWJ (emoji sequencing)\"\n    if cp == 0x200C:\n        return \"ZWNJ\"\n    if cat == \"Cc\":\n        return \"C0/C1 control\"\n    return \"other invisible\"\n\n\ndef is_literal_invisible(ch: str) -> bool:\n    \"\"\"The predicate from tests/test_readme_invisible_characters.py.\"\"\"\n    return ch not in ALLOWED and unicodedata.category(ch) in {\"Cf\", \"Cc\"}\n\n\ndef main() -> int:\n    by_class: collections.Counter[str] = collections.Counter()\n    files: dict[str, collections.Counter[str]] = collections.defaultdict(collections.Counter)\n    by_codepoint: collections.Counter[str] = collections.Counter()\n\n    for path in sorted(pathlib.Path(\".\").rglob(\"*\")):\n        if any(part in SKIP_DIRS for part in path.parts):\n            continue\n        if not path.is_file() or path.suffix not in EXTENSIONS:\n            continue\n        try:\n            text = path.read_text(encoding=\"utf-8\")\n        except (UnicodeDecodeError, OSError):\n            continue\n        for ch in text:\n            if is_literal_invisible(ch):\n                cls = risk_class(ch)\n                by_class[cls] += 1\n                files[cls][str(path)] += 1\n                by_codepoint[f\"U+{ord(ch):04X} {unicodedata.name(ch, 'unnamed')}\"] += 1\n\n    total = sum(by_class.values())\n    all_files = {f for c in files.values() for f in c}\n    print(f\"{total} literal invisible characters in {len(all_files)} files\\n\")\n    for cls, count in by_class.most_common():\n        print(f\"=== {cls}: {count} in {len(files[cls])} files\")\n        for name, n in files[cls].most_common():\n            print(f\"    {n:4}  {name}\")\n        print()\n    print(\"by code point:\")\n    for name, n in by_codepoint.most_common():\n        print(f\"  {n:4}  {name}\")\n    return 0\n\n\nif __name__ == \"__main__\":\n    raise SystemExit(main())\n", "creation_timestamp": "2026-08-31T12:23:10.984667Z"}