Normalize fullwidth punctuation to ASCII across codebase.

Add scripts/normalize_ambiguous_unicode.py; fix corrupted patch_instance_theme_templates.py. Preserves curly quotes in string literals; removes Git homoglyph warnings on .env.example. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-08 23:42:26 +08:00
parent aaa72c7961
commit b733e551a0
392 changed files with 71522 additions and 71369 deletions
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""将全角/易混淆标点规范为半角 ASCII(注释, 文档, 配置模板).
+
+不转换弯引号 “ ” ‘ ’,避免破坏 Python/JS 字符串字面量.
+"""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[1]
+
+SKIP_DIRS = frozenset({
+    ".git",
+    ".venv",
+    "node_modules",
+    "__pycache__",
+    ".cursor",
+    "agent-transcripts",
+})
+
+SCAN_SUFFIXES = frozenset({
+    ".py",
+    ".js",
+    ".html",
+    ".md",
+    ".sh",
+    ".css",
+    ".json",
+    ".cjs",
+    ".txt",
+    ".yml",
+    ".yaml",
+    ".example",
+})
+
+AMBIGUOUS_CHARS = frozenset(
+    "\uff08\uff09\uff1a\uff0c\uff1b\uff1f\uff01\u3002\u3001\u00a0"
+)
+
+TRANSLATION = str.maketrans(
+    {
+        "\uff08": "(",
+        "\uff09": ")",
+        "\uff1a": ":",
+        "\uff0c": ",",
+        "\uff1b": ";",
+        "\uff1f": "?",
+        "\uff01": "!",
+        "\u3002": ".",
+        "\u3001": ",",
+        "\u00a0": " ",
+    }
+)
+
+
+def should_scan(path: Path) -> bool:
+    if not path.is_file():
+        return False
+    if any(part in SKIP_DIRS for part in path.parts):
+        return False
+    if path.name == ".env.example" or path.name.endswith(".env.example"):
+        return True
+    return path.suffix in SCAN_SUFFIXES
+
+
+def normalize_text(text: str) -> tuple[str, int]:
+    count = sum(1 for ch in text if ch in AMBIGUOUS_CHARS)
+    if not count:
+        return text, 0
+    return text.translate(TRANSLATION), count
+
+
+def read_text_strip_bom(path: Path) -> tuple[str, bool]:
+    raw = path.read_text(encoding="utf-8")
+    if raw.startswith("\ufeff"):
+        return raw.lstrip("\ufeff"), True
+    return raw, False
+
+
+def iter_targets(root: Path) -> list[Path]:
+    return sorted(p for p in root.rglob("*") if should_scan(p))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Normalize ambiguous Unicode punctuation")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--root", default=str(REPO))
+    args = parser.parse_args()
+
+    root = Path(args.root)
+    files_changed = 0
+    chars_changed = 0
+
+    for path in iter_targets(root):
+        try:
+            original, had_bom = read_text_strip_bom(path)
+        except (OSError, UnicodeDecodeError):
+            continue
+        normalized, n = normalize_text(original)
+        if not n and not had_bom:
+            continue
+        rel = path.relative_to(root)
+        if args.dry_run:
+            print(f"[dry-run] {rel}: {n} chars")
+        else:
+            # 保持原换行风格, 仅替换标点
+            path.write_text(normalized, encoding="utf-8", newline="")
+        files_changed += 1
+        chars_changed += n
+
+    print(f"done: {files_changed} files, {chars_changed} replacements")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())