"""Original data-only merge of reference lookup aliases into Unicode candidates.

No reference JavaScript implementation is used. Unicode fields are preserved;
reference readings/ranking are marked separately as compatibility lookup data.
"""
from pathlib import Path
import json,re,copy,gzip,hashlib,shutil,collections
ROOT=Path(__file__).parent
BASE=ROOT/'generated'
OUT=BASE/'merged'
LICENSES=OUT/'licenses'
LICENSES.mkdir(parents=True,exist_ok=True)

def read(path):return json.loads(path.read_text(encoding='utf-8-sig'))
def distinct(values):return list(dict.fromkeys(values))
def append(record,key,values):record[key]=distinct(record.get(key,[])+values)
def emit(name,data,folder=OUT):
 b=(json.dumps(data,ensure_ascii=False,separators=(',',':'))+'\n').encode()
 (folder/name).write_bytes(b)
 with gzip.GzipFile(filename=str(folder/(name+'.gz')),mode='wb',mtime=0) as f:f.write(b)
 return {'records':len(data),'bytes':len(b),'gzipBytes':(folder/(name+'.gz')).stat().st_size}
def norm_pinyin(value):return value.strip().lower().replace('u:','ü').replace('v','ü')
def plain_pinyin(value):return re.sub('[1-5]$','',norm_pinyin(value))
def marked_syllable(match):
 letters,tone=match[1],int(match[2]); accents={'a':'āáǎà','e':'ēéěè','i':'īíǐì','o':'ōóǒò','u':'ūúǔù','ü':'ǖǘǚǜ'}
 if tone==5:return letters
 place=next((letters.index(v) for v in ['a','e'] if v in letters),None)
 if place is None and 'ou' in letters:place=letters.index('o')
 if place is None:
  positions=[i for i,c in enumerate(letters) if c in accents]
  place=positions[-1] if positions else None
 if place is None:return letters+str(tone)
 return letters[:place]+accents[letters[place]][tone-1]+letters[place+1:]
def marked(value):return re.sub(r'([a-züê]+)([1-5])',marked_syllable,norm_pinyin(value))

han={r['char']:r for r in read(BASE/'han-radicals.json')}
provenance={'generatedAt':'2026-10-15','base':'Official Unihan 18.0.0; see Unicode license and original provenance.','purpose':'Keep every reference candidate output and lookup alias available while retaining the wider official Unicode coverage. Reference aliases are compatibility data, not a claim that every spelling or reading is linguistically authoritative.','referenceCodeIncluded':False,'sources':{},'stats':{},'files':{}}

for language,refname in [('chinese','chinois'),('japanese','japonais'),('cantonese','cantonais')]:
 original=read(BASE/(language+'-candidates.json'))
 data=copy.deepcopy(original); bychar={r['char']:r for r in data}; count_before=len(data)
 reference=read(ROOT/('reference-'+refname+'.json'))
 missing=[]; input_pairs=[]; aliases=0; placeholder_count=0; supplements=[]
 if isinstance(reference,dict):
  rows=[{'char':chr(int(cp,16)),'latin':[code],'order':i} for i,(code,cps) in enumerate(reference.items()) for cp in cps]
 else:
  rows=[dict(row,char=chr(int(row['unicode'],16)),order=row.get('ordre',i)) for i,row in enumerate(reference)]
 for row in rows:
  char=row['char']
  if char not in bychar:
   metadata=copy.deepcopy(han.get(char,{'char':char,'radicals':[],'strokes':[]}))
   if language=='chinese':metadata.update(pinyin=[],pinyinNumber=[],pinyinPlain=[])
   elif language=='japanese':metadata.update(japanese=[],kun=[],on=[])
   else:metadata.update(cantonese=[],cantonesePlain=[])
   bychar[char]=metadata;data.append(metadata);missing.append(char)
  record=bychar[char]; record['referenceOrder']=min(record.get('referenceOrder',row['order']),row['order'])
  append(record,'referenceReadings',row['latin'])
  supplement={'char':char}; new=[]
  if language=='chinese':
   for raw in row['latin']:
    q=norm_pinyin(raw);input_pairs.append((char,q))
    if q not in record['pinyinNumber']:
     new.append(q);aliases+=1
    if raw=='-':placeholder_count+=1
    append(record,'pinyinNumber',[q]);append(record,'pinyinPlain',[plain_pinyin(raw)]);append(record,'pinyin',[marked(raw)])
   if new:supplement.update(pinyinNumber=new,pinyinPlain=distinct(plain_pinyin(v) for v in new),pinyin=distinct(marked(v) for v in new))
  elif language=='japanese':
   for q in row['latin']:
    input_pairs.append((char,q))
    if q not in record['japanese']:new.append(q);aliases+=1
    append(record,'japanese',[q]);append(record,'on' if q.isupper() else 'kun',[q])
   # Preserve reference filter metadata separately; never overwrite Unicode facts.
   record['referenceRadical']=row['cles'];record['referenceStrokes']=row['traits']
   if new:supplement.update(japanese=new,kun=[v for v in new if not v.isupper()],on=[v for v in new if v.isupper()])
  else:
   for raw in row['latin']:
    # Preserve raw compound label plus split alternatives, giving every
    # pronunciation first-class lookup while retaining source prefix queries.
    readings=distinct([raw]+[v.strip() for v in raw.split(',') if v.strip()])
    for q in readings:
     input_pairs.append((char,q))
     if q not in record['cantonese']:new.append(q);aliases+=1
     append(record,'cantonese',[q]);append(record,'cantonesePlain',[re.sub(r'\d','',q)])
   if new:supplement.update(cantonese=new,cantonesePlain=distinct(re.sub(r'\d','',q) for q in new))
  if new:supplements.append(supplement)
 # One output may occupy several reference rows; keep one merged supplemental record.
 merged_supp={}
 for row in supplements:
  item=merged_supp.setdefault(row['char'],{'char':row['char']})
  for k,vs in row.items():
   if k!='char':append(item,k,vs)
 supplements=list(merged_supp.values())
 field={'chinese':'pinyinNumber','japanese':'japanese','cantonese':'cantonese'}[language]
 absent=[(c,r) for c,r in input_pairs if r not in bychar[c][field]]
 assert not absent,(language,absent[:10])
 assert len(bychar)==len(data)
 for record in original:
  target=bychar[record['char']]
  for k,v in record.items():
   if isinstance(v,list):assert all(x in target[k] for x in v),(language,record['char'],k)
   else:assert v==target[k],(language,record['char'],k)
 # Reference subset first, then original Unicode commonness ordering.
 data.sort(key=lambda r:r.get('referenceOrder',10**9))
 filename=language+'-candidates.json'
 provenance['files'][filename]=emit(filename,data)
 emit(language+'-reference-supplement.json',supplements,LICENSES)
 provenance['sources'][language]={'url':'https://www.lexilogos.com/json/'+refname+'.json','referencePage':'https://www.lexilogos.com/keyboard/'+language+'.htm','sha256':hashlib.sha256((ROOT/('reference-'+refname+'.json')).read_bytes()).hexdigest()}
 provenance['stats'][language]={'originalRecords':count_before,'mergedRecords':len(data),'sourceRecords':len(reference),'sourceDistinctOutputs':len(set(r['char'] for r in rows)),'sourceLookupPairs':len(set(input_pairs)),'addedOutputs':len(missing),'addedAliases':aliases,'supplementalRecords':len(supplements),'missingReferencePairsAfterMerge':0,'maxOutputLength':max(len(r['char']) for r in rows)}
 if language=='chinese':provenance['stats'][language]['unknownReadingPlaceholders']=placeholder_count
 if language=='japanese':
  differences=[{'char':r['char'],'referenceStrokes':r['traits'],'unicodeStrokes':bychar[r['char']]['strokes']} for r in rows if r['traits'] not in bychar[r['char']]['strokes']]
  provenance['stats'][language]['strokeMetadataDifferences']=len(differences)
  emit('japanese-reference-stroke-differences.json',differences,LICENSES)
 if language=='cantonese':provenance['sources'][language]['upstream']='https://github.com/gwinterstein/Cifu'

shutil.copy2(BASE/'UNICODE-LICENSE.txt',LICENSES/'UNICODE-LICENSE.txt')
shutil.copy2(BASE/'provenance.json',LICENSES/'unihan-provenance.json')
shutil.copy2(ROOT/'cifu-LICENSE',LICENSES/'CIFU-GPL-3.0.txt')
shutil.copy2(ROOT/'cifu-README_md',LICENSES/'CIFU-README.md')
(LICENSES/'REFERENCE-PARITY-NOTICE.txt').write_text('Chinese and Japanese lookup supplements contain factual character/readings/ordering from the referenced Lexilogos data endpoints, accessed 2026-10-15. No reference implementation, page design, or instructional prose is copied. These aliases supplement, and do not alter, Unicode 18.0.0 character facts.\nCantonese reference identifies Cifu (Regine Lai and Grégoire Winterstein, 2020) as its underlying data. The Cantonese supplementary data are supplied under GPL-3.0; full license, attribution, source URLs and readable JSON supplement are included.\nChinese source placeholder - is retained only for reference lookup coverage; it is not a pronunciation. Japanese short kun stems and long-vowel variants are retained as lookup aliases. Two Japanese combined lookup strings containing spaces are retained verbatim; they do not signify multi-character output.\n')
provenance['behaviorNotes']=[
 'Japanese: query ! requests exact match; = requests case-sensitive match (uppercase on versus lowercase kun). Preserve uppercase rather than unconditionally lowercasing. Long-vowel query aliases should handle oo/ou/ō/ô, aa/ā/â, ii/ī/î, uu/ū/û, and ei/ee/ē/ê.',
 'Japanese all 2030 reference outputs and Chinese all 9934 reference outputs are single codepoints. No hidden phrase dictionary in these two sources.',
 'Japanese referenceRadical/referenceStrokes are included only as source filter metadata. Some differ from official Unicode data; do not overwrite Unicode facts.',
 'Chinese v and u: normalize to ü; keep numbered and plain fields. Reference source alias ge1ge5 is associated with the single character 歌 and preserved as data.',
 'Cantonese raw reading labels sometimes contain comma-separated alternatives. Merged arrays contain both original raw labels and each split alternative. Some unusual source aliases (e.g. cī, zehw) retained for compatibility.',
 'Reference order is supplied per output. Every source query/output pair is present; exact output ranking among additional Unicode results may differ unless UI uses referenceOrder.']
(LICENSES/'reference-parity-provenance.json').write_text(json.dumps(provenance,ensure_ascii=False,indent=2)+'\n')
shutil.copy2(__file__,LICENSES/'build_reference_parity.py')
print(json.dumps(provenance['stats'],ensure_ascii=False,indent=2))
print(json.dumps(provenance['files'],ensure_ascii=False,indent=2))