#!/usr/bin/env python3
"""USGS splib07a reference subset and exploratory, abstaining mineral-group matching.
No customer ground-truth mineral labels were supplied. Outputs are candidates, never ore grades.
"""
from pathlib import Path
import argparse, re, json, zipfile, io, hashlib
from collections import Counter, defaultdict
import numpy as np
from scipy.ndimage import gaussian_filter1d
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
from PIL import Image

ROOT=Path(__file__).resolve().parent.parent
BASE=ROOT.parent.parent
P=argparse.ArgumentParser()
P.add_argument('--archive',type=Path,default=BASE/'tmp/usgs_reference_download/ASCIIdata_splib07a.zip')
P.add_argument('--source',type=Path,default=ROOT.parent/'调整后目标')
P.add_argument('--output',type=Path,default=ROOT)
args=P.parse_args();out=args.output;out.mkdir(parents=True,exist_ok=True)
for sub in ['参考库/原始谱','图表','附件']:(out/sub).mkdir(parents=True,exist_ok=True)

GROUPS={
'Carbonate':['Calcite','Dolomite','Siderite','Magnesite','Ankerite','Aragonite'],
'Chlorite':['Chlorite','Clinochlore','Prochlorite','Thuringite','Chamosite'],
'White-mica / illite':['Muscovite','Illite','Paragonite','Phengite'],
'Kaolin':['Kaolinite','Dickite','Nacrite','Halloysite','Endellite'],
'Smectite':['Montmorillonite','Beidellite','Nontronite','Saponite','Hectorite'],
'Talc / serpentine':['Talc','Serpentine','Antigorite','Lizardite','Chrysotile'],
'Amphibole':['Actinolite','Tremolite','Hornblende','Anthophyllite'],
'Epidote':['Epidote','Zoisite'],
'Dark mica':['Biotite','Phlogopite'],
'Sulfate':['Alunite','Jarosite','Gypsum','Kieserite'],
'Other Al-OH':['Pyrophyllite','Gibbsite','Diaspore','Prehnite'],
'Low-specificity control':['Quartz','Albite','Microcline','Orthoclase','Magnetite','Hematite','Goethite','Pyrite']}
ZH={'Carbonate':'碳酸盐组','Chlorite':'绿泥石组','White-mica / illite':'白云母／伊利石组','Kaolin':'高岭石组',
'Smectite':'蒙皂石组','Talc / serpentine':'滑石／蛇纹石组','Amphibole':'闪石组','Epidote':'绿帘石组',
'Dark mica':'黑云母／金云母组','Sulfate':'硫酸盐组','Other Al-OH':'其他含铝羟基矿物','Low-specificity control':'低特异性对照'}
lookup={m:g for g,ms in GROUPS.items() for m in ms}
settings={'match_window_nm':[2050,2450],'fwhm_scenarios_nm':[12.,18.,24.],
 'max_residual_fraction':0.40,'min_cosine':0.92,'min_group_margin':0.025,
 'min_absorption_depth':0.03,'brightness_bounds':[0.06,0.85],
 'max_peak_difference_nm':12.,'max_missing_band_fraction':0.10,
 'min_library_holdout_families':3,'min_library_macro_family_agreement':0.70,
 'roi_y_centers':[300,700,1050],'roi_x_centers':[80,180,270],
 'patch_height':48,'patch_width':24,'point_cell_size':4,
 'threshold_status':'exploratory heuristics, not calibrated probabilities or customer accuracy thresholds'}


def load_ascii(z,name):
    text=z.read(name).decode('utf-8',errors='replace')
    return text.splitlines()[0].strip(),np.loadtxt(io.StringIO(text),skiprows=1)


def hdr(p):
    d={}
    for m in re.finditer(r'^\s*([^=\n]+?)\s*=\s*(\{[^}]*\}|[^\n]*)',p.read_text(),re.M):
        k=m[1].strip().lower();v=m[2].strip()
        if k in ['wavelength','fwhm']:v=[float(s) for s in v.strip('{}').split(',') if s.strip()]
        d[k]=v
    return d


def continuum(w,r):
    # Piecewise linear upper convex envelope. It cannot create absorptions in a flat spectrum.
    hull=[]
    for k in range(len(w)):
        while len(hull)>=2:
            i,j=hull[-2:]
            if (r[j]-r[i])/(w[j]-w[i]) <= (r[k]-r[j])/(w[k]-w[j]):hull.pop()
            else:break
        hull.append(k)
    return np.interp(w,w[hull],r[hull])


def feature(w,r):
    c=continuum(w,r)
    if not np.all(c>0):return np.full(len(w),np.nan)
    return np.clip(1-r/c,0,None)


def gaussian_resample(w,r,source_fwhm,target_w,target_fwhm):
    result=np.full(len(target_w),np.nan)
    for k,t in enumerate(target_w):
        if t<w[0] or t>w[-1]:continue
        native=float(np.interp(t,w,source_fwhm))
        extra=np.sqrt(max(target_fwhm**2-native**2,0.))/2.354820045
        if extra<.1:
            j=np.searchsorted(w,t);j=min(max(j,1),len(w)-1)
            if np.isfinite(r[j-1:j+1]).all() and w[j]-w[j-1]<10:result[k]=np.interp(t,w,r)
            continue
        ix=np.abs(w-t)<=3*extra
        if not ix.any():continue
        weights=np.exp(-.5*((w[ix]-t)/extra)**2)
        good=np.isfinite(r[ix])
        if weights[good].sum()<.95*weights.sum():continue
        result[k]=np.sum(weights[good]*r[ix][good])/weights[good].sum()
    return result


def specimen_family(name):
    # Group by specimen family before holding out all sizes / instrument repeats.
    m=re.search(r'(NMNH\d+|HS\d+|GDS\d+|WS\d+|CU\d+-\d+|ML\d+-\d+|IL\d+)',name)
    return m.group(0) if m else name.rsplit('_',2)[0]

folders=sorted(p for p in args.source.iterdir() if p.is_dir())
first=hdr(folders[0]/'capture'/f'{folders[0].name}.hdr');target=np.array(first['wavelength'])
window=(target>=2050)&(target<=2450);wave=target[window]
z=zipfile.ZipFile(args.archive)
archive_md5=hashlib.md5(args.archive.read_bytes()).hexdigest()
assert archive_md5=='bfe74068d85811e52e5e07d017720a17'
wname=next(n for n in z.namelist() if 'Wavelengths_ASD' in n);_,sw=load_ascii(z,wname);sw*=1000
bandpasses={}
for key,part in [('ASDFR','ASDFR_StandardResolution'),('ASDNG','ASDNG_High-Res_NextGen'),('ASDHR','ASDHR_High-Resolution')]:
    name=next(n for n in z.namelist() if 'Bandpass_(FWHM)_'+part in n)
    _,bandpasses[key]=load_ascii(z,name);bandpasses[key]*=1000
    (out/'参考库/原始谱'/Path(name).name).write_bytes(z.read(name))
(out/'参考库/原始谱'/Path(wname).name).write_bytes(z.read(wname))
refs=[];rejected=[];arrays=defaultdict(list);features=defaultdict(list)
for name in z.namelist():
    if '/ChapterM_Minerals/' not in name or not name.endswith('_AREF.txt'):continue
    mineral=Path(name).name.split('_')[1]
    if mineral not in lookup:continue
    m=re.search(r'_(ASDFR|ASDNG|ASDHR)([a-z])_AREF',name)
    if not m:
        rejected.append({'source':name,'reason':'not ASD reflectance measurement; not in this first uniform-instrument subset'});continue
    instrument,purity=m.groups()
    if purity not in ['a','b']:
        rejected.append({'source':name,'reason':f'purity {purity} excluded from first matching subset'});continue
    title,r=load_ascii(z,name)
    assert len(r)==len(sw)
    r[(r<=0)|(r>1.5)|(~np.isfinite(r))]=np.nan
    forms={f:gaussian_resample(sw,r,bandpasses[instrument],target,f) for f in settings['fwhm_scenarios_nm']}
    if any(not np.isfinite(v[window]).all() for v in forms.values()):
        rejected.append({'source':name,'reason':'missing / invalid source channels influence matching window'});continue
    item={'id':len(refs),'title':title,'filename':Path(name).name,'source_archive_path':name,
        'mineral':mineral,'group':lookup[mineral],'group_zh':ZH[lookup[mineral]],'instrument':instrument,
        'purity_code':purity,'specimen_family':specimen_family(Path(name).name),
        'sha256':hashlib.sha256(z.read(name)).hexdigest(),'sample_description_review':'pending individual HTML review'}
    refs.append(item);(out/'参考库/原始谱'/Path(name).name).write_bytes(z.read(name))
    for f,v in forms.items():arrays[f].append(v);features[f].append(feature(wave,v[window]))
assert len(refs)>30
for f in arrays:arrays[f]=np.array(arrays[f]);features[f]=np.array(features[f])
groups=sorted(GROUPS);group_idx=np.array([groups.index(r['group']) for r in refs]);families=np.array([r['specimen_family'] for r in refs])
# Non-diagnostic controls remain visible but are never accepted as positive mineral IDs.
active=np.array([r['group']!='Low-specificity control' for r in refs])&(features[12.].max(1)>.015)
fnorms={f:np.linalg.norm(q,axis=1) for f,q in features.items()}
qnormalized={f:q/np.maximum(fnorms[f][:,None],1e-10) for f,q in features.items()}


def match(obs,fwhm=12.,exclude_family=None):
    q=feature(wave,obs);qn=np.linalg.norm(q)
    if qn<1e-10 or not np.isfinite(q).all():return None
    cos=(q/qn)@qnormalized[fwhm].T
    allowed=active.copy()
    if exclude_family is not None:allowed&=families!=exclude_family
    cos[~allowed]=-np.inf
    order=np.argsort(cos)[::-1];best=int(order[0])
    if not np.isfinite(cos[best]):return None
    alt=next((int(i) for i in order if refs[i]['group']!=refs[best]['group'] and np.isfinite(cos[i])),None)
    scale=max(0.,float(q@features[fwhm][best]/max(fnorms[fwhm][best]**2,1e-12)))
    fitted=scale*features[fwhm][best];res=np.linalg.norm(q-fitted)/qn
    return {'best_ref':best,'best_group':refs[best]['group'],'best_mineral':refs[best]['mineral'],
        'cosine':float(cos[best]),'residual_fraction':float(res),'depth_scale':scale,
        'group_margin':float(cos[best]-cos[alt]) if alt is not None else None,
        'alternative_ref':alt,'alternative_group':refs[alt]['group'] if alt is not None else None,
        'obs_max_depth':float(q.max()),'peak_difference_nm':float(abs(wave[q.argmax()]-wave[features[fwhm][best].argmax()])),
        'top5':[{'ref':int(i),'mineral':refs[i]['mineral'],'group':refs[i]['group'],'cosine':float(cos[i])} for i in order[:5] if np.isfinite(cos[i])]}


def assess(m,brightness,missing,sensitivity,library_gate=True):
    reasons=[]
    if not settings['brightness_bounds'][0]<=brightness<=settings['brightness_bounds'][1]:reasons.append('brightness / background gate')
    if missing>settings['max_missing_band_fraction']:reasons.append('too many bad-band sites')
    if m is None:return ['no valid diagnostic shape']
    if m['obs_max_depth']<settings['min_absorption_depth']:reasons.append('weak absorption')
    if m['cosine']<settings['min_cosine']:reasons.append('shape similarity below gate')
    if m['residual_fraction']>settings['max_residual_fraction']:reasons.append('large unexplained feature residual')
    if m['group_margin'] is None or m['group_margin']<settings['min_group_margin']:reasons.append('competing mineral groups')
    if m['peak_difference_nm']>settings['max_peak_difference_nm']:reasons.append('strongest feature position mismatch')
    if any(t is None or t['best_group']!=m['best_group'] for t in sensitivity):reasons.append('bandwidth-sensitive group')
    if library_gate and not group_reliability[m['best_group']]['screen_pass']:reasons.append('insufficient library-only group discrimination')
    return reasons

# Method controls: exact library recoverability is not geological validation.
control_flat=feature(wave,np.ones(len(wave)))
control_tri=feature(np.array([1.,2.,3.]),np.array([1.,.8,1.]))
assert np.allclose(control_flat,0) and np.allclose(control_tri,[0,.2,0])
self_correct=0;holdout=[]
for i,r in enumerate(refs):
    if not active[i]:continue
    obs=arrays[12.][i,window]
    same=match(obs);self_correct+=same['best_group']==r['group']
    if not np.any(active&(group_idx==group_idx[i])&(families!=families[i])):continue
    m=match(obs,exclude_family=families[i]);holdout.append({'ref':i,'actual_library_group':r['group'],
        'held_out_family':families[i],'predicted_group':m['best_group'],'cosine':m['cosine'],
        'correct':m['best_group']==r['group']})
controls={'flat_spectrum_has_zero_absorption':True,'known_three_point_feature_correct':True,
    'self_retrieval_group_correct':int(self_correct),'self_retrieval_group_total':int(active.sum()),
    'specimen_family_holdout_total':len(holdout),'specimen_family_holdout_correct':sum(r['correct'] for r in holdout),
    'holdout_details':holdout,'warning':'Library-only computational controls; not accuracy on customer cores.'}

# Conservative engineering screen based on reference-family holdout, not customer accuracy.
group_reliability={}
for group in groups:
    rows=[r for r in holdout if r['actual_library_group']==group]
    family_ids=sorted(set(r['held_out_family'] for r in rows))
    rates=[np.mean([r['correct'] for r in rows if r['held_out_family']==family]) for family in family_ids]
    macro=float(np.mean(rates)) if rates else None
    group_reliability[group]={'tested_spectra':len(rows),'families':len(family_ids),'macro_family_agreement':macro,
        'screen_pass':len(family_ids)>=settings['min_library_holdout_families'] and macro is not None and macro>=settings['min_library_macro_family_agreement']}
controls['group_reliability']=group_reliability

# Small-area sampling: 18 windows, each 48 lines x 24 columns. Each plotted point = median of a 4x4 pixel cell.
points=[];patches=[];obs_all=[];previews={};reference_diagnostics=[]
for folder in folders:
    h=hdr(folder/'capture'/f'{folder.name}.hdr');L,B,X=[int(h[k]) for k in ['lines','bands','samples']]
    assert np.array_equal(target,h['wavelength'])
    raw=np.memmap(folder/'capture'/f'{folder.name}.raw',dtype='<u2',mode='r',shape=(L,B,X))
    read_ref=lambda tag:np.memmap(folder/'capture'/f'{tag}_{folder.name}.raw',dtype='<u2',mode='r',shape=(100,B,X)).mean(0,dtype='f4')
    dark=read_ref('DARKREF');white=read_ref('WHITEREF');den=white-dark
    bad=np.fromfile(folder/'calibrations/bpr/bprmap.bpr',dtype='<u2').reshape(B,X).astype(bool)
    box=int(re.search(r'SWIR_b_(\d+)',folder.name)[1]);previews[box]=np.asarray(Image.open(folder/f'{folder.name}.png'))
    for row in settings['roi_y_centers']:
      for lane,col in enumerate(settings['roi_x_centers'],1):
        y0,y1=row-24,row+24;x0,x1=col-12,col+12;pid=f'B{box}_L{lane}_Y{row}'
        cube=(raw[y0:y1,:,x0:x1].astype('f8')-dark[:,x0:x1])/den[:,x0:x1]
        localbad=bad[:,x0:x1];cube[:,localbad]=np.nan
        spectrum=np.nanmedian(cube,axis=(0,2));full=spectrum.copy()
        finite=np.isfinite(full);full[~finite]=np.interp(target[~finite],target[finite],full[finite])
        ms=match(full[window]);sens=[match(full[window],f) for f in [18.,24.]]
        br=float(np.nanmedian(full[(target>=1100)&(target<=2400)]));missing=float(np.isnan(spectrum[window]).mean())
        reasons=assess(ms,br,missing,sens)
        patch={'id':pid,'box':box,'lane':lane,'y_center':row,'x_center':col,'y_bounds':[y0,y1],'x_bounds':[x0,x1],
            'raw_pixel_count':48*24,'brightness':br,'matching':ms,'bandwidth_18':sens[0],'bandwidth_24':sens[1],
            'status':'candidate' if not reasons else 'unresolved','reasons':reasons}
        patches.append(patch);obs_all.append(full)
        for yy in range(0,48,4):
          for xx in range(0,24,4):
            sp=np.nanmedian(cube[yy:yy+4,:,xx:xx+4],axis=(0,2));valid=np.isfinite(sp)
            miss=float(np.isnan(sp[window]).mean());sp[~valid]=np.interp(target[~valid],target[valid],sp[valid])
            brightness=float(np.nanmedian(sp[(target>=1100)&(target<=2400)]))
            m=match(sp[window]);sensitivity=[match(sp[window],f) for f in [18.,24.]]
            failed=assess(m,brightness,miss,sensitivity)
            # A shifted-spectrum reference is a negative control, not a competing mineral hypothesis.
            shifted=np.interp(wave+40,wave,sp[window],left=sp[window][0],right=sp[window][-1])
            shifted_match=match(shifted)
            points.append({'box':box,'patch':pid,'x':x0+xx+1.5,'y':y0+yy+1.5,'cell_size':4,
                'brightness':brightness,'matching':m,'status':'candidate' if not failed else 'unresolved',
                'reasons':failed,'initial_shape_gate_status':'candidate' if not assess(m,brightness,miss,sensitivity,library_gate=False) else 'unresolved','bandwidth_18_group':sensitivity[0]['best_group'] if sensitivity[0] else None,
                'bandwidth_24_group':sensitivity[1]['best_group'] if sensitivity[1] else None,
                'shift_40nm_control_cosine':shifted_match['cosine'] if shifted_match else None})

# Serialize native spectra provenance and common-axis templates before drawing.
manifest={'name':'USGS splib07a ASD SWIR reference subset','source_doi':'10.5066/F7RR1WDJ',
    'source_item':'https://www.sciencebase.gov/catalog/item/586e8c88e4b0f5ce109fccae',
    'archive':'ASCIIdata_splib07a.zip','archive_md5':archive_md5,
    'archive_sha256':hashlib.sha256(args.archive.read_bytes()).hexdigest(),
    'selection':'ChapterM minerals; explicit target and confounder list; ASD AREF; spectral purity a/b; valid matching window',
    'groups':GROUPS,'group_names_zh':ZH,'reference_count':len(refs),'refs':refs,'excluded':rejected,
    'settings':settings,'active_matching_reference_count':int(active.sum()),
    'target_wavelength_nm':target.tolist(),'match_band_indices_zero_based':np.where(window)[0].tolist(),
    'no_extrapolation':'The last camera band at 2504.55 nm exceeds ASD 2500 nm coverage and remains NaN; not used for matching.',
    'bandwidth_status':'12/18/24 nm are assumptions, not measured camera FWHM; source bandpass is accounted for approximately by Gaussian quadrature.'}
np.savez_compressed(out/'参考库/参考子库_仪器波长对齐.npz',wavelength_nm=target,match_wavelength_nm=wave,
    spectra_fwhm12=arrays[12.],spectra_fwhm18=arrays[18.],spectra_fwhm24=arrays[24.],
    absorption_fwhm12=features[12.],active=active,reference_ids=np.arange(len(refs)))
np.savez_compressed(out/'附件/18窗口中位光谱.npz',wavelength_nm=target,spectra=np.array(obs_all),patch_ids=np.array([p['id'] for p in patches]))
(out/'参考库/参考子库清单.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2))
(out/'附件/方法控制验证.json').write_text(json.dumps(controls,ensure_ascii=False,indent=2))
results={'settings':settings,'no_ground_truth':True,'patches':patches,'points':points,
    'reference_count':len(refs),'active_reference_count':int(active.sum()),
    'patch_summary':dict(Counter(p['status'] for p in patches)),
    'point_summary':dict(Counter(p['status'] for p in points)),
    'accepted_groups':dict(Counter(p['matching']['best_group'] for p in points if p['status']=='candidate')),
    'unresolved_reasons':dict(Counter(r for p in points for r in p['reasons'])),
    'initial_shape_gate_summary':dict(Counter(p['initial_shape_gate_status'] for p in points)),
    'library_group_screen':group_reliability}
(out/'附件/小范围匹配结果.json').write_text(json.dumps(results,ensure_ascii=False,indent=2))

plt.rcParams.update({'font.size':9,'axes.spines.top':False,'axes.spines.right':False})
# Figure 1: exact tested windows in supplied image coordinates.
fig,axs=plt.subplots(1,2,figsize=(9,12))
for ax,box in zip(axs,[4,5]):
    ax.imshow(previews[box]);ax.set(title=f'Box {box} | nominal {box}-{box+1} m',xlabel='Source image x',ylabel='Scan line y; NOT borehole depth')
    for p in patches:
        if p['box']!=box:continue
        x0,x1=p['x_bounds'];y0,y1=p['y_bounds'];ax.add_patch(plt.Rectangle((x0,y0),x1-x0,y1-y0,fill=False,color='#ffb000',lw=1.3))
        ax.text(x0-2,y0-9,p['id'].replace(f'B{box}_',''),color='#ffb000',fontsize=7)
fig.suptitle('18 test windows only | 48 lines x 24 pixels per window');fig.tight_layout(rect=[0,0,1,.97]);fig.savefig(out/'图表/01_两箱小范围验证窗口.png',dpi=160);plt.close(fig)

# Figure 2: unresolved grey, candidate groups use shared legend.
accepted_groups=sorted(set(p['matching']['best_group'] for p in points if p['status']=='candidate'))
all_used_groups=sorted(set(p['matching']['best_group'] for p in points if p['matching']))
palette=['#0072B2','#009E73','#8C510A','#E69F00','#CC79A7','#56B4E9','#D55E00','#7B2CBF','#BCBD22','#1B9E77','#D73027','#7570B3']
color={g:palette[i] for i,g in enumerate(groups)}
fig,axs=plt.subplots(1,2,figsize=(10,11))
for ax,box in zip(axs,[4,5]):
    ax.imshow(previews[box],alpha=.5)
    for status in ['unresolved','candidate']:
        for group in (all_used_groups if status=='candidate' else [None]):
            ps=[p for p in points if p['box']==box and p['status']==status and (group is None or p['matching']['best_group']==group)]
            if ps:ax.scatter([p['x'] for p in ps],[p['y'] for p in ps],s=5,c=[color[group] if status=='candidate' else '#888888'],marker='s',linewidths=0)
    ax.set(title=f'Box {box} | 648 cell medians tested',xlabel='Source image x',ylabel='Scan line y; NOT depth')
legend=[Line2D([],[],marker='s',ls='',color='#888888',label='Unresolved / failed gates')]+[Line2D([],[],marker='s',ls='',color=color[g],label=g+' candidate') for g in accepted_groups]
fig.legend(handles=legend,ncol=2,loc='lower center',bbox_to_anchor=(.5,.015),fontsize=8)
fig.suptitle('USGS reference-matched mineral-GROUP CANDIDATES | small windows only\nNo customer mineral ground truth; colors are NOT confirmed mineral occurrence',fontsize=11)
fig.tight_layout(rect=[0,.07,1,.94]);fig.savefig(out/'图表/02_USGS候选矿物组点图_未实证.png',dpi=170);plt.close(fig)

# Figure 3: observed continuum-removed absorption versus fitted USGS spectrum for each patch.
fig,axs=plt.subplots(6,3,figsize=(14,19))
for ax,p,full in zip(axs.flat,patches,obs_all):
    m=p['matching'];q=feature(wave,full[window]);ax.plot(wave,q,color='black',lw=1.5,label='Core median feature')
    if m:
        idx=m['best_ref'];fit=features[12.][idx]*m['depth_scale'];ax.plot(wave,fit,color='#0072B2',lw=1,label=refs[idx]['mineral']+' best ref')
        alt=m['alternative_ref']
        if alt is not None:
            f=features[12.][alt];scale=max(0,q@f/max(f@f,1e-12));ax.plot(wave,scale*f,color='#D55E00',lw=.8,ls='--',label=refs[alt]['mineral']+' competing')
        ax.set_title(f"{p['id']} | {p['status']}\n{m['best_group']}; cos={m['cosine']:.3f}, margin={m['group_margin']:.3f}",fontsize=8)
    ax.set(xlabel='Wavelength (nm)',ylabel='Continuum-removed depth');ax.legend(fontsize=6,loc='upper left')
fig.suptitle('18 ROI median spectra: observed feature, scaled best reference and competing group\nFitted depth scale is NOT mineral abundance',fontsize=12,y=.995)
fig.tight_layout(rect=[0,0,1,.974]);fig.savefig(out/'图表/03_18窗口光谱匹配与竞争候选.png',dpi=150);plt.close(fig)
# Figure 4: zoom the 18 windows so individual 4x4-cell decisions can be inspected.
fig,axs=plt.subplots(3,6,figsize=(14,8))
for p in patches:
    row=settings['roi_y_centers'].index(p['y_center']);col=p['lane']-1+(3 if p['box']==5 else 0)
    ax=axs[row,col];x0,x1=p['x_bounds'];y0,y1=p['y_bounds']
    ax.imshow(previews[p['box']],alpha=.35)
    ps=[r for r in points if r['patch']==p['id']]
    ax.scatter([r['x'] for r in ps],[r['y'] for r in ps],c=[color[r['matching']['best_group']] if r['status']=='candidate' else '#aaaaaa' for r in ps],s=18,marker='s',linewidths=0)
    ax.set(xlim=(x0,x1),ylim=(y1,y0),title=p['id']);ax.tick_params(labelsize=6)
    ax.set_aspect('equal')
fig.legend(handles=legend,ncol=2,loc='lower center',bbox_to_anchor=(.5,.01),fontsize=8)
fig.suptitle('Zoomed candidate point decisions | left 3 columns: Box 4; right 3 columns: Box 5\nGrey = unresolved; colored = provisional group candidate, NOT ground truth',fontsize=11)
fig.tight_layout(rect=[0,.09,1,.91]);fig.savefig(out/'图表/04_18窗口候选点图放大.png',dpi=160);plt.close(fig)

# Figure 5: reference-only family holdout confusion; does not measure core identification accuracy.
valid_groups=[g for g in groups if g!='Low-specificity control']
matrix=np.zeros((len(valid_groups),len(valid_groups)),int)
for row in holdout:
    matrix[valid_groups.index(row['actual_library_group']),valid_groups.index(row['predicted_group'])]+=1
fig,ax=plt.subplots(figsize=(11,8));im=ax.imshow(matrix,cmap='Blues');fig.colorbar(im,ax=ax,label='Number of held-out reference spectra')
ax.set_xticks(range(len(valid_groups)),valid_groups,rotation=55,ha='right',fontsize=8);ax.set_yticks(range(len(valid_groups)),valid_groups,fontsize=8)
for i in range(len(valid_groups)):
    for k in range(len(valid_groups)):
        if matrix[i,k]:ax.text(k,i,str(matrix[i,k]),ha='center',va='center',color='white' if matrix[i,k]>7 else 'black')
ax.set(xlabel='Closest predicted group',ylabel='USGS reference group',title='Reference-library specimen-family holdout | 2050-2450 nm only\nThis is NOT measured accuracy on the customer cores')
fig.tight_layout();fig.savefig(out/'图表/05_参考库留出检查_非客户准确率.png',dpi=150);plt.close(fig)

print('REFERENCE COUNTS',len(refs),'active',int(active.sum()),dict(Counter(r['group'] for r in refs)))
print('CONTROLS',controls['self_retrieval_group_correct'],controls['self_retrieval_group_total'],'holdout',controls['specimen_family_holdout_correct'],controls['specimen_family_holdout_total'])
print('PATCHES',results['patch_summary'],'POINTS',results['point_summary'],'ACCEPTED',results['accepted_groups'])
for p in patches:
 m=p['matching'];print(p['id'],p['status'],m['best_mineral'],m['best_group'],round(m['cosine'],3),round(m['group_margin'],3),p['reasons'])
