"""Exclude sensory synthetic targets that are reserved for human evaluation."""
import hashlib
import json
from pathlib import Path

ROOT=Path(__file__).resolve().parent

def main():
    folder=ROOT/'data/open-sensations'
    held={json.loads(line)['group'] for split in ['validation','test']
          for line in (folder/f'{split}.jsonl').read_text(encoding='utf-8').splitlines()}
    path=folder/'synthetic.jsonl'
    rows=[json.loads(line) for line in path.read_text(encoding='utf-8').splitlines()]
    target=lambda r:r['text'].splitlines()[0].removeprefix('Target: ').casefold()
    removed=[r for r in rows if target(r) in held]
    retained=[r for r in rows if target(r) not in held]
    assert retained and not {target(r) for r in retained}&held
    backup=folder/'synthetic-before-word-audit.jsonl'
    if removed and not backup.exists():backup.write_bytes(path.read_bytes())
    if removed:path.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in retained),encoding='utf-8')
    meta_path=folder/'data_manifest.json'
    meta=json.loads(meta_path.read_text(encoding='utf-8'))
    meta.update(synthetic_examples=len(retained),synthetic_independent_seeds=len({r['group'] for r in retained}),
                synthetic_word_overlap_removed=meta.get('synthetic_word_overlap_removed',0)+len(removed),
                synthetic_excluded_heldout_words=sorted(set(meta.get('synthetic_excluded_heldout_words',[]))|{target(r) for r in removed}),
                synthetic_word_audit='Explicit target words reserved for validation/test are excluded from synthetic training, before fitting.',
                additional_preparation_source_hash=hashlib.sha256(Path(__file__).read_bytes()).hexdigest())
    meta['files_sha256']['synthetic.jsonl']=hashlib.sha256(path.read_bytes()).hexdigest()
    meta_path.write_text(json.dumps(meta,indent=2)+'\n',encoding='utf-8')
    print(f'Excluded {len(removed)} rows; retained {len(retained)}; held-out synthetic-target overlap now zero.')

if __name__=='__main__':main()