recipes/glm47_native_transformers.py

#!/usr/bin/env python3
"""Validated GLM-4.7 native pretokenized Trainer recipe; no rendering or truncation."""
from __future__ import annotations
import argparse,torch
from datasets import load_dataset
from transformers import AutoModelForCausalLM,AutoTokenizer,Trainer,TrainingArguments
def main():
 p=argparse.ArgumentParser();p.add_argument('--repo',default='r0b0tlab/qwen3.8-max-glm5.2-distillation-51389');p.add_argument('--model',default='zai-org/GLM-4.7-Flash');p.add_argument('--output',default='outputs/glm47-native');p.add_argument('--max-steps',type=int,default=-1);a=p.parse_args();ds=load_dataset(a.repo,'glm47_native');tok=AutoTokenizer.from_pretrained(a.model);tok.pad_token=tok.pad_token or tok.eos_token
 def collate(batch):
  n=max(len(x['input_ids']) for x in batch);ids=[];labels=[];mask=[]
  for x in batch:
   if len(x['input_ids'])!=len(x['labels']):raise ValueError('input/label length mismatch')
   if not any(v!=-100 for v in x['labels']):raise ValueError('zero supervised tokens')
   pad=n-len(x['input_ids']);ids.append(x['input_ids']+[tok.pad_token_id]*pad);labels.append(x['labels']+[-100]*pad);mask.append([1]*len(x['input_ids'])+[0]*pad)
  return {'input_ids':torch.tensor(ids),'labels':torch.tensor(labels),'attention_mask':torch.tensor(mask)}
 model=AutoModelForCausalLM.from_pretrained(a.model);args=TrainingArguments(output_dir=a.output,per_device_train_batch_size=1,gradient_accumulation_steps=8,learning_rate=2e-5,num_train_epochs=1,max_steps=a.max_steps,bf16=True,logging_steps=1,save_steps=250,remove_unused_columns=False);Trainer(model=model,args=args,train_dataset=ds['train'],eval_dataset=ds.get('validation'),data_collator=collate).train()
if __name__=='__main__':main()