""" The Void -- Buleyean RL Demo Interactive side-by-side comparison of Buleyean RL-trained models against their base counterparts. """ import gradio as gr import numpy as np import torch from transformers import AutoModelForCausalLM, AutoTokenizer from scipy.stats import entropy as scipy_entropy # --------------------------------------------------------------------------- # Model registry # --------------------------------------------------------------------------- MODEL_REGISTRY = { "SmolLM2-360M": { "base": "HuggingFaceTB/SmolLM2-360M-Instruct", "buleyean": "forkjoin-ai/buleyean-smollm2-360m", }, "Qwen2.5-0.5B": { "base": "Qwen/Qwen2.5-0.5B-Instruct", "buleyean": "forkjoin-ai/buleyean-qwen2.5-0.5b", }, } # Cache loaded models so we only load once per session _model_cache: dict[str, tuple] = {} # --------------------------------------------------------------------------- # Generation parameters (CPU-friendly) # --------------------------------------------------------------------------- MAX_NEW_TOKENS = 200 TEMPERATURE = 0.7 TOP_P = 0.9 REPETITION_PENALTY = 1.15 # --------------------------------------------------------------------------- # Model loading # --------------------------------------------------------------------------- def _load_model(repo_id: str, is_buleyean: bool = False): """Load a model and tokenizer, with optional LoRA adapter merge.""" if repo_id in _model_cache: return _model_cache[repo_id] if is_buleyean: # The Buleyean checkpoint is a LoRA adapter on top of the base model. # Try loading via peft; fall back to base-only if the adapter is not # yet published (early development). try: from peft import PeftModel # Determine the base model id from the registry base_id = None for entry in MODEL_REGISTRY.values(): if entry["buleyean"] == repo_id: base_id = entry["base"] break if base_id is None: base_id = repo_id # fallback tokenizer = AutoTokenizer.from_pretrained(base_id, trust_remote_code=True) base_model = AutoModelForCausalLM.from_pretrained( base_id, torch_dtype=torch.float32, device_map="cpu", trust_remote_code=True, ) model = PeftModel.from_pretrained(base_model, repo_id) model = model.merge_and_unload() except Exception: # Adapter not available yet -- fall back to base model so the # Space still runs during early development. tokenizer = AutoTokenizer.from_pretrained(repo_id.replace("forkjoin-ai/buleyean-", ""), trust_remote_code=True) # Find the matching base id base_id = None for entry in MODEL_REGISTRY.values(): if entry["buleyean"] == repo_id: base_id = entry["base"] break if base_id is None: base_id = repo_id tokenizer = AutoTokenizer.from_pretrained(base_id, trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained( base_id, torch_dtype=torch.float32, device_map="cpu", trust_remote_code=True, ) else: tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained( repo_id, torch_dtype=torch.float32, device_map="cpu", trust_remote_code=True, ) if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token model.eval() _model_cache[repo_id] = (model, tokenizer) return model, tokenizer # --------------------------------------------------------------------------- # Metrics # --------------------------------------------------------------------------- def compute_bule_entropy(logits: torch.Tensor) -> float: """Compute Bule entropy -- the mean Shannon entropy over the generated token probability distributions, measured in nats then converted to bits.""" probs = torch.softmax(logits, dim=-1).detach().cpu().numpy() entropies = [] for t in range(probs.shape[1]): dist = probs[0, t] h = scipy_entropy(dist, base=2) entropies.append(h) return float(np.mean(entropies)) if entropies else 0.0 def compute_optimality_gap(base_entropy: float, buleyean_entropy: float) -> float: """Optimality gap: relative entropy reduction achieved by Buleyean training. A positive value means Buleyean is more decisive (lower entropy).""" if base_entropy == 0: return 0.0 return (base_entropy - buleyean_entropy) / base_entropy # --------------------------------------------------------------------------- # Inference # --------------------------------------------------------------------------- def generate_with_metrics(model, tokenizer, prompt: str): """Run generation and return (text, logits_for_metrics, token_count).""" messages = [{"role": "user", "content": prompt}] # Use chat template if available, otherwise raw prompt try: text_input = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) except Exception: text_input = f"User: {prompt}\nAssistant:" inputs = tokenizer(text_input, return_tensors="pt", truncation=True, max_length=512) with torch.no_grad(): outputs = model.generate( **inputs, max_new_tokens=MAX_NEW_TOKENS, temperature=TEMPERATURE, top_p=TOP_P, repetition_penalty=REPETITION_PENALTY, do_sample=True, return_dict_in_generate=True, output_logits=True, ) generated_ids = outputs.sequences[0][inputs["input_ids"].shape[1] :] response_text = tokenizer.decode(generated_ids, skip_special_tokens=True).strip() token_count = len(generated_ids) # Stack per-step logits for entropy calculation if outputs.logits: logits = torch.stack(outputs.logits, dim=1) # (1, seq_len, vocab) else: logits = torch.zeros(1, 1, 1) return response_text, logits, token_count # --------------------------------------------------------------------------- # Main comparison function # --------------------------------------------------------------------------- def compare(prompt: str, model_name: str): """Generate from both base and Buleyean models, compute metrics.""" if not prompt or not prompt.strip(): return ( "Please enter a prompt.", "Please enter a prompt.", "N/A", "N/A", "N/A", "N/A", "N/A", ) entry = MODEL_REGISTRY[model_name] # --- Base model --- base_model, base_tok = _load_model(entry["base"], is_buleyean=False) base_text, base_logits, base_tokens = generate_with_metrics( base_model, base_tok, prompt ) base_entropy = compute_bule_entropy(base_logits) # --- Buleyean model --- bule_model, bule_tok = _load_model(entry["buleyean"], is_buleyean=True) bule_text, bule_logits, bule_tokens = generate_with_metrics( bule_model, bule_tok, prompt ) bule_entropy = compute_bule_entropy(bule_logits) # --- Metrics --- gap = compute_optimality_gap(base_entropy, bule_entropy) base_entropy_str = f"{base_entropy:.4f} bits" bule_entropy_str = f"{bule_entropy:.4f} bits" gap_str = f"{gap:+.2%}" base_tokens_str = str(base_tokens) bule_tokens_str = str(bule_tokens) return ( base_text, bule_text, base_entropy_str, bule_entropy_str, gap_str, base_tokens_str, bule_tokens_str, ) # --------------------------------------------------------------------------- # Example prompts # --------------------------------------------------------------------------- EXAMPLES = [ ["Explain entropy in information theory in simple terms."], ["Write a short poem about uncertainty and probability."], ["What are three practical uses of reinforcement learning?"], ["Describe the relationship between order and chaos in nature."], ["Summarize the key idea behind the exploration-exploitation tradeoff."], ] # --------------------------------------------------------------------------- # About text # --------------------------------------------------------------------------- ABOUT_MD = """ ## What is Buleyean RL? **Buleyean Reinforcement Learning** is a training methodology grounded in Buleyean probability theory. Where standard RLHF optimizes for human preference scores, Buleyean RL optimizes for *entropy reduction* -- transforming high-entropy (uncertain, diffuse) model outputs into low-entropy (decisive, structured) responses. ### Core Concepts - **Bule Entropy**: The mean Shannon entropy (in bits) across the token probability distributions during generation. Lower entropy means the model is more confident and decisive in its token choices. - **Optimality Gap**: The relative reduction in entropy between the base model and the Buleyean-trained variant. A positive gap indicates the trained model has learned to collapse probability mass onto better token sequences. - **The Void**: In Buleyean probability, "the void" represents the space of unrealized possibilities -- the complement of what the model chooses. Buleyean RL teaches models to navigate the void efficiently, selecting paths that minimize wasted probability mass. ### How It Works 1. **Start with a base model** (e.g., SmolLM2-360M) 2. **Train a LoRA adapter** using Buleyean reward signals that penalize high-entropy distributions and reward decisive, coherent outputs 3. **Merge the adapter** to produce a model that generates with lower entropy while maintaining (or improving) output quality ### Why It Matters Standard language models spread probability across many plausible continuations. Buleyean RL teaches them to *commit* -- to find the low-entropy attractor states that correspond to clear, structured communication. The result is less hedging, less repetition, and more information-dense outputs. """ # --------------------------------------------------------------------------- # Gradio UI # --------------------------------------------------------------------------- def build_app(): with gr.Blocks( title="The Void -- Buleyean RL Demo", theme=gr.themes.Base( primary_hue="indigo", neutral_hue="slate", ), css=""" .void-header { text-align: center; margin-bottom: 1rem; } .void-header h1 { font-size: 2.2rem; font-weight: 700; } .void-header p { opacity: 0.7; font-size: 1.05rem; } .metrics-row { gap: 1rem; } """, ) as demo: gr.HTML( """

The Void

Buleyean RL: from high entropy to low entropy

""" ) with gr.Row(): with gr.Column(scale=3): prompt_input = gr.Textbox( label="Prompt", placeholder="Enter a prompt to compare base vs Buleyean-trained outputs...", lines=3, ) with gr.Column(scale=1): model_dropdown = gr.Dropdown( choices=list(MODEL_REGISTRY.keys()), value="SmolLM2-360M", label="Model", ) generate_btn = gr.Button("Generate", variant="primary", size="lg") with gr.Row(): with gr.Column(): gr.Markdown("### Base Model") base_output = gr.Textbox( label="Base Response", lines=10, interactive=False, ) with gr.Column(): gr.Markdown("### Buleyean-Trained") bule_output = gr.Textbox( label="Buleyean Response", lines=10, interactive=False, ) gr.Markdown("### Metrics") with gr.Row(elem_classes="metrics-row"): base_entropy_box = gr.Textbox(label="Base Bule Entropy", interactive=False) bule_entropy_box = gr.Textbox(label="Buleyean Bule Entropy", interactive=False) gap_box = gr.Textbox(label="Optimality Gap", interactive=False) base_tokens_box = gr.Textbox(label="Base Tokens", interactive=False) bule_tokens_box = gr.Textbox(label="Buleyean Tokens", interactive=False) generate_btn.click( fn=compare, inputs=[prompt_input, model_dropdown], outputs=[ base_output, bule_output, base_entropy_box, bule_entropy_box, gap_box, base_tokens_box, bule_tokens_box, ], ) gr.Examples( examples=EXAMPLES, inputs=[prompt_input], label="Example Prompts", ) with gr.Accordion("What is Buleyean RL?", open=False): gr.Markdown(ABOUT_MD) return demo if __name__ == "__main__": app = build_app() app.launch()