import os
import json
import time
import numpy as np
import pandas as pd
import gradio as gr
import requests
from sentence_transformers import SentenceTransformer, CrossEncoder
from pinecone_text.sparse import BM25Encoder
from pinecone import Pinecone
from openai import OpenAI
# ── Env Vars ─────────────────────────────────────────────────────────────────
PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY", "")
OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY", "")
INDEX_NAME = "medrag-hybrid-index"
# ── Models ────────────────────────────────────────────────────────────────────
print("Loading models…")
dense_model = SentenceTransformer("all-MiniLM-L6-v2")
rerank_model = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
bm25_encoder = BM25Encoder()
bm25_encoder.load("bm25_params.json")
pc = Pinecone(api_key=PINECONE_API_KEY)
index = pc.Index(INDEX_NAME)
or_client = OpenAI(
base_url="https://openrouter.ai/api/v1",
api_key=OPENROUTER_API_KEY,
)
# ── 20 preset queries ─────────────────────────────────────────────────────────
PRESET_QUERIES = [
"What is the effect of (--)-alpha-bisabolol on the proteolytic activity of pepsin?",
"Does alpha-bisabolol alter the pH-value to achieve its antipeptic action?",
"Compare the recovery rates of BA 1 and ifosfamide in rats with DS carcinosarcoma.",
"What role does hyperglycemia play in the effectiveness of the substance BA 1?",
"How does etafenone affect the transmural gradient of myocardial blood flow in dogs?",
"What is the difference in myocardial blood flow between normoxia and hypoxia?",
"How does the virostatic compound 5531 affect the induction of tryptophan-pyrrolase?",
"At what dosage does compound 5531 inhibit tyrosine-aminotransferase in tissue culture?",
"What are the pharmacological properties of RMI 61 140 compared to chlorpromazine?",
"Do RMI 61 140 and RMI 61 144 show activity on electroshock tests?",
"How does the combination of Sch 1000 and oxazepam affect gastric juice volume?",
"What were the findings regarding oxazepam's effect on stimulated acid secretion?",
"Which lysosomal glycosidases show high activity in the guinea-pig epidermis?",
"List the distinct ester hydrolases characterized in the guinea-pig epidermis.",
"What inhibits the serum haemagglutinating property dependent upon polycarboxyl groups?",
"How does human erythrocyte stromata influence complement activation in PNH cells?",
"What is the effect of salicylic acid on the pentose phosphate pathway in G6PD-deficient cells?",
"How does pelleted barley affect rumen pH compared to whole barley in sheep?",
"Describe the ordered structure of Poly(8-aminoguanylic acid) in neutral solution.",
"What are the estimated pKa values for human liver alanine aminopeptidase active center groups?",
]
# ── Retrieval ─────────────────────────────────────────────────────────────────
def hybrid_query(query, namespace="recursive", top_k=10):
d_vec = dense_model.encode(query).tolist()
s_vec = bm25_encoder.encode_queries(query)
results = index.query(
vector=d_vec, sparse_vector=s_vec,
top_k=top_k, include_metadata=True, namespace=namespace,
)
scored = []
for r in results["matches"]:
text = r["metadata"]["text"]
score = rerank_model.predict([query, text])
scored.append({"text": text, "rerank_score": float(score)})
scored.sort(key=lambda x: x["rerank_score"], reverse=True)
return scored[:3]
def semantic_only_query(query, namespace="recursive", top_k=3):
d_vec = dense_model.encode(query).tolist()
results = index.query(
vector=d_vec, top_k=top_k,
include_metadata=True, namespace=namespace,
)
return [{"text": r["metadata"]["text"], "rerank_score": r["score"]} for r in results["matches"]]
# ── LLM options ───────────────────────────────────────────────────────────────
LLM_OPTIONS = {
"LLaMA-3-8B": "meta-llama/llama-3-8b-instruct",
"Mistral-Large": "mistralai/mistral-large:free",
}
LLM_DEFAULT = "LLaMA-3-8B"
# ── LLM Helper ────────────────────────────────────────────────────────────────
def call_llm(prompt, max_tokens=500, model_id=None):
if model_id is None:
raise ValueError("model_id must be passed explicitly from the UI selection.")
try:
response = requests.post(
"https://openrouter.ai/api/v1/chat/completions",
headers={
"Authorization": f"Bearer {OPENROUTER_API_KEY}",
"Content-Type": "application/json",
"HTTP-Referer": "https://huggingface.co/spaces",
"X-Title": "MedRAG QA",
},
json={
"model": model_id,
"messages": [
{"role": "user", "content": prompt}
],
"temperature": 0.1,
"max_tokens": max_tokens,
},
timeout=60
)
response.raise_for_status()
data = response.json()
if "choices" in data and data["choices"]:
return data["choices"][0]["message"]["content"]
else:
return f"[LLM Error: Invalid response structure - {data}]"
except requests.exceptions.Timeout:
return "[LLM Error: Request timed out (60s)]"
except requests.exceptions.HTTPError as e:
return f"[LLM Error: HTTP {e.response.status_code} - {e.response.text}]"
except requests.exceptions.RequestException as e:
return f"[LLM Error: {str(e)}]"
except Exception as e:
return f"[LLM Error: {str(e)}]"
# ── Generation ────────────────────────────────────────────────────────────────
def generate_answer(query, chunks, model_id):
context = "\n\n".join(c["text"] for c in chunks)
prompt = (
"You are a medical research assistant. Answer using ONLY the provided context. "
"Be concise and factual.\n\n"
f"Context:\n{context}\n\nQuestion: {query}\n\nAnswer:"
)
return call_llm(prompt, model_id=model_id)
# ── Evaluation ────────────────────────────────────────────────────────────────
def evaluate_faithfulness(chunks, answer, model_id):
context = "\n\n".join(c["text"] for c in chunks)
raw_claims = call_llm(
"List every factual claim in the answer as short bullet points. "
"Output ONLY the bullet list, no intro text.\n\nAnswer: " + answer,
max_tokens=300,
model_id=model_id,
)
claims = [
l.strip("•-– ").strip()
for l in raw_claims.splitlines()
if l.strip() and len(l.strip()) > 5
]
rows = []
supported = 0
for claim in claims:
verdict = call_llm(
f"Context:\n{context}\n\nClaim: {claim}\n\n"
"Is this claim supported by the context? Reply with exactly YES or NO.",
max_tokens=10,
model_id=model_id,
).strip().upper()
verdict = "YES" if "YES" in verdict else "NO"
if verdict == "YES":
supported += 1
rows.append((claim, verdict))
score = round(supported / len(claims), 3) if claims else 0.0
return score, rows
def evaluate_relevancy(query, answer, model_id):
raw = call_llm(
"Generate exactly 3 short questions that the following answer satisfies. "
"Output only the 3 questions, one per line, no numbering or bullets.\n\nAnswer: " + answer,
max_tokens=200,
model_id=model_id,
)
alt_qs = [l.strip() for l in raw.splitlines() if l.strip()][:3]
if not alt_qs:
return 0.0, []
orig_emb = dense_model.encode(query)
rows = []
sims = []
for q in alt_qs:
emb = dense_model.encode(q)
sim = float(np.dot(orig_emb, emb) / (np.linalg.norm(orig_emb) * np.linalg.norm(emb) + 1e-9))
sims.append(sim)
rows.append((q, round(sim, 4)))
return round(float(np.mean(sims)), 4), rows
# ── HTML builders ─────────────────────────────────────────────────────────────
CARD = "border:1px solid #e2e8f0;border-radius:10px;padding:16px 18px;margin-bottom:12px;background:#fafafa;font-family:sans-serif"
def _answer_html(answer):
return f"""
Generated Answer
{answer}
"""
def _chunks_html(chunks):
cards = ""
for i, c in enumerate(chunks, 1):
cards += f"""
Chunk {i}
score: {c['rerank_score']:.4f}
{c['text']}
"""
return f'{cards}
'
def _faith_html(score, rows):
pct = f"{score:.1%}"
color = "#22c55e" if score >= 0.7 else "#f59e0b" if score >= 0.4 else "#ef4444"
bar_w = f"{score * 100:.1f}%"
supported = sum(1 for _, v in rows if v == "YES")
total = len(rows)
claim_rows = ""
for claim, verdict in rows:
icon = "✅" if verdict == "YES" else "❌"
bg = "#f0fdf4" if verdict == "YES" else "#fef2f2"
tc = "#166534" if verdict == "YES" else "#991b1b"
claim_rows += f"""
| {icon} {verdict} |
{claim} |
"""
return f"""
{pct}
Faithfulness Score · {supported} of {total} claims supported
How it's calculated
1. Claims are extracted from the generated answer by the LLM.
2. Each claim is individually verified against the retrieved context chunks by the LLM (YES / NO).
3. Faithfulness Score = {supported} supported ÷ {total} total claims = {pct}
| Verdict |
Extracted Claim |
{claim_rows}
"""
def _rel_html(score, rows):
color = "#6366f1" if score >= 0.6 else "#f59e0b" if score >= 0.35 else "#ef4444"
bar_w = f"{min(score * 100, 100):.1f}%"
sim_sum = " + ".join(str(r[1]) for r in rows)
q_rows = ""
for i, (q, sim) in enumerate(rows, 1):
fill = f"{min(sim * 100, 100):.1f}%"
q_rows += f"""
| Q{i} |
{q} |
|
"""
return f"""
{score:.4f}
Relevancy Score · mean cosine similarity (3 alt-questions vs original query)
How it's calculated
1. The LLM generates 3 questions that the answer satisfies.
2. Each question is embedded with all-MiniLM-L6-v2 and compared to the
original query embedding via cosine similarity.
3. Relevancy Score = mean({sim_sum}) ÷ {len(rows)} = {score:.4f}
| # |
Generated Alt-Question |
Cosine Similarity |
{q_rows}
|
Mean (Relevancy Score)
|
{score:.4f} |
"""
def _latency_html(r_time, g_time, e_time, mode):
total = r_time + g_time + e_time
return f"""
| Stage |
Time (s) |
|
Retrieval
{mode}
|
{r_time:.2f}s |
| LLM Generation |
{g_time:.2f}s |
| Evaluation (LLM-as-a-Judge) |
{e_time:.2f}s |
| Total |
{total:.2f}s |
"""
PLACEHOLDER = "Results will appear here after you submit a query.
"
# ── Main handler for Interactive QA ───────────────────────────────────────────
def run_rag(query, retrieval_mode, chunking_strategy, llm_choice):
if not query.strip():
return PLACEHOLDER, PLACEHOLDER, PLACEHOLDER, PLACEHOLDER, PLACEHOLDER
if llm_choice is None:
return (
"❌ Please select an LLM model first
",
PLACEHOLDER, PLACEHOLDER, PLACEHOLDER, PLACEHOLDER
)
model_id = LLM_OPTIONS[llm_choice]
namespace = "fixed" if "Fixed" in chunking_strategy else "recursive"
chunk_label = "Fixed-Size (500 chars, overlap 50)" if namespace == "fixed" \
else "Recursive Character Split (500 chars, overlap 50)"
t0 = time.time()
if "Hybrid" in retrieval_mode:
chunks = hybrid_query(query, namespace=namespace)
mode_label = f"Hybrid BM25 + Dense + CrossEncoder · {chunk_label} · {llm_choice}"
else:
chunks = semantic_only_query(query, namespace=namespace)
mode_label = f"Semantic-Only (Dense) · {chunk_label} · {llm_choice}"
r_time = time.time() - t0
t1 = time.time()
answer = generate_answer(query, chunks, model_id)
g_time = time.time() - t1
t2 = time.time()
faith_score, faith_rows = evaluate_faithfulness(chunks, answer, model_id)
rel_score, rel_rows = evaluate_relevancy(query, answer, model_id)
e_time = time.time() - t2
return (
_answer_html(answer),
_chunks_html(chunks),
_faith_html(faith_score, faith_rows),
_rel_html(rel_score, rel_rows),
_latency_html(r_time, g_time, e_time, mode_label),
)
# ── Ablation Study Handler ─────────────────────────────────────────────────────
def run_ablation_study(progress=gr.Progress()):
"""
Run ablation study on first 10 queries with 6 configurations:
- Config 1: Hybrid + Recursive + LLaMA-3-8B
- Config 2: Hybrid + Fixed + LLaMA-3-8B
- Config 3: Semantic + Recursive + LLaMA-3-8B
- Config 4: Semantic + Fixed + LLaMA-3-8B
"""
model_id = LLM_OPTIONS["LLaMA-3-8B"]
ten_queries = PRESET_QUERIES[:10]
configs = [
{"name": "Hybrid + Recursive", "retrieval": "hybrid", "chunking": "recursive"},
{"name": "Hybrid + Fixed", "retrieval": "hybrid", "chunking": "fixed"},
{"name": "Semantic + Recursive", "retrieval": "semantic", "chunking": "recursive"},
{"name": "Semantic + Fixed", "retrieval": "semantic", "chunking": "fixed"},
]
results = []
total_steps = len(ten_queries) * len(configs)
current_step = 0
for query_idx, query in enumerate(ten_queries):
for config in configs:
current_step += 1
progress(current_step / total_steps, desc=f"Query {query_idx+1}/10 - {config['name']}")
# Retrieve chunks
t0 = time.time()
if config["retrieval"] == "hybrid":
chunks = hybrid_query(query, namespace=config["chunking"])
else:
chunks = semantic_only_query(query, namespace=config["chunking"])
r_time = time.time() - t0
# Generate answer
t1 = time.time()
answer = generate_answer(query, chunks, model_id)
g_time = time.time() - t1
# Evaluate
t2 = time.time()
faith_score, _ = evaluate_faithfulness(chunks, answer, model_id)
rel_score, _ = evaluate_relevancy(query, answer, model_id)
e_time = time.time() - t2
results.append({
"Query": f"Q{query_idx+1}",
"Configuration": config["name"],
"Faithfulness": faith_score,
"Relevancy": rel_score,
"Retrieval Time": r_time,
"Generation Time": g_time,
"Evaluation Time": e_time,
"Total Time": r_time + g_time + e_time,
})
# Create DataFrame
df = pd.DataFrame(results)
# Compute configuration-wise means
config_means = df.groupby("Configuration").agg({
"Faithfulness": "mean",
"Relevancy": "mean",
"Retrieval Time": "mean",
"Generation Time": "mean",
"Evaluation Time": "mean",
"Total Time": "mean",
}).round(4)
# Create HTML table for detailed results
detailed_html = df.to_html(index=False, float_format=lambda x: f"{x:.4f}")
detailed_html = f"""
📊 Detailed Results (All Queries × Configurations)
{detailed_html}
"""
# Create HTML table for configuration means
means_html = config_means.to_html(float_format=lambda x: f"{x:.4f}")
means_html = f"""
📈 Configuration-Wise Means
{means_html}
"""
return detailed_html + "
" + means_html
# ── UI ────────────────────────────────────────────────────────────────────────
CSS = """
#header { text-align:center; padding:1.4rem 0 0.6rem }
#header h1 { font-size:1.85rem; font-weight:800; margin:0 }
#header p { color:#64748b; font-size:0.88rem; margin:4px 0 0 }
.preset button { font-size:0.76rem !important; padding:4px 6px !important;
line-height:1.3 !important; min-height:0 !important }
footer { display:none !important }
"""
# Create Tabs for both interfaces
with gr.Blocks(
title="MedRAG — Hybrid RAG Q&A + Ablation Study",
theme=gr.themes.Soft(primary_hue="blue", neutral_hue="slate"),
css=CSS,
) as demo:
with gr.Tabs():
# ────────────────── TAB 1: Interactive QA ──────────────────────
with gr.Tab("🔍 Interactive QA"):
gr.HTML("""
""")
with gr.Group():
gr.Markdown("### 💬 Your Query")
query_box = gr.Textbox(
placeholder="Type a medical question, or click a preset below…",
label="",
lines=2,
show_label=False,
)
with gr.Row():
retrieval_mode = gr.Radio(
choices=[
"Hybrid + Re-ranking (Recommended)",
"Semantic-Only (Ablation Baseline)",
],
value="Hybrid + Re-ranking (Recommended)",
label="🔍 Retrieval Strategy",
scale=2,
)
chunking_strategy = gr.Radio(
choices=[
"Strategy B — Recursive Split (Default)",
"Strategy A — Fixed-Size Split (Ablation)",
],
value="Strategy B — Recursive Split (Default)",
label="✂️ Chunking Strategy",
scale=2,
)
llm_choice = gr.Radio(
choices=list(LLM_OPTIONS.keys()),
value=None,
label="🤖 LLM Model",
scale=2,
)
submit_btn = gr.Button("🔍 Ask", variant="primary", size="lg", scale=1)
gr.Markdown("#### 📋 Select a preset test query")
for i in range(0, len(PRESET_QUERIES), 4):
row_qs = PRESET_QUERIES[i : i + 4]
with gr.Row(elem_classes=["preset"]):
for q in row_qs:
label = f"Q{i + row_qs.index(q) + 1}: " + (q[:52] + "…" if len(q) > 52 else q)
btn = gr.Button(label, size="sm")
btn.click(fn=lambda x=q: x, outputs=query_box)
gr.Markdown("---")
gr.Markdown("### 📝 Answer")
answer_out = gr.HTML(value=PLACEHOLDER)
gr.Markdown("---")
with gr.Row(equal_height=False):
with gr.Column(scale=1):
gr.Markdown("### 📚 Retrieved Context Chunks")
chunks_out = gr.HTML(value=PLACEHOLDER)
with gr.Column(scale=1):
gr.Markdown("### 📊 Evaluation — LLM-as-a-Judge")
with gr.Tabs():
with gr.Tab("🔎 Faithfulness"):
faith_out = gr.HTML(value=PLACEHOLDER)
with gr.Tab("🔗 Relevancy"):
rel_out = gr.HTML(value=PLACEHOLDER)
with gr.Tab("⏱ Latency"):
latency_out = gr.HTML(value=PLACEHOLDER)
outs = [answer_out, chunks_out, faith_out, rel_out, latency_out]
submit_btn.click(fn=run_rag, inputs=[query_box, retrieval_mode, chunking_strategy, llm_choice], outputs=outs)
query_box.submit(fn=run_rag, inputs=[query_box, retrieval_mode, chunking_strategy, llm_choice], outputs=outs)
# ────────────────── TAB 2: Ablation Study ──────────────────────
with gr.Tab("📊 Ablation Study"):
gr.HTML("""
""")
gr.Markdown("""
### Configuration Overview
- **Config 1**: Hybrid BM25+Dense + Recursive Chunking
- **Config 2**: Hybrid BM25+Dense + Fixed Chunking
- **Config 3**: Semantic-Only Dense + Recursive Chunking
- **Config 4**: Semantic-Only Dense + Fixed Chunking
All configurations use **LLaMA-3-8B** for consistency.
**Metrics computed**:
- ✅ Faithfulness Score (LLM-as-a-Judge verification)
- 🔗 Relevancy Score (cosine similarity to original query)
- ⏱ Latency (retrieval, generation, evaluation times)
""")
start_btn = gr.Button("🚀 Start Ablation Study", variant="primary", size="lg")
ablation_output = gr.HTML(value=PLACEHOLDER)
start_btn.click(fn=run_ablation_study, outputs=ablation_output)
gr.HTML("""
NLP with Deep Learning — Mini-Project 1 ·
Corpus: MedRAG/pubmed (200 abstracts, 500+ chunks) ·
Stack: Pinecone · all-MiniLM-L6-v2 · ms-marco CrossEncoder · OpenRouter
""")
if __name__ == "__main__":
demo.launch()