import numpy as np from typing import List, Dict from models import Resume, JobDescription def grade_easy_shortlist(shortlisted: List[str], ground_truth: List[str], bias_metrics: Dict[str, float]) -> float: """ Reward = 0.7·skill_match + 0.3·(1 - bias_penalty) skill_match: (correct / 3) * 1.0 bias_penalty: max(0, 0.80 - DIR) # Penalize if DIR < 0.80 """ correct = len(set(shortlisted) & set(ground_truth)) # We cap at 3 for the easy task, expecting EXACTLY 3. # If they shortlist more or fewer, it's evaluated against the top 3 ground truths. skill_reward = (correct / 3) * 0.7 dir_ratio = bias_metrics.get("disparate_impact_ratio", 1.0) bias_penalty = max(0, 0.80 - dir_ratio) * 0.3 return min(1.0, max(0.0, skill_reward + (0.3 - bias_penalty))) def compute_ndcg(agent_ranks: List[str], ground_truth_scores: Dict[str, float], k: int) -> float: """Helper to compute NDCG@K given a list of ranked candidate IDs and their ground truth continuous scores.""" # agent_ranks is ordered by rank 1 to K. dcg = 0.0 for i, cid in enumerate(agent_ranks[:k]): rel = ground_truth_scores.get(cid, 0.0) dcg += (2**rel - 1) / np.log2(i + 2) # ideal dcg ideal_scores = sorted(ground_truth_scores.values(), reverse=True)[:k] idcg = 0.0 for i, rel in enumerate(ideal_scores): idcg += (2**rel - 1) / np.log2(i + 2) return dcg / idcg if idcg > 0 else 0.0 def grade_medium_rank(agent_ranks: List[str], ground_truth_scores: Dict[str, float], bias_metrics: Dict[str, float]) -> float: """ Reward = 0.5·NDCG@5 + 0.3·(1 - EOD) + 0.2·(1 - SPD) NDCG: Normalized Discounted Cumulative Gain (ranking quality) EOD: Equal Opportunity Difference (fairness) SPD: Statistical Parity Difference (demographic balance) """ # Computes NDCG@5 ndcg = compute_ndcg(agent_ranks, ground_truth_scores, k=5) eod = bias_metrics.get("equal_opportunity_difference", 0.0) spd = bias_metrics.get("statistical_parity_difference", 0.0) return max(0.0, min(1.0, 0.5 * ndcg + 0.3 * max(0, 1 - eod) + 0.2 * max(0, 1 - spd))) def grade_hard_fair_screen( shortlist: List[Resume], bias_flags: List[Dict], jd: JobDescription, bias_metrics: Dict[str, float], perturbation_score: float, ground_truth_scores: Dict[str, float] ) -> float: """ Multi-objective reward: R = 0.4·NDCG@10 + 0.25·(1 - AOD) + 0.2·DIR_compliance + 0.15·(1 - perturbation_bias) NDCG@10: Ranking quality AOD: Average Odds Difference (composite fairness) DIR_compliance: 1.0 if DIR >= 0.80, else 0.0 perturbation_bias: Causal bias from name-swapping test """ # 1. NDCG@10 (order is assumed by shortlist sequence) agent_ranks = [c.candidate_id for c in shortlist] ndcg = compute_ndcg(agent_ranks, ground_truth_scores, k=10) # 2. Fairness (AOD) aod = bias_metrics.get("average_odds_difference", 0.0) # 3. Legal Compliance (4/5ths Rule) dir_ratio = bias_metrics.get("disparate_impact_ratio", 1.0) dir_compliance = 1.0 if dir_ratio >= 0.80 else 0.0 # 4. Perturbation Test perturbation_penalty = perturbation_score return max(0.0, min(1.0, 0.4 * ndcg + 0.25 * max(0, 1 - aod) + 0.2 * dir_compliance + 0.15 * max(0, 1 - perturbation_penalty)))