Upload 11 files

.env.example

@@ -0,0 +1,2 @@
+AGENT_API_KEY=your_api_key_here
+AGENT_API_URL=https://api.openai.com/v1/chat/completions

Dataset.json

@@ -0,0 +1,452 @@
+[
+  {
+    "log": "Failed password for invalid user admin from 185.234.217.92 port 49822 ssh2",
+    "system": "SSH",
+    "expected": {
+      "category": "brute_force",
+      "severity": "medium",
+      "action": "block IP and enable rate limiting"
+    }
+  },
+  {
+    "log": "Accepted publickey for ubuntu from 10.0.0.14 port 42111 ssh2",
+    "system": "SSH",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "POST /api/auth HTTP/1.1 401 Unauthorized from 203.91.112.44",
+    "system": "web server",
+    "expected": {
+      "category": "brute_force",
+      "severity": "medium",
+      "action": "enable rate limiting and captcha"
+    }
+  },
+  {
+    "log": "GET /home HTTP/1.1 200 OK from 192.168.0.22",
+    "system": "web server",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "User root authentication failures exceeded threshold from 45.77.23.11",
+    "system": "SSH",
+    "expected": {
+      "category": "brute_force",
+      "severity": "high",
+      "action": "disable root login and block source IP"
+    }
+  },
+  {
+    "log": "Suspicious upload detected: .php file via /images endpoint",
+    "system": "web server",
+    "expected": {
+      "category": "malware",
+      "severity": "high",
+      "action": "remove file and restrict upload types"
+    }
+  },
+  {
+    "log": "Database connection established from app node 10.1.2.3",
+    "system": "database",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "Outbound SMTP traffic spike detected for user sales@company.com",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "high",
+      "action": "suspend account and inspect sent emails"
+    }
+  },
+  {
+    "log": "High frequency requests to /search endpoint from 103.56.98.21",
+    "system": "web server",
+    "expected": {
+      "category": "dos_attack",
+      "severity": "medium",
+      "action": "apply rate limiting and block IP"
+    }
+  },
+  {
+    "log": "User analyst logged into database from 10.2.3.5",
+    "system": "database",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "Email attachment with macro blocked from unknown sender",
+    "system": "email",
+    "expected": {
+      "category": "malware",
+      "severity": "medium",
+      "action": "quarantine email and alert user"
+    }
+  },
+  {
+    "log": "Repeated AUTH failures for mailbox admin@corp.com from 51.158.32.9",
+    "system": "email",
+    "expected": {
+      "category": "brute_force",
+      "severity": "medium",
+      "action": "block IP and enforce MFA"
+    }
+  },
+  {
+    "log": "GET /admin HTTP/1.1 403 Forbidden from 198.18.1.2",
+    "system": "web server",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "monitor for repeated attempts"
+    }
+  },
+  {
+    "log": "Slow query detected: SELECT * FROM payments exceeding threshold",
+    "system": "database",
+    "expected": {
+      "category": "normal",
+      "severity": "medium",
+      "action": "optimize query and indexes"
+    }
+  },
+  {
+    "log": "SSH connection attempt using deprecated protocol version 1",
+    "system": "SSH",
+    "expected": {
+      "category": "malware",
+      "severity": "medium",
+      "action": "disable legacy protocol support"
+    }
+  },
+  {
+    "log": "Inbound email flagged for spoofed domain billing@secure-payments.co",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "high",
+      "action": "block sender domain and notify users"
+    }
+  },
+  {
+    "log": "GET /api/status HTTP/1.1 200 OK from 10.0.0.7",
+    "system": "web server",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "Multiple DB connections opened from 192.168.3.14 in short interval",
+    "system": "database",
+    "expected": {
+      "category": "dos_attack",
+      "severity": "medium",
+      "action": "limit connections per IP"
+    }
+  },
+  {
+    "log": "Account locked after consecutive failed SSH logins",
+    "system": "SSH",
+    "expected": {
+      "category": "brute_force",
+      "severity": "medium",
+      "action": "investigate IP and enable MFA"
+    }
+  },
+  {
+    "log": "Outbound email rate anomaly detected for hr@company.com",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "high",
+      "action": "disable account and audit activity"
+    }
+  },
+  {
+    "log": "Checksum mismatch detected for /usr/sbin/sshd",
+    "system": "SSH",
+    "expected": {
+      "category": "malware",
+      "severity": "high",
+      "action": "restore binary and investigate compromise"
+    }
+  },
+  {
+    "log": "GET /robots.txt HTTP/1.1 200 OK from 66.249.65.10",
+    "system": "web server",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "Database login failed for admin from 77.91.12.33",
+    "system": "database",
+    "expected": {
+      "category": "brute_force",
+      "severity": "medium",
+      "action": "restrict access and rotate credentials"
+    }
+  },
+  {
+    "log": "Email contains suspicious shortened URL",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "medium",
+      "action": "block URL and warn recipient"
+    }
+  },
+  {
+    "log": "Traffic surge detected on /login endpoint",
+    "system": "web server",
+    "expected": {
+      "category": "dos_attack",
+      "severity": "medium",
+      "action": "enable throttling and monitoring"
+    }
+  },
+  {
+    "log": "SSH session closed for user ec2-user",
+    "system": "SSH",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "Execution of unknown binary from /tmp/.x9",
+    "system": "database",
+    "expected": {
+      "category": "malware",
+      "severity": "high",
+      "action": "remove binary and scan system"
+    }
+  },
+  {
+    "log": "Incoming email rejected due to SPF validation failure",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "medium",
+      "action": "update SPF rules and monitor sender"
+    }
+  },
+  {
+    "log": "GET /login HTTP/1.1 200 OK from 172.16.1.2",
+    "system": "web server",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "Too many DB connections from 10.10.5.6 causing slowdown",
+    "system": "database",
+    "expected": {
+      "category": "dos_attack",
+      "severity": "high",
+      "action": "block IP and enforce connection limits"
+    }
+  },
+  {
+    "log": "SSH login attempt with invalid key for user admin",
+    "system": "SSH",
+    "expected": {
+      "category": "brute_force",
+      "severity": "medium",
+      "action": "disable password auth and monitor attempts"
+    }
+  },
+  {
+    "log": "Injected script detected in HTTP response payload",
+    "system": "web server",
+    "expected": {
+      "category": "malware",
+      "severity": "high",
+      "action": "sanitize inputs and deploy WAF"
+    }
+  },
+  {
+    "log": "Email impersonation attempt detected for ceo@company.com",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "high",
+      "action": "alert users and block sender"
+    }
+  },
+  {
+    "log": "Long running UPDATE query detected in orders table",
+    "system": "database",
+    "expected": {
+      "category": "normal",
+      "severity": "medium",
+      "action": "optimize query performance"
+    }
+  },
+  {
+    "log": "SYN flood pattern detected from 91.200.12.5",
+    "system": "web server",
+    "expected": {
+      "category": "dos_attack",
+      "severity": "high",
+      "action": "enable SYN protection and block IP"
+    }
+  },
+  {
+    "log": "SSH key authentication successful for user deploy",
+    "system": "SSH",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "Unauthorized schema change detected in production DB",
+    "system": "database",
+    "expected": {
+      "category": "malware",
+      "severity": "high",
+      "action": "audit changes and restore if needed"
+    }
+  },
+  {
+    "log": "Executable attachment blocked in incoming email",
+    "system": "email",
+    "expected": {
+      "category": "malware",
+      "severity": "medium",
+      "action": "quarantine attachment"
+    }
+  },
+  {
+    "log": "Repeated POST requests to /wp-login.php from multiple IPs",
+    "system": "web server",
+    "expected": {
+      "category": "brute_force",
+      "severity": "high",
+      "action": "block IPs and enable login protection"
+    }
+  },
+  {
+    "log": "Database connection timeout from application server",
+    "system": "database",
+    "expected": {
+      "category": "normal",
+      "severity": "medium",
+      "action": "check DB load and connectivity"
+    }
+  },
+  {
+    "log": "Email link redirecting to suspicious domain detected",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "high",
+      "action": "block link and notify users"
+    }
+  },
+  {
+    "log": "Unusual connection pattern on SSH port 22",
+    "system": "SSH",
+    "expected": {
+      "category": "dos_attack",
+      "severity": "medium",
+      "action": "limit connections and enable firewall"
+    }
+  },
+  {
+    "log": "GET /healthcheck HTTP/1.1 200 OK from 127.0.0.1",
+    "system": "web server",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "no action required"
+    }
+  },
+  {
+    "log": "Unauthorized SELECT attempt on restricted table payroll",
+    "system": "database",
+    "expected": {
+      "category": "malware",
+      "severity": "high",
+      "action": "revoke permissions and audit logs"
+    }
+  },
+  {
+    "log": "High bounce rate detected for outgoing emails",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "medium",
+      "action": "investigate account compromise"
+    }
+  },
+  {
+    "log": "Failed SSH login for user test from 82.102.44.2",
+    "system": "SSH",
+    "expected": {
+      "category": "brute_force",
+      "severity": "low",
+      "action": "monitor activity"
+    }
+  },
+  {
+    "log": "GET /api/data HTTP/1.1 500 Internal Server Error",
+    "system": "web server",
+    "expected": {
+      "category": "normal",
+      "severity": "medium",
+      "action": "check application logs and fix error"
+    }
+  },
+  {
+    "log": "Scheduled database backup completed successfully",
+    "system": "database",
+    "expected": {
+      "category": "normal",
+      "severity": "low",
+      "action": "verify backup integrity"
+    }
+  },
+  {
+    "log": "Email flagged due to mismatched sender domain",
+    "system": "email",
+    "expected": {
+      "category": "phishing",
+      "severity": "medium",
+      "action": "block sender and educate users"
+    }
+  },
+  {
+    "log": "CPU spike observed during HTTP traffic surge",
+    "system": "web server",
+    "expected": {
+      "category": "dos_attack",
+      "severity": "high",
+      "action": "scale resources and filter traffic"
+    }
+  }
+]

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.10-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,10 +1,92 @@
----
-title: Security Log Analysis OpenENV
-emoji: 🏃
-colorFrom: gray
-colorTo: pink
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Meta Hackathon OpenEnv - Cyber Security Log Analysis
+
+This project exposes a small cybersecurity log-analysis environment with a FastAPI
+server, a local environment class, and an agent evaluation script.
+
+## Output contract
+
+The agent must return only JSON with exactly these keys:
+
+```json
+{
+  "category": "brute_force",
+  "severity": "high",
+  "action": "block source IP and enable rate limiting"
+}
+```
+
+Allowed `category` values:
+
+- `brute_force`
+- `malware`
+- `phishing`
+- `dos_attack`
+- `normal`
+
+Allowed `severity` values:
+
+- `low`
+- `medium`
+- `high`
+
+`action` should be a short, concrete mitigation step.
+
+## API endpoints
+
+- `GET /reset` returns a random sample plus `instructions`, `allowed_categories`,
+  `allowed_severities`, `response_example`, and `agent_prompt`.
+- `POST /step` accepts the agent JSON payload and returns the normalized reward.
+- `GET /state` returns the current step count.
+- `GET /tasks` describes the task tiers and output contract.
+- `POST /grader` scores a `predicted` payload against an `expected` payload.
+- `GET /baseline` runs one simple baseline action against a fresh sample.
+
+## Local setup
+
+```bash
+python -m venv .venv
+. .venv/Scripts/activate
+pip install -r requirements.txt
+uvicorn app:app --host 0.0.0.0 --port 7860
+```
+
+For PowerShell activation, use:
+
+```powershell
+.venv\Scripts\Activate.ps1
+```
+
+## Agent evaluation runner
+
+`test_grader.py` is the local runner that calls a chat completions API, parses the
+model output, grades it, and appends a record to `agent_eval_log.jsonl`.
+
+Create a local `.env` file with:
+
+```env
+AGENT_API_KEY=your_api_key
+AGENT_API_URL=https://api.openai.com/v1/chat/completions
+```
+
+Then run:
+
+```bash
+python test_grader.py
+```
+
+## Docker
+
+Build and run the API container with:
+
+```bash
+docker build -t security-log-env .
+docker run --rm -p 7860:7860 security-log-env
+```
+
+## Scoring
+
+Scoring uses cosine similarity between vectorized predicted and expected responses.
+The raw cosine value is mapped from `[-1, 1]` into the reward range `[0, 1]`:
+
+- aligned vectors score `1`
+- opposite vectors score `0`

agent_contract.py

@@ -0,0 +1,40 @@
+import json
+
+ALLOWED_CATEGORIES = ["brute_force", "malware", "phishing", "dos_attack", "normal"]
+ALLOWED_SEVERITIES = ["low", "medium", "high"]
+RESPONSE_EXAMPLE = {
+  "category": "normal",
+  "severity": "low",
+  "action": "monitor activity"
+}
+
+BASE_AGENT_INSTRUCTIONS = (
+    "You are a cybersecurity log analysis agent. Analyze the provided system name "
+    "and log entry, then return only a JSON object with exactly three keys: "
+    "`category`, `severity`, and `action`. "
+    f"`category` must be one of {ALLOWED_CATEGORIES}. "
+    f"`severity` must be one of {ALLOWED_SEVERITIES}. "
+    "`action` should be a short, concrete mitigation step. "
+    "Do not add markdown, code fences, explanations, or extra keys."
+)
+
+
+def build_agent_prompt(log, system):
+    example_json = json.dumps(RESPONSE_EXAMPLE)
+    return (
+        f"{BASE_AGENT_INSTRUCTIONS}\n\n"
+        f"System: {system}\n"
+        f"Log: {log}\n\n"
+        "Return a response in this shape:\n"
+        f"{example_json}"
+    )
+
+
+def build_agent_context(log, system):
+    return {
+        "instructions": BASE_AGENT_INSTRUCTIONS,
+        "allowed_categories": ALLOWED_CATEGORIES,
+        "allowed_severities": ALLOWED_SEVERITIES,
+        "response_example": RESPONSE_EXAMPLE,
+        "agent_prompt": build_agent_prompt(log, system),
+    }

app.py

@@ -0,0 +1,87 @@
+from fastapi import FastAPI
+
+from agent_contract import (
+    ALLOWED_CATEGORIES,
+    ALLOWED_SEVERITIES,
+    BASE_AGENT_INSTRUCTIONS,
+    RESPONSE_EXAMPLE,
+)
+from environment import SecurityEnv
+from grader import grade_response
+
+
+app = FastAPI()
+env = SecurityEnv()
+
+
+@app.get("/reset")
+def reset():
+    return env.reset()
+
+
+@app.post("/step")
+def step(action: dict):
+    # ✅ Safe extraction (prevents crashes)
+    safe_action = {
+        "category": str(action.get("category", "")).strip().lower(),
+        "severity": str(action.get("severity", "")).strip().lower(),
+        "action": str(action.get("action", "")).strip()
+    }
+
+    return env.step(safe_action)
+
+
+@app.get("/state")
+def state():
+    return env.state()
+
+
+@app.get("/tasks")
+def tasks():
+    return {
+        "tasks": [
+            {
+                "name": "easy",
+                "description": "Detect normal vs attack",
+            },
+            {
+                "name": "medium",
+                "description": "Classify category",
+            },
+            {
+                "name": "hard",
+                "description": "Category + severity + action",
+            },
+        ],
+        "output_contract": {
+            "instructions": BASE_AGENT_INSTRUCTIONS,
+            "allowed_categories": ALLOWED_CATEGORIES,
+            "allowed_severities": ALLOWED_SEVERITIES,
+            "response_example": RESPONSE_EXAMPLE,
+        },
+    }
+
+
+@app.post("/grader")
+def grader(data: dict):
+    predicted = data["predicted"]
+    expected = data["expected"]
+    score = grade_response(predicted, expected)
+
+    return {"score": score}
+
+
+@app.get("/baseline")
+def baseline():
+    sample = env.reset()
+    action = {
+        "category": "normal",
+        "severity": "low",
+        "action": "monitor",
+    }
+    result = env.step(action)
+
+    return {
+        "observation": sample,
+        "result": result,
+    }

environment.py

@@ -0,0 +1,58 @@
+import json
+import random
+from pathlib import Path
+
+from agent_contract import build_agent_context
+from grader import grade_response
+
+
+BASE_DIR = Path(__file__).resolve().parent
+DEFAULT_DATASET_PATH = BASE_DIR / "Dataset.json"
+
+
+class SecurityEnv:
+    def __init__(self, dataset_path=None):
+        dataset_file = Path(dataset_path) if dataset_path else DEFAULT_DATASET_PATH
+        if not dataset_file.is_absolute():
+            dataset_file = BASE_DIR / dataset_file
+
+        with dataset_file.open("r", encoding="utf-8") as handle:
+            self.data = json.load(handle)
+
+        self.current_sample = None
+        self.step_count = 0
+
+    def reset(self):
+        self.current_sample = random.choice(self.data)
+        self.step_count = 0
+
+        observation = {
+            "log": self.current_sample["log"],
+            "system": self.current_sample["system"],
+        }
+        observation.update(
+            build_agent_context(
+                log=self.current_sample["log"],
+                system=self.current_sample["system"],
+            )
+        )
+        return observation
+
+    def step(self, action):
+        expected = self.current_sample["expected"]
+        reward = grade_response(action, expected)
+        self.step_count += 1
+
+        return {
+            "observation": None,
+            "reward": reward,
+            "done": True,
+            "info": {
+                "expected": expected,
+            },
+        }
+
+    def state(self):
+        return {
+            "step_count": self.step_count,
+        }

grader.py

@@ -0,0 +1,129 @@
+import hashlib
+import json
+import math
+import re
+
+
+CATEGORIES = ["brute_force", "malware", "phishing", "dos_attack", "normal"]
+SEVERITY_ANGLES = {
+    "low": math.radians(150),
+    "medium": math.radians(90),
+    "high": math.radians(30),
+}
+
+CATEGORY_WEIGHT = 0.3
+SEVERITY_WEIGHT = 0.2
+ACTION_WEIGHT = 0.5
+ACTION_VECTOR_SIZE = 128
+TOKEN_PATTERN = re.compile(r"[a-z0-9_]+")
+
+
+def _safe_text(value):
+    return str(value or "").strip().lower()
+
+
+def _normalize_payload(payload):
+    if isinstance(payload, str):
+        try:
+            payload = json.loads(payload)
+        except json.JSONDecodeError:
+            payload = {"action": payload}
+
+    if not isinstance(payload, dict):
+        payload = {}
+
+    return {
+        "category": _safe_text(payload.get("category")),
+        "severity": _safe_text(payload.get("severity")),
+        "action": _safe_text(payload.get("action")),
+    }
+
+
+def _normalize_vector(values):
+    norm = math.sqrt(sum(value * value for value in values))
+    if norm == 0.0:
+        return values
+
+    return [value / norm for value in values]
+
+
+def _centered_one_hot(value, vocabulary):
+    if value not in vocabulary:
+        return [0.0] * len(vocabulary)
+
+    off_value = -1.0 / (len(vocabulary) - 1)
+    vector = [off_value] * len(vocabulary)
+    vector[vocabulary.index(value)] = 1.0
+    return _normalize_vector(vector)
+
+
+def _severity_vector(value):
+    angle = SEVERITY_ANGLES.get(value)
+    if angle is None:
+        return [0.0, 0.0]
+
+    return [math.cos(angle), math.sin(angle)]
+
+
+def _hash_feature(feature):
+    digest = hashlib.sha256(feature.encode("utf-8")).digest()
+    index = int.from_bytes(digest[:4], "big") % ACTION_VECTOR_SIZE
+    sign = 1.0 if digest[4] % 2 == 0 else -1.0
+    return index, sign
+
+
+def _action_vector(action):
+    tokens = TOKEN_PATTERN.findall(action)
+    if not tokens:
+        return [0.0] * ACTION_VECTOR_SIZE
+
+    vector = [0.0] * ACTION_VECTOR_SIZE
+
+    for token in tokens:
+        index, sign = _hash_feature(token)
+        vector[index] += sign
+
+    for left, right in zip(tokens, tokens[1:]):
+        index, sign = _hash_feature(f"{left}_{right}")
+        vector[index] += 0.5 * sign
+
+    return _normalize_vector(vector)
+
+
+def response_to_vector(payload):
+    normalized = _normalize_payload(payload)
+    category_vector = [
+        value * CATEGORY_WEIGHT
+        for value in _centered_one_hot(normalized["category"], CATEGORIES)
+    ]
+    severity_vector = [
+        value * SEVERITY_WEIGHT
+        for value in _severity_vector(normalized["severity"])
+    ]
+    action_vector = [
+        value * ACTION_WEIGHT
+        for value in _action_vector(normalized["action"])
+    ]
+
+    return category_vector + severity_vector + action_vector
+
+
+def cosine_similarity(left, right):
+    dot = sum(left_value * right_value for left_value, right_value in zip(left, right))
+    left_norm = math.sqrt(sum(value * value for value in left))
+    right_norm = math.sqrt(sum(value * value for value in right))
+
+    if left_norm == 0.0 or right_norm == 0.0:
+        return 0.0
+
+    cosine = dot / (left_norm * right_norm)
+    return max(-1.0, min(1.0, cosine))
+
+
+def grade_response(predicted, expected):
+    predicted_vector = response_to_vector(predicted)
+    expected_vector = response_to_vector(expected)
+    cosine = cosine_similarity(predicted_vector, expected_vector)
+
+    # Maps cosine similarity from [-1, 1] into the required reward range [0, 1].
+    return round((cosine + 1.0) / 2.0, 4)

openenv.yaml

@@ -0,0 +1,68 @@
+name: security-log-env
+description: >
+  A reinforcement learning environment for cybersecurity log analysis.
+  The agent must analyze system logs and return a JSON answer with the keys
+  category, severity, and action. Scoring is based on cosine similarity
+  between vectorized expected and predicted responses, normalized to [0, 1].
+
+version: 1.0.0
+
+tasks:
+  - name: easy
+    description: Detect whether the log is normal or an attack
+
+  - name: medium
+    description: Classify the type of attack
+
+  - name: hard
+    description: Predict category, severity, and mitigation action
+
+action_space:
+  type: object
+  properties:
+    category:
+      type: string
+      enum: [brute_force, malware, phishing, dos_attack, normal]
+      description: Must match one of the allowed category labels.
+    severity:
+      type: string
+      enum: [low, medium, high]
+      description: Must match one of the allowed severity labels.
+    action:
+      type: string
+      description: A short, concrete mitigation action.
+
+observation_space:
+  type: object
+  properties:
+    log:
+      type: string
+    system:
+      type: string
+    instructions:
+      type: string
+      description: Output contract for the agent.
+    allowed_categories:
+      type: array
+      items:
+        type: string
+      description: Allowed category labels for the response.
+    allowed_severities:
+      type: array
+      items:
+        type: string
+      description: Allowed severity labels for the response.
+    agent_prompt:
+      type: string
+      description: Ready-to-send prompt for an OpenAI agent.
+    response_example:
+      type: object
+      properties:
+        category:
+          type: string
+        severity:
+          type: string
+        action:
+          type: string
+
+reward_range: [0, 1]

requirements.txt

@@ -0,0 +1,2 @@
+fastapi>=0.110,<1.0
+uvicorn>=0.29,<1.0

test_grader.py

@@ -0,0 +1,359 @@
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib import error, request
+from urllib.parse import urlparse
+
+from environment import SecurityEnv
+
+
+ENV_FILE = Path(__file__).resolve().parent / ".env"
+PREFERRED_MODELS = [
+    "llama-3.3-70b-versatile",
+    "groq/compound-mini",
+    "openai/gpt-oss-120b",
+    "llama-3.1-8b-instant",
+    "groq/compound",
+    "openai/gpt-oss-20b",
+]
+NON_CHAT_MODEL_MARKERS = [
+    "whisper",
+    "tts",
+    "transcribe",
+    "transcription",
+    "speech",
+    "vision-preview",
+]
+_RESOLVED_MODEL = None
+
+
+def load_dotenv(dotenv_path):
+    if not dotenv_path.exists():
+        return
+
+    for raw_line in dotenv_path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+
+        key, value = line.split("=", 1)
+        key = key.strip()
+        value = value.strip().strip("\"'")
+        os.environ.setdefault(key, value)
+
+
+load_dotenv(ENV_FILE)
+
+
+AGENT_CONFIG = {
+    "api_url": os.getenv("AGENT_API_URL", "https://api.openai.com/v1/chat/completions"),
+    "api_key": os.getenv("AGENT_API_KEY", ""),
+    "temperature": 0,
+    "max_tokens": 300,
+    "timeout_seconds": 60,
+    "send_feedback_to_agent": True,
+    "runs": 1,
+    "log_file": Path(__file__).resolve().parent / "agent_eval_log.jsonl",
+}
+
+
+def normalize_chat_completions_url(api_url):
+    normalized = api_url.rstrip("/")
+    if normalized.endswith("/chat/completions"):
+        return normalized
+    if normalized.endswith("/v1"):
+        return f"{normalized}/chat/completions"
+    if normalized.endswith("/openai/v1"):
+        return f"{normalized}/chat/completions"
+    return normalized
+
+
+def build_models_url(api_url):
+    chat_url = normalize_chat_completions_url(api_url)
+    if chat_url.endswith("/chat/completions"):
+        return f"{chat_url[:-len('/chat/completions')]}/models"
+    return f"{chat_url}/models"
+
+
+def get_provider_host():
+    return urlparse(normalize_chat_completions_url(AGENT_CONFIG["api_url"])).netloc.lower()
+
+
+def build_headers():
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "User-Agent": "Meta-Hackathon-OpenEnv/1.0",
+    }
+    if AGENT_CONFIG["api_key"]:
+        headers["Authorization"] = f"Bearer {AGENT_CONFIG['api_key']}"
+    return headers
+
+
+def is_chat_model(model_id):
+    lowered = model_id.lower()
+    return not any(marker in lowered for marker in NON_CHAT_MODEL_MARKERS)
+
+
+def choose_model(model_items):
+    active_ids = [item["id"] for item in model_items if item.get("active")]
+    if not active_ids:
+        raise RuntimeError("The provider returned no active models.")
+
+    for model_id in PREFERRED_MODELS:
+        if model_id in active_ids:
+            return model_id
+
+    for model_id in active_ids:
+        if is_chat_model(model_id):
+            return model_id
+
+    return active_ids[0]
+
+
+def resolve_model():
+    global _RESOLVED_MODEL
+    if _RESOLVED_MODEL:
+        return _RESOLVED_MODEL
+
+    models_url = build_models_url(AGENT_CONFIG["api_url"])
+    http_request = request.Request(
+        models_url,
+        headers=build_headers(),
+        method="GET",
+    )
+
+    try:
+        with request.urlopen(http_request, timeout=AGENT_CONFIG["timeout_seconds"]) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except error.HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"Model discovery failed with HTTP {exc.code}: {body}") from exc
+    except error.URLError as exc:
+        raise RuntimeError(f"Could not reach model discovery endpoint: {exc}") from exc
+
+    _RESOLVED_MODEL = choose_model(payload.get("data", []))
+    return _RESOLVED_MODEL
+
+
+def build_messages(observation):
+    return [
+        {
+            "role": "system",
+            "content": observation["instructions"],
+        },
+        {
+            "role": "user",
+            "content": (
+                f"System: {observation['system']}\n"
+                f"Log: {observation['log']}\n\n"
+                "Return only JSON with this shape:\n"
+                f"{json.dumps(observation['response_example'])}"
+            ),
+        },
+    ]
+
+
+def build_feedback_message(observation, expected, predicted, score):
+    return {
+        "role": "user",
+        "content": (
+            "Your previous answer has been graded.\n"
+            f"Original system: {observation['system']}\n"
+            f"Original log: {observation['log']}\n"
+            f"Expected answer: {json.dumps(expected)}\n"
+            f"Your answer: {json.dumps(predicted)}\n"
+            f"Score: {score}\n\n"
+            "Use this feedback to improve future answers while keeping the exact same JSON-only format."
+        ),
+    }
+
+
+def build_request_payload(messages):
+    payload = {
+        "model": resolve_model(),
+        "messages": messages,
+        "temperature": AGENT_CONFIG["temperature"],
+    }
+
+    if "groq.com" in get_provider_host():
+        payload["max_completion_tokens"] = AGENT_CONFIG["max_tokens"]
+        payload["response_format"] = {"type": "json_object"}
+    else:
+        payload["max_tokens"] = AGENT_CONFIG["max_tokens"]
+
+    return payload
+
+
+def call_agent(messages):
+    payload = json.dumps(build_request_payload(messages)).encode("utf-8")
+    http_request = request.Request(
+        normalize_chat_completions_url(AGENT_CONFIG["api_url"]),
+        data=payload,
+        headers=build_headers(),
+        method="POST",
+    )
+
+    try:
+        with request.urlopen(
+            http_request,
+            timeout=AGENT_CONFIG["timeout_seconds"],
+        ) as response:
+            return json.loads(response.read().decode("utf-8"))
+    except error.HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"Agent API returned HTTP {exc.code}: {body}") from exc
+    except error.URLError as exc:
+        raise RuntimeError(f"Could not reach agent API: {exc}") from exc
+
+
+def extract_response_text(response_json):
+    choices = response_json.get("choices") or []
+    if not choices:
+        raise RuntimeError(f"Agent response does not contain choices: {response_json}")
+
+    message = choices[0].get("message") or {}
+    content = message.get("content", "")
+
+    if isinstance(content, str):
+        return content.strip()
+
+    if isinstance(content, list):
+        text_parts = []
+        for item in content:
+            if isinstance(item, dict) and item.get("type") == "text":
+                text_parts.append(item.get("text", ""))
+        return "\n".join(text_parts).strip()
+
+    return str(content).strip()
+
+
+def extract_first_json_block(text):
+    start_index = text.find("{")
+    if start_index == -1:
+        return None
+
+    depth = 0
+    for index in range(start_index, len(text)):
+        char = text[index]
+        if char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start_index:index + 1]
+
+    return None
+
+
+def parse_prediction(raw_text):
+    cleaned = raw_text.strip()
+
+    try:
+        parsed = json.loads(cleaned)
+        if isinstance(parsed, dict):
+            return parsed, None
+    except json.JSONDecodeError:
+        pass
+
+    json_block = extract_first_json_block(cleaned)
+    if json_block:
+        try:
+            parsed = json.loads(json_block)
+            if isinstance(parsed, dict):
+                return parsed, None
+        except json.JSONDecodeError:
+            pass
+
+    fallback = {
+        "category": "",
+        "severity": "",
+        "action": cleaned,
+    }
+    return fallback, "Agent response was not valid JSON; using raw text as action."
+
+
+def append_log(record):
+    log_file = AGENT_CONFIG["log_file"]
+    log_file.parent.mkdir(parents=True, exist_ok=True)
+    with log_file.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+def validate_config():
+    if not AGENT_CONFIG["api_url"]:
+        raise RuntimeError("Set AGENT_CONFIG['api_url'] before running this script.")
+    if not AGENT_CONFIG["api_key"]:
+        raise RuntimeError("Set AGENT_API_KEY in .env before running this script.")
+
+
+def run_single_evaluation():
+    env = SecurityEnv()
+    observation = env.reset()
+    expected = env.current_sample["expected"]
+    selected_model = resolve_model()
+
+    messages = build_messages(observation)
+    response_json = call_agent(messages)
+    raw_response = extract_response_text(response_json)
+    predicted, parse_warning = parse_prediction(raw_response)
+
+    result = env.step(predicted)
+    score = result["reward"]
+
+    feedback_response = None
+    if AGENT_CONFIG["send_feedback_to_agent"]:
+        feedback_messages = messages + [
+            {
+                "role": "assistant",
+                "content": raw_response,
+            },
+            build_feedback_message(observation, expected, predicted, score),
+        ]
+        feedback_json = call_agent(feedback_messages)
+        feedback_response = extract_response_text(feedback_json)
+
+    record = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "query": {
+            "system": observation["system"],
+            "log": observation["log"],
+        },
+        "prompt": observation["agent_prompt"],
+        "expected": expected,
+        "predicted": predicted,
+        "raw_response": raw_response,
+        "score": score,
+        "parse_warning": parse_warning,
+        "feedback_response": feedback_response,
+        "provider": get_provider_host(),
+        "model_used": selected_model,
+    }
+    append_log(record)
+    return record
+
+
+def main():
+    validate_config()
+
+    for run_number in range(1, AGENT_CONFIG["runs"] + 1):
+        record = run_single_evaluation()
+        print(f"Run {run_number}")
+        print(f"Provider: {record['provider']}")
+        print(f"Model used: {record['model_used']}")
+        print(f"Query system: {record['query']['system']}")
+        print(f"Query log: {record['query']['log']}")
+        print(f"Expected: {json.dumps(record['expected'])}")
+        print(f"Predicted: {json.dumps(record['predicted'])}")
+        print(f"Score: {record['score']}")
+        print(f"Log file: {AGENT_CONFIG['log_file']}")
+        if record["parse_warning"]:
+            print(f"Warning: {record['parse_warning']}")
+        if record["feedback_response"]:
+            print(f"Feedback response: {record['feedback_response']}")
+        print("-" * 60)
+
+
+if __name__ == "__main__":
+    main()