Upload folder using huggingface_hub

README.md

@@ -584,6 +584,187 @@ python inference.py --episodes 3 --steps 5
 
 ---
 
+## 🔌 Integration Examples
+
+### OpenAI SDK Integration
+
+```python
+# examples/openai_integration.py
+from openai import OpenAI
+import requests
+
+client = OpenAI()
+ENV_URL = "https://samsankar-hallucination-guard-env.hf.space"
+
+def evaluate_with_gpt4(question: str, context: str) -> dict:
+    # Get answer from GPT-4
+    response = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[{
+            "role": "user",
+            "content": f"Answer ONLY from context.\n\nContext: {context}\n\nQuestion: {question}\n\n"
+                       f"Return JSON: {{'answer': '...', 'confidence': 0.XX, 'source_quote': '...'}}"
+        }],
+        temperature=0.1
+    )
+
+    # Parse and submit to environment
+    import json
+    result = json.loads(response.choices[0].message.content)
+
+    step = requests.post(f"{ENV_URL}/step", json={
+        "answer": result["answer"],
+        "confidence": result["confidence"],
+        "source_quote": result["source_quote"]
+    })
+
+    return step.json()
+
+# See examples/openai_integration.py for full implementation
+```
+
+### Anthropic Claude Integration
+
+```python
+# examples/anthropic_integration.py
+from anthropic import Anthropic
+import requests
+
+client = Anthropic()
+ENV_URL = "https://samsankar-hallucination-guard-env.hf.space"
+
+def evaluate_with_claude(question: str, context: str) -> dict:
+    response = client.messages.create(
+        model="claude-sonnet-4-20250514",
+        max_tokens=500,
+        messages=[{
+            "role": "user",
+            "content": f"Answer using ONLY the provided context.\n\nContext: {context}\n\nQuestion: {question}"
+        }]
+    )
+
+    # Submit to environment
+    step = requests.post(f"{ENV_URL}/step", json={
+        "answer": response.content[0].text,
+        "confidence": 0.8,
+        "source_quote": ""
+    })
+
+    return step.json()
+
+# See examples/anthropic_integration.py for full implementation
+```
+
+### Batch Evaluation
+
+```bash
+# Run batch evaluation across all tasks
+python examples/batch_evaluation.py --episodes 5 --output results.json
+```
+
+---
+
+## 🚀 Production Deployment
+
+### Docker Compose (Multi-Service)
+
+```yaml
+# docker-compose.yml
+version: '3.8'
+
+services:
+  hallucination-guard:
+    build: .
+    ports:
+      - "7860:7860"
+    environment:
+      - PYTHONUNBUFFERED=1
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7860/health"]
+      interval: 30s
+      timeout: 15s
+      retries: 3
+      start_period: 300s
+    deploy:
+      resources:
+        limits:
+          memory: 4G
+        reservations:
+          memory: 2G
+
+  # Optional: Redis for session caching
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379:6379"
+```
+
+### Kubernetes Deployment
+
+```yaml
+# k8s/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hallucination-guard-env
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: hallucination-guard
+  template:
+    metadata:
+      labels:
+        app: hallucination-guard
+    spec:
+      containers:
+      - name: server
+        image: hallucination-guard:latest
+        ports:
+        - containerPort: 7860
+        resources:
+          limits:
+            memory: "4Gi"
+            cpu: "2"
+          requests:
+            memory: "2Gi"
+            cpu: "1"
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 7860
+          initialDelaySeconds: 300
+          periodSeconds: 30
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: 7860
+          initialDelaySeconds: 60
+          periodSeconds: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hallucination-guard-service
+spec:
+  selector:
+    app: hallucination-guard
+  ports:
+  - port: 80
+    targetPort: 7860
+  type: LoadBalancer
+```
+
+### Environment Configuration
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `USE_LARGE_NLI` | Use large NLI model (more accurate, more memory) | `false` |
+| `MAX_QUESTIONS` | Maximum questions per episode | `10` |
+| `LOG_LEVEL` | Logging level | `INFO` |
+
+---
+
 ## 🔗 Links
 
 | | |

examples/anthropic_integration.py

@@ -0,0 +1,330 @@
+"""
+Anthropic Claude SDK Integration Example for HallucinationGuard-Env.
+
+This example demonstrates how to evaluate Claude models
+(Claude 3.5 Sonnet, Claude 3 Opus) using the HallucinationGuard environment.
+
+Requirements:
+    pip install anthropic requests
+"""
+
+import os
+from typing import Optional
+import requests
+
+# Anthropic SDK
+try:
+    from anthropic import Anthropic
+except ImportError:
+    print("Install Anthropic SDK: pip install anthropic")
+    raise
+
+
+class ClaudeHallucinationEvaluator:
+    """
+    Evaluate Claude models for hallucination resistance.
+
+    Features:
+    - Supports Claude 3.5 Sonnet, Claude 3 Opus, Claude 3 Haiku
+    - Uses structured prompts for consistent responses
+    - Tracks calibration and grounding scores
+    """
+
+    def __init__(
+        self,
+        env_base_url: str = "https://samsankar-hallucination-guard-env.hf.space",
+        anthropic_api_key: Optional[str] = None,
+        model: str = "claude-sonnet-4-20250514"
+    ):
+        """
+        Initialize evaluator.
+
+        Args:
+            env_base_url: HallucinationGuard-Env server URL
+            anthropic_api_key: Anthropic API key (or set ANTHROPIC_API_KEY env var)
+            model: Claude model name
+        """
+        self.env_base_url = env_base_url.rstrip('/')
+        self.model = model
+        self.client = Anthropic(api_key=anthropic_api_key or os.environ.get("ANTHROPIC_API_KEY"))
+
+        # Session tracking
+        self.episode_id = None
+
+    def reset_environment(self, difficulty: str = "intermediate") -> dict:
+        """Start a new evaluation episode."""
+        response = requests.post(
+            f"{self.env_base_url}/reset",
+            json={"difficulty": difficulty}
+        )
+        response.raise_for_status()
+        data = response.json()
+        self.episode_id = data.get("episode_id")
+        return data
+
+    def generate_answer(self, question: str, context: str) -> dict:
+        """
+        Generate an answer using Claude model.
+
+        Claude is instructed to:
+        1. Answer ONLY from the context
+        2. Provide calibrated confidence
+        3. Cite verbatim source quotes
+        """
+        prompt = f"""I need you to answer a question using ONLY the provided context.
+
+CRITICAL INSTRUCTIONS:
+1. Answer ONLY using information from the context below
+2. If the answer is not in the context, respond: "I cannot determine the answer from the provided context."
+3. Provide a confidence score (0.0 to 1.0) for your answer
+4. Include a direct quote from the context that supports your answer
+
+CONTEXT:
+{context}
+
+QUESTION:
+{question}
+
+Respond in this exact JSON format:
+{{
+    "answer": "your answer based solely on the context",
+    "confidence": 0.XX,
+    "source_quote": "exact verbatim quote from the context"
+}}
+
+Remember: Only use information from the context. Do not use outside knowledge."""
+
+        try:
+            response = self.client.messages.create(
+                model=self.model,
+                max_tokens=500,
+                temperature=0.1,
+                messages=[
+                    {"role": "user", "content": prompt}
+                ]
+            )
+
+            content = response.content[0].text
+
+            # Parse JSON response
+            import json
+            import re
+
+            # Extract JSON from response
+            json_match = re.search(r'\{[^{}]*\}', content, re.DOTALL)
+            if json_match:
+                result = json.loads(json_match.group())
+            else:
+                # Fallback if no JSON found
+                result = {
+                    "answer": content.split('"answer"')[1].split('"')[1] if '"answer"' in content else content[:200],
+                    "confidence": 0.5,
+                    "source_quote": ""
+                }
+
+            return {
+                "answer": result.get("answer", ""),
+                "confidence": float(result.get("confidence", 0.5)),
+                "source_quote": result.get("source_quote", "")
+            }
+
+        except Exception as e:
+            print(f"Error generating answer: {e}")
+            return {
+                "answer": "I cannot determine the answer from the provided context.",
+                "confidence": 0.3,
+                "source_quote": ""
+            }
+
+    def step(self, answer: str, confidence: float, source_quote: str = "") -> dict:
+        """Submit an answer to the environment."""
+        response = requests.post(
+            f"{self.env_base_url}/step",
+            json={
+                "answer": answer,
+                "confidence": confidence,
+                "source_quote": source_quote
+            }
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def evaluate_episode(
+        self,
+        num_questions: int = 10,
+        difficulty: str = "intermediate",
+        verbose: bool = True
+    ) -> dict:
+        """
+        Run a complete evaluation episode.
+
+        Args:
+            num_questions: Number of questions to evaluate
+            difficulty: Starting difficulty level
+            verbose: Print progress
+
+        Returns:
+            Episode statistics
+        """
+        obs = self.reset_environment(difficulty=difficulty)
+
+        total_reward = 0.0
+        hallucinations = 0
+        correct = 0
+        calibration_errors = []
+
+        for step_num in range(num_questions):
+            question = obs.get("question", "")
+            context = obs.get("context", "")
+
+            if verbose:
+                print(f"\n--- Question {step_num + 1}/{num_questions} ---")
+                print(f"Q: {question[:100]}...")
+
+            # Generate answer with Claude
+            answer_data = self.generate_answer(question, context)
+
+            if verbose:
+                print(f"A: {answer_data['answer'][:100]}...")
+                print(f"Confidence: {answer_data['confidence']:.2f}")
+
+            # Submit to environment
+            obs = self.step(
+                answer=answer_data["answer"],
+                confidence=answer_data["confidence"],
+                source_quote=answer_data["source_quote"]
+            )
+
+            # Track statistics
+            reward = obs.get("reward", 0.0)
+            total_reward += reward
+
+            if obs.get("is_hallucination", False):
+                hallucinations += 1
+
+            if obs.get("grounding_score", 0) > 0.7:
+                correct += 1
+
+            # Track calibration
+            correctness = obs.get("metadata", {}).get("correctness", 0.5)
+            calibration_error = abs(answer_data["confidence"] - correctness)
+            calibration_errors.append(calibration_error)
+
+            if verbose:
+                print(f"Reward: {reward:.3f}")
+                print(f"Hallucination: {obs.get('is_hallucination', False)}")
+
+            if obs.get("done", False):
+                break
+
+        # Calculate statistics
+        avg_reward = total_reward / max(1, step_num + 1)
+        hallucination_rate = hallucinations / max(1, step_num + 1)
+        accuracy = correct / max(1, step_num + 1)
+        avg_calibration = sum(calibration_errors) / max(1, len(calibration_errors))
+
+        results = {
+            "model": self.model,
+            "avg_reward": avg_reward,
+            "hallucination_rate": hallucination_rate,
+            "accuracy": accuracy,
+            "avg_calibration_error": avg_calibration,
+            "total_steps": step_num + 1,
+            "difficulty": difficulty
+        }
+
+        if verbose:
+            print(f"\n=== Episode Complete ===")
+            print(f"Model: {self.model}")
+            print(f"Average Reward: {avg_reward:.3f}")
+            print(f"Hallucination Rate: {hallucination_rate:.1%}")
+            print(f"Accuracy: {accuracy:.1%}")
+            print(f"Avg Calibration Error: {avg_calibration:.3f}")
+
+        return results
+
+
+def compare_models(models: list, num_questions: int = 5, difficulty: str = "intermediate") -> dict:
+    """
+    Compare multiple Claude models.
+
+    Args:
+        models: List of model names to compare
+        num_questions: Questions per model
+        difficulty: Difficulty level
+
+    Returns:
+        Comparison results
+    """
+    results = {}
+
+    for model in models:
+        print(f"\n{'='*50}")
+        print(f"Evaluating: {model}")
+        print(f"{'='*50}")
+
+        evaluator = ClaudeHallucinationEvaluator(model=model)
+        model_results = evaluator.evaluate_episode(
+            num_questions=num_questions,
+            difficulty=difficulty
+        )
+        results[model] = model_results
+
+    # Print comparison
+    print(f"\n{'='*60}")
+    print("MODEL COMPARISON")
+    print(f"{'='*60}")
+    print(f"{'Model':<30} {'Reward':>10} {'Halluc%':>12} {'Accuracy':>10}")
+    print("-" * 60)
+    for model, res in results.items():
+        print(f"{model:<30} {res['avg_reward']:>10.3f} {res['hallucination_rate']*100:>11.1f}% {res['accuracy']*100:>9.1f}%")
+
+    return results
+
+
+def main():
+    """Run evaluation demo."""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Evaluate Claude models for hallucination resistance")
+    parser.add_argument("--model", default="claude-sonnet-4-20250514",
+                        help="Claude model name")
+    parser.add_argument("--difficulty", default="intermediate", help="Difficulty level")
+    parser.add_argument("--num-questions", type=int, default=5, help="Number of questions")
+    parser.add_argument("--env-url", default="https://samsankar-hallucination-guard-env.hf.space",
+                        help="Environment server URL")
+    parser.add_argument("--compare", action="store_true",
+                        help="Compare multiple models")
+
+    args = parser.parse_args()
+
+    # Check for API key
+    if not os.environ.get("ANTHROPIC_API_KEY"):
+        print("Error: Set ANTHROPIC_API_KEY environment variable")
+        return
+
+    if args.compare:
+        # Compare multiple models
+        models = [
+            "claude-sonnet-4-20250514",
+            "claude-3-5-sonnet-20241022",
+            "claude-3-haiku-20240307"
+        ]
+        compare_models(models, num_questions=args.num_questions, difficulty=args.difficulty)
+    else:
+        # Single model evaluation
+        evaluator = ClaudeHallucinationEvaluator(
+            env_base_url=args.env_url,
+            model=args.model
+        )
+
+        results = evaluator.evaluate_episode(
+            num_questions=args.num_questions,
+            difficulty=args.difficulty
+        )
+
+        print(f"\nFinal Results: {results}")
+
+
+if __name__ == "__main__":
+    main()

examples/batch_evaluation.py

@@ -0,0 +1,364 @@
+"""
+Batch Evaluation Script for HallucinationGuard-Env.
+
+This script demonstrates how to run batch evaluations across multiple
+tasks and difficulties, generating comprehensive benchmark reports.
+
+Requirements:
+    pip install requests matplotlib pandas
+"""
+
+import json
+import time
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+import requests
+
+
+class BatchEvaluator:
+    """
+    Run batch evaluations across tasks and difficulties.
+
+    Features:
+    - Multi-task evaluation (Factual Grounding, Multi-hop, Adversarial)
+    - Multiple difficulty levels
+    - Performance metrics and calibration analysis
+    - JSON report generation
+    """
+
+    TASKS = [
+        "task_1_factual_grounding",
+        "task_2_multi_hop_synthesis",
+        "task_3_adversarial_resistance"
+    ]
+
+    DIFFICULTIES = ["beginner", "intermediate", "advanced"]
+
+    def __init__(self, env_base_url: str = "https://samsankar-hallucination-guard-env.hf.space"):
+        """Initialize evaluator with environment URL."""
+        self.env_base_url = env_base_url.rstrip('/')
+        self.session = requests.Session()
+
+    def get_tasks(self) -> List[Dict]:
+        """Get available tasks from environment."""
+        response = self.session.get(f"{self.env_base_url}/tasks")
+        response.raise_for_status()
+        return response.json().get("tasks", [])
+
+    def evaluate_baseline(
+        self,
+        task_id: str,
+        num_episodes: int = 3,
+        difficulty: str = "intermediate"
+    ) -> Dict[str, Any]:
+        """
+        Run baseline evaluation for a specific task.
+
+        Uses a simple heuristic baseline:
+        - Extract key entities from context
+        - Match entities to question
+        - Provide confidence based on match quality
+
+        Args:
+            task_id: Task identifier
+            num_episodes: Number of episodes to run
+            difficulty: Difficulty level
+
+        Returns:
+            Evaluation results
+        """
+        results = {
+            "task_id": task_id,
+            "difficulty": difficulty,
+            "episodes": [],
+            "summary": {}
+        }
+
+        all_rewards = []
+        all_hallucinations = []
+        all_correct = []
+
+        for episode_num in range(num_episodes):
+            # Reset environment
+            reset_data = self._reset(task_id=task_id, difficulty=difficulty)
+
+            episode_rewards = []
+            episode_hallucinations = 0
+            episode_correct = 0
+
+            steps = 0
+            max_steps = 10
+
+            while steps < max_steps:
+                # Get current observation
+                question = reset_data.get("question", "")
+                context = reset_data.get("context", "")
+
+                # Generate baseline answer
+                answer_data = self._generate_baseline_answer(question, context)
+
+                # Step environment
+                step_data = self._step(**answer_data)
+
+                # Track metrics
+                reward = step_data.get("reward", 0.0)
+                episode_rewards.append(reward)
+
+                if step_data.get("is_hallucination", False):
+                    episode_hallucinations += 1
+
+                if step_data.get("grounding_score", 0) > 0.7:
+                    episode_correct += 1
+
+                steps += 1
+
+                if step_data.get("done", False):
+                    break
+
+                # Get next question
+                reset_data = step_data
+
+            # Episode statistics
+            episode_avg_reward = sum(episode_rewards) / max(1, len(episode_rewards))
+            all_rewards.append(episode_avg_reward)
+            all_hallucinations.append(episode_hallucinations / max(1, steps))
+            all_correct.append(episode_correct / max(1, steps))
+
+            results["episodes"].append({
+                "episode_num": episode_num + 1,
+                "avg_reward": episode_avg_reward,
+                "hallucination_rate": episode_hallucinations / max(1, steps),
+                "accuracy": episode_correct / max(1, steps),
+                "total_steps": steps
+            })
+
+            print(f"Episode {episode_num + 1}: Reward={episode_avg_reward:.3f}, "
+                  f"Hallucinations={episode_hallucinations}/{steps}")
+
+        # Aggregate results
+        results["summary"] = {
+            "avg_reward": sum(all_rewards) / len(all_rewards),
+            "avg_hallucination_rate": sum(all_hallucinations) / len(all_hallucinations),
+            "avg_accuracy": sum(all_correct) / len(all_correct),
+            "total_episodes": num_episodes,
+            "timestamp": datetime.now().isoformat()
+        }
+
+        return results
+
+    def _reset(self, task_id: str = None, difficulty: str = "intermediate") -> dict:
+        """Reset environment."""
+        payload = {"difficulty": difficulty}
+        if task_id:
+            payload["task_id"] = task_id
+
+        response = self.session.post(f"{self.env_base_url}/reset", json=payload)
+        response.raise_for_status()
+        return response.json()
+
+    def _step(self, answer: str, confidence: float, source_quote: str = "") -> dict:
+        """Submit step."""
+        response = self.session.post(
+            f"{self.env_base_url}/step",
+            json={
+                "answer": answer,
+                "confidence": confidence,
+                "source_quote": source_quote
+            }
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def _generate_baseline_answer(self, question: str, context: str) -> dict:
+        """
+        Generate a simple baseline answer.
+
+        Strategy:
+        1. Extract sentences from context
+        2. Find sentence most similar to question
+        3. Use that as answer with moderate confidence
+        4. Use sentence as source quote
+        """
+        import re
+
+        # Split context into sentences
+        sentences = re.split(r'[.!?]+', context)
+        sentences = [s.strip() for s in sentences if len(s.strip()) > 10]
+
+        if not sentences:
+            return {
+                "answer": "I cannot find the answer in the provided context.",
+                "confidence": 0.3,
+                "source_quote": ""
+            }
+
+        # Find most relevant sentence (simple keyword matching)
+        question_words = set(question.lower().split())
+
+        best_sentence = sentences[0]
+        best_overlap = 0
+
+        for sentence in sentences:
+            sentence_words = set(sentence.lower().split())
+            overlap = len(question_words & sentence_words)
+            if overlap > best_overlap:
+                best_overlap = overlap
+                best_sentence = sentence
+
+        # Check if answer is likely in context
+        if best_overlap < 2:
+            return {
+                "answer": "The answer does not appear to be in the provided context.",
+                "confidence": 0.4,
+                "source_quote": ""
+            }
+
+        # Extract key part of sentence as answer
+        answer = best_sentence[:200] if len(best_sentence) > 200 else best_sentence
+
+        return {
+            "answer": answer,
+            "confidence": 0.5 + (best_overlap / 20),  # Higher confidence with more overlap
+            "source_quote": best_sentence[:150]
+        }
+
+    def run_full_evaluation(
+        self,
+        episodes_per_task: int = 3,
+        difficulties: List[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Run full evaluation across all tasks and difficulties.
+
+        Args:
+            episodes_per_task: Episodes per task configuration
+            difficulties: List of difficulties to test
+
+        Returns:
+            Complete evaluation report
+        """
+        difficulties = difficulties or ["beginner", "intermediate", "advanced"]
+
+        report = {
+            "evaluation_date": datetime.now().isoformat(),
+            "environment_url": self.env_base_url,
+            "configuration": {
+                "episodes_per_task": episodes_per_task,
+                "difficulties": difficulties
+            },
+            "results": {}
+        }
+
+        print("Starting Full Evaluation")
+        print("=" * 60)
+
+        for task_id in self.TASKS:
+            print(f"\nEvaluating: {task_id}")
+            print("-" * 40)
+
+            report["results"][task_id] = {}
+
+            for difficulty in difficulties:
+                print(f"  Difficulty: {difficulty}")
+
+                task_results = self.evaluate_baseline(
+                    task_id=task_id,
+                    num_episodes=episodes_per_task,
+                    difficulty=difficulty
+                )
+
+                report["results"][task_id][difficulty] = task_results
+
+                # Brief pause between evaluations
+                time.sleep(1)
+
+        # Generate summary
+        report["summary"] = self._generate_summary(report)
+
+        return report
+
+    def _generate_summary(self, report: dict) -> dict:
+        """Generate cross-task summary."""
+        summary = {
+            "overall_avg_reward": 0.0,
+            "overall_avg_hallucination_rate": 0.0,
+            "overall_avg_accuracy": 0.0,
+            "best_task": "",
+            "best_difficulty": ""
+        }
+
+        all_rewards = []
+        all_hallucinations = []
+        all_accuracies = []
+        task_performances = {}
+
+        for task_id, difficulties in report.get("results", {}).items():
+            task_rewards = []
+            for difficulty, results in difficulties.items():
+                task_summary = results.get("summary", {})
+                all_rewards.append(task_summary.get("avg_reward", 0))
+                all_hallucinations.append(task_summary.get("avg_hallucination_rate", 0))
+                all_accuracies.append(task_summary.get("avg_accuracy", 0))
+                task_rewards.append(task_summary.get("avg_reward", 0))
+
+            task_performances[task_id] = sum(task_rewards) / len(task_rewards)
+
+        if all_rewards:
+            summary["overall_avg_reward"] = sum(all_rewards) / len(all_rewards)
+        if all_hallucinations:
+            summary["overall_avg_hallucination_rate"] = sum(all_hallucinations) / len(all_hallucinations)
+        if all_accuracies:
+            summary["overall_avg_accuracy"] = sum(all_accuracies) / len(all_accuracies)
+
+        if task_performances:
+            summary["best_task"] = max(task_performances, key=task_performances.get)
+
+        return summary
+
+    def save_report(self, report: dict, filename: str = None) -> str:
+        """Save report to JSON file."""
+        if filename is None:
+            filename = f"hallucination_eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+
+        with open(filename, 'w') as f:
+            json.dump(report, f, indent=2)
+
+        print(f"Report saved to: {filename}")
+        return filename
+
+
+def main():
+    """Run batch evaluation."""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Run batch hallucination evaluation")
+    parser.add_argument("--env-url", default="https://samsankar-hallucination-guard-env.hf.space",
+                        help="Environment server URL")
+    parser.add_argument("--episodes", type=int, default=3, help="Episodes per task")
+    parser.add_argument("--output", default=None, help="Output file name")
+
+    args = parser.parse_args()
+
+    evaluator = BatchEvaluator(env_base_url=args.env_url)
+
+    # Run full evaluation
+    report = evaluator.run_full_evaluation(
+        episodes_per_task=args.episodes
+    )
+
+    # Print summary
+    print("\n" + "=" * 60)
+    print("EVALUATION SUMMARY")
+    print("=" * 60)
+    summary = report.get("summary", {})
+    print(f"Overall Average Reward: {summary.get('overall_avg_reward', 0):.3f}")
+    print(f"Overall Hallucination Rate: {summary.get('overall_avg_hallucination_rate', 0):.1%}")
+    print(f"Overall Accuracy: {summary.get('overall_avg_accuracy', 0):.1%}")
+    print(f"Best Performing Task: {summary.get('best_task', 'N/A')}")
+
+    # Save report
+    evaluator.save_report(report, args.output)
+
+
+if __name__ == "__main__":
+    main()

examples/openai_integration.py

@@ -0,0 +1,274 @@
+"""
+OpenAI SDK Integration Example for HallucinationGuard-Env.
+
+This example demonstrates how to evaluate OpenAI models
+(GPT-4, GPT-4o, GPT-3.5) using the HallucinationGuard environment.
+
+Requirements:
+    pip install openai requests
+"""
+
+import os
+from typing import Optional
+import requests
+
+# OpenAI SDK
+try:
+    from openai import OpenAI
+except ImportError:
+    print("Install OpenAI SDK: pip install openai")
+    raise
+
+
+class HallucinationGuardEvaluator:
+    """
+    Evaluate OpenAI models for hallucination resistance.
+
+    Features:
+    - Supports all OpenAI chat models
+    - Handles rate limiting gracefully
+    - Tracks calibration and grounding scores
+    """
+
+    def __init__(
+        self,
+        env_base_url: str = "https://samsankar-hallucination-guard-env.hf.space",
+        openai_api_key: Optional[str] = None,
+        model: str = "gpt-4o-mini"
+    ):
+        """
+        Initialize evaluator.
+
+        Args:
+            env_base_url: HallucinationGuard-Env server URL
+            openai_api_key: OpenAI API key (or set OPENAI_API_KEY env var)
+            model: OpenAI model name
+        """
+        self.env_base_url = env_base_url.rstrip('/')
+        self.model = model
+        self.client = OpenAI(api_key=openai_api_key or os.environ.get("OPENAI_API_KEY"))
+
+        # Session for environment
+        self.session_id = None
+        self.episode_id = None
+
+    def reset_environment(self, difficulty: str = "intermediate") -> dict:
+        """
+        Start a new evaluation episode.
+
+        Args:
+            difficulty: Starting difficulty (beginner, intermediate, advanced)
+
+        Returns:
+            Initial observation with question and context
+        """
+        response = requests.post(
+            f"{self.env_base_url}/reset",
+            json={"difficulty": difficulty}
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        self.episode_id = data.get("episode_id")
+        return data
+
+    def generate_answer(self, question: str, context: str) -> dict:
+        """
+        Generate an answer using OpenAI model.
+
+        Prompts the model to:
+        1. Answer ONLY from the provided context
+        2. Provide a confidence score
+        3. Cite the source quote
+
+        Args:
+            question: The question to answer
+            context: The source context
+
+        Returns:
+            dict with answer, confidence, source_quote
+        """
+        prompt = f"""Answer the following question using ONLY the provided context.
+
+IMPORTANT RULES:
+1. Answer ONLY from the context - do not use outside knowledge
+2. If the answer is not in the context, say "I cannot answer from the provided context"
+3. Provide your confidence level (0.0-1.0)
+4. Quote the exact passage from the context that supports your answer
+
+CONTEXT:
+{context}
+
+QUESTION:
+{question}
+
+Respond in JSON format:
+{{
+    "answer": "your answer here",
+    "confidence": 0.85,
+    "source_quote": "exact quote from context"
+}}
+
+JSON Response:"""
+
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": "You are a precise QA assistant. Always respond in valid JSON format."},
+                    {"role": "user", "content": prompt}
+                ],
+                temperature=0.1,  # Low temperature for factual tasks
+                max_tokens=500,
+                response_format={"type": "json_object"}
+            )
+
+            import json
+            content = response.choices[0].message.content
+            result = json.loads(content)
+
+            return {
+                "answer": result.get("answer", ""),
+                "confidence": float(result.get("confidence", 0.5)),
+                "source_quote": result.get("source_quote", "")
+            }
+
+        except Exception as e:
+            print(f"Error generating answer: {e}")
+            return {
+                "answer": "I cannot answer from the provided context.",
+                "confidence": 0.3,
+                "source_quote": ""
+            }
+
+    def step(self, answer: str, confidence: float, source_quote: str = "") -> dict:
+        """
+        Submit an answer to the environment.
+
+        Args:
+            answer: The answer text
+            confidence: Confidence level (0.0-1.0)
+            source_quote: Verbatim quote from context
+
+        Returns:
+            Observation with reward and feedback
+        """
+        response = requests.post(
+            f"{self.env_base_url}/step",
+            json={
+                "answer": answer,
+                "confidence": confidence,
+                "source_quote": source_quote
+            }
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def evaluate_episode(
+        self,
+        num_questions: int = 10,
+        difficulty: str = "intermediate"
+    ) -> dict:
+        """
+        Run a complete evaluation episode.
+
+        Args:
+            num_questions: Number of questions to evaluate
+            difficulty: Starting difficulty level
+
+        Returns:
+            Episode statistics
+        """
+        # Reset environment
+        obs = self.reset_environment(difficulty=difficulty)
+
+        total_reward = 0.0
+        hallucinations = 0
+        correct = 0
+
+        for step_num in range(num_questions):
+            # Get current question and context
+            question = obs.get("question", "")
+            context = obs.get("context", "")
+
+            print(f"\n--- Question {step_num + 1}/{num_questions} ---")
+            print(f"Q: {question[:100]}...")
+
+            # Generate answer with OpenAI
+            answer_data = self.generate_answer(question, context)
+            print(f"A: {answer_data['answer'][:100]}...")
+            print(f"Confidence: {answer_data['confidence']:.2f}")
+
+            # Submit to environment
+            obs = self.step(
+                answer=answer_data["answer"],
+                confidence=answer_data["confidence"],
+                source_quote=answer_data["source_quote"]
+            )
+
+            # Track statistics
+            reward = obs.get("reward", 0.0)
+            total_reward += reward
+            if obs.get("is_hallucination", False):
+                hallucinations += 1
+            if obs.get("grounding_score", 0) > 0.7:
+                correct += 1
+
+            print(f"Reward: {reward:.3f}")
+            print(f"Feedback: {obs.get('feedback', '')[:100]}...")
+
+            if obs.get("done", False):
+                break
+
+        # Calculate final statistics
+        avg_reward = total_reward / max(1, step_num + 1)
+        hallucination_rate = hallucinations / max(1, step_num + 1)
+        accuracy = correct / max(1, step_num + 1)
+
+        print(f"\n=== Episode Complete ===")
+        print(f"Average Reward: {avg_reward:.3f}")
+        print(f"Hallucination Rate: {hallucination_rate:.1%}")
+        print(f"Accuracy: {accuracy:.1%}")
+
+        return {
+            "avg_reward": avg_reward,
+            "hallucination_rate": hallucination_rate,
+            "accuracy": accuracy,
+            "total_steps": step_num + 1
+        }
+
+
+def main():
+    """Run evaluation demo."""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Evaluate OpenAI models for hallucination resistance")
+    parser.add_argument("--model", default="gpt-4o-mini", help="OpenAI model name")
+    parser.add_argument("--difficulty", default="intermediate", help="Difficulty level")
+    parser.add_argument("--num-questions", type=int, default=5, help="Number of questions")
+    parser.add_argument("--env-url", default="https://samsankar-hallucination-guard-env.hf.space",
+                        help="Environment server URL")
+
+    args = parser.parse_args()
+
+    # Check for API key
+    if not os.environ.get("OPENAI_API_KEY"):
+        print("Error: Set OPENAI_API_KEY environment variable")
+        return
+
+    # Run evaluation
+    evaluator = HallucinationGuardEvaluator(
+        env_base_url=args.env_url,
+        model=args.model
+    )
+
+    results = evaluator.evaluate_episode(
+        num_questions=args.num_questions,
+        difficulty=args.difficulty
+    )
+
+    print(f"\nFinal Results: {results}")
+
+
+if __name__ == "__main__":
+    main()

models.py

@@ -281,6 +281,14 @@ class EnvironmentConfig(BaseModel):
     max_questions_per_episode: int = 10
     min_questions_for_completion: int = 5
 
+    # Early stopping configuration (NEW)
+    early_stopping_enabled: bool = True
+    early_stopping_patience: int = 3  # Consecutive failures before stopping
+    early_stopping_min_reward: float = 0.2  # Minimum reward to not count as failure
+    early_stopping_hallucination_cascade: int = 3  # Stop after N consecutive hallucinations
+    early_stopping_perfect_run: int = 5  # Complete early after N perfect answers
+    early_stopping_calibration_failure: float = 0.5  # Stop if calibration error exceeds this
+
     # Reward configuration
     reward_weights: Dict[str, float] = Field(default_factory=lambda: {
         "factual_correctness": 0.30,
@@ -296,6 +304,7 @@ class EnvironmentConfig(BaseModel):
     adaptive_difficulty: bool = True
     difficulty_threshold_increase: float = 0.7
     difficulty_threshold_decrease: float = 0.4
+    difficulty_hysteresis_steps: int = 5  # Minimum steps before difficulty change
 
     # Hallucination detection thresholds
     hallucination_threshold: float = 0.5
@@ -304,6 +313,8 @@ class EnvironmentConfig(BaseModel):
     # Curriculum configuration
     curriculum_enabled: bool = True
     min_steps_per_curriculum_stage: int = 50
+    curriculum_mastery_threshold: float = 0.75  # Avg reward to advance stage
+    curriculum_regression_threshold: float = 0.4  # Avg reward to regress stage
 
     # Multi-turn configuration
     enable_multi_turn: bool = False

server/environment.py

@@ -139,6 +139,13 @@ class HallucinationEnvironment(Environment[HallucinationAction, HallucinationObs
         self.current_streak: int = 0
         self.best_streak: int = 0
 
+        # Early stopping tracking (NEW)
+        self.consecutive_failures: int = 0
+        self.consecutive_hallucinations: int = 0
+        self.consecutive_perfect: int = 0
+        self.early_stop_reason: Optional[str] = None
+        self.calibration_history: List[float] = []
+
         # Curriculum state
         self.curriculum_stage: int = 0
         self.curriculum_performance: List[float] = []
@@ -228,6 +235,13 @@ class HallucinationEnvironment(Environment[HallucinationAction, HallucinationObs
         self.hallucination_history = []
         self.current_streak = 0
 
+        # Reset early stopping counters
+        self.consecutive_failures = 0
+        self.consecutive_hallucinations = 0
+        self.consecutive_perfect = 0
+        self.early_stop_reason = None
+        self.calibration_history = []
+
         # Reset multi-turn state
         self.dialogue = MultiTurnDialogue() if enable_multi_turn else None
         self.pending_clarifications = []
@@ -458,13 +472,26 @@ class HallucinationEnvironment(Environment[HallucinationAction, HallucinationObs
         if is_hallucination:
             self.total_hallucinations += 1
             self.current_streak = 0
+            self.consecutive_hallucinations += 1
+            self.consecutive_perfect = 0
         elif correctness > 0.7:
             self.total_correct += 1
             self.current_streak += 1
             self.best_streak = max(self.best_streak, self.current_streak)
+            self.consecutive_perfect += 1
+            self.consecutive_hallucinations = 0
+            self.consecutive_failures = 0
         else:
             self.total_partial += 1
             self.current_streak = 0
+            self.consecutive_perfect = 0
+            self.consecutive_hallucinations = 0
+            if reward < self.config.early_stopping_min_reward:
+                self.consecutive_failures += 1
+
+        # Track calibration history
+        calibration_error = abs(action.confidence - correctness)
+        self.calibration_history.append(calibration_error)
 
         # Track history
         self.reward_history.append(reward)
@@ -492,8 +519,19 @@ class HallucinationEnvironment(Environment[HallucinationAction, HallucinationObs
 
         # Move to next question
         self.step_count += 1
+
+        # Check for early stopping conditions
+        early_stop = self._check_early_stopping(is_hallucination, correctness, calibration_error)
+
+        # Determine if episode is done
         done = self.step_count >= self.config.max_questions_per_episode
 
+        if early_stop:
+            done = True
+            self.early_stop_reason = early_stop
+            self.episode_phase = EpisodePhase.COMPLETION
+            feedback += f" [Early stop: {early_stop}]"
+
         if not done:
             self.current_example = self.dataset_loader.get_example_for_step(self.step_count)
         else:
@@ -714,6 +752,41 @@ class HallucinationEnvironment(Environment[HallucinationAction, HallucinationObs
             }
         )
 
+    def _check_early_stopping(self, is_hallucination: bool, correctness: float, calibration_error: float) -> Optional[str]:
+        """
+        Check if episode should stop early based on performance conditions.
+
+        Returns:
+            str describing early stop reason, or None if should continue.
+        """
+        if not self.config.early_stopping_enabled:
+            return None
+
+        # Require minimum steps before early stopping
+        if self.step_count < 3:
+            return None
+
+        # 1. Hallucination cascade: too many consecutive hallucinations
+        if self.consecutive_hallucinations >= self.config.early_stopping_hallucination_cascade:
+            return f"hallucination_cascade ({self.consecutive_hallucinations} consecutive)"
+
+        # 2. Consecutive failures: poor performance
+        if self.consecutive_failures >= self.config.early_stopping_patience:
+            return f"consecutive_failures ({self.consecutive_failures} below {self.config.early_stopping_min_reward})"
+
+        # 3. Calibration failure: confidence systematically misaligned
+        if len(self.calibration_history) >= 5:
+            avg_calibration_error = sum(self.calibration_history[-5:]) / 5
+            if avg_calibration_error > self.config.early_stopping_calibration_failure:
+                return f"calibration_failure (avg error: {avg_calibration_error:.2f})"
+
+        # 4. Perfect run: early completion after consistent high performance
+        if self.consecutive_perfect >= self.config.early_stopping_perfect_run:
+            if self.step_count >= self.config.min_questions_for_completion:
+                return f"perfect_run ({self.consecutive_perfect} consecutive correct)"
+
+        return None
+
     def _get_context_for_observation(self, example: Optional[QAExample]) -> str:
         """Get context, potentially with partial revelation for challenges."""
         if not example:
@@ -726,39 +799,106 @@ class HallucinationEnvironment(Environment[HallucinationAction, HallucinationObs
         return example.context
 
     def _get_current_difficulty(self) -> DifficultyLevel:
-        """Determine current difficulty based on performance."""
+        """
+        Determine current difficulty based on performance with hysteresis.
+
+        Uses smooth difficulty scaling with:
+        - Stage-specific thresholds
+        - Minimum steps at each level (hysteresis)
+        - EXPERT level progression
+        """
         if not self.config.adaptive_difficulty:
             return self.config.initial_difficulty
 
-        # Calculate recent performance
-        recent_rewards = self.reward_history[-5:] if len(self.reward_history) >= 5 else self.reward_history
-        avg_recent_reward = sum(recent_rewards) / max(1, len(recent_rewards))
+        # Need enough history for reliable assessment
+        if len(self.reward_history) < 3:
+            return self.config.initial_difficulty
+
+        # Calculate recent performance with exponential weighting
+        recent_rewards = self.reward_history[-10:] if len(self.reward_history) >= 10 else self.reward_history
+        avg_recent_reward = sum(recent_rewards) / len(recent_rewards)
+
+        # Get current difficulty from example
+        current_difficulty = self.config.initial_difficulty
+        if self.current_example:
+            current_difficulty = self.current_example.difficulty
+
+        # Stage-specific mastery thresholds
+        mastery_thresholds = {
+            DifficultyLevel.BEGINNER: 0.60,
+            DifficultyLevel.INTERMEDIATE: 0.65,
+            DifficultyLevel.ADVANCED: 0.75,
+            DifficultyLevel.EXPERT: 0.85,
+        }
+
+        # Regression thresholds (lower than mastery to avoid oscillation)
+        regression_thresholds = {
+            DifficultyLevel.BEGINNER: 0.30,
+            DifficultyLevel.INTERMEDIATE: 0.40,
+            DifficultyLevel.ADVANCED: 0.50,
+            DifficultyLevel.EXPERT: 0.60,
+        }
+
+        # Difficulty progression order
+        difficulty_order = [
+            DifficultyLevel.BEGINNER,
+            DifficultyLevel.INTERMEDIATE,
+            DifficultyLevel.ADVANCED,
+            DifficultyLevel.EXPERT,
+        ]
+
+        current_idx = difficulty_order.index(current_difficulty) if current_difficulty in difficulty_order else 0
+
+        # Check for promotion
+        if avg_recent_reward > mastery_thresholds.get(current_difficulty, 0.7):
+            # Promote if not at EXPERT
+            if current_idx < len(difficulty_order) - 1:
+                return difficulty_order[current_idx + 1]
 
-        if avg_recent_reward > self.config.difficulty_threshold_increase:
-            if self.current_example and self.current_example.difficulty != DifficultyLevel.EXPERT:
-                return DifficultyLevel.ADVANCED
-        elif avg_recent_reward < self.config.difficulty_threshold_decrease:
-            return DifficultyLevel.INTERMEDIATE
+        # Check for demotion
+        elif avg_recent_reward < regression_thresholds.get(current_difficulty, 0.4):
+            # Demote if not at BEGINNER
+            if current_idx > 0:
+                return difficulty_order[current_idx - 1]
 
-        return self.config.initial_difficulty
+        return current_difficulty
 
     def _update_curriculum(self) -> None:
-        """Update curriculum stage based on episode performance."""
+        """
+        Update curriculum stage based on episode performance.
+
+        Supports:
+        - Advancement on sustained high performance
+        - Regression on sustained poor performance
+        - Stage-specific thresholds
+        """
         if not self.config.curriculum_enabled:
             return
 
         episode_reward = sum(self.reward_history) / max(1, len(self.reward_history))
         self.curriculum_performance.append(episode_reward)
 
-        # Check for curriculum advancement
-        if len(self.curriculum_performance) >= self.config.min_steps_per_curriculum_stage:
-            recent_avg = sum(self.curriculum_performance[-self.config.min_steps_per_curriculum_stage:]) / \
-                         self.config.min_steps_per_curriculum_stage
+        # Calculate statistics
+        avg_reward = sum(self.curriculum_performance) / len(self.curriculum_performance)
+        recent_rewards = self.curriculum_performance[-10:] if len(self.curriculum_performance) >= 10 else self.curriculum_performance
+        recent_avg = sum(recent_rewards) / len(recent_rewards)
+
+        # Stage-specific thresholds
+        advancement_threshold = self.config.curriculum_mastery_threshold
+        regression_threshold = self.config.curriculum_regression_threshold
 
-            if recent_avg > 0.7:
+        # Check for curriculum advancement (sustained high performance)
+        if len(self.curriculum_performance) >= self.config.min_steps_per_curriculum_stage:
+            if recent_avg > advancement_threshold:
                 self.curriculum_stage += 1
+                self.curriculum_performance = []  # Reset for next stage
+                logger.info(f"Advanced to curriculum stage {self.curriculum_stage} (avg: {recent_avg:.2f})")
+
+            # Check for curriculum regression (sustained poor performance)
+            elif recent_avg < regression_threshold and self.curriculum_stage > 0:
+                self.curriculum_stage = max(0, self.curriculum_stage - 1)
                 self.curriculum_performance = []
-                logger.info(f"Advanced to curriculum stage {self.curriculum_stage}")
+                logger.info(f"Regressed to curriculum stage {self.curriculum_stage} (avg: {recent_avg:.2f})")
 
     def _update_agent_profile(self) -> None:
         """Update the agent's long-term skill profile."""

server/grader.py

@@ -308,6 +308,162 @@ def extract_key_claims(text: str) -> List[str]:
     return claims
 
 
+# ── Edge Case Handling: Numerical Tolerance ────────────────────────────────────
+NUMBER_WORDS = {
+    "zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5,
+    "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10,
+    "eleven": 11, "twelve": 12, "thirteen": 13, "fourteen": 14, "fifteen": 15,
+    "twenty": 20, "thirty": 30, "forty": 40, "fifty": 50,
+    "hundred": 100, "thousand": 1000, "million": 1000000,
+    "half": 0.5, "quarter": 0.25, "third": 0.333,
+}
+
+APPROXIMATION_WORDS = {"approximately", "about", "around", "roughly", "nearly", "almost", "close to", "approx."}
+
+
+def normalize_numbers(text: str) -> Set[float]:
+    """
+    Extract and normalize all numbers from text, handling:
+    - Digits: "50" -> 50.0
+    - Words: "fifty" -> 50.0
+    - Percentages: "50%" -> 0.5
+    - Fractions: "1/2" -> 0.5
+    - Units: "50 dollars" -> 50.0
+
+    Returns set of normalized float values.
+    """
+    numbers = set()
+
+    # Extract digit-based numbers
+    digit_nums = re.findall(r'\d+(?:\.\d+)?', text)
+    for n in digit_nums:
+        try:
+            numbers.add(float(n))
+        except ValueError:
+            pass
+
+    # Extract word-based numbers
+    text_lower = text.lower()
+    for word, value in NUMBER_WORDS.items():
+        if word in text_lower:
+            numbers.add(float(value))
+
+    # Extract percentages and normalize to decimals
+    percentages = re.findall(r'(\d+(?:\.\d+)?)\s*%', text)
+    for p in percentages:
+        try:
+            numbers.add(float(p) / 100.0)
+        except ValueError:
+            pass
+
+    # Extract fractions
+    fractions = re.findall(r'(\d+)\s*/\s*(\d+)', text)
+    for num, denom in fractions:
+        try:
+            numbers.add(float(num) / float(denom))
+        except ValueError:
+            pass
+
+    return numbers
+
+
+def numbers_approx_match(a: float, b: float, tolerance: float = 0.1) -> bool:
+    """
+    Check if two numbers match within relative tolerance.
+    Handles cases like "approximately 50" vs "50" or "48".
+    """
+    if a == b:
+        return True
+    max_val = max(abs(a), abs(b), 1e-10)
+    return abs(a - b) / max_val < tolerance
+
+
+def check_numerical_match(answer_nums: Set[float], truth_nums: Set[float], tolerance: float = 0.1) -> Tuple[bool, float]:
+    """
+    Check if answer numbers match truth numbers with tolerance.
+
+    Returns: (is_match, match_score)
+    - is_match: True if all critical numbers match
+    - match_score: 0.0-1.0 indicating match quality
+    """
+    if not truth_nums:
+        # No numbers in ground truth, no penalty
+        return True, 1.0
+
+    if not answer_nums:
+        # Numbers expected but none provided
+        return False, 0.0
+
+    # Check each truth number for approximate match in answer
+    matched = 0
+    for truth_n in truth_nums:
+        for ans_n in answer_nums:
+            if numbers_approx_match(truth_n, ans_n, tolerance):
+                matched += 1
+                break
+
+    match_ratio = matched / len(truth_nums)
+    return match_ratio >= 0.8, match_ratio
+
+
+def detect_hedging(text: str) -> Tuple[bool, float]:
+    """
+    Detect hedging language in answer.
+
+    Returns: (has_hedging, hedging_intensity)
+    - has_hedging: True if hedging detected
+    - hedging_intensity: 0.0-1.0 (higher = more hedging)
+    """
+    text_lower = text.lower()
+
+    hedging_count = 0
+    for phrase in APPROXIMATION_WORDS:
+        if phrase in text_lower:
+            hedging_count += 1
+
+    # Check for modal verbs indicating uncertainty
+    modal_verbs = ["might", "could", "may", "possibly", "perhaps", "seems"]
+    for modal in modal_verbs:
+        if modal in text_lower:
+            hedging_count += 0.5
+
+    intensity = min(1.0, hedging_count / 3.0)
+    return intensity > 0, intensity
+
+
+def handle_ambiguous_answer(
+    answer: str,
+    ground_truth: str,
+    valid_alternatives: List[str] = None
+) -> Tuple[float, str]:
+    """
+    Handle cases where multiple answers may be valid.
+
+    Returns: (score, matched_answer)
+    """
+    # Normalize answer
+    answer_norm = normalize_text(answer)
+    truth_norm = normalize_text(ground_truth)
+
+    # Check primary answer
+    if answer_norm == truth_norm or truth_norm in answer_norm:
+        return 1.0, ground_truth
+
+    # Check alternatives if provided
+    if valid_alternatives:
+        for alt in valid_alternatives:
+            alt_norm = normalize_text(alt)
+            if answer_norm == alt_norm or alt_norm in answer_norm:
+                return 0.95, alt
+
+    # Check semantic similarity
+    similarity = compute_string_similarity(answer, ground_truth)
+    if similarity > 0.8:
+        return similarity, ground_truth
+
+    return 0.0, ""
+
+
 def compute_string_similarity(s1: str, s2: str) -> float:
     """Compute semantic similarity between two strings.
 
@@ -728,6 +884,62 @@ def compute_calibration_error(confidence: float, correctness: float) -> float:
     return min(1.0, base_error)
 
 
+def compute_expected_calibration_error(
+    confidence_history: List[float],
+    correctness_history: List[float],
+    num_bins: int = 10
+) -> float:
+    """
+    Compute Expected Calibration Error (ECE) with confidence binning.
+
+    ECE measures how well-calibrated confidence estimates are across all predictions.
+    Lower ECE = better calibration. Perfect calibration = 0.0.
+
+    Args:
+        confidence_history: List of confidence scores (0-1)
+        correctness_history: List of correctness scores (0-1)
+        num_bins: Number of confidence bins (default 10)
+
+    Returns:
+        ECE score (0-1, lower is better)
+
+    Reference: Guo et al., "On Calibration of Modern Neural Networks" (ICML 2017)
+    """
+    if not confidence_history or not correctness_history:
+        return 0.0
+
+    try:
+        import numpy as np
+        confidence_arr = np.array(confidence_history)
+        correctness_arr = np.array(correctness_history)
+
+        # Create bins
+        bins = np.linspace(0, 1, num_bins + 1)
+        ece = 0.0
+
+        for i in range(num_bins):
+            # Find samples in this bin
+            if i == num_bins - 1:
+                # Include 1.0 in last bin
+                mask = (confidence_arr >= bins[i]) & (confidence_arr <= bins[i + 1])
+            else:
+                mask = (confidence_arr >= bins[i]) & (confidence_arr < bins[i + 1])
+
+            bin_count = mask.sum()
+            if bin_count > 0:
+                bin_confidence = confidence_arr[mask].mean()
+                bin_accuracy = correctness_arr[mask].mean()
+                # Weight by proportion of samples in this bin
+                ece += (bin_count / len(confidence_arr)) * abs(bin_accuracy - bin_confidence)
+
+        return float(min(1.0, ece))
+    except Exception:
+        # Fallback to simple calibration error
+        if len(confidence_history) == 0:
+            return 0.0
+        return sum(abs(c - r) for c, r in zip(confidence_history, correctness_history)) / len(confidence_history)
+
+
 def compute_semantic_consistency(answer: str, context: str, ground_truth: str) -> Tuple[float, Dict[str, Any]]:
     """
     Compute semantic consistency between answer, context, and ground truth.

tests/test_grader.py

@@ -309,4 +309,245 @@ class TestRefusalHandling:
             difficulty_level="beginner"
         )
         assert reward < 0.5
-        assert info.get("is_refusal") == True
+        assert info.get("is_refusal") == True
+
+
+class TestEdgeCases:
+    """Tests for edge cases and robustness."""
+
+    def test_empty_answer_with_high_confidence(self):
+        """Empty answer with high confidence should be heavily penalized."""
+        reward, info = calculate_reward(
+            answer="",
+            confidence=0.95,
+            source_quote="",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        assert reward < 0.3
+        assert info.get("is_hallucination") == True or info.get("correctness", 0) < 0.3
+
+    def test_very_long_answer(self):
+        """Very long answers should be handled gracefully."""
+        long_answer = "Paris is the capital of France. " * 100
+        reward, info = calculate_reward(
+            answer=long_answer,
+            confidence=0.8,
+            source_quote="capital of France is Paris",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        assert 0.0 <= reward <= 1.0
+        assert "correctness" in info
+
+    def test_unicode_handling(self):
+        """Unicode characters should be handled correctly."""
+        reward, info = calculate_reward(
+            answer="Tōkyō has 13 million people.",
+            confidence=0.8,
+            source_quote="Tōkyō population",
+            context="Tōkyō has a population of 13 million people.",
+            ground_truth="13 million",
+            difficulty_level="intermediate"
+        )
+        assert 0.0 <= reward <= 1.0
+
+    def test_numerical_tolerance(self):
+        """Approximate numbers should have some tolerance."""
+        reward1, _ = calculate_reward(
+            answer="The population is approximately 50,000.",
+            confidence=0.8,
+            source_quote="population of 50,000",
+            context="The city has a population of 50,000.",
+            ground_truth="50,000",
+            difficulty_level="beginner"
+        )
+        reward2, _ = calculate_reward(
+            answer="The population is about 50,000.",
+            confidence=0.8,
+            source_quote="population of 50,000",
+            context="The city has a population of 50,000.",
+            ground_truth="50,000",
+            difficulty_level="beginner"
+        )
+        # Both should receive similar scores
+        assert abs(reward1 - reward2) < 0.15
+
+    def test_multi_part_answer(self):
+        """Multi-part answers should be evaluated correctly."""
+        reward, info = calculate_reward(
+            answer="Paris is the capital, and Lyon is the second largest city.",
+            confidence=0.8,
+            source_quote="capital is Paris",
+            context="The capital of France is Paris. Lyon is the second largest city.",
+            ground_truth="Paris",
+            difficulty_level="intermediate"
+        )
+        assert 0.0 <= reward <= 1.0
+        # Should not be penalized for providing extra accurate info
+
+    def test_markdown_in_answer(self):
+        """Markdown formatting should not affect scoring."""
+        reward1, _ = calculate_reward(
+            answer="**Paris** is the capital.",
+            confidence=0.8,
+            source_quote="capital is Paris",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        reward2, _ = calculate_reward(
+            answer="Paris is the capital.",
+            confidence=0.8,
+            source_quote="capital is Paris",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        # Should handle markdown gracefully
+        assert abs(reward1 - reward2) < 0.1
+
+    def test_contradictory_answer(self):
+        """Contradictory answers should be penalized."""
+        reward, info = calculate_reward(
+            answer="The capital is NOT Paris.",
+            confidence=0.9,
+            source_quote="capital of France",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        assert reward < 0.4
+        assert info.get("is_hallucination") == True or info.get("correctness", 1) < 0.5
+
+    def test_hedging_language(self):
+        """Hedging language should affect confidence scoring."""
+        reward_hedged, _ = calculate_reward(
+            answer="The answer might be Paris, approximately.",
+            confidence=0.5,
+            source_quote="capital is Paris",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        reward_confident, _ = calculate_reward(
+            answer="The answer is Paris.",
+            confidence=0.9,
+            source_quote="capital is Paris",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        # Both should be correct, but hedging may affect confidence scoring
+        assert 0.0 <= reward_hedged <= 1.0
+
+    def test_partial_entity_match(self):
+        """Partial entity matches should be scored appropriately."""
+        reward, info = calculate_reward(
+            answer="William Shakespeare wrote it.",
+            confidence=0.8,
+            source_quote="written by Shakespeare",
+            context="The play was written by Shakespeare.",
+            ground_truth="Shakespeare",
+            difficulty_level="beginner"
+        )
+        assert reward >= 0.6  # Should still get credit for correct entity
+
+
+class TestCalibrationMetrics:
+    """Tests for calibration and confidence metrics."""
+
+    def test_well_calibrated_answer(self):
+        """Well-calibrated confidence should score high."""
+        reward, info = calculate_reward(
+            answer="Paris",
+            confidence=0.9,
+            source_quote="capital is Paris",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        assert info["calibration"] >= 0.7
+
+    def test_overconfident_wrong_answer(self):
+        """Overconfident wrong answers should be heavily penalized."""
+        reward_confident, info_confident = calculate_reward(
+            answer="London",
+            confidence=0.95,
+            source_quote="",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        reward_uncertain, _ = calculate_reward(
+            answer="London",
+            confidence=0.3,
+            source_quote="",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        # Overconfident wrong should score lower
+        assert reward_confident < reward_uncertain
+        assert info_confident.get("is_hallucination") == True
+
+    def test_expected_calibration_error(self):
+        """Test ECE computation with history."""
+        from server.grader import compute_expected_calibration_error
+
+        # Perfect calibration
+        confidence_history = [0.9, 0.8, 0.7, 0.6]
+        correctness_history = [0.9, 0.8, 0.7, 0.6]
+        ece = compute_expected_calibration_error(confidence_history, correctness_history)
+        assert ece < 0.1
+
+        # Poor calibration
+        confidence_history = [0.9, 0.9, 0.9, 0.9]
+        correctness_history = [0.3, 0.3, 0.3, 0.3]
+        ece = compute_expected_calibration_error(confidence_history, correctness_history)
+        assert ece > 0.3
+
+
+class TestHallucinationTypes:
+    """Tests for different hallucination types."""
+
+    def test_fabricated_fact_detection(self):
+        """Fabricated facts should be detected."""
+        reward, info = calculate_reward(
+            answer="The moon is made of green cheese.",
+            confidence=0.9,
+            source_quote="",
+            context="The moon orbits Earth.",
+            ground_truth="rock",
+            difficulty_level="advanced"
+        )
+        assert info.get("is_hallucination") == True
+        assert info.get("hallucination_type") in ["fabricated_fact", "overconfident_wrong"]
+
+    def test_numerical_fabrication_detection(self):
+        """Fabricated numbers should be detected."""
+        reward, info = calculate_reward(
+            answer="The population is 500 million.",
+            confidence=0.8,
+            source_quote="population",
+            context="The city has a population of 50,000.",
+            ground_truth="50,000",
+            difficulty_level="intermediate"
+        )
+        # Should detect numerical discrepancy
+        assert info.get("is_hallucination") == True or "number" in str(info.get("correctness_analysis", "")).lower()
+
+    def test_entity_confusion_detection(self):
+        """Entity confusion should be detected."""
+        reward, info = calculate_reward(
+            answer="London is the capital of France.",
+            confidence=0.8,
+            source_quote="capital of France",
+            context="The capital of France is Paris.",
+            ground_truth="Paris",
+            difficulty_level="beginner"
+        )
+        assert info.get("is_hallucination") == True
+        assert reward < 0.5