| 1 | import contextlib |
| 2 | import io |
| 3 | import json |
| 4 | import os |
| 5 | import tempfile |
| 6 | import unittest |
| 7 | from pathlib import Path |
| 8 | from unittest import mock |
| 9 | |
| 10 | import evaluate_search_quality as evaluator |
| 11 | |
| 12 | |
| 13 | class EvaluatorV3Tests(unittest.TestCase): |
| 14 | def test_build_ranked_items_uses_multi_source_provenance_and_best_date(self): |
| 15 | report = { |
| 16 | "ranked_candidates": [ |
| 17 | { |
| 18 | "candidate_id": "c1", |
| 19 | "item_id": "i1", |
| 20 | "source": "grounding", |
| 21 | "sources": ["grounding", "reddit"], |
| 22 | "title": "Title", |
| 23 | "url": "https://example.com", |
| 24 | "snippet": "Snippet", |
| 25 | "subquery_labels": ["primary"], |
| 26 | "native_ranks": {"primary:grounding": 1}, |
| 27 | "local_relevance": 0.8, |
| 28 | "freshness": 90, |
| 29 | "engagement": None, |
| 30 | "source_quality": 1.0, |
| 31 | "rrf_score": 0.02, |
| 32 | "final_score": 88.0, |
| 33 | "source_items": [ |
| 34 | {"item_id": "i1", "source": "grounding", "title": "Title", "body": "Body", "url": "https://example.com", "published_at": "2026-03-10"}, |
| 35 | {"item_id": "i2", "source": "reddit", "title": "Title", "body": "Body", "url": "https://example.com", "published_at": "2026-03-12"}, |
| 36 | ], |
| 37 | } |
| 38 | ] |
| 39 | } |
| 40 | items = evaluator.build_ranked_items(report, 10) |
| 41 | self.assertEqual(["grounding", "reddit"], items[0]["sources"]) |
| 42 | self.assertEqual("grounding, reddit", items[0]["source"]) |
| 43 | self.assertEqual("2026-03-12", items[0]["date"]) |
| 44 | |
| 45 | grouped = evaluator.source_sets(report, 10) |
| 46 | self.assertEqual({"c1"}, grouped["grounding"]) |
| 47 | self.assertEqual({"c1"}, grouped["reddit"]) |
| 48 | |
| 49 | def test_write_failure_summary_persists_failures(self): |
| 50 | with tempfile.TemporaryDirectory() as tmp: |
| 51 | output_dir = Path(tmp) |
| 52 | evaluator.write_failure_summary( |
| 53 | output_dir, |
| 54 | "HEAD~1", |
| 55 | "HEAD", |
| 56 | summaries=[{ |
| 57 | "topic": "test topic", |
| 58 | "baseline": {"precision_at_5": 0.5, "ndcg_at_5": 0.6, "source_coverage_recall": 1.0}, |
| 59 | "candidate": {"precision_at_5": 0.7, "ndcg_at_5": 0.8, "source_coverage_recall": 1.0}, |
| 60 | "stability": {"overall_jaccard": 0.4, "overall_retention_vs_baseline": 0.9}, |
| 61 | }], |
| 62 | failures=[{"topic": "broken topic", "error": "timeout"}], |
| 63 | ) |
| 64 | metrics = json.loads((output_dir / "metrics.json").read_text()) |
| 65 | summary = (output_dir / "summary.md").read_text() |
| 66 | self.assertEqual(1, len(metrics["failures"])) |
| 67 | self.assertIn("broken topic", summary) |
| 68 | self.assertIn("## Failures", summary) |
| 69 | |
| 70 | def test_resolve_repo_dir_keeps_live_worktree(self): |
| 71 | repo_dir, is_temp = evaluator.resolve_repo_dir("WORKTREE") |
| 72 | self.assertEqual(evaluator.REPO_ROOT, repo_dir) |
| 73 | self.assertFalse(is_temp) |
| 74 | |
| 75 | def test_resolve_repo_dir_materializes_git_ref_in_temp_worktree(self): |
| 76 | fake_dir = Path("/tmp/last30days-eval-fake") |
| 77 | with mock.patch.object(evaluator, "create_worktree", return_value=fake_dir) as create_worktree: |
| 78 | repo_dir, is_temp = evaluator.resolve_repo_dir("HEAD~2") |
| 79 | create_worktree.assert_called_once_with("HEAD~2") |
| 80 | self.assertEqual(fake_dir, repo_dir) |
| 81 | self.assertTrue(is_temp) |
| 82 | |
| 83 | def test_metric_helpers_cover_empty_and_ranked_cases(self): |
| 84 | ranking = [ |
| 85 | {"key": "a", "sources": ["grounding"]}, |
| 86 | {"key": "b", "sources": ["reddit"]}, |
| 87 | ] |
| 88 | judged = [{"key": "a", "sources": ["grounding"]}, {"key": "b", "sources": ["reddit"]}] |
| 89 | judgments = {"a": 3, "b": 1} |
| 90 | |
| 91 | self.assertEqual(1.0, evaluator.jaccard(set(), set())) |
| 92 | self.assertEqual(1.0, evaluator.retention(set(), {"a"})) |
| 93 | self.assertEqual(0.5, evaluator.precision_at_k(ranking, judgments, 2)) |
| 94 | self.assertGreater(evaluator.ndcg_at_k(ranking, judgments, 2, judged), 0.0) |
| 95 | self.assertEqual(1.0, evaluator.source_coverage_recall(ranking, judged, judgments)) |
| 96 | self.assertEqual(0.0, evaluator.precision_at_k([], judgments, 5)) |
| 97 | self.assertEqual(0.0, evaluator.ndcg_at_k([], judgments, 5, judged)) |
| 98 | |
| 99 | def test_resolve_google_judge_api_key_prefers_google_key(self): |
| 100 | with mock.patch.dict("os.environ", {"GOOGLE_API_KEY": "google", "GEMINI_API_KEY": "gemini"}, clear=False): |
| 101 | self.assertEqual("google", evaluator.resolve_google_judge_api_key({})) |
| 102 | with mock.patch.dict("os.environ", {k: "" for k in ("GOOGLE_API_KEY", "GEMINI_API_KEY", "GOOGLE_GENAI_API_KEY")}, clear=False): |
| 103 | for k in ("GOOGLE_API_KEY", "GEMINI_API_KEY", "GOOGLE_GENAI_API_KEY"): |
| 104 | os.environ.pop(k, None) |
| 105 | self.assertEqual("fallback", evaluator.resolve_google_judge_api_key({"GOOGLE_GENAI_API_KEY": "fallback"})) |
| 106 | |
| 107 | def test_extract_gemini_text_raises_when_missing(self): |
| 108 | self.assertEqual( |
| 109 | "hello", |
| 110 | evaluator.extract_gemini_text({"candidates": [{"content": {"parts": [{"text": "hello"}]}}]}), |
| 111 | ) |
| 112 | with self.assertRaises(ValueError): |
| 113 | evaluator.extract_gemini_text({"candidates": [{"content": {"parts": [{}]}}]}) |
| 114 | |
| 115 | def test_get_judgments_uses_cache_and_skips_when_not_configured(self): |
| 116 | with tempfile.TemporaryDirectory() as tmp: |
| 117 | output_dir = Path(tmp) |
| 118 | cache_dir = output_dir / "judgments" |
| 119 | cache_dir.mkdir() |
| 120 | (cache_dir / "topic.json").write_text( |
| 121 | json.dumps( |
| 122 | { |
| 123 | "judge_model": "gemini-3.1-flash-lite", |
| 124 | "judgments": [{"id": "a", "grade": 3}], |
| 125 | } |
| 126 | ) |
| 127 | ) |
| 128 | cached = evaluator.get_judgments( |
| 129 | output_dir=output_dir, |
| 130 | slug="topic", |
| 131 | topic="test topic", |
| 132 | query_type="general", |
| 133 | items=[{"key": "a"}], |
| 134 | judge_model="gemini-3.1-flash-lite", |
| 135 | gemini_api_key="key", |
| 136 | ) |
| 137 | self.assertEqual({"a": 3}, cached) |
| 138 | |
| 139 | skipped = evaluator.get_judgments( |
| 140 | output_dir=output_dir, |
| 141 | slug="fresh", |
| 142 | topic="test topic", |
| 143 | query_type="general", |
| 144 | items=[], |
| 145 | judge_model="gemini-3.1-flash-lite", |
| 146 | gemini_api_key=None, |
| 147 | ) |
| 148 | self.assertEqual({}, skipped) |
| 149 | |
| 150 | def test_get_judgments_remisses_on_judge_model_change(self): |
| 151 | """A cache written by a different judge model must not be reused; a |
| 152 | --judge-model change forces a re-judge instead of returning stale grades.""" |
| 153 | with tempfile.TemporaryDirectory() as tmp: |
| 154 | output_dir = Path(tmp) |
| 155 | cache_dir = output_dir / "judgments" |
| 156 | cache_dir.mkdir() |
| 157 | (cache_dir / "topic.json").write_text( |
| 158 | json.dumps( |
| 159 | { |
| 160 | "judge_model": "gemini-3.1-flash-lite", |
| 161 | "judgments": [{"id": "a", "grade": 3}], |
| 162 | } |
| 163 | ) |
| 164 | ) |
| 165 | # Same slug, different model, no API key to re-judge: the stale |
| 166 | # grades must NOT come back — an empty result signals "re-judge |
| 167 | # needed" rather than silently wrong numbers, and the discard is |
| 168 | # announced on stderr instead of failing silently. |
| 169 | stderr = io.StringIO() |
| 170 | with contextlib.redirect_stderr(stderr): |
| 171 | result = evaluator.get_judgments( |
| 172 | output_dir=output_dir, |
| 173 | slug="topic", |
| 174 | topic="test topic", |
| 175 | query_type="general", |
| 176 | items=[{"key": "a"}], |
| 177 | judge_model="gemini-2.5-pro", |
| 178 | gemini_api_key=None, |
| 179 | ) |
| 180 | self.assertEqual({}, result) |
| 181 | self.assertIn("different", stderr.getvalue()) |
| 182 | |
| 183 | def test_create_eval_env_and_run_last30days(self): |
| 184 | credential_env = { |
| 185 | key: "" |
| 186 | for key in evaluator.EVAL_CREDENTIAL_ENV_KEYS |
| 187 | } |
| 188 | credential_env.update({"PATH": "/bin", "GOOGLE_API_KEY": "env-google"}) |
| 189 | with mock.patch.object(evaluator.envlib, "get_config", return_value={"OPENAI_API_KEY": "config-openai"}): |
| 190 | with mock.patch.dict("os.environ", credential_env, clear=False): |
| 191 | created = evaluator.create_eval_env() |
| 192 | self.assertEqual("/bin", created["PATH"]) |
| 193 | self.assertEqual("env-google", created["GOOGLE_API_KEY"]) |
| 194 | self.assertEqual("config-openai", created["OPENAI_API_KEY"]) |
| 195 | self.assertEqual("", created["LAST30DAYS_CONFIG_DIR"]) |
| 196 | |
| 197 | with tempfile.TemporaryDirectory() as tmp: |
| 198 | repo_dir = Path(tmp) |
| 199 | engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py" |
| 200 | engine.parent.mkdir(parents=True) |
| 201 | engine.write_text('parser.add_argument("--json-profile")') |
| 202 | completed = mock.Mock(returncode=0, stdout='{"topic":"x"}', stderr="") |
| 203 | with mock.patch.object(evaluator.subprocess, "run", return_value=completed) as run: |
| 204 | payload = evaluator.run_last30days( |
| 205 | repo_dir, |
| 206 | "topic", |
| 207 | search="reddit", |
| 208 | timeout_seconds=30, |
| 209 | quick=True, |
| 210 | mock=True, |
| 211 | env={"PATH": "/bin"}, |
| 212 | ) |
| 213 | self.assertEqual("x", payload["topic"]) |
| 214 | self.assertIn("--json-profile=raw", run.call_args.args[0]) |
| 215 | |
| 216 | with mock.patch.object(evaluator.subprocess, "run", return_value=mock.Mock(returncode=2, stdout="", stderr="bad run")): |
| 217 | with self.assertRaises(RuntimeError): |
| 218 | evaluator.run_last30days( |
| 219 | Path("/tmp/repo"), |
| 220 | "topic", |
| 221 | search="reddit", |
| 222 | timeout_seconds=30, |
| 223 | quick=False, |
| 224 | mock=False, |
| 225 | env={"PATH": "/bin"}, |
| 226 | ) |
| 227 | |
| 228 | def test_run_last30days_keeps_legacy_engine_implicit_raw_profile(self): |
| 229 | with tempfile.TemporaryDirectory() as tmp: |
| 230 | repo_dir = Path(tmp) |
| 231 | engine = repo_dir / "skills" / "last30days" / "scripts" / "last30days.py" |
| 232 | engine.parent.mkdir(parents=True) |
| 233 | engine.write_text('parser.add_argument("--emit")') |
| 234 | completed = mock.Mock(returncode=0, stdout='{"topic":"x"}', stderr="") |
| 235 | with mock.patch.object(evaluator.subprocess, "run", return_value=completed) as run: |
| 236 | evaluator.run_last30days( |
| 237 | repo_dir, |
| 238 | "topic", |
| 239 | search="reddit", |
| 240 | timeout_seconds=30, |
| 241 | quick=False, |
| 242 | mock=False, |
| 243 | env={"PATH": "/bin"}, |
| 244 | ) |
| 245 | |
| 246 | self.assertNotIn("--json-profile=raw", run.call_args.args[0]) |
| 247 | |
| 248 | def test_parse_topics_file_and_summary_writer(self): |
| 249 | with tempfile.TemporaryDirectory() as tmp: |
| 250 | tmp_path = Path(tmp) |
| 251 | topics_path = tmp_path / "topics.json" |
| 252 | topics_path.write_text(json.dumps([{"topic": "topic a", "query_type": "comparison"}, {"topic": "topic b"}])) |
| 253 | self.assertEqual( |
| 254 | [("topic a", "comparison"), ("topic b", "general")], |
| 255 | evaluator.parse_topics_file(topics_path), |
| 256 | ) |
| 257 | |
| 258 | evaluator.write_summary( |
| 259 | tmp_path, |
| 260 | "HEAD~1", |
| 261 | "WORKTREE", |
| 262 | [ |
| 263 | { |
| 264 | "topic": "topic a", |
| 265 | "baseline": {"precision_at_5": 0.1, "ndcg_at_5": 0.2, "source_coverage_recall": 0.5}, |
| 266 | "candidate": {"precision_at_5": 0.3, "ndcg_at_5": 0.4, "source_coverage_recall": 0.8}, |
| 267 | "stability": {"overall_jaccard": 0.6, "overall_retention_vs_baseline": 0.7}, |
| 268 | } |
| 269 | ], |
| 270 | ) |
| 271 | summary = (tmp_path / "summary.md").read_text() |
| 272 | metrics = json.loads((tmp_path / "metrics.json").read_text()) |
| 273 | self.assertIn("| topic a | 0.10 | 0.30 |", summary) |
| 274 | self.assertEqual("HEAD~1", metrics["baseline"]) |
| 275 | |
| 276 | if __name__ == "__main__": |
| 277 | unittest.main() |
| 278 |