"""Hermetic tests for the eval farm intake (issue #8 acceptance). MockGitea (in-memory repos/issues/comments/labels), a scripted emitter (retry/backoff verified via call counts), and a real BillingCore on a tmp DB (funded vs unfunded gating is real). No network, no sleeps (a noop ``sleep_fn`` is injected everywhere). Run: python3 -m pytest tests/test_intake_eval.py -q """ import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import pytest from gitea_saas.billing_core import BillingCore from gitea_saas.intake_eval import EvalIntake, EvalIntakeError, EVAL_LABEL class Config: """Tiny config object for the intake (db_path + optional overrides).""" def __init__(self, db_path, **kwargs): self.db_path = Path(db_path) for k, v in kwargs.items(): setattr(self, k, v) class MockGitea: """In-memory Gitea seam recording every mutation.""" def __init__(self): self.repos = {} # repo -> {"issues": {n: dict}, "comments": {n: [str]}} self.next_number = 0 # global issue numbering self.created_repos = [] # repo names created self.created_issues = [] # (repo, title, body, labels) def _repo(self, name): if name not in self.repos: self.repos[name] = {"issues": {}, "comments": {}} return self.repos[name] def repo_exists(self, name): return name in self.repos def create_org_repo(self, name): self._repo(name) self.created_repos.append(name) def create_issue(self, repo, title, body, labels=None): r = self._repo(repo) self.next_number += 1 n = self.next_number r["issues"][n] = {"title": title, "body": body, "state": "open", "labels": list(labels or [])} r["comments"][n] = [] self.created_issues.append((repo, title, body, list(labels or []))) # Canonical client contract (issue #4/!17): returns the issue number. return n def get_issue(self, repo, number): r = self._repo(repo) issue = r["issues"].get(number) if issue is None: return None return dict(issue) def list_issue_comments(self, repo, number): return [{"body": b} for b in self._repo(repo)["comments"].get(number, [])] # -- test helpers: simulate the lane draining a task ------------------- def add_comment(self, repo, number, body): self._repo(repo)["comments"].get(number, []).append(body) def close_issue(self, repo, number): self._repo(repo)["issues"][number]["state"] = "closed" class MockEmitter: """Scripted emitter: fails the first ``fail_first`` pushes.""" def __init__(self, fail_first=0): self.fail_first = fail_first self.push_count = 0 self.push_payloads = [] # (report, url) def push(self, report, url): self.push_count += 1 if self.push_count <= self.fail_first: return False self.push_payloads.append((report, url)) return True def noop_sleep(_seconds): return None @pytest.fixture def funded(tmp_path): core = BillingCore(tmp_path / "billing.db") ok, c = core.create_client("eval-client") assert ok ok, _ = core.credit(c["id"], 10_000, "fund-1") assert ok return core, c["id"] @pytest.fixture def unfunded(tmp_path): core = BillingCore(tmp_path / "billing.db") ok, c = core.create_client("poor-client") assert ok return core, c["id"] def make_intake(core, tmp_path, gitea=None, emitter=None, **config_kwargs): gitea = gitea or MockGitea() cfg = Config(tmp_path / "saas.db", **config_kwargs) return EvalIntake(core, gitea, cfg, emitter=emitter, sleep_fn=noop_sleep) TASKS = [ {"task_id": "t1", "title": "Eval task one", "body": "## Spec\nrun X\n## Acceptance\n- [ ] a\n- [ ] b"}, {"task_id": "t2", "title": "Eval task two", "body": "## Spec\nrun Y"}, ] # -------------------------------------------------------------------------- # # Submission # # -------------------------------------------------------------------------- # class TestSubmitBatch: def test_funded_batch_creates_one_issue_per_task(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() intake = make_intake(core, tmp_path, gitea) r = intake.submit_batch(client_id, TASKS) assert r["ok"] is True assert r["batch_id"] > 0 assert r["repo"] == "eval-runs" assert [i["task_id"] for i in r["issues"]] == ["t1", "t2"] # shared repo exists; two issues; each carries the eval-task label assert "eval-runs" in gitea.created_repos assert len(gitea.created_issues) == 2 for repo, title, body, labels in gitea.created_issues: assert repo == "eval-runs" assert labels == [EVAL_LABEL] assert f"batch: {r['batch_id']}" in body # both tasks persisted as pending batch = intake.get_batch(r["batch_id"]) assert batch["ok"] and len(batch["tasks"]) == 2 assert all(t["status"] == "pending" for t in batch["tasks"]) assert batch["batch"]["status"] == "open" def test_unfunded_batch_402_and_no_gitea_mutations(self, unfunded, tmp_path): core, client_id = unfunded gitea = MockGitea() intake = make_intake(core, tmp_path, gitea) r = intake.submit_batch(client_id, TASKS) assert r["ok"] is False assert r["reason"] == "unfunded" assert r["status"] == 402 assert gitea.created_issues == [] # no issues created assert gitea.created_repos == [] # no repo created either assert gitea.next_number == 0 def test_unknown_client_is_fail_closed_unfunded(self, tmp_path): core = BillingCore(tmp_path / "billing.db") gitea = MockGitea() intake = make_intake(core, tmp_path, gitea) r = intake.submit_batch(9999, TASKS) assert r["ok"] is False and r["reason"] == "unfunded" assert r["status"] == 402 assert gitea.created_issues == [] def test_duplicate_task_id_rejected(self, funded, tmp_path): core, client_id = funded intake = make_intake(core, tmp_path) with pytest.raises(EvalIntakeError): intake.submit_batch(client_id, [TASKS[0], dict(TASKS[0])]) def test_intake_failure_creates_no_partial_batch(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() class FlakyGitea(MockGitea): def create_issue(self, *a, **kw): if self.next_number >= 1: # second issue fails raise RuntimeError("gitea hiccup") return super().create_issue(*a, **kw) intake = make_intake(core, tmp_path, FlakyGitea()) r = intake.submit_batch(client_id, TASKS) assert r["ok"] is False and r["reason"] == "intake_failed" # no batch row left behind assert intake.get_batch(1)["ok"] is False # -------------------------------------------------------------------------- # # Polling # # -------------------------------------------------------------------------- # class TestPollCompleted: def test_poll_stores_closed_results_and_skips_open(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() intake = make_intake(core, tmp_path, gitea) r = intake.submit_batch(client_id, TASKS) n1 = r["issues"][0]["issue_number"] gitea.add_comment("eval-runs", n1, "lane report: all checks green") gitea.close_issue("eval-runs", n1) # task 1 drained; task 2 still open out = intake.poll_completed(r["batch_id"]) assert out["done"] == 1 and out["pending"] == 1 batch = intake.get_batch(r["batch_id"]) done_task = [t for t in batch["tasks"] if t["task_id"] == "t1"][0] pending_task = [t for t in batch["tasks"] if t["task_id"] == "t2"][0] assert done_task["status"] == "done" result = done_task["result_json"] assert result["summary"] == "lane report: all checks green" assert result["labels"] == [EVAL_LABEL] assert pending_task["status"] == "pending" assert batch["batch"]["status"] == "open" # not all done yet def test_poll_marks_batch_done_when_all_closed(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() intake = make_intake(core, tmp_path, gitea) r = intake.submit_batch(client_id, TASKS) for issue in r["issues"]: gitea.add_comment("eval-runs", issue["issue_number"], "ok") gitea.close_issue("eval-runs", issue["issue_number"]) out = intake.poll_completed(r["batch_id"]) assert out["done"] == 2 and out["pending"] == 0 assert intake.get_batch(r["batch_id"])["batch"]["status"] == "done" def test_poll_unknown_batch(self, funded, tmp_path): core, client_id = funded intake = make_intake(core, tmp_path) out = intake.poll_completed(123) assert out["ok"] is False and out["error"] == "unknown_batch" # -------------------------------------------------------------------------- # # Emitting # # -------------------------------------------------------------------------- # class TestEmit: def test_emit_posts_report_to_callback_with_retry(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() # fails twice (500s), succeeds on the third attempt emitter = MockEmitter(fail_first=2) intake = make_intake(core, tmp_path, gitea, emitter=emitter) r = intake.submit_batch(client_id, TASKS, callback_url="https://client.example/hook") for issue in r["issues"]: gitea.add_comment("eval-runs", issue["issue_number"], "done: all checks passed") gitea.close_issue("eval-runs", issue["issue_number"]) intake.poll_completed(r["batch_id"]) out = intake.emit(r["batch_id"]) assert out["ok"] is True and out["emitted"] is True assert emitter.push_count == 3 # retry x3 total assert len(emitter.push_payloads) == 1 # delivered once report, url = emitter.push_payloads[0] assert url == "https://client.example/hook" assert report["batch_id"] == r["batch_id"] assert report["client_id"] == client_id assert [res["task_id"] for res in report["results"]] == ["t1", "t2"] assert all(res["status"] == "done" for res in report["results"]) assert all("summary" in res for res in report["results"]) assert intake.get_batch(r["batch_id"])["batch"]["status"] == "emitted" def test_emit_file_mode_writes_json_drop(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() intake = make_intake(core, tmp_path, gitea, output_dir=tmp_path / "drops") r = intake.submit_batch(client_id, TASKS) # no callback_url for issue in r["issues"]: gitea.close_issue("eval-runs", issue["issue_number"]) intake.poll_completed(r["batch_id"]) out = intake.emit(r["batch_id"]) assert out["ok"] is True and out["emitted"] is True drop = tmp_path / "drops" / f"{r['batch_id']}.json" assert drop.exists() payload = json.loads(drop.read_text(encoding="utf-8")) assert payload["batch_id"] == r["batch_id"] assert len(payload["results"]) == 2 def test_emit_refuses_before_all_tasks_done(self, funded, tmp_path): core, client_id = funded intake = make_intake(core, tmp_path) r = intake.submit_batch(client_id, TASKS) out = intake.emit(r["batch_id"]) assert out["ok"] is False and out["reason"] == "tasks_pending" assert intake.get_batch(r["batch_id"])["batch"]["status"] == "open" def test_reemit_is_idempotent_no_double_post(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() emitter = MockEmitter() intake = make_intake(core, tmp_path, gitea, emitter=emitter) r = intake.submit_batch(client_id, TASKS, callback_url="https://cb.example/x") for issue in r["issues"]: gitea.close_issue("eval-runs", issue["issue_number"]) intake.poll_completed(r["batch_id"]) assert intake.emit(r["batch_id"])["ok"] is True pushes_after_first = emitter.push_count again = intake.emit(r["batch_id"]) assert again["ok"] is True and again.get("idempotent") is True assert emitter.push_count == pushes_after_first # no second POST def test_callback_failure_keeps_batch_emittable(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() emitter = MockEmitter(fail_first=99) # always fails intake = make_intake(core, tmp_path, gitea, emitter=emitter) r = intake.submit_batch(client_id, TASKS, callback_url="https://cb.example/x") for issue in r["issues"]: gitea.close_issue("eval-runs", issue["issue_number"]) intake.poll_completed(r["batch_id"]) out = intake.emit(r["batch_id"]) assert out["ok"] is False and out["reason"] == "emit_failed" assert out["attempts"] == 3 assert intake.get_batch(r["batch_id"])["batch"]["status"] == "done" # -------------------------------------------------------------------------- # # Persistence + config # # -------------------------------------------------------------------------- # class TestPersistence: def test_batch_survives_intake_reopen(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() intake = make_intake(core, tmp_path, gitea) r = intake.submit_batch(client_id, TASKS) intake2 = make_intake(core, tmp_path, gitea) # fresh connection batch = intake2.get_batch(r["batch_id"]) assert batch["ok"] and batch["batch"]["client_id"] == client_id assert len(batch["tasks"]) == 2 def test_shared_repo_name_from_config(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() intake = make_intake(core, tmp_path, gitea, eval_repo="qa-runs") r = intake.submit_batch(client_id, TASKS) assert r["repo"] == "qa-runs" assert "qa-runs" in gitea.created_repos def test_per_batch_repo_mode(self, funded, tmp_path): core, client_id = funded gitea = MockGitea() intake = make_intake(core, tmp_path, gitea, per_batch_repos=True) r = intake.submit_batch(client_id, TASKS) assert r["repo"] == f"eval-{client_id}-{r['batch_id']}" assert r["repo"] in gitea.created_repos def test_real_client_list_issue_comments_builds_correct_request(monkeypatch): """The REAL GiteaClient (not the in-memory fake) must request GET /api/v1/repos/{owner}/{repo}/issues/{n}/comments with the token. This guards the exact bug class the !20 bounce caught: the fake carried list_issue_comments while the real (canonical, issue #4) client did not, and the suite stayed green because nothing exercised the real HTTP path. Network-free: urlopen is captured, never invoked. NOTE: ``gitea_saas/gitea_client.py`` is owned by issue #4 (!17), not #8; this branch must not fork it. If the canonical client has not yet grown ``list_issue_comments`` (extension requested on #4), this guard skips rather than failing — it re-activates automatically once !17 lands. """ try: from gitea_saas.gitea_client import GiteaClient except ImportError: pytest.skip("gitea_saas/gitea_client.py ships via issue #4 (!17); " "not present until !17 lands") if not hasattr(GiteaClient, "list_issue_comments"): pytest.skip("canonical client (issue #4) lacks list_issue_comments; " "extension requested on #4 — guard activates once landed") captured = {} class FakeResponse: status = 200 def __enter__(self): return self def __exit__(self, *exc): return False def read(self): return b'[{"body": "lane report", "id": 1}]' def fake_urlopen(req, timeout=None): captured["req"] = req return FakeResponse() monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) client = GiteaClient(base_url="https://gitea.example", token="tok123", owner="pbwraith") comments = client.list_issue_comments("eval-runs", 42) assert comments == [{"body": "lane report", "id": 1}] req = captured["req"] assert req.method == "GET" assert req.full_url == ( "https://gitea.example/api/v1/repos/pbwraith/eval-runs/" "issues/42/comments") assert req.headers.get("Authorization") == "token tok123" assert req.headers.get("Accept") == "application/json"