b30a358ff2
Bounce #2 (2026-08-25 22:38): gitea_saas/gitea_client.py is not in #8's Files-allowed set. It was reassigned to issue #4 (sole owner) by !23 which closed #21. Removed from this branch entirely; the import resolves from main once !17 lands (reviewer: "only who ships the module changes"). The canonical client (issue-4 branch) is missing list_issue_comments — the exact 8-line delta the bounce predicted. Per bounce instruction #3, that is NOT forked here; the missing method is being requested on issue #4. test_real_client_list_issue_comments_builds_correct_request now skips with an explicit reason until the canonical client grows the method, so the gate stays green and the guard re-activates automatically once !17 lands.
423 lines
17 KiB
Python
423 lines
17 KiB
Python
"""Hermetic tests for the eval farm intake (issue #8 acceptance).
|
|
|
|
MockGitea (in-memory repos/issues/comments/labels), a scripted emitter
|
|
(retry/backoff verified via call counts), and a real BillingCore on a tmp
|
|
DB (funded vs unfunded gating is real). No network, no sleeps (a noop
|
|
``sleep_fn`` is injected everywhere).
|
|
|
|
Run: python3 -m pytest tests/test_intake_eval.py -q
|
|
"""
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
import pytest
|
|
|
|
from gitea_saas.billing_core import BillingCore
|
|
from gitea_saas.intake_eval import EvalIntake, EvalIntakeError, EVAL_LABEL
|
|
|
|
|
|
class Config:
|
|
"""Tiny config object for the intake (db_path + optional overrides)."""
|
|
|
|
def __init__(self, db_path, **kwargs):
|
|
self.db_path = Path(db_path)
|
|
for k, v in kwargs.items():
|
|
setattr(self, k, v)
|
|
|
|
|
|
class MockGitea:
|
|
"""In-memory Gitea seam recording every mutation."""
|
|
|
|
def __init__(self):
|
|
self.repos = {} # repo -> {"issues": {n: dict}, "comments": {n: [str]}}
|
|
self.next_number = 0 # global issue numbering
|
|
self.created_repos = [] # repo names created
|
|
self.created_issues = [] # (repo, title, body, labels)
|
|
|
|
def _repo(self, name):
|
|
if name not in self.repos:
|
|
self.repos[name] = {"issues": {}, "comments": {}}
|
|
return self.repos[name]
|
|
|
|
def repo_exists(self, name):
|
|
return name in self.repos
|
|
|
|
def create_org_repo(self, name):
|
|
self._repo(name)
|
|
self.created_repos.append(name)
|
|
|
|
def create_issue(self, repo, title, body, labels=None):
|
|
r = self._repo(repo)
|
|
self.next_number += 1
|
|
n = self.next_number
|
|
r["issues"][n] = {"title": title, "body": body,
|
|
"state": "open", "labels": list(labels or [])}
|
|
r["comments"][n] = []
|
|
self.created_issues.append((repo, title, body, list(labels or [])))
|
|
# Canonical client contract (issue #4/!17): returns the issue number.
|
|
return n
|
|
|
|
def get_issue(self, repo, number):
|
|
r = self._repo(repo)
|
|
issue = r["issues"].get(number)
|
|
if issue is None:
|
|
return None
|
|
return dict(issue)
|
|
|
|
def list_issue_comments(self, repo, number):
|
|
return [{"body": b} for b in self._repo(repo)["comments"].get(number, [])]
|
|
|
|
# -- test helpers: simulate the lane draining a task -------------------
|
|
def add_comment(self, repo, number, body):
|
|
self._repo(repo)["comments"].get(number, []).append(body)
|
|
|
|
def close_issue(self, repo, number):
|
|
self._repo(repo)["issues"][number]["state"] = "closed"
|
|
|
|
|
|
class MockEmitter:
|
|
"""Scripted emitter: fails the first ``fail_first`` pushes."""
|
|
|
|
def __init__(self, fail_first=0):
|
|
self.fail_first = fail_first
|
|
self.push_count = 0
|
|
self.push_payloads = [] # (report, url)
|
|
|
|
def push(self, report, url):
|
|
self.push_count += 1
|
|
if self.push_count <= self.fail_first:
|
|
return False
|
|
self.push_payloads.append((report, url))
|
|
return True
|
|
|
|
|
|
def noop_sleep(_seconds):
|
|
return None
|
|
|
|
|
|
@pytest.fixture
|
|
def funded(tmp_path):
|
|
core = BillingCore(tmp_path / "billing.db")
|
|
ok, c = core.create_client("eval-client")
|
|
assert ok
|
|
ok, _ = core.credit(c["id"], 10_000, "fund-1")
|
|
assert ok
|
|
return core, c["id"]
|
|
|
|
|
|
@pytest.fixture
|
|
def unfunded(tmp_path):
|
|
core = BillingCore(tmp_path / "billing.db")
|
|
ok, c = core.create_client("poor-client")
|
|
assert ok
|
|
return core, c["id"]
|
|
|
|
|
|
def make_intake(core, tmp_path, gitea=None, emitter=None, **config_kwargs):
|
|
gitea = gitea or MockGitea()
|
|
cfg = Config(tmp_path / "saas.db", **config_kwargs)
|
|
return EvalIntake(core, gitea, cfg, emitter=emitter, sleep_fn=noop_sleep)
|
|
|
|
|
|
TASKS = [
|
|
{"task_id": "t1", "title": "Eval task one",
|
|
"body": "## Spec\nrun X\n## Acceptance\n- [ ] a\n- [ ] b"},
|
|
{"task_id": "t2", "title": "Eval task two", "body": "## Spec\nrun Y"},
|
|
]
|
|
|
|
|
|
# -------------------------------------------------------------------------- #
|
|
# Submission #
|
|
# -------------------------------------------------------------------------- #
|
|
|
|
class TestSubmitBatch:
|
|
def test_funded_batch_creates_one_issue_per_task(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea)
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
assert r["ok"] is True
|
|
assert r["batch_id"] > 0
|
|
assert r["repo"] == "eval-runs"
|
|
assert [i["task_id"] for i in r["issues"]] == ["t1", "t2"]
|
|
# shared repo exists; two issues; each carries the eval-task label
|
|
assert "eval-runs" in gitea.created_repos
|
|
assert len(gitea.created_issues) == 2
|
|
for repo, title, body, labels in gitea.created_issues:
|
|
assert repo == "eval-runs"
|
|
assert labels == [EVAL_LABEL]
|
|
assert f"batch: {r['batch_id']}" in body
|
|
# both tasks persisted as pending
|
|
batch = intake.get_batch(r["batch_id"])
|
|
assert batch["ok"] and len(batch["tasks"]) == 2
|
|
assert all(t["status"] == "pending" for t in batch["tasks"])
|
|
assert batch["batch"]["status"] == "open"
|
|
|
|
def test_unfunded_batch_402_and_no_gitea_mutations(self, unfunded, tmp_path):
|
|
core, client_id = unfunded
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea)
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
assert r["ok"] is False
|
|
assert r["reason"] == "unfunded"
|
|
assert r["status"] == 402
|
|
assert gitea.created_issues == [] # no issues created
|
|
assert gitea.created_repos == [] # no repo created either
|
|
assert gitea.next_number == 0
|
|
|
|
def test_unknown_client_is_fail_closed_unfunded(self, tmp_path):
|
|
core = BillingCore(tmp_path / "billing.db")
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea)
|
|
r = intake.submit_batch(9999, TASKS)
|
|
assert r["ok"] is False and r["reason"] == "unfunded"
|
|
assert r["status"] == 402
|
|
assert gitea.created_issues == []
|
|
|
|
def test_duplicate_task_id_rejected(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
intake = make_intake(core, tmp_path)
|
|
with pytest.raises(EvalIntakeError):
|
|
intake.submit_batch(client_id, [TASKS[0], dict(TASKS[0])])
|
|
|
|
def test_intake_failure_creates_no_partial_batch(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
|
|
class FlakyGitea(MockGitea):
|
|
def create_issue(self, *a, **kw):
|
|
if self.next_number >= 1: # second issue fails
|
|
raise RuntimeError("gitea hiccup")
|
|
return super().create_issue(*a, **kw)
|
|
|
|
intake = make_intake(core, tmp_path, FlakyGitea())
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
assert r["ok"] is False and r["reason"] == "intake_failed"
|
|
# no batch row left behind
|
|
assert intake.get_batch(1)["ok"] is False
|
|
|
|
|
|
# -------------------------------------------------------------------------- #
|
|
# Polling #
|
|
# -------------------------------------------------------------------------- #
|
|
|
|
class TestPollCompleted:
|
|
def test_poll_stores_closed_results_and_skips_open(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea)
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
n1 = r["issues"][0]["issue_number"]
|
|
gitea.add_comment("eval-runs", n1, "lane report: all checks green")
|
|
gitea.close_issue("eval-runs", n1) # task 1 drained; task 2 still open
|
|
out = intake.poll_completed(r["batch_id"])
|
|
assert out["done"] == 1 and out["pending"] == 1
|
|
batch = intake.get_batch(r["batch_id"])
|
|
done_task = [t for t in batch["tasks"] if t["task_id"] == "t1"][0]
|
|
pending_task = [t for t in batch["tasks"] if t["task_id"] == "t2"][0]
|
|
assert done_task["status"] == "done"
|
|
result = done_task["result_json"]
|
|
assert result["summary"] == "lane report: all checks green"
|
|
assert result["labels"] == [EVAL_LABEL]
|
|
assert pending_task["status"] == "pending"
|
|
assert batch["batch"]["status"] == "open" # not all done yet
|
|
|
|
def test_poll_marks_batch_done_when_all_closed(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea)
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
for issue in r["issues"]:
|
|
gitea.add_comment("eval-runs", issue["issue_number"], "ok")
|
|
gitea.close_issue("eval-runs", issue["issue_number"])
|
|
out = intake.poll_completed(r["batch_id"])
|
|
assert out["done"] == 2 and out["pending"] == 0
|
|
assert intake.get_batch(r["batch_id"])["batch"]["status"] == "done"
|
|
|
|
def test_poll_unknown_batch(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
intake = make_intake(core, tmp_path)
|
|
out = intake.poll_completed(123)
|
|
assert out["ok"] is False and out["error"] == "unknown_batch"
|
|
|
|
|
|
# -------------------------------------------------------------------------- #
|
|
# Emitting #
|
|
# -------------------------------------------------------------------------- #
|
|
|
|
class TestEmit:
|
|
def test_emit_posts_report_to_callback_with_retry(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
# fails twice (500s), succeeds on the third attempt
|
|
emitter = MockEmitter(fail_first=2)
|
|
intake = make_intake(core, tmp_path, gitea, emitter=emitter)
|
|
r = intake.submit_batch(client_id, TASKS,
|
|
callback_url="https://client.example/hook")
|
|
for issue in r["issues"]:
|
|
gitea.add_comment("eval-runs", issue["issue_number"],
|
|
"done: all checks passed")
|
|
gitea.close_issue("eval-runs", issue["issue_number"])
|
|
intake.poll_completed(r["batch_id"])
|
|
|
|
out = intake.emit(r["batch_id"])
|
|
assert out["ok"] is True and out["emitted"] is True
|
|
assert emitter.push_count == 3 # retry x3 total
|
|
assert len(emitter.push_payloads) == 1 # delivered once
|
|
report, url = emitter.push_payloads[0]
|
|
assert url == "https://client.example/hook"
|
|
assert report["batch_id"] == r["batch_id"]
|
|
assert report["client_id"] == client_id
|
|
assert [res["task_id"] for res in report["results"]] == ["t1", "t2"]
|
|
assert all(res["status"] == "done" for res in report["results"])
|
|
assert all("summary" in res for res in report["results"])
|
|
assert intake.get_batch(r["batch_id"])["batch"]["status"] == "emitted"
|
|
|
|
def test_emit_file_mode_writes_json_drop(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea,
|
|
output_dir=tmp_path / "drops")
|
|
r = intake.submit_batch(client_id, TASKS) # no callback_url
|
|
for issue in r["issues"]:
|
|
gitea.close_issue("eval-runs", issue["issue_number"])
|
|
intake.poll_completed(r["batch_id"])
|
|
out = intake.emit(r["batch_id"])
|
|
assert out["ok"] is True and out["emitted"] is True
|
|
drop = tmp_path / "drops" / f"{r['batch_id']}.json"
|
|
assert drop.exists()
|
|
payload = json.loads(drop.read_text(encoding="utf-8"))
|
|
assert payload["batch_id"] == r["batch_id"]
|
|
assert len(payload["results"]) == 2
|
|
|
|
def test_emit_refuses_before_all_tasks_done(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
intake = make_intake(core, tmp_path)
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
out = intake.emit(r["batch_id"])
|
|
assert out["ok"] is False and out["reason"] == "tasks_pending"
|
|
assert intake.get_batch(r["batch_id"])["batch"]["status"] == "open"
|
|
|
|
def test_reemit_is_idempotent_no_double_post(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
emitter = MockEmitter()
|
|
intake = make_intake(core, tmp_path, gitea, emitter=emitter)
|
|
r = intake.submit_batch(client_id, TASKS,
|
|
callback_url="https://cb.example/x")
|
|
for issue in r["issues"]:
|
|
gitea.close_issue("eval-runs", issue["issue_number"])
|
|
intake.poll_completed(r["batch_id"])
|
|
assert intake.emit(r["batch_id"])["ok"] is True
|
|
pushes_after_first = emitter.push_count
|
|
again = intake.emit(r["batch_id"])
|
|
assert again["ok"] is True and again.get("idempotent") is True
|
|
assert emitter.push_count == pushes_after_first # no second POST
|
|
|
|
def test_callback_failure_keeps_batch_emittable(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
emitter = MockEmitter(fail_first=99) # always fails
|
|
intake = make_intake(core, tmp_path, gitea, emitter=emitter)
|
|
r = intake.submit_batch(client_id, TASKS,
|
|
callback_url="https://cb.example/x")
|
|
for issue in r["issues"]:
|
|
gitea.close_issue("eval-runs", issue["issue_number"])
|
|
intake.poll_completed(r["batch_id"])
|
|
out = intake.emit(r["batch_id"])
|
|
assert out["ok"] is False and out["reason"] == "emit_failed"
|
|
assert out["attempts"] == 3
|
|
assert intake.get_batch(r["batch_id"])["batch"]["status"] == "done"
|
|
|
|
|
|
# -------------------------------------------------------------------------- #
|
|
# Persistence + config #
|
|
# -------------------------------------------------------------------------- #
|
|
|
|
class TestPersistence:
|
|
def test_batch_survives_intake_reopen(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea)
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
intake2 = make_intake(core, tmp_path, gitea) # fresh connection
|
|
batch = intake2.get_batch(r["batch_id"])
|
|
assert batch["ok"] and batch["batch"]["client_id"] == client_id
|
|
assert len(batch["tasks"]) == 2
|
|
|
|
def test_shared_repo_name_from_config(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea, eval_repo="qa-runs")
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
assert r["repo"] == "qa-runs"
|
|
assert "qa-runs" in gitea.created_repos
|
|
|
|
def test_per_batch_repo_mode(self, funded, tmp_path):
|
|
core, client_id = funded
|
|
gitea = MockGitea()
|
|
intake = make_intake(core, tmp_path, gitea, per_batch_repos=True)
|
|
r = intake.submit_batch(client_id, TASKS)
|
|
assert r["repo"] == f"eval-{client_id}-{r['batch_id']}"
|
|
assert r["repo"] in gitea.created_repos
|
|
|
|
|
|
def test_real_client_list_issue_comments_builds_correct_request(monkeypatch):
|
|
"""The REAL GiteaClient (not the in-memory fake) must request
|
|
GET /api/v1/repos/{owner}/{repo}/issues/{n}/comments with the token.
|
|
|
|
This guards the exact bug class the !20 bounce caught: the fake carried
|
|
list_issue_comments while the real (canonical, issue #4) client did not,
|
|
and the suite stayed green because nothing exercised the real HTTP path.
|
|
Network-free: urlopen is captured, never invoked.
|
|
|
|
NOTE: ``gitea_saas/gitea_client.py`` is owned by issue #4 (!17), not #8;
|
|
this branch must not fork it. If the canonical client has not yet grown
|
|
``list_issue_comments`` (extension requested on #4), this guard skips
|
|
rather than failing — it re-activates automatically once !17 lands.
|
|
"""
|
|
try:
|
|
from gitea_saas.gitea_client import GiteaClient
|
|
except ImportError:
|
|
pytest.skip("gitea_saas/gitea_client.py ships via issue #4 (!17); "
|
|
"not present until !17 lands")
|
|
if not hasattr(GiteaClient, "list_issue_comments"):
|
|
pytest.skip("canonical client (issue #4) lacks list_issue_comments; "
|
|
"extension requested on #4 — guard activates once landed")
|
|
|
|
captured = {}
|
|
|
|
class FakeResponse:
|
|
status = 200
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def read(self):
|
|
return b'[{"body": "lane report", "id": 1}]'
|
|
|
|
def fake_urlopen(req, timeout=None):
|
|
captured["req"] = req
|
|
return FakeResponse()
|
|
|
|
monkeypatch.setattr("urllib.request.urlopen", fake_urlopen)
|
|
|
|
client = GiteaClient(base_url="https://gitea.example",
|
|
token="tok123", owner="pbwraith")
|
|
comments = client.list_issue_comments("eval-runs", 42)
|
|
|
|
assert comments == [{"body": "lane report", "id": 1}]
|
|
req = captured["req"]
|
|
assert req.method == "GET"
|
|
assert req.full_url == (
|
|
"https://gitea.example/api/v1/repos/pbwraith/eval-runs/"
|
|
"issues/42/comments")
|
|
assert req.headers.get("Authorization") == "token tok123"
|
|
assert req.headers.get("Accept") == "application/json"
|