08febd56ca
The HTML operator page served the same clients/ledger/queue/review data as the authenticated JSON summary, but with no auth at all. The loopback requirement it relied on existed only as prose (page note + docstring); it was never enforced by mount_dashboard, which does not own the app and cannot constrain the bind address. Put /dashboard behind the same _guard as /api/dashboard/summary: - no X-API-Key -> 401, no billing data - no configured key -> 503 (fail-closed) - valid key -> 200 HTML (unchanged rendering) Also drop the now-false 'loopback-only' security claims from the page note and the mount_dashboard docstring, and update the tests: - test_dashboard_page_open_and_html now sends a valid key - new test_dashboard_page_guard_blocks_anonymous_access: no key -> 401, body contains no client data (mutation check: fails if guard removed) - new test_dashboard_wrong_key_same_length_rejected: wrong same-length key -> 401 on HTML page too - test_summary_fail_closed_without_configured_key now asserts the HTML page also fails closed (503), not open - test_dashboard_mounts_onto_existing_app sends a valid key /api/dashboard/summary behaviour unchanged. Suite: 160 passed, 1 skipped (before: 158 passed, 1 skipped).
420 lines
18 KiB
Python
420 lines
18 KiB
Python
"""Hermetic tests for the SaaS operator dashboard (issue #9 acceptance).
|
|
|
|
Seeds a tmp SQLite DB with the billing core (clients/ledger) plus the
|
|
best-effort partner tables (pending_issues/factory_locks/reviews/
|
|
eval_batches/eval_tasks — the same DDL the owning modules use), then
|
|
asserts every dashboard section and the JSON/HTML endpoints share the
|
|
same data. No network.
|
|
|
|
Server contract (bounce-fix, review of !22): this issue does NOT define a
|
|
second Flask app. issue #2 owns ``gitea_saas/server.py`` (the app factory
|
|
and the API-key discipline, per ARCHITECTURE.md); the dashboard MOUNTS onto
|
|
an existing app via ``mount_dashboard(app, core, db_path, api_key)``. The
|
|
endpoint tests below build a throwaway app with Flask's test client the
|
|
same way the app factory will when issue #2 lands.
|
|
|
|
Run: python3 -m pytest tests/test_dashboard.py -q
|
|
"""
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
import pytest
|
|
|
|
import flask
|
|
|
|
from gitea_saas.billing_core import BillingCore
|
|
from gitea_saas.dashboard import (Dashboard, EMPTY, render_html,
|
|
mount_dashboard, API_KEY_HEADER)
|
|
|
|
|
|
def _iso(dt):
|
|
return dt.isoformat()
|
|
|
|
|
|
# Partner-table DDL (mirrors issues #4/#6/#8 so the dashboard reads work
|
|
# against the real tables when those land).
|
|
EXTRA_DDL = """
|
|
CREATE TABLE IF NOT EXISTS pending_issues (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
client_id INTEGER NOT NULL,
|
|
repo TEXT NOT NULL,
|
|
title TEXT NOT NULL,
|
|
body_json TEXT NOT NULL,
|
|
labels_json TEXT NOT NULL DEFAULT '[]',
|
|
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
|
status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'shipped'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS factory_locks (
|
|
client_id INTEGER PRIMARY KEY,
|
|
locked_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS reviews (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
repo TEXT NOT NULL,
|
|
pr INTEGER NOT NULL,
|
|
head_sha TEXT NOT NULL,
|
|
verdict TEXT NOT NULL,
|
|
reviewed_at TEXT NOT NULL,
|
|
UNIQUE (repo, pr, head_sha)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS eval_batches (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
client_id INTEGER NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'open',
|
|
callback_url TEXT,
|
|
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS eval_tasks (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
batch_id INTEGER NOT NULL REFERENCES eval_batches(id),
|
|
repo TEXT NOT NULL,
|
|
issue_number INTEGER NOT NULL,
|
|
task_id TEXT NOT NULL,
|
|
payload_json TEXT NOT NULL DEFAULT '{}',
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
result_json TEXT,
|
|
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
|
UNIQUE (batch_id, task_id)
|
|
);
|
|
"""
|
|
|
|
|
|
def _seed(tmp_path):
|
|
"""Billing core + partner tables with fully controlled timestamps."""
|
|
now = datetime.now(timezone.utc)
|
|
yesterday = now - timedelta(days=1)
|
|
ten_days = now - timedelta(days=10)
|
|
forty_days = now - timedelta(days=40)
|
|
|
|
core = BillingCore(tmp_path / "saas.db")
|
|
core.create_client("Acme", tier="gold")
|
|
core.create_client("Poorco", tier="free")
|
|
# issue #16 API: create_client/get_client return FLAT records keyed by
|
|
# "id" (never nested, never "client_id").
|
|
acme = core.get_client(1)[1]["id"]
|
|
poor = core.get_client(2)[1]["id"]
|
|
|
|
conn = sqlite3.connect(tmp_path / "saas.db")
|
|
conn.executescript(EXTRA_DDL)
|
|
c = conn.cursor()
|
|
# Ledger with explicit timestamps (direct inserts, schema-legal).
|
|
c.execute(
|
|
"INSERT INTO ledger (client_id, kind, amount_minor, currency,"
|
|
" idempotency_key, ref_type, ref_id, created_at)"
|
|
" VALUES (?, 'credit', 10000, 'USD', 'seed-fund', NULL, NULL, ?)",
|
|
(acme, _iso(now)))
|
|
c.execute(
|
|
"INSERT INTO ledger (client_id, kind, amount_minor, currency,"
|
|
" idempotency_key, ref_type, ref_id, created_at)"
|
|
" VALUES (?, 'debit', 100, 'USD', 'seed-del-1', 'delivery',"
|
|
" 'acme-repo#7', ?)", (acme, _iso(now)))
|
|
c.execute(
|
|
"INSERT INTO ledger (client_id, kind, amount_minor, currency,"
|
|
" idempotency_key, ref_type, ref_id, created_at)"
|
|
" VALUES (?, 'debit', 100, 'USD', 'seed-del-2', 'delivery',"
|
|
" 'acme-repo#6', ?)", (acme, _iso(ten_days)))
|
|
c.execute(
|
|
"INSERT INTO ledger (client_id, kind, amount_minor, currency,"
|
|
" idempotency_key, ref_type, ref_id, created_at)"
|
|
" VALUES (?, 'debit', 100, 'USD', 'seed-del-3', 'delivery',"
|
|
" 'acme-repo#5', ?)", (acme, _iso(forty_days)))
|
|
c.execute(
|
|
"INSERT INTO ledger (client_id, kind, amount_minor, currency,"
|
|
" idempotency_key, ref_type, ref_id, created_at)"
|
|
" VALUES (?, 'credit', 50, 'USD', 'seed-poor-fund', NULL, NULL, ?)",
|
|
(poor, _iso(yesterday)))
|
|
# Queue + locks (issue #4 tables).
|
|
c.execute(
|
|
"INSERT INTO pending_issues (client_id, repo, title, body_json,"
|
|
" labels_json, created_at, status) VALUES (?, 'poorco-repo',"
|
|
" 'Unpaid banner request', '{}', '[]', ?, 'queued')",
|
|
(poor, _iso(yesterday)))
|
|
c.execute("INSERT INTO factory_locks (client_id) VALUES (?)", (poor,))
|
|
# Review gate (issue #6 table): one pair matches a delivery debit
|
|
# exactly one day before, one verdict has no debit.
|
|
c.execute(
|
|
"INSERT INTO reviews (repo, pr, head_sha, verdict, reviewed_at)"
|
|
" VALUES ('acme-repo', 7, 'deadbeefcafe0001', 'APPROVE', ?)",
|
|
(_iso(now - timedelta(days=1)),))
|
|
c.execute(
|
|
"INSERT INTO reviews (repo, pr, head_sha, verdict, reviewed_at)"
|
|
" VALUES ('acme-repo', 6, 'deadbeefcafe0002', 'BOUNCE', ?)",
|
|
(_iso(yesterday - timedelta(hours=2)),))
|
|
# Eval farm (issue #8 tables).
|
|
c.execute(
|
|
"INSERT INTO eval_batches (id, client_id, status, callback_url,"
|
|
" created_at) VALUES (1, ?, 'done', NULL, ?)", (acme, _iso(ten_days)))
|
|
c.execute(
|
|
"INSERT INTO eval_tasks (batch_id, repo, issue_number, task_id,"
|
|
" payload_json, status, result_json, created_at)"
|
|
" VALUES (1, 'eval-runs', 11, 'eval-a', '{}', 'done',"
|
|
" '{\"summary\": \"ok\"}', ?)", (_iso(ten_days),))
|
|
conn.commit()
|
|
conn.close()
|
|
return core, tmp_path / "saas.db", acme, poor
|
|
|
|
|
|
@pytest.fixture
|
|
def seeded(tmp_path):
|
|
return _seed(tmp_path)
|
|
|
|
|
|
@pytest.fixture
|
|
def dash(seeded):
|
|
core, db, _acme, _poor = seeded
|
|
return Dashboard(core, db)
|
|
|
|
|
|
@pytest.fixture
|
|
def app(seeded):
|
|
"""Throwaway Flask app with the dashboard mounted — the same way issue
|
|
#2's app factory will mount it when it lands (mount, never define)."""
|
|
core, db, _acme, _poor = seeded
|
|
a = flask.Flask(__name__)
|
|
mount_dashboard(a, core=core, db_path=db, api_key="sekret")
|
|
return a
|
|
|
|
|
|
# -------------------------------------------------------------------------- #
|
|
# Sections #
|
|
# -------------------------------------------------------------------------- #
|
|
|
|
class TestClients:
|
|
def test_clients_section(self, dash, seeded):
|
|
_core, _db, acme, poor = seeded
|
|
by_name = {c["name"]: c for c in dash.summary()["clients"]}
|
|
ac = by_name["Acme"]
|
|
assert ac["tier"] == "gold"
|
|
assert ac["balance"] == 9700 # 10000 - 3x100
|
|
assert ac["status"] == "active"
|
|
assert ac["credits_this_month"] == 10000
|
|
# Today's debit is always in this month; the 10-day-old one depends
|
|
# on where the month boundary falls — assert the deterministic part.
|
|
assert ac["debits_this_month"] == 200 or ac["debits_this_month"] == 100
|
|
pc = by_name["Poorco"]
|
|
assert pc["balance"] == 50
|
|
assert pc["status"] == "locked" # lock row wins over overdue
|
|
|
|
|
|
class TestQueue:
|
|
def test_queue_count_and_flags(self, dash, seeded):
|
|
_core, _db, _acme, poor = seeded
|
|
q = dash.summary()["queue"]
|
|
assert q["count"] == 1
|
|
item = q["items"][0]
|
|
assert item["client_id"] == poor
|
|
assert item["client_name"] == "Poorco"
|
|
assert item["locked"] is True
|
|
assert item["unfunded"] is True # 50 < 500 floor
|
|
|
|
|
|
class TestThroughput:
|
|
def test_throughput_windows_and_evals(self, dash):
|
|
by_name = {t["client_name"]: t for t in dash.summary()["throughput"]}
|
|
ac = by_name["Acme"]
|
|
assert ac["merged_7d"] == 1 # today's delivery debit
|
|
assert ac["merged_30d"] == 2 # today + 10 days ago
|
|
assert ac["evals_completed"] == 1
|
|
assert ac["avg_review_to_merge_days"] == 1.0 # review 1d before debit
|
|
poor = by_name["Poorco"]
|
|
assert poor["merged_7d"] == 0 and poor["merged_30d"] == 0
|
|
|
|
|
|
class TestReviewGate:
|
|
def test_last_verdicts_short_head(self, dash):
|
|
gate = dash.summary()["review_gate"]
|
|
verdicts = [(g["verdict"], g["pr"]) for g in gate]
|
|
assert ("APPROVE", 7) in verdicts
|
|
assert ("BOUNCE", 6) in verdicts
|
|
entry = next(g for g in gate if g["pr"] == 7)
|
|
assert entry["repo"] == "acme-repo"
|
|
assert len(entry["head_sha"]) == 8 # shortened
|
|
|
|
|
|
class TestLedgerFlash:
|
|
def test_latest_rows_desc(self, dash):
|
|
flash = dash.summary()["ledger_flash"]
|
|
assert len(flash) == 5 # 5 ledger seed rows
|
|
ids = [l["id"] for l in flash]
|
|
assert ids == sorted(ids, reverse=True)
|
|
first = flash[0]
|
|
assert first["client_name"] == "Poorco" # newest row first
|
|
assert first["kind"] == "credit"
|
|
assert first["amount_minor"] == 50
|
|
# a delivery debit row is present with its ref rendered
|
|
deliveries = [l for l in flash if l["kind"] == "debit"
|
|
and "delivery" in l["ref"]]
|
|
assert len(deliveries) >= 2
|
|
assert any("acme-repo#7" in d["ref"] for d in deliveries)
|
|
|
|
|
|
# -------------------------------------------------------------------------- #
|
|
# Empty state + HTML #
|
|
# -------------------------------------------------------------------------- #
|
|
|
|
class TestEmptyAndHtml:
|
|
def test_empty_db_renders_dash_empty_states(self, tmp_path):
|
|
core = BillingCore(tmp_path / "empty.db")
|
|
core.create_client("Nothing", tier="free")
|
|
d = Dashboard(core, tmp_path / "empty.db")
|
|
s = d.summary()
|
|
assert s["clients"] and s["clients"][0]["name"] == "Nothing"
|
|
assert s["queue"] == {"count": 0, "items": []}
|
|
assert s["review_gate"] == [] and s["ledger_flash"] == []
|
|
page = render_html(s)
|
|
assert EMPTY in page # empty-state em dash rendered
|
|
|
|
def test_html_has_all_sections_and_data(self, dash, seeded):
|
|
s = dash.summary()
|
|
page = render_html(s)
|
|
for section in ("1 \u00b7 Clients", "2 \u00b7 Queue", "3 \u00b7 Throughput",
|
|
"4 \u00b7 Review gate", "5 \u00b7 Ledger flash"):
|
|
assert section in page
|
|
assert "Acme" in page and "Poorco" in page
|
|
assert "APPROVE" in page and "BOUNCE" in page
|
|
assert "deadbeef" in page # short head sha
|
|
assert "listening" not in page.lower() or True # page renders
|
|
|
|
|
|
# -------------------------------------------------------------------------- #
|
|
# Mount contract (issue #2 owns the app factory) #
|
|
# -------------------------------------------------------------------------- #
|
|
|
|
class TestMountContract:
|
|
def test_dashboard_mounts_onto_existing_app(self, app):
|
|
"""The dashboard registers its routes on a caller-supplied app — it
|
|
does NOT define or run a second Flask app (review finding: #2 owns
|
|
the app factory; three-way server.py conflict removed)."""
|
|
client = app.test_client()
|
|
# /dashboard is key-protected (issue #25): open with a valid key.
|
|
assert client.get("/dashboard",
|
|
headers={API_KEY_HEADER: "sekret"}).status_code == 200
|
|
assert client.get("/healthz").status_code == 200
|
|
|
|
def test_no_second_server_module_in_this_issue(self):
|
|
"""Issue #9 must not stand up a second Flask app: #2 owns the app
|
|
factory and the dashboard only mounts onto it.
|
|
|
|
This originally asserted that `gitea_saas/server.py` was absent from
|
|
the tree. That held only until #2 merged -- afterwards the guard fired
|
|
against #2's own file rather than against anything #9 shipped, so it
|
|
could never pass again (it is the single failure when this branch is
|
|
merged onto main). The invariant that actually matters is unchanged and
|
|
is now checked directly: this module exposes a mount seam and never
|
|
constructs an app of its own.
|
|
"""
|
|
import ast
|
|
|
|
import gitea_saas.dashboard as d
|
|
assert callable(getattr(d, "mount_dashboard", None))
|
|
|
|
# Parse rather than grep: mount_dashboard's docstring contains a
|
|
# `app = Flask(__name__)` usage example for callers, so a substring
|
|
# check reports a violation that does not exist.
|
|
src = Path(d.__file__).resolve().read_text(encoding="utf-8")
|
|
constructs_app = [
|
|
n for n in ast.walk(ast.parse(src))
|
|
if isinstance(n, ast.Call)
|
|
and getattr(n.func, "id", None) == "Flask"
|
|
]
|
|
assert not constructs_app, \
|
|
"dashboard.py must mount onto the app owned by #2, not create its own"
|
|
|
|
|
|
# -------------------------------------------------------------------------- #
|
|
# Server endpoints (via the mount, same as the app factory will use) #
|
|
# -------------------------------------------------------------------------- #
|
|
|
|
class TestServer:
|
|
def test_summary_json_requires_api_key(self, dash, app):
|
|
client = app.test_client()
|
|
r = client.get("/api/dashboard/summary")
|
|
assert r.status_code == 401
|
|
r = client.get("/api/dashboard/summary",
|
|
headers={"X-API-Key": "wrong"})
|
|
assert r.status_code == 401
|
|
r = client.get("/api/dashboard/summary",
|
|
headers={API_KEY_HEADER: "sekret"})
|
|
assert r.status_code == 200
|
|
payload = r.get_json()
|
|
assert payload is not None
|
|
assert payload.pop("generated_at") # live timestamp, differs per call
|
|
expected = dash.summary()
|
|
expected.pop("generated_at")
|
|
assert payload == expected # same queries as the HTML
|
|
|
|
def test_wrong_key_of_same_length_rejected(self, dash, app):
|
|
"""A wrong key of the SAME length as the configured key must be
|
|
rejected with 401.
|
|
|
|
This is the test that distinguishes the comparison implementations:
|
|
plain ``str !=`` returns on the first differing byte, while
|
|
``hmac.compare_digest`` is constant-time. A length-DIFFERING wrong
|
|
key ("wrong" vs the configured "sekret") is rejected by both and is
|
|
therefore not evidence either way (bounce finding, 2026-08-26)."""
|
|
client = app.test_client()
|
|
configured = "sekret" # api_key in the `app` fixture
|
|
wrong_same_length = "hunter" # 6 chars, differs in every byte
|
|
assert len(wrong_same_length) == len(configured)
|
|
r = client.get("/api/dashboard/summary",
|
|
headers={API_KEY_HEADER: wrong_same_length})
|
|
assert r.status_code == 401
|
|
|
|
def test_dashboard_page_open_and_html(self, app, seeded):
|
|
client = app.test_client()
|
|
r = client.get("/dashboard", headers={API_KEY_HEADER: "sekret"})
|
|
assert r.status_code == 200
|
|
assert "text/html" in r.content_type
|
|
assert "SaaS dashboard" in r.get_data(as_text=True)
|
|
assert "Acme" in r.get_data(as_text=True)
|
|
|
|
def test_dashboard_page_guard_blocks_anonymous_access(self, app, seeded):
|
|
"""GET /dashboard with NO X-API-Key must NOT return billing data
|
|
(issue #25): same fail-closed guard as the JSON summary — 401 and no
|
|
client/ledger/queue content in the body."""
|
|
client = app.test_client()
|
|
r = client.get("/dashboard")
|
|
# no auth → 401, and no billing data leaks in the body
|
|
assert r.status_code == 401
|
|
body = r.get_data(as_text=True)
|
|
assert "Acme" not in body
|
|
assert "Poorco" not in body
|
|
assert "SaaS dashboard" not in body
|
|
|
|
def test_dashboard_wrong_key_same_length_rejected(self, app):
|
|
"""A wrong key of the SAME length as the configured key must be
|
|
rejected with 401 on the HTML page too (matches the JSON endpoint's
|
|
constant-time comparison contract, issue #25)."""
|
|
client = app.test_client()
|
|
configured = "sekret"
|
|
wrong_same_length = "hunter"
|
|
assert len(wrong_same_length) == len(configured)
|
|
r = client.get("/dashboard",
|
|
headers={API_KEY_HEADER: wrong_same_length})
|
|
assert r.status_code == 401
|
|
assert "Acme" not in r.get_data(as_text=True)
|
|
|
|
def test_summary_fail_closed_without_configured_key(self, seeded,
|
|
monkeypatch):
|
|
monkeypatch.delenv("SAAS_API_KEY", raising=False)
|
|
core, db, _a, _p = seeded
|
|
a = flask.Flask(__name__)
|
|
mount_dashboard(a, core=core, db_path=db, api_key=None)
|
|
client = a.test_client()
|
|
r = client.get("/api/dashboard/summary")
|
|
assert r.status_code == 503 # fail-closed, not anonymous
|
|
# HTML page fails closed too (issue #25): no key configured → 503,
|
|
# and no billing data is returned either way.
|
|
r2 = client.get("/dashboard")
|
|
assert r2.status_code == 503
|
|
assert "Acme" not in r2.get_data(as_text=True)
|
|
|
|
def test_healthz(self, app):
|
|
r = app.test_client().get("/healthz")
|
|
assert r.status_code == 200 and r.get_json() == {"ok": True}
|