রুব্রিক, রিগ্রেশন ও রিলিজ গেট

def toy_judge(first: str, second: str) -> str:
    """A STAND-IN, not a real judge: it prefers more words, and gives the first answer a 3-word head start."""
    a, b = len(first.split()) + 3, len(second.split())
    return "first" if a > b else "second" if b > a else "tie"


short, padded = "Returns take 30 days.", "Returns take 30 days. Returns take 30 days. We are happy to explain our returns."
equal_a, equal_b = "Returns: 30 days.", "Refunds: 14 days."  # same length, but only the first is current policy
for name, (x, y) in {"position": (equal_a, equal_b), "length": (short, padded)}.items():
    print(f"{name:<8} x first: {toy_judge(x, y):<6} y first: {toy_judge(y, x)}")

position x first: first  y first: first
length   x first: second y first: first

# file: kit/data/rubric.json
{
 "name": "evidencedesk-answer-rubric",
 "version": 1,
 "criteria": [
  {"name": "relevance", "levels": {
    "0": "The answer is about something other than the question.",
    "1": "The answer touches the topic but leaves the exact question open.",
    "2": "The answer states the specific fact asked for, in the customer's language."}},
  {"name": "evidence_use", "levels": {
    "0": "A claim has no citation, or the cited source does not contain it.",
    "1": "Cited sources exist, but a number or condition is not in the cited text.",
    "2": "Every claim, numbers included, appears in a cited source the model was shown."}},
  {"name": "format", "levels": {
    "0": "The reply is not valid JSON, or a required field is missing.",
    "1": "The JSON parses, but a rule is broken (status answered with no cited ids).",
    "2": "The JSON parses, has all fields and obeys every rule of the contract."}}
 ]
}

# file: kit/evidencedesk/gate.py (new file)
"""Rubric, pairwise comparison, regression compare, report and release gate. Nothing here calls a model."""
from __future__ import annotations

import argparse
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable

from pydantic import BaseModel, ConfigDict, model_validator

from evidencedesk.citations import MIN_MARGIN
from evidencedesk.evals import CaseModel, Dataset, Scores, build_retriever, load_dataset, load_results, run_suite
from evidencedesk.llm import LLMRequest, LLMResponse
from evidencedesk.rag import MIN_SCORE
from evidencedesk.tools import default_registry
from evidencedesk.usage import request_tokens

KIT = Path(__file__).resolve().parent.parent
DATA, MANIFEST = KIT / "data", KIT / "data" / "eval_manifest_v2.json"
METRICS = ("recall_at_k", "citation_validity", "abstention_correct", "tool_pass", "format_pass", "pass_rate")  # all: higher is better
VAGUE = {"good", "nice", "great", "sounds", "better", "appropriate", "quality"}


# --- 1. A rubric: observable criteria, three levels, an anchor sentence for each level ---
class Criterion(BaseModel):
    model_config = ConfigDict(extra="forbid", frozen=True)

    name: str
    levels: dict[int, str]

    @model_validator(mode="after")
    def anchored(self) -> Criterion:
        if sorted(self.levels) != [0, 1, 2] or len(set(self.levels.values())) != 3:
            raise ValueError(f"{self.name}: levels 0, 1 and 2 each need their own anchor")
        for level, text in self.levels.items():
            if len(text) < 25 or VAGUE & set(re.findall(r"[a-z]+", text.lower())):
                raise ValueError(f"{self.name} level {level}: describe what a rater can see, not 'good' or 'nice'")
        return self


class Rubric(BaseModel):
    model_config = ConfigDict(extra="forbid", frozen=True)

    name: str
    version: int
    criteria: tuple[Criterion, ...]

    def score(self, ratings: dict[str, int]) -> Scores:
        if set(ratings) != {c.name for c in self.criteria}:
            raise ValueError("rate every criterion, and only those")
        return Scores(**ratings)  # the criteria are the Scores fields

    def explain(self, ratings: dict[str, int]) -> list[str]:
        return [f"{c.name}={ratings[c.name]}: {c.levels[ratings[c.name]]}" for c in self.criteria]


def load_rubric(path: Path = DATA / "rubric.json") -> Rubric:
    return Rubric.model_validate_json(path.read_text(encoding="utf-8"))

from pydantic import ValidationError

from evidencedesk.gate import Criterion, load_rubric

rubric = load_rubric()  # kit/data/rubric.json
answer = "You have 30 days from delivery, and we also refund the delivery fee. [kc-returns-v3:c0]"
ratings = {"relevance": 2, "evidence_use": 1, "format": 2}  # my judgment, written down with its reason below
print(rubric.score(ratings))
print("\n".join(rubric.explain(ratings)))
try:
    Criterion(name="relevance", levels={0: "Not on topic at all.", 1: "Partly on the topic.", 2: "The answer sounds good."})
except ValidationError as error:
    print("rejected:", error.errors()[0]["msg"])

Scores(relevance=2, evidence_use=1, format=2)
relevance=2: The answer states the specific fact asked for, in the customer's language.
evidence_use=1: Cited sources exist, but a number or condition is not in the cited text.
format=2: The JSON parses, has all fields and obeys every rule of the contract.
rejected: Value error, relevance level 0: describe what a rater can see, not 'good' or 'nice'

# file: kit/evidencedesk/gate.py (add)
# --- 2. Pairwise comparison: no reference answer, and the order is swapped ---
Judge = Callable[[str, str], str]  # (first, second) -> "first" | "second" | "tie"
_AS_X = {"first": "x", "second": "y", "tie": "tie"}
_SWAPPED = {"first": "y", "second": "x", "tie": "tie"}


def compare_pair(judge: Judge, x: str, y: str) -> tuple[str, bool]:
    """('x' | 'y' | 'tie', order_disagreement): x wins only if the judge picks it in BOTH orders."""
    one, two = _AS_X[judge(x, y)], _SWAPPED[judge(y, x)]
    return (one, False) if one == two else ("tie", True)


def compare_pairs(judge: Judge, pairs: list[tuple[str, str]]) -> dict[str, int]:
    """x = new output, y = old. win/tie/loss are for x; disagree = verdicts that flipped with the order."""
    done = [compare_pair(judge, x, y) for x, y in pairs]
    return {"win": sum(v == "x" for v, _ in done), "tie": sum(v == "tie" for v, _ in done),
            "loss": sum(v == "y" for v, _ in done), "disagree": sum(d for _, d in done), "n": len(done)}

from evidencedesk.gate import compare_pairs

pairs = [(short, padded), (padded, short), (equal_a, equal_b), ("Returns take 30 days.", "Returns take 30 days.")]  # (new, old)
print("one order only, wins for new:", sum(toy_judge(x, y) == "first" for x, y in pairs), "of", len(pairs))
print("both orders:", compare_pairs(toy_judge, pairs))

one order only, wins for new: 3 of 4
both orders: {'win': 1, 'tie': 2, 'loss': 1, 'disagree': 2, 'n': 4}

{"id": "lg01", "kind": "rag", "input": "ডেলিভারি ফ্রি পেতে কত টাকার অর্ডার করতে হয়?", "expect": "answer", "doc_ids": ["kc-shipping-v1-bn"], "reason": "Bangla question, Bangla page."}
{"id": "lg04", "kind": "rag", "input": "Express delivery কত দিনে পৌঁছায়?", "expect": "answer", "doc_ids": ["kc-shipping-v1", "kc-shipping-v1-bn"], "reason": "Mixed: either shipping page is right."}

# file: kit/evidencedesk/gate.py (add)
# --- 3. Slices: the same run, cut by language (or any other key) ---
def language_of(text: str) -> str:
    """'bn' (Bangla letters only), 'mixed' (Bangla and Latin letters) or 'en' (no Bangla letters)."""
    bangla, latin = bool(re.search(r"[\u0980-\u09ff]", text)), bool(re.search("[A-Za-z]", text))
    return "mixed" if bangla and latin else "bn" if bangla else "en"


def slice_results(dataset: Dataset, result: dict[str, Any]) -> dict[str, dict]:
    """Pass rate per language of the question, and the unanswerable questions that were answered ('unsafe')."""
    groups: dict[str, list[str]] = {}
    for case in dataset.of_kind("rag"):
        groups.setdefault(language_of(case.input), []).append(case.id)
    expects = {c.id: c.expect for c in dataset.cases}
    return {name: {"cases": len(ids), "pass_rate": round(sum(result["cases"][i]["passed"] for i in ids) / len(ids), 3),
                   "failures": sorted(i for i in ids if not result["cases"][i]["passed"]),
                   "unsafe": sorted(i for i in ids if expects[i] == "abstain" and not result["cases"][i]["checks"]["abstention"])}
            for name, ids in sorted(groups.items())}

# file: kit/evidencedesk/gate.py (add)
# --- 3b. The fixed suite: dataset version 2, hand-written replies, default tools ---
def hand_written_client(dataset: Dataset) -> CaseModel:
    from tests.test_citations import REPLIES  # hand-written replies from RAG Reliability and Evaluation Data

    return CaseModel({c.input: REPLIES[c.id] for c in dataset.cases if c.id in REPLIES})


def run_gate_suite(min_score: float = MIN_SCORE, min_margin: float = MIN_MARGIN, k: int = 3, client=None) -> dict[str, Any]:
    dataset = load_dataset(MANIFEST)
    return run_suite(dataset, build_retriever(KIT / "corpus", min_score=min_score, k=k), client or hand_written_client(dataset),
                     default_registry(KIT / "corpus"), k=k, min_margin=min_margin)

from pathlib import Path

from evidencedesk.evals import load_dataset
from evidencedesk.gate import MANIFEST, language_of, run_gate_suite, slice_results

dataset = load_dataset(MANIFEST)  # kit/data/eval_manifest_v2.json
print(len(dataset.cases), "cases, version", dataset.version, [language_of(c.input) for c in dataset.cases if c.id in ("lg01", "lg04", "rc01")])
result = run_gate_suite()
print("pass_rate", result["metrics"]["pass_rate"], "failures", result["failures"])
for name, s in slice_results(dataset, result).items():
    print(f"{name:<6}{s['pass_rate']:>6} of {s['cases']:>2} cases   failing: {', '.join(s['failures']) or '-':<22} unsafe: {', '.join(s['unsafe']) or '-'}")

33 cases, version 2 ['bn', 'mixed', 'en']
pass_rate 0.848 failures ['lg03', 'rc03', 'rc06', 'rc08', 'rc13']
bn       0.8 of  5 cases   failing: lg03                   unsafe: lg03
en     0.714 of 14 cases   failing: rc03, rc06, rc08, rc13 unsafe: -
mixed    1.0 of  3 cases   failing: -                      unsafe: -

# file: kit/evidencedesk/gate.py (add)
# --- 4. Regression: compare two saved runs ---
class NotComparable(ValueError):
    """The runs used different datasets."""


def compare_runs(baseline: dict[str, Any], current: dict[str, Any]) -> dict[str, Any]:
    if baseline["dataset"] != current["dataset"]:
        was, now = baseline["dataset"], current["dataset"]
        raise NotComparable(f"dataset v{was['version']} {was['sha256'][:8]} vs v{now['version']} {now['sha256'][:8]}")
    cases = {}
    for cid in sorted(current["cases"]):
        before, after = baseline["cases"][cid]["passed"], current["cases"][cid]["passed"]
        cases[cid] = "improved" if after and not before else "regressed" if before and not after else "unchanged"
    metrics = {}
    for name in sorted(current["metrics"]):
        before, after = baseline["metrics"][name], current["metrics"][name]
        rank = lambda v: -1.0 if v is None else v  # a metric that can no longer be measured is a regression
        status = "improved" if rank(after) > rank(before) else "regressed" if rank(after) < rank(before) else "unchanged"
        metrics[name] = {"before": before, "after": after, "delta": None if None in (before, after) else round(after - before, 3), "status": status}
    return {"cases": cases, "metrics": metrics,
            "regressed": [c for c, s in cases.items() if s == "regressed"], "improved": [c for c, s in cases.items() if s == "improved"]}

import json

from evidencedesk.evals import load_results
from evidencedesk.gate import NotComparable, compare_runs, hand_written_client

try:
    compare_runs(load_results(Path("kit/data/baseline_v1.json")), result)
except NotComparable as error:
    print("not comparable:", error)
baseline = load_results(Path("kit/data/baseline_v2.json"))


class NewSchema:
    """A STAND-IN for 'we changed the prompt': the new reply format calls the citation list 'sources'."""

    def __init__(self, inner):
        self.inner = inner

    def complete(self, request):
        response = self.inner.complete(request)
        body = json.loads(response.text)
        if "cited_ids" in body:
            body["sources"] = body.pop("cited_ids")
        return type(response)(json.dumps(body), response.model, response.usage, response.stop_reason)


changed = run_gate_suite(client=NewSchema(hand_written_client(dataset)))
diff = compare_runs(baseline, changed)
print("regressed:", len(diff["regressed"]), "improved:", diff["improved"], "unchanged:", sum(s == "unchanged" for s in diff["cases"].values()))
print("lg01 now:", changed["cases"]["lg01"]["detail"])
for name, m in diff["metrics"].items():
    if m["status"] != "unchanged":
        print(f"{name:<20}{m['before']!s:>6} -> {m['after']!s:<6} {m['status']}")
tuned = run_gate_suite(min_score=0.25)
better = compare_runs(baseline, tuned)
print("min_score 0.25: improved", better["improved"], "regressed", better["regressed"])

not comparable: dataset v1 a575ab37 vs v2 4bbacf85
regressed: 13 improved: ['lg03'] unchanged: 19
lg01 now: invalid_schema
abstention_correct   0.773 -> 0.227  regressed
citation_validity      1.0 -> None   regressed
pass_rate            0.848 -> 0.485  regressed
min_score 0.25: improved ['rc03'] regressed []

# file: kit/evidencedesk/gate.py (add)
# --- 5. A stable report: quality, cost and latency are separate sections ---
def report(result: dict[str, Any], slices: dict | None = None, cost: dict | None = None, latency: dict | None = None) -> dict[str, Any]:
    """Only 'quality' comes from the evaluation cases; cost and latency are passed in and never mixed into it."""
    return {"run": {"dataset": result["dataset"], "config": result["config"]},
            "quality": {"metrics": result["metrics"], "failures": result["failures"], "slices": slices or {}},
            "cost": cost or {}, "latency": latency or {}}


def report_text(rep: dict[str, Any]) -> str:
    ds, q = rep["run"]["dataset"], rep["quality"]
    lines = [f"dataset {ds['name']} v{ds['version']} {ds['sha256'][:8]}",
             "config " + " ".join(f"{k}={v}" for k, v in sorted(rep["run"]["config"].items())), "QUALITY"]
    lines += [f"  {k:<24}{v}" for k, v in q["metrics"].items()]
    lines += [f"  slice {k:<6} {v['pass_rate']} of {v['cases']}, failing: {', '.join(v['failures']) or '-'}; unsafe: {', '.join(v['unsafe']) or '-'}"
              for k, v in q["slices"].items()]
    lines.append("  failures: " + (", ".join(q["failures"]) or "-"))
    for section in ("cost", "latency"):
        lines += [section.upper()] + [f"  {k:<24}{v}" for k, v in sorted(rep[section].items())] + ([] if rep[section] else ["  not measured"])
    return "\n".join(lines)


class Metered:
    """Wraps a client and counts calls and estimated input tokens: a cost figure, never a quality metric."""

    def __init__(self, inner) -> None:
        self.inner, self.calls, self.input_tokens = inner, 0, 0

    def complete(self, request: LLMRequest) -> LLMResponse:
        self.calls, self.input_tokens = self.calls + 1, self.input_tokens + request_tokens(request)
        return self.inner.complete(request)

from evidencedesk.gate import Metered, report

runs = {}
for k in (3, 1):
    meter = Metered(hand_written_client(dataset))
    res = run_gate_suite(k=k, client=meter)
    runs[k] = (res, report(res, slice_results(dataset, res), {"model_calls": meter.calls, "input_tokens_estimated": meter.input_tokens}))
(big, rep3), (small, rep1) = runs[3], runs[1]
tokens3, tokens1 = rep3["cost"]["input_tokens_estimated"], rep1["cost"]["input_tokens_estimated"]
print(f"input tokens k=3: {tokens3}, k=1: {tokens1} ({tokens1 / tokens3 - 1:+.1%})")
for name in ("recall_at_k", "citation_validity", "pass_rate"):
    print(f"{name:<20}{big['metrics'][name]:>6} -> {small['metrics'][name]}")
print("unanswerable answered:", big["confusion"]["unanswerable"]["answered"], "->", small["confusion"]["unanswerable"]["answered"])
Path("report_k1.json").write_text(json.dumps(rep1, indent=1, sort_keys=True) + "\n", encoding="utf-8")
print("saved sections:", sorted(rep1), "| cost:", rep1["cost"])

input tokens k=3: 3919, k=1: 3304 (-15.7%)
recall_at_k          0.882 -> 0.824
citation_validity      1.0 -> 0.824
pass_rate            0.848 -> 0.788
unanswerable answered: 1 -> 2
saved sections: ['cost', 'latency', 'quality', 'run'] | cost: {'model_calls': 17, 'input_tokens_estimated': 3304}

# file: kit/evidencedesk/gate.py (add)
# --- 6. Provider fixtures: assert the contract, not the wording ---
def contract_problems(response: LLMResponse) -> list[str]:
    """What must hold for ANY reply, recorded or live. Wording is not checked."""
    found = []
    if response.stop_reason == "end" and not response.text.strip():
        found.append("a finished reply needs text")
    if response.stop_reason not in ("end", "length", "other"):
        found.append(f"unknown stop_reason {response.stop_reason!r}")
    return found

from evidencedesk.gate import contract_problems
from evidencedesk.llm import FakeClient, LLMRequest, LLMResponse, Message, Usage

# FakeClient replays kit/fixtures/responses.json: the same bytes on every run
recorded = FakeClient().complete(LLMRequest((Message("user", "How long do I have to return a new bike?"),)))
reworded = LLMResponse("Thirty days from delivery.", "other-model", Usage(30, 6))  # a live provider may say it differently
for name, r in {"recorded": recorded, "reworded": reworded, "empty": LLMResponse("", "m", Usage(3, 0)),
                "odd stop": LLMResponse("x", "m", Usage(3, 1), "weird")}.items():  # type: ignore[arg-type]
    print(f"{name:<9} wording check: {r.text.startswith('You can return')!s:<5} contract: {contract_problems(r) or 'ok'}")

recorded  wording check: True  contract: ok
reworded  wording check: False contract: ok
empty     wording check: False contract: ['a finished reply needs text']
odd stop  wording check: False contract: ["unknown stop_reason 'weird'"]

# file: kit/data/gate_policy.json
{
 "name": "evidencedesk-release-gate",
 "version": 1,
 "critical_cases": ["rc09", "rc14", "rc15", "rc16", "lg06", "tl03", "tl06", "fm03"],
 "block_below": {"citation_validity": 0.9, "pass_rate": 0.8},
 "review_below": {"recall_at_k": 0.85, "abstention_correct": 0.7},
 "block_on_drop": ["citation_validity", "tool_pass", "format_pass"],
 "review_on_drop": ["recall_at_k", "abstention_correct", "pass_rate"]
}

# file: kit/evidencedesk/gate.py (add)
# --- 7. The release gate ---
class GatePolicy(BaseModel):
    model_config = ConfigDict(extra="forbid", frozen=True)

    name: str
    version: int
    critical_cases: list[str]  # any failure blocks, whatever the averages say
    block_below: dict[str, float] = {}  # absolute floors
    review_below: dict[str, float] = {}
    block_on_drop: list[str] = []  # metrics that may not fall below the baseline
    review_on_drop: list[str] = []

    @model_validator(mode="after")
    def known_metrics(self) -> GatePolicy:
        named = {*self.block_below, *self.review_below, *self.block_on_drop, *self.review_on_drop}
        if unknown := named - set(METRICS):
            raise ValueError(f"unknown metric {sorted(unknown)}")
        return self


@dataclass(frozen=True)
class Decision:
    verdict: str  # "go" | "review" | "block"
    blockers: tuple[str, ...]
    reviews: tuple[str, ...]

    def exit_code(self) -> int:  # review does not fail the build; it asks a person
        return int(self.verdict == "block")

    def text(self) -> str:
        return "\n".join([f"GATE: {self.verdict.upper()}", *(f"  block:  {r}" for r in self.blockers), *(f"  review: {r}" for r in self.reviews)])


def load_policy(path: Path = DATA / "gate_policy.json") -> GatePolicy:
    return GatePolicy.model_validate_json(path.read_text(encoding="utf-8"))


def decide(policy: GatePolicy, current: dict[str, Any], baseline: dict[str, Any] | None = None) -> Decision:
    block: list[str] = []
    review: list[str] = []
    for cid in policy.critical_cases:
        case = current["cases"].get(cid)
        if case is None:
            block.append(f"critical case {cid} is missing from the run")
        elif not case["passed"]:
            block.append(f"critical case {cid} failed its {', '.join(k for k, ok in case['checks'].items() if not ok)} check")
    for floors, out in ((policy.block_below, block), (policy.review_below, review)):
        out += [f"{n} {current['metrics'][n]} is below the floor {f}" for n, f in sorted(floors.items()) if (current["metrics"][n] or 0) < f]
    if baseline is not None:
        try:
            diff = compare_runs(baseline, current)
        except NotComparable as error:
            review.append(f"baseline not comparable: {error}")
        else:
            for n in policy.block_on_drop + policy.review_on_drop:
                if (m := diff["metrics"][n])["status"] == "regressed":
                    (block if n in policy.block_on_drop else review).append(f"{n} fell from {m['before']} to {m['after']}")
            review += [f"case {c} passed in the baseline and fails now" for c in diff["regressed"] if c not in policy.critical_cases]
    return Decision("block" if block else "review" if review else "go", tuple(block), tuple(review))

from evidencedesk.gate import decide, load_policy

policy = load_policy()
candidates = {"baseline": result, "min_score 0.25": tuned, "new schema": changed, "margin 0": run_gate_suite(min_margin=0.0),
              "k=1 (cheaper)": small}
for name, run in candidates.items():
    d = decide(policy, run, baseline)
    print(f"{name:<15}{d.verdict:<7} pass_rate={run['metrics']['pass_rate']:<6}", (d.blockers + d.reviews + ("-",))[0])

baseline       go      pass_rate=0.848  -
min_score 0.25 go      pass_rate=0.879  -
new schema     block   pass_rate=0.485  critical case rc09 failed its abstention check
margin 0       block   pass_rate=0.848  critical case rc15 failed its abstention check
k=1 (cheaper)  block   pass_rate=0.788  critical case rc15 failed its abstention check

# file: kit/evidencedesk/gate.py (add)
# --- 8. The command ---
def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(prog="python -m evidencedesk.gate", description="Run the fixed suite and apply the release gate.")
    ap.add_argument("--min-score", type=float, default=MIN_SCORE)
    ap.add_argument("--min-margin", type=float, default=MIN_MARGIN)
    args = ap.parse_args(argv)
    metered = Metered(hand_written_client(load_dataset(MANIFEST)))
    result = run_gate_suite(args.min_score, args.min_margin, client=metered)
    rep = report(result, slice_results(load_dataset(MANIFEST), result), {"model_calls": metered.calls, "input_tokens_estimated": metered.input_tokens})
    decision = decide(load_policy(), result, load_results(DATA / "baseline_v2.json"))
    print(report_text(rep), decision.text(), sep="\n")
    return decision.exit_code()


if __name__ == "__main__":
    raise SystemExit(main())

python -m evidencedesk.gate; echo "exit code: $?"

dataset evidencedesk-eval v2 4bbacf85
config chunks=36 k=3 min_margin=0.05 min_score=0.3
QUALITY
  recall_at_k             0.882
  citation_validity       1.0
  abstention_correct      0.773
  tool_pass               1.0
  format_pass             1.0
  pass_rate               0.848
  slice bn     0.8 of 5, failing: lg03; unsafe: lg03
  slice en     0.714 of 14, failing: rc03, rc06, rc08, rc13; unsafe: -
  slice mixed  1.0 of 3, failing: -; unsafe: -
  failures: lg03, rc03, rc06, rc08, rc13
COST
  input_tokens_estimated  3919
  model_calls             14
LATENCY
  not measured
GATE: GO
exit code: 0

python -m evidencedesk.gate --min-margin 0 | sed -n '/slice en/p;/^GATE/,$p'; echo "exit code: ${PIPESTATUS[0]}"

  slice en     0.714 of 14, failing: rc03, rc06, rc08, rc15; unsafe: rc15
GATE: BLOCK
  block:  critical case rc15 failed its abstention check
  block:  citation_validity fell from 1.0 to 0.941
exit code: 1

python -m pytest -q kit/tests/test_gate.py | sed 's/ in [0-9.]*s//'

....................                                                     [100%]
20 passed

GATE_MIN_MARGIN=0 python -m pytest -q --tb=no kit/tests/test_gate.py::test_release_gate | sed 's/ in [0-9.]*s//'; echo "exit code: ${PIPESTATUS[0]}"

F                                                                        [100%]
=========================== short test summary info ============================
FAILED kit/tests/test_gate.py::test_release_gate - AssertionError: critical c...
1 failed
exit code: 1

print(rubric.score({"relevance": 2, "evidence_use": 0, "format": 2}))
print(rubric.explain({"relevance": 2, "evidence_use": 0, "format": 2})[1])

Scores(relevance=2, evidence_use=0, format=2)
evidence_use=0: A claim has no citation, or the cited source does not contain it.

for k in (2, 1):
    meter = Metered(hand_written_client(dataset))
    res = run_gate_suite(k=k, client=meter)
    d = decide(policy, res, baseline)
    print(f"k={k}: {d.verdict:<6} input tokens {meter.input_tokens}, pass_rate {res['metrics']['pass_rate']}")

k=2: go     input tokens 3344, pass_rate 0.848
k=1: block  input tokens 3304, pass_rate 0.788

from evidencedesk.gate import GatePolicy

strict = GatePolicy(**{**policy.model_dump(), "critical_cases": policy.critical_cases + ["lg03"]})
print(decide(strict, baseline, baseline).text())

GATE: BLOCK
  block:  critical case lg03 failed its abstention check