মজবুত মেট্রিক ও রিলিজ গেট

from evidencedesk.evalkit.suite import BASELINE, CANDIDATE, Config, run_report

old, new = run_report(BASELINE), run_report(CANDIDATE)  # version 1 and version 2 of the previous page
print("dataset", new.manifest.dataset, "| cases", len(new.passed), "| strict passes", sum(old.passed.values()), "->", sum(new.passed.values()))
print(f"{'measure':<20}{'old':>7}{'new':>7}   new rate, 95% interval")
for name in ("answer_correct", "false_abstain_rate", "abstain_correct", "false_answer_rate", "citation_present", "citation_support", "recall_at_3"):
    a, b = old.counts[name], new.counts[name]
    print(f"{name:<20}{f'{a.k}/{a.n}':>7}{f'{b.k}/{b.n}':>7}   {b.rate} {b.interval}")

dataset 8a2bc76d5867 | cases 31 | strict passes 23 -> 26
measure                 old    new   new rate, 95% interval
answer_correct        20/25  22/25   0.88 (0.7, 0.958)
false_abstain_rate     2/25   1/25   0.04 (0.007, 0.195)
abstain_correct         3/6    4/6   0.667 (0.3, 0.903)
false_answer_rate       3/6    2/6   0.333 (0.097, 0.7)
citation_present      23/23  24/24   1.0 (0.862, 1.0)
citation_support      21/21  23/23   1.0 (0.857, 1.0)
recall_at_3           24/25  24/25   0.96 (0.805, 0.993)

# file: evidencedesk/evalkit/measures.py (excerpt)
def miscite(observed: list[Observed]) -> list[Observed]:
    """A model that writes the right sentence but cites a neighbour: the id is real and was shown, the text does not hold the claim."""
    def swap(o: Observed) -> Observed:
        other = [i for i in o.answer.retrieved_ids if i not in o.cited]
        return replace(o, cited=(other[0],)) if o.answered and other else o
    return [swap(o) for o in observed]

from pathlib import Path

from evidencedesk.evalkit.dataset import load_cases
from evidencedesk.evalkit.measures import measure, miscite, observe
from evidencedesk.evals import CaseModel, build_retriever

cases = load_cases()
obs = observe(cases, build_retriever(Path("kit/corpus"), "sentence", 0.3), CaseModel({}), 0.05)
shifted = miscite(obs)  # same sentences, but each cites the next retrieved chunk
g01, g01_shifted = next(o for o in obs if o.case.id == "g01"), next(o for o in shifted if o.case.id == "g01")
print("g01 cites", g01.cited, "->", g01_shifted.cited)
for label, run in (("as written", obs), ("neighbour cited", shifted)):
    m = measure(run)
    print(f"{label:<16}" + "  ".join(f"{k} {m[k].k}/{m[k].n}" for k in ("citation_present", "citation_support", "answer_correct")))

g01 cites ('kc-returns-v3:c0',) -> ('kc-returns-v3:c1',)
as written      citation_present 24/24  citation_support 23/23  answer_correct 22/25
neighbour cited citation_present 24/24  citation_support 0/23  answer_correct 22/25

for o in obs:
    if o.case.expect == "abstain" and o.answered:
        print(o.case.id, "| asked:", o.case.question, "| answered from", o.cited[0], "|", o.case.why)
for expect in ("answer", "abstain"):
    rows = [o for o in obs if o.case.expect == expect]
    print(f"{expect:<8} answered {sum(o.answered for o in rows):>2}  abstained {sum(not o.answered for o in rows):>2}")

a02 | asked: What is the warranty on electric motors? | answered from kc-warranty-v2:c0 | Shares words with the warranty page but the topic is absent.
a05 | asked: How do I return a bike to a shop in Brazil? | answered from kc-returns-v3:c0 | Plausible, but the documents do not name the country.
answer   answered 24  abstained  1
abstain  answered  2  abstained  4

from evidencedesk.evalkit.tooluse import full_registry, load_tool_cases, run_tool_case, tool_measures

tools = load_tool_cases()
print(f"{'case':<5}{'selection':>10}{'args':>6}{'permission':>11}{'complete':>9}  note")
show = lambda v: "-" if v is None else "ok" if v else "FAIL"
for c in tools:
    r = run_tool_case(c, full_registry())
    print(f"{c.id:<5}{show(r['selection']):>10}{show(r['args']):>6}{show(r['permission']):>11}{show(r['complete']):>9}  {c.note[:34]}")
print({k: f"{v.k}/{v.n}" for k, v in tool_measures(tools).items()})

case  selection  args permission complete  note
t01          ok    ok          -       ok  right tool, right arguments
t02          ok  FAIL          -       ok  same sum written in another order:
t03          ok  FAIL          -     FAIL  wrong arguments that the tool itse
t04          ok    ok          -       ok  right tool, right arguments
t05        FAIL     -         ok        -  wrong tool, and it is not allowed 
t06        FAIL     -         ok        -  a greeting needs no tool; calling 
t07          ok     -          -        -  correctly calls nothing
t08          ok    ok          -       ok  an id that does not exist: the too
{'tool_selection': '6/8', 'tool_args': '3/5', 'tool_permission': '2/2', 'tool_complete': '4/5'}

from evidencedesk.evalkit.slices import by_suite

for name, report in (("old", old), ("new", new)):
    print(name, {k: f"{v.k}/{v.n}" for k, v in by_suite(report.passed, cases).items()})
print("failing in new, by set:", {s: sorted(c.id for c in cases if c.set == s and not new.passed[c.id]) for s in ("golden", "production_like", "adversarial")})

old {'regression': '19/24', 'adversarial': '4/7'}
new {'regression': '21/24', 'adversarial': '5/7'}
failing in new, by set: {'golden': ['g08'], 'production_like': ['p05', 'p08'], 'adversarial': ['a02', 'a05']}

from evidencedesk.evalkit.stats import Count, flips_needed, min_detectable, paired_diff_ci, sample_size
from evidencedesk.retrieval.metrics import paired

ids = sorted(new.passed)
a, b = [float(old.passed[i]) for i in ids], [float(new.passed[i]) for i in ids]
d = paired(dict(zip(ids, a)), dict(zip(ids, b)))
print("old", Count(k=int(sum(a)), n=31).text())
print("new", Count(k=int(sum(b)), n=31).text())
print("better", list(d.better), "worse", list(d.worse), "unchanged", d.tied, "sign-test p", d.p)
print("difference in pass rate", round((sum(b) - sum(a)) / 31, 3), "95% interval", paired_diff_ci(a, b))
print("changed cases needed, all one way, for p < 0.05:", flips_needed())

print("smallest true difference found 8 times in 10, by cases per group:", "  ".join(f"{n}: {min_detectable(n):.2f}" for n in (31, 100, 300, 1000)))
print("cases per group needed to detect 0.10:", sample_size(0.10), "| 0.05:", sample_size(0.05))

old 0.742 (23/31, 95% 0.568-0.863)
new 0.839 (26/31, 95% 0.674-0.929)
better ['a06', 'p02', 'p09'] worse [] unchanged 28 sign-test p 0.25
difference in pass rate 0.097 95% interval (0.0, 0.226)
changed cases needed, all one way, for p < 0.05: 6
smallest true difference found 8 times in 10, by cases per group: 31: 0.28  100: 0.16  300: 0.09  1000: 0.05
cases per group needed to detect 0.10: 251 | 0.05: 1004

import itertools

from evidencedesk.evalkit.flaky import FlakyClient
from evidencedesk.evalkit.stats import family_error, holm
from evidencedesk.evals import CaseModel

flat = lambda passed: {i: float(ok) for i, ok in passed.items()}
noisy = lambda seed: run_report(Config("sentence", 0.35, 0.05), client=FlakyClient(CaseModel({}), 0.4, seed)).passed
grid = {}  # 48 settings tried against the same 31 cases
for chunking, score, margin in itertools.product(("paragraph", "sentence"), (0.2, 0.3, 0.35, 0.4, 0.45, 0.5), (0.0, 0.05, 0.1, 0.15)):
    r = run_report(Config(chunking, score, margin))
    grid[f"{chunking[:4]}-{score}-{margin}"] = (sum(r.passed.values()), paired(flat(old.passed), flat(r.passed)))
best = max(grid, key=lambda k: grid[k][0])
d = grid[best][1]
print(f"best of {len(grid)}: {best} passes {grid[best][0]}/31 (old {sum(old.passed.values())}); better {len(d.better)} worse {len(d.worse)} p {d.p}")
print("p < 0.05 before correction:", sum(g[1].p < 0.05 for g in grid.values()), "| after Holm:", sum(holm({k: g[1].p for k, g in grid.items()}).values()))
ref = noisy(1000)  # twenty copies of the SAME system, each compared with one reference run
p = {f"copy{s:02d}": paired(flat(ref), flat(noisy(2000 + s))).p for s in range(20)}
print("identical systems that look different (p < 0.05):", sum(v < 0.05 for v in p.values()), "of 20 | after Holm:", sum(holm(p).values()), "| chance of at least one:", family_error(20))

best of 48: sent-0.35-0.05 passes 27/31 (old 23); better 4 worse 0 p 0.125
p < 0.05 before correction: 0 | after Holm: 0
identical systems that look different (p < 0.05): 1 of 20 | after Holm: 0 | chance of at least one: 0.642

from evidencedesk.evalkit.flaky import repeat_summary

runs = [run_report(Config("sentence", 0.35, 0.05), client=FlakyClient(CaseModel({}), 0.1, seed)).passed for seed in range(10)]
s = repeat_summary(runs)
print({k: s[k] for k in ("runs", "mean", "sd", "min", "max", "stable_pass")}, "always failing:", s["stable_fail"])
print(len(s["flaky"]), "flaky cases, the least stable:", {k: v for k, v in s["flaky"].items() if int(v[0]) <= 7})

{'runs': 10, 'mean': 0.787, 'sd': 0.053, 'min': 0.71, 'max': 0.871, 'stable_pass': 12} always failing: ['a05', 'g08', 'p05', 'p08']
15 flaky cases, the least stable: {'g05': '6/10', 'g07': '7/10'}

from evidencedesk.evalkit.slices import judge_slices
from evidencedesk.evalkit.stats import Count

overall = Count(k=sum(new.passed.values()), n=len(new.passed))
rows = judge_slices(new.slices, overall)
print("overall", overall.rate, "| judged slices:")
for r in rows:
    if r["status"] != "few":
        print(f"  {r['slice']:<22}{r['k']:>3}/{r['n']:<3} {r['rate']}  {r['interval']}  {r['status']}")
print("not judged (n < 8):", ", ".join(f"{r['slice']} {r['k']}/{r['n']}" for r in rows if r["status"] == "few"))

overall 0.839 | judged slices:
  lang=en                23/28  0.821  (0.644, 0.921)  ok
  task=lookup            12/12  1.0  (0.757, 1.0)  ok
  set=golden             13/14  0.929  (0.685, 0.987)  ok
  set=production_like     8/10  0.8  (0.49, 0.943)  ok
not judged (n < 8): lang=bn 3/3, task=exact_term 3/3, task=multi_fact 5/6, task=paraphrase 2/4, task=unanswerable 4/6, set=adversarial 5/7

from evidencedesk.evalkit.goodhart import disagreements, gaming_flags

harsh = run_report(Config("sentence", 0.5, 0.05))
print("version 2 -> min_score 0.5:", [d for d in disagreements(new.counts, harsh.counts) if "_rate" not in d])
trivial = {"always abstains": run_report(Config("sentence", 9.0, 0.0)), "never abstains": run_report(Config("sentence", 0.0, -1.0))}
for name, r in trivial.items():
    c = r.counts
    print(f"{name:<16} strict passes {sum(r.passed.values()):>2}/31  answer_correct {c['answer_correct'].rate}  abstain_correct {c['abstain_correct'].rate}", gaming_flags(new.counts, c))

version 2 -> min_score 0.5: ['abstain_correct +0.33 but answer_correct -0.20']
always abstains  strict passes  6/31  answer_correct 0.0  abstain_correct 1.0 ['abstain_correct rose +0.33 while answer_correct fell -0.88']
never abstains   strict passes 22/31  answer_correct 0.88  abstain_correct 0.0 []

from evidencedesk.evalkit.gates import evaluate, load_policy

policy = load_policy()
candidates = {"old (v1)": BASELINE, "new (v2)": CANDIDATE, "score 0.35": Config("sentence", 0.35, 0.05), "score 0.5": Config("sentence", 0.5, 0.05)}
reports = {name: run_report(cfg) for name, cfg in candidates.items()}
for name, r in list(reports.items())[1:]:  # the baseline is not judged against itself
    d = evaluate(policy, r, old)
    print(f"{name:<11}{d.verdict.upper()}")
    for x in d.reasons:
        if x.level != "note":
            print(f"   {x.level:<7}{x.rule}: {x.text}")
print("trivial systems:", {n: evaluate(policy, r, old).verdict for n, r in trivial.items()})

new (v2)   BLOCK
   block  critical: case a02 failed
   block  floor: abstain_correct 0.667 (4/6) is below the floor 0.8
score 0.35 REVIEW
   review band: abstain_correct 0.833 (5/6) is inside the review band above the floor 0.8
score 0.5  BLOCK
   block  critical: case g05 failed
   block  floor: answer_correct 0.68 (17/25) is below the floor 0.8
   block  regress: answer_correct fell from 0.8 to 0.68 (allowed drop 0.05)
   review flips: 3 cases passed in the baseline and fail now: a01, g07, p06
trivial systems: {'always abstains': 'block', 'never abstains': 'block'}

from evidencedesk.evalkit.stats import Count

for times in (1, 10):
    c = Count(k=22 * times, n=25 * times)
    print(f"answer_correct 0.88 from {c.n} cases: interval {c.interval} -> floor 0.8 proven: {c.interval[0] >= 0.8}")

answer_correct 0.88 from 25 cases: interval (0.7, 0.958) -> floor 0.8 proven: False
answer_correct 0.88 from 250 cases: interval (0.834, 0.915) -> floor 0.8 proven: True

from evidencedesk.evalkit.versions import changed, not_comparable

mid = reports["score 0.35"]
m = mid.manifest
print("manifest", m.sha, "| dataset", m.dataset, "| prompt", m.prompt_sha, "| model", m.model, "| retrieval", m.retrieval)
print("changed against the baseline:", changed(old.manifest, m))
print("numbers comparable:", not_comparable(old.manifest, m) == [], "| after editing one case:", not_comparable(old.manifest, m.model_copy(update={"dataset": "0123456789ab"})))

manifest 5cb05a77bd | dataset 8a2bc76d5867 | prompt c3bcffe3e0 | model fake-case-model | retrieval {'chunking': 'sentence', 'min_score': 0.35, 'min_margin': 0.05, 'k': 3}
changed against the baseline: {'retrieval.chunking': ('paragraph', 'sentence'), 'retrieval.min_score': (0.2, 0.35)}
numbers comparable: True | after editing one case: ['dataset differs (8a2bc76d5867 vs 0123456789ab)']

from evidencedesk.evalkit.evidence import collect, review_queue, scrub, with_note
from evidencedesk.evalkit.judge import DEMO_JUDGES, load_pairs, load_prompts

pairs, prompts = load_pairs(), load_prompts()
ev = collect(DEMO_JUDGES["mixed"], prompts[1], pairs)
e = ev[4]
print(e.pair_id, e.lang, "| order a-b:", e.order_ab, "| order b-a:", e.order_ba, "| judged:", e.judged, "| human:", e.human, "|", e.why, "| text hash", e.input_sha)
queue = review_queue(ev)
print(len(queue), "of", len(ev), "pairs go to a person:", {r: sum(r in why for why in queue.values()) for r in ("order flip", "judge vs human")})
note = "call " + "+44 7911 " + "123456 or " + "ana" + "@" + "example.org " + "token" + "=" + "fake-value-123"  # bait made at run time, never stored
print(with_note(ev[4], note).note)

clear-5 bn | order a-b: tie | order b-a: b | judged: tie | human: a | score a 0.15 b 0.1 | text hash 84e5cd8fc5a1
17 of 30 pairs go to a person: {'order flip': 17, 'judge vs human': 12}
call +[number] or [email] [secret]

print(f"{'candidate':<12}{'tokens/case':>12}{'calls/case':>11}{'p95 ms (formula)':>17}{'answer_correct':>16}  gate")
for name, r in reports.items():
    s = r.scalars
    print(f"{name:<12}{s['input_tokens_per_case']:>12}{s['model_calls_per_case']:>11}{s['latency_p95_ms']:>17}{r.counts['answer_correct'].rate:>16}  {evaluate(policy, r, old).verdict}")

candidate    tokens/case calls/case p95 ms (formula)  answer_correct  gate
old (v1)           241.9      0.839           1065.0             0.8  block
new (v2)           223.2      0.839           1005.0            0.88  block
score 0.35         205.5      0.806           1003.0            0.88  review
score 0.5          128.4      0.581            987.0            0.68  block

python -m evidencedesk.evalkit.ci | grep -vE "  note|tool_|input_"; echo "exit ${PIPESTATUS[0]}"

policy evidencedesk-release v1  candidate 0f9236807b  baseline c4a8abc04e
changed: retrieval.chunking paragraph->sentence, retrieval.min_score 0.2->0.3
metric                     value  95% interval   baseline
answer_correct              0.88  0.7-0.958           0.8
abstain_correct            0.667  0.3-0.903           0.5
citation_support             1.0  0.857-1.0           1.0
recall_at_3                 0.96  0.805-0.993        0.96
latency_p95_ms            1005.0                   1065.0
GATE: BLOCK
  block  critical: case a02 failed
  block  floor: abstain_correct 0.667 (4/6) is below the floor 0.8
exit 1

python -m evidencedesk.evalkit.ci --min-score 0.35 | grep -E "^GATE|  (block|review)"; echo "exit ${PIPESTATUS[0]}"
python -m evidencedesk.evalkit.ci --min-score 0.5 | grep -E "^GATE|  (block|review)"; echo "exit ${PIPESTATUS[0]}"

GATE: REVIEW
  review band: abstain_correct 0.833 (5/6) is inside the review band above the floor 0.8
exit 0
GATE: BLOCK
  block  critical: case g05 failed
  block  floor: answer_correct 0.68 (17/25) is below the floor 0.8
  block  regress: answer_correct fell from 0.8 to 0.68 (allowed drop 0.05)
  review flips: 3 cases passed in the baseline and fail now: a01, g07, p06
exit 1

python -m pytest kit/tests/test_robust_stats.py kit/tests/test_robust_measures.py kit/tests/test_robust_gates.py | sed 's/ in [0-9.]*s//'

..........................                                               [100%]
26 passed

a = {f"c{i:02d}": 0.0 for i in range(31)}
for fixed in (5, 6):
    b = {k: 1.0 if int(k[1:]) < fixed else 0.0 for k in a}
    print(f"{fixed} cases fixed, none broken: p = {paired(a, b).p}")

5 cases fixed, none broken: p = 0.062
6 cases fixed, none broken: p = 0.031

from evidencedesk.evalkit.slices import judge_slices
from evidencedesk.evalkit.stats import Count

for n in range(8, 20):
    row = judge_slices({"lang=bn": Count(k=round(0.6 * n), n=n)}, Count(k=26, n=31))[0]
    print(n, f"{row['k']}/{n}", row["status"])
    if row["status"] == "low":
        break

8 5/8 weak
9 5/9 low

from evidencedesk.evalkit.calibrate import calibrate, is_stale
from evidencedesk.evalkit.gates import Decision, Reason


def with_judge(decision, record, prompt, pairs, min_kappa=0.6):
    if is_stale(record, prompt, pairs):
        extra = Reason("review", "judge", "calibration is stale")
    elif record["kappa"] < min_kappa:
        extra = Reason("block", "judge", f"kappa {record['kappa']} is below {min_kappa}")
    else:
        return decision
    reasons = decision.reasons + (extra,)
    return Decision("block" if any(r.level == "block" for r in reasons) else "review", reasons)


mid = evaluate(policy, reports["score 0.35"], old)
for judge, version in (("mixed", 1), ("clean", 1), ("clean", 2)):
    record = calibrate(pairs, DEMO_JUDGES[judge], prompts[1], swap=False)  # measured with prompt 1
    d = with_judge(mid, record, prompts[version], pairs)
    print(f"{judge} judge, prompt v{version}:", d.verdict, [r.text for r in d.reasons if r.rule == "judge"])

mixed judge, prompt v1: block ['kappa 0.456 is below 0.6']
clean judge, prompt v1: review []
clean judge, prompt v2: review ['calibration is stale']