মূল্যায়ন ডেটাসেট ও নির্ধারিত টেস্ট

# file: kit/evidencedesk/evals.py (new file)
"""Evaluation: cases, a versioned dataset with a drift check, deterministic runners and metrics, a saved baseline. No live model."""
from __future__ import annotations

import hashlib
import json
import re
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Literal

from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator

from evidencedesk.chunking import chunk_corpus
from evidencedesk.citations import MIN_MARGIN, RagCase, check_citations, guarded_answer, load_rag_cases
from evidencedesk.llm import LLMRequest, LLMResponse, Usage
from evidencedesk.rag import MIN_SCORE, RecordedReplies, Retrieved, Retriever, reply
from evidencedesk.results import parse_answer
from evidencedesk.tools import ToolCall, ToolRegistry

KIT = Path(__file__).resolve().parent.parent
REASONS = {  # why a RagCase matters, by its first tag (the RAG file has no reason column)
    "direct": "The plain question customers ask most: a miss here is a basic failure.",
    "exact_term": "An exact identifier or number: hard for embeddings.",
    "paraphrase": "Different words, same meaning: retrieval beyond shared vocabulary.",
    "bangla": "A Bangla question must find a Bangla document.",
    "stale": "A superseded policy says something else: the old version must never answer.",
    "unanswerable": "The corpus has no answer: the assistant must say so, not guess.",
}


# --- 1. One evaluation case: input, expected behaviour, and why it matters ---
class EvalCase(BaseModel):
    model_config = ConfigDict(extra="forbid", frozen=True)

    id: str
    kind: Literal["rag", "tool", "format"]
    input: str | dict[str, Any]  # rag: the question. tool: {name, arguments}. format: {text, stop_reason}
    expect: str  # rag: answer | abstain. tool: ok or an error code. format: ok or a parse code
    doc_ids: list[str] = []  # rag: documents that hold the answer
    must_cite: bool = False
    tags: list[str] = []
    reason: str = Field(min_length=10)  # why this case exists

    @model_validator(mode="after")
    def consistent(self) -> EvalCase:
        if self.kind == "rag":
            if self.expect not in ("answer", "abstain") or not isinstance(self.input, str):
                raise ValueError("a rag case needs a question and expect answer or abstain")
            if (self.expect == "answer") != bool(self.doc_ids) or (self.must_cite and self.expect != "answer"):
                raise ValueError("answer needs doc_ids (and only then), and only an answer can require a citation")
        elif not isinstance(self.input, dict):
            raise ValueError(f"a {self.kind} case needs an object as input")
        return self

    @classmethod
    def from_rag(cls, case: RagCase, reason: str | None = None) -> EvalCase:
        return cls(id=case.id, kind="rag", input=case.question, expect="answer" if case.answerable else "abstain",
                   doc_ids=case.expected_doc_ids, must_cite=case.must_cite, tags=list(case.tags),
                   reason=reason or REASONS[case.tags[0]])

from pathlib import Path

from pydantic import ValidationError

from evidencedesk.citations import load_rag_cases
from evidencedesk.evals import EvalCase

stale = EvalCase.from_rag(load_rag_cases(Path("kit/data/rag_cases.jsonl"))[8])  # rc09, the stale trap
print(stale.model_dump_json(indent=1))
try:
    EvalCase(id="x1", kind="rag", input="Can I pay in instalments?", expect="abstain", reason="because")
except ValidationError as error:
    print("rejected:", error.errors()[0]["msg"])

{
 "id": "rc09",
 "kind": "rag",
 "input": "Are refunds paid as store credit?",
 "expect": "answer",
 "doc_ids": [
  "kc-returns-v3"
 ],
 "must_cite": true,
 "tags": [
  "stale"
 ],
 "reason": "A superseded policy says something else: the old version must never answer."
}
rejected: String should have at least 10 characters

# file: kit/evidencedesk/evals.py (add)
# --- 2. A fixed, versioned dataset: a manifest names the files and their content hashes ---
class DatasetDrift(ValueError):
    """A dataset file changed without a new version."""


@dataclass(frozen=True)
class Dataset:
    name: str
    version: int
    sha256: str  # short hash over all file hashes
    cases: tuple[EvalCase, ...]

    def of_kind(self, kind: str) -> list[EvalCase]:
        return [c for c in self.cases if c.kind == kind]


def file_hash(path: Path) -> str:
    return hashlib.sha256(path.read_bytes().replace(b"\r\n", b"\n")).hexdigest()[:16]  # same hash on Windows and Linux


def write_manifest(folder: Path, name: str, version: int, files: dict[str, str]) -> Path:
    """files: {file name: 'rag' | 'eval'}. Run it on purpose, together with a version bump."""
    listed = {f: {"format": fmt, "sha256": file_hash(folder / f)} for f, fmt in sorted(files.items())}
    path = folder / "eval_manifest.json"
    path.write_text(json.dumps({"name": name, "version": version, "files": listed}, indent=1) + "\n", encoding="utf-8")
    return path


def load_dataset(manifest: Path, doc_ids: set[str] | None = None) -> Dataset:
    meta = json.loads(manifest.read_text(encoding="utf-8"))
    cases: list[EvalCase] = []
    for fname, info in meta["files"].items():
        path = manifest.parent / fname
        if (found := file_hash(path)) != info["sha256"]:
            raise DatasetDrift(f"{fname} changed (manifest {info['sha256']}, file {found}): bump the version, update the manifest")
        if info["format"] == "rag":
            cases += [EvalCase.from_rag(c) for c in load_rag_cases(path, doc_ids)]
            continue
        for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
            try:
                cases.append(EvalCase.model_validate_json(line))
            except ValidationError as error:
                raise ValueError(f"{fname} line {number}: {error.errors()[0]['msg']}") from None
    if len({c.id for c in cases}) != len(cases):
        raise ValueError("case ids must be unique")
    combined = "".join(f"{f}:{i['sha256']}" for f, i in meta["files"].items())
    return Dataset(meta["name"], meta["version"], hashlib.sha256(combined.encode()).hexdigest()[:16], tuple(cases))

import json
import shutil

from evidencedesk.evals import DatasetDrift, load_dataset, write_manifest

data = Path("kit/data")
files = {"rag_cases.jsonl": "rag", "eval_extra.jsonl": "eval"}
write_manifest(data, "evidencedesk-eval", 1, files)  # the same bytes as the committed manifest
dataset = load_dataset(data / "eval_manifest.json")
print(len(dataset.cases), "cases:", {k: len(dataset.of_kind(k)) for k in ("rag", "tool", "format")}, "version", dataset.version)
print({f: i["sha256"][:8] for f, i in json.loads((data / "eval_manifest.json").read_text())["files"].items()})
shutil.copytree(data, "drift")
changed = Path("drift/rag_cases.jsonl")
changed.write_text(changed.read_text(encoding="utf-8").replace("Which frame size", "What frame size"), encoding="utf-8")
try:
    load_dataset(Path("drift/eval_manifest.json"))
except DatasetDrift as error:
    print("drift:", error)

27 cases: {'rag': 16, 'tool': 6, 'format': 5} version 1
{'eval_extra.jsonl': '72ed5a9a', 'rag_cases.jsonl': '3e0cd4d2'}
drift: rag_cases.jsonl changed (manifest 3e0cd4d2a0b6c963, file 59adee656bacd701): bump the version, update the manifest

# file: kit/evidencedesk/evals.py (add)
# --- 3. Fake models for suites (stand-ins: they prove the plumbing, not the quality of a real model) ---
class CaseModel(RecordedReplies):
    """The hand-written reply if there is one; otherwise the WORST case: copy the top evidence, never abstain."""

    def complete(self, request: LLMRequest) -> LLMResponse:
        question = re.search(r'<question source="user-input">\n(.*?)\n</question>', request.messages[-1].content, re.S)
        if question and question.group(1) in self.replies:
            return super().complete(request)
        self.calls.append(request)
        block = re.search(r'<evidence source="([^"]+)">\n(.*?)\n</evidence>', request.messages[-1].content, re.S)
        body = reply(block.group(2)[:200], block.group(1)) if block else reply("Not stated.")
        return LLMResponse(json.dumps(body), "recorded", Usage(0, 0))


class Tape:
    """Wraps a client and keeps its replies, to score what the model said before any guard acted on it."""

    def __init__(self, inner) -> None:
        self.inner, self.replies = inner, []

    def complete(self, request: LLMRequest) -> LLMResponse:
        self.replies.append(self.inner.complete(request))
        return self.replies[-1]

# file: kit/evidencedesk/evals.py (add)
# --- 4. Metrics on one question or one reply ---
def docs_in_top(retriever: Retriever, question: str, k: int = 3, where: dict | None = None) -> list[str]:
    """Documents of the top-k chunks, best first. where=None -> current chunks only (production); {} -> no filter."""
    hits = retriever.index.search(question, retriever.model, k, {"status": "current"} if where is None else where)
    return list(dict.fromkeys(retriever.chunks[h.passage_id].doc_id for h in hits))


def retrieval_recall(retriever: Retriever, cases: list[EvalCase], k: int = 3, where: dict | None = None) -> tuple[float, list[str]]:
    """Share of answerable rag cases whose expected document is in the top k, and the ids of the misses."""
    asked = [c for c in cases if c.kind == "rag" and c.doc_ids]
    missed = [c.id for c in asked if not set(c.doc_ids) & set(docs_in_top(retriever, c.input, k, where))]
    return (1 - len(missed) / len(asked) if asked else 1.0), missed

from evidencedesk.chunking import chunk_corpus
from evidencedesk.evals import docs_in_top, retrieval_recall
from evidencedesk.rag import MIN_SCORE, Retriever

# paragraph chunks, a fixed clock (build_retriever, below, wraps this)
retriever = Retriever(chunk_corpus(Path("kit/corpus")), clock=lambda: "2026-10-13T09:00:00Z", k=3, min_score=MIN_SCORE)
cases = list(dataset.cases)
for k in (1, 3):
    value, missed = retrieval_recall(retriever, cases, k)
    print(f"recall@{k} = {value:.3f} on 13 answerable cases; missed: {', '.join(missed)}")
for case_id in ("rc06", "rc08"):
    case = next(c for c in cases if c.id == case_id)
    print(f"{case_id} wanted {case.doc_ids[0]}, top 3 documents: {docs_in_top(retriever, case.input)}")
stale = [c for c in cases if "stale" in c.tags]
for label, where in (("with status filter", None), ("without filter", {})):
    print(f"{label:<19} rc09 recall@1 = {retrieval_recall(retriever, stale, 1, where)[0]:.1f}, top document: {docs_in_top(retriever, stale[0].input, 1, where)}")

recall@1 = 0.769 on 13 answerable cases; missed: rc02, rc06, rc08
recall@3 = 0.846 on 13 answerable cases; missed: rc06, rc08
rc06 wanted kc-returns-v3, top 3 documents: ['kc-privacy-v1', 'kc-shipping-v1', 'kc-returns-v3-bn']
rc08 wanted kc-giftcards-v1, top 3 documents: ['kc-sizing-v1', 'kc-privacy-v1', 'kc-shipping-v1']
with status filter  rc09 recall@1 = 1.0, top document: ['kc-returns-v3']
without filter      rc09 recall@1 = 0.0, top document: ['kc-returns-v2']

# file: kit/evidencedesk/evals.py (add)
def citation_validity(cited: list[str], shown: list[Retrieved], known: set[str]) -> tuple[int, int]:
    """(cited ids that resolve to a chunk the model was shown, distinct cited ids)."""
    report = check_citations("", cited, shown, known)
    total = len(set(cited))
    return total - len(report.outside) - len(report.invented), total

from evidencedesk.evals import citation_validity

question = "How long do I have to return a bike?"
shown, known = retriever.retrieve(question), set(retriever.chunks)
for label, cited in (("all resolve", ["kc-returns-v3:c0"]), ("one real but not shown", ["kc-returns-v3:c0", "kc-recalls-v1:c0"]),
                     ("one invented", ["kc-returns-v3:c0", "kc-returns-v9:c0"]), ("no citation", [])):
    valid, total = citation_validity(cited, shown, known)
    print(f"{label:<24} {valid} of {total}")

all resolve              1 of 1
one real but not shown   1 of 2
one invented             1 of 2
no citation              0 of 0

# file: kit/evidencedesk/evals.py (add)
# --- 5. Running cases ---
@dataclass(frozen=True)
class CaseResult:
    id: str
    kind: str
    checks: dict[str, bool]
    detail: str = ""
    valid: int = 0  # cited ids that resolved
    cited: int = 0  # distinct cited ids

    @property
    def passed(self) -> bool:
        return all(self.checks.values())


def run_rag_case(case: EvalCase, retriever: Retriever, client, k: int = 3, min_margin: float = MIN_MARGIN) -> CaseResult:
    tape = Tape(client)
    answer = guarded_answer(tape, retriever, case.input, min_margin)
    checks = {"abstention": answer.abstained == (case.expect == "abstain")}
    if case.doc_ids:
        checks["retrieved"] = bool(set(case.doc_ids) & set(docs_in_top(retriever, case.input, k)))
    cited: list[str] = []
    if tape.replies and (outcome := parse_answer(tape.replies[-1])).ok:
        cited = list(outcome.value.cited_ids)
    valid, total = citation_validity(cited, retriever.retrieve(case.input), set(retriever.chunks))
    if total:
        checks["citations_resolve"] = valid == total
    if case.must_cite and not answer.abstained:
        checks["cites_expected"] = bool(answer.citations) and all(c.doc_id in case.doc_ids for c in answer.citations)
    return CaseResult(case.id, "rag", dict(sorted(checks.items())), answer.reason, valid, total)


def run_tool_case(case: EvalCase, registry: ToolRegistry) -> CaseResult:
    result = registry.execute(ToolCall(case.id, case.input["name"], case.input["arguments"]))
    got = "ok" if result.ok else result.error["code"]
    return CaseResult(case.id, "tool", {"outcome": got == case.expect}, got)


def run_format_case(case: EvalCase) -> CaseResult:
    reply_ = LLMResponse(case.input["text"], "recorded", Usage(0, 0), case.input.get("stop_reason", "end"))
    got = parse_answer(reply_).code
    return CaseResult(case.id, "format", {"outcome": got == case.expect}, got)


def build_retriever(corpus: Path, strategy: str = "paragraph", min_score: float = MIN_SCORE, k: int = 3) -> Retriever:
    return Retriever(chunk_corpus(corpus, strategy), clock=lambda: "2026-10-13T09:00:00Z", k=k, min_score=min_score)


def _share(done: int, total: int) -> float | None:
    return round(done / total, 3) if total else None


def run_suite(dataset: Dataset, retriever: Retriever, client, registry: ToolRegistry, k: int = 3, min_margin: float = MIN_MARGIN,
              notes: dict[str, Any] | None = None) -> dict[str, Any]:
    """Run every case. The result is plain JSON-able data: the same input gives the same bytes."""
    results = [run_rag_case(c, retriever, client, k, min_margin) if c.kind == "rag" else run_tool_case(c, registry)
               if c.kind == "tool" else run_format_case(c) for c in dataset.cases]
    rag, answerable = [r for r in results if r.kind == "rag"], {c.id for c in dataset.cases if c.doc_ids}
    table = {row: {"answered": 0, "abstained": 0} for row in ("answerable", "unanswerable")}
    for r in rag:
        table["answerable" if r.id in answerable else "unanswerable"]["answered" if r.detail == "ok" else "abstained"] += 1
    passed = lambda kind=None: _share(sum(r.passed for r in results if kind in (None, r.kind)), sum(kind in (None, r.kind) for r in results))
    metrics = {"recall_at_k": round(retrieval_recall(retriever, list(dataset.cases), k)[0], 3),
               "citation_validity": _share(sum(r.valid for r in rag), sum(r.cited for r in rag)),
               "abstention_correct": _share(sum(r.checks["abstention"] for r in rag), len(rag)),
               "tool_pass": passed("tool"), "format_pass": passed("format"), "pass_rate": passed()}
    config = {"k": k, "min_score": retriever.min_score, "min_margin": min_margin, "chunks": len(retriever.chunks), **(notes or {})}
    return {"dataset": {"name": dataset.name, "version": dataset.version, "sha256": dataset.sha256}, "config": config,
            "metrics": metrics, "confusion": table,
            "cases": {r.id: {"passed": r.passed, "checks": r.checks, "detail": r.detail} for r in results},
            "failures": sorted(r.id for r in results if not r.passed)}

from evidencedesk.evals import CaseModel, run_suite
from evidencedesk.tools import default_registry
from tests.test_citations import REPLIES  # hand-written replies from RAG Reliability and Evaluation Data

registry = default_registry(Path("kit/corpus"))
client = CaseModel({c.input: REPLIES[c.id] for c in dataset.cases if c.id in REPLIES})
result = run_suite(dataset, retriever, client, registry, notes={"chunking": "paragraph"})
print(f"{'':<14}{'answered':>9}{'abstained':>10}")
for row, counts in result["confusion"].items():
    print(f"{row:<14}{counts['answered']:>9}{counts['abstained']:>10}")
print("failures:", {i: result["cases"][i]["detail"] for i in result["failures"]})
worst = run_suite(dataset, retriever, CaseModel({}), registry)  # a model that never abstains
print("worst-case model, unanswerable:", worst["confusion"]["unanswerable"])

               answered abstained
answerable            9         4
unanswerable          0         3
failures: {'rc03': 'weak', 'rc06': 'weak', 'rc08': 'ambiguous', 'rc13': 'ambiguous'}
worst-case model, unanswerable: {'answered': 0, 'abstained': 3}

# file: kit/evidencedesk/evals.py (add)
@dataclass(frozen=True)
class Scores:
    """A manual rubric in miniature: three judgments, each 0 (fails), 1 (partly) or 2 (meets), kept apart on purpose."""

    relevance: int
    evidence_use: int
    format: int

    def __post_init__(self) -> None:
        if any(v not in (0, 1, 2) for v in asdict(self).values()):
            raise ValueError("each score is 0, 1 or 2")

from evidencedesk.evals import Scores

rated = {  # one human rater, three answers to "How long do I have to return a bike?"
    "30 days, cites the returns policy": Scores(relevance=2, evidence_use=2, format=2),
    "30 days, no citation": Scores(relevance=2, evidence_use=0, format=2),
    "correct text, but not the JSON contract": Scores(relevance=2, evidence_use=2, format=0),
}
for answer, s in rated.items():
    print(f"relevance={s.relevance} evidence_use={s.evidence_use} format={s.format}  {answer}")

relevance=2 evidence_use=2 format=2  30 days, cites the returns policy
relevance=2 evidence_use=0 format=2  30 days, no citation
relevance=2 evidence_use=2 format=0  correct text, but not the JSON contract

# file: kit/evidencedesk/evals.py (add)
# --- 6. Saved results (the baseline) ---
def save_results(results: dict[str, Any], path: Path) -> None:
    path.write_text(json.dumps(results, indent=1, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8")


def load_results(path: Path) -> dict[str, Any]:
    return json.loads(path.read_text(encoding="utf-8"))

from evidencedesk.evals import load_results, save_results

path = Path("kit/data/baseline_v1.json")
save_results(result, path)
saved = load_results(path)
print({k: saved["dataset"][k] for k in ("name", "version")}, saved["config"])
print(saved["metrics"])
rerun = run_suite(dataset, retriever, client, registry, notes={"chunking": "paragraph"})
save_results(rerun, Path("again.json"))
print("second run identical, byte for byte:", Path("again.json").read_bytes() == path.read_bytes())

{'name': 'evidencedesk-eval', 'version': 1} {'chunking': 'paragraph', 'chunks': 36, 'k': 3, 'min_margin': 0.05, 'min_score': 0.3}
{'abstention_correct': 0.75, 'citation_validity': 1.0, 'format_pass': 1.0, 'pass_rate': 0.852, 'recall_at_k': 0.846, 'tool_pass': 1.0}
second run identical, byte for byte: True

python -m pytest -q kit/tests/test_evals.py kit/tests/test_rag.py kit/tests/test_tools.py kit/tests/test_results.py kit/tests/test_citations.py | sed 's/ in [0-9.]*s//'

........................................................................ [ 88%]
.........                                                                [100%]
81 passed

from evidencedesk.evals import build_retriever

options = {"default": (0.3, 0.05), "lenient": (0.2, 0.0)}  # (min_score, min_margin)
runs = {name: run_suite(dataset, build_retriever(Path("kit/corpus"), min_score=score), client, registry, min_margin=margin,
                        notes={"strategy": name}) for name, (score, margin) in options.items()}
a, b = runs["default"], runs["lenient"]
for metric in a["metrics"]:
    print(f"{metric:<20}{a['metrics'][metric]:>8}{b['metrics'][metric]:>8}")
print("unanswerable answered:", a["confusion"]["unanswerable"]["answered"], "->", b["confusion"]["unanswerable"]["answered"])
print("new failures:", sorted(set(b["failures"]) - set(a["failures"])))

recall_at_k            0.846   0.846
citation_validity        1.0   0.875
abstention_correct      0.75   0.688
tool_pass                1.0     1.0
format_pass              1.0     1.0
pass_rate              0.852   0.815
unanswerable answered: 0 -> 3
new failures: ['rc14', 'rc15', 'rc16']

# file: kit/tests/test_lab_regression.py
from pathlib import Path

import pytest
from test_citations import REPLIES

from evidencedesk.evals import CaseModel, build_retriever, load_dataset, run_suite
from evidencedesk.tools import default_registry

CORPUS = Path(__file__).resolve().parent.parent / "corpus"
DATASET = load_dataset(CORPUS.parent / "data" / "eval_manifest.json")
CLIENT = CaseModel({c.input: REPLIES[c.id] for c in DATASET.cases if c.id in REPLIES})
STRATEGIES = {"default": (0.3, 0.05), "lenient": (0.2, 0.0)}  # (min_score, min_margin)
KNOWN = pytest.mark.xfail(strict=True, reason="known regression: the lenient setting answers the unanswerable")


def run(name):
    score, margin = STRATEGIES[name]
    return run_suite(DATASET, build_retriever(CORPUS, min_score=score), CLIENT, default_registry(CORPUS), min_margin=margin)


@pytest.mark.parametrize("name", ["default", pytest.param("lenient", marks=KNOWN)])
def test_critical_unanswerable_questions_are_never_answered(name):
    assert run(name)["confusion"]["unanswerable"]["answered"] == 0

python -m pytest -q kit/tests/test_lab_regression.py | sed 's/ in [0-9.]*s//'

.x                                                                       [100%]
1 passed, 1 xfailed

from evidencedesk.evals import run_tool_case

shutil.copytree(data, "easy")
line = {"id": "tl07", "kind": "tool", "input": {"name": "calculator", "arguments": {"expression": "2 ** 3"}}, "expect": "bad_expression",
        "tags": ["tool"], "reason": "Powers could build huge numbers, so the calculator must refuse them."}
with open("easy/eval_extra.jsonl", "a", encoding="utf-8") as f:
    f.write(json.dumps(line) + "\n")
try:
    load_dataset(Path("easy/eval_manifest.json"))
except DatasetDrift:
    print("drift caught: the manifest was not updated")
write_manifest(Path("easy"), "evidencedesk-eval", 2, files)
bigger = load_dataset(Path("easy/eval_manifest.json"))
new = next(c for c in bigger.cases if c.id == "tl07")
print(f"version {bigger.version}, {len(bigger.cases)} cases, tl07:", run_tool_case(new, registry).detail)

drift caught: the manifest was not updated
version 2, 28 cases, tl07: bad_expression

from evidencedesk.index import recall_at_k

labels = [(c.input, c.doc_ids[0]) for c in cases if c.kind == "rag" and c.doc_ids]
for k in (1, 3):
    plain, _ = recall_at_k(retriever.index, retriever.model, labels, k)
    print(f"k={k}: recall_at_k (no filter) {plain:.3f}, retrieval_recall (current only) {retrieval_recall(retriever, cases, k)[0]:.3f}")

k=1: recall_at_k (no filter) 0.692, retrieval_recall (current only) 0.769
k=3: recall_at_k (no filter) 0.846, retrieval_recall (current only) 0.846

for strategy in ("paragraph", "fixed"):
    r = build_retriever(Path("kit/corpus"), strategy)
    res = run_suite(dataset, r, client, registry, notes={"chunking": strategy})
    m = res["metrics"]
    print(f"{strategy:<10} chunks={res['config']['chunks']} recall={m['recall_at_k']} citation_validity={m['citation_validity']} pass_rate={m['pass_rate']}")

paragraph  chunks=36 recall=0.846 citation_validity=1.0 pass_rate=0.852
fixed      chunks=38 recall=0.846 citation_validity=0.889 pass_rate=0.815