RAG Reliability and Evaluation Data

# file: evidencedesk/citations.py
"""RAG reliability: citation checks, weak-retrieval guard, versions, saved index, evaluation cases."""
from __future__ import annotations

import hashlib
import json
import re
from dataclasses import asdict, dataclass, replace
from pathlib import Path
from typing import Callable, Literal

from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator

from evidencedesk.bm25 import tokens
from evidencedesk.chunking import Chunk, Location, chunk_corpus
from evidencedesk.context import NO_EVIDENCE
from evidencedesk.context_quality import family
from evidencedesk.index import Model, VectorIndex, ensure_index
from evidencedesk.llm import LLMClient
from evidencedesk.rag import MIN_SCORE, RagAnswer, Retrieved, Retriever, build_request, citations_for
from evidencedesk.results import parse_answer

MIN_MARGIN = 0.05  # a starting guess, like MIN_SCORE

# file: evidencedesk/citations.py (append)
# --- 1. Citations: resolve and support ---
def numbers(text: str) -> set[str]:
    return {str(int(n)) for n in re.findall(r"\d+", text)}  # also reads Bangla digits: '৩০' -> '30'


def stem(word: str) -> str:
    return word[:6].removesuffix("s")  # crude: 'returned' and 'returns' both become 'return'


@dataclass(frozen=True)
class Support:
    numbers_missing: tuple[str, ...]  # numbers of the claim that the evidence lacks
    share: float  # share of the claim's key words found in the evidence

    @property
    def ok(self) -> bool:
        return not self.numbers_missing and self.share >= 0.6


def check_support(claim: str, evidence: list[str]) -> Support:
    """Do the claim's numbers and key words appear in the cited text? Words only, not meaning."""
    text = " ".join(evidence)
    have = {stem(t) for t in tokens(text)}
    terms = [t for t in dict.fromkeys(tokens(claim)) if not any(ch.isdigit() for ch in t)]
    share = round(sum(stem(t) in have for t in terms) / len(terms), 3) if terms else 1.0
    return Support(tuple(sorted(numbers(claim) - numbers(text))), share)


@dataclass(frozen=True)
class CitationReport:
    outside: tuple[str, ...]  # real chunk ids that were not retrieved
    invented: tuple[str, ...]  # ids that exist nowhere
    support: Support | None  # None when no cited id was retrieved

    @property
    def ok(self) -> bool:
        return not self.outside and not self.invented and self.support is not None and self.support.ok


def check_citations(claim: str, cited: list[str], retrieved: list[Retrieved], known: set[str]) -> CitationReport:
    """`known` = every chunk id. Unresolved ids are split in two: they have different causes."""
    shown = {r.chunk.id: r.chunk.text for r in retrieved}
    ids = list(dict.fromkeys(cited))
    outside = tuple(i for i in ids if i not in shown and i in known)
    invented = tuple(i for i in ids if i not in shown and i not in known)
    found = [shown[i] for i in ids if i in shown]
    return CitationReport(outside, invented, check_support(claim, found) if found else None)

from pathlib import Path

from evidencedesk.chunking import chunk_corpus
from evidencedesk.citations import check_citations
from evidencedesk.rag import Retriever

chunks = chunk_corpus(Path("corpus"))
retriever = Retriever(chunks, clock=lambda: "2026-10-12T09:00:00Z")
Q = "How long do I have to return a bike?"
found, known = retriever.retrieve(Q), set(retriever.chunks)
RIGHT, C0 = "You can return a new, unridden bike within 30 days of delivery.", "kc-returns-v3:c0"
TRIES = [("correct", RIGHT, C0), ("wrong number", RIGHT.replace("30", "60"), C0),
         ("off-topic claim", "Orders above 500 ship free.", C0), ("real id, not retrieved", RIGHT, "kc-recalls-v1:c0"),
         ("invented id", RIGHT, "kc-returns-v9:c0"),
         ("fair paraphrase", "A bike can be sent back for a full refund up to a month after it arrives.", C0)]
print(f"{'case':<24}outside invented  numbers missing  share  ok")
for label, claim, cited in TRIES:
    r = check_citations(claim, [cited], found, known)
    nums, share = (list(r.support.numbers_missing), r.support.share) if r.support else ("-", "-")
    print(f"{label:<24}{len(r.outside):>7}{len(r.invented):>9}  {nums!s:<16}{share:>5}  {r.ok}")

case                    outside invented  numbers missing  share  ok
correct                       0        0  []                1.0  True
wrong number                  0        0  ['60']            1.0  False
off-topic claim               0        0  ['500']           0.0  False
real id, not retrieved        1        0  -                   -  False
invented id                   0        1  -                   -  False
fair paraphrase               0        0  []              0.444  False

# file: evidencedesk/citations.py (append)
# --- 2. Weak retrieval ---
@dataclass(frozen=True)
class Assessment:
    state: str  # ok | no_result | weak | ambiguous
    top: float
    margin: float  # best score minus second best


def candidates(retriever: Retriever, question: str) -> list[Retrieved]:
    """Like retriever.retrieve, without the score threshold."""
    hits = retriever.index.search(question, retriever.model, retriever.k, {"status": "current"})
    return [Retrieved(retriever.chunks[h.passage_id], h.score) for h in hits]


def assess(found: list[Retrieved], min_score: float = MIN_SCORE, min_margin: float = MIN_MARGIN) -> Assessment:
    """weak: best score below min_score. ambiguous: the two best hits are from different documents and closer than
    min_margin (two hits from one document are no doubt)."""
    if not found:
        return Assessment("no_result", 0.0, 0.0)
    top = found[0].score
    margin = top - found[1].score if len(found) > 1 else top
    if top < min_score:
        return Assessment("weak", top, margin)
    if len(found) > 1 and found[0].chunk.doc_id != found[1].chunk.doc_id and margin < min_margin:
        return Assessment("ambiguous", top, margin)
    return Assessment("ok", top, margin)


def guarded_answer(client: LLMClient, retriever: Retriever, question: str, min_margin: float = MIN_MARGIN) -> RagAnswer:
    """Abstain on weak retrieval, unresolved citations and unsupported claims."""
    if not question.strip():
        raise ValueError("empty question")
    found = candidates(retriever, question)
    no = lambda reason: RagAnswer(NO_EVIDENCE, (), True, reason, tuple(r.chunk.id for r in found))
    state = assess(found, retriever.min_score, min_margin).state
    if state != "ok":
        return no(state)  # the model is not called
    kept = [r for r in found if r.score >= retriever.min_score]
    outcome = parse_answer(client.complete(build_request(question, kept)))
    if not outcome.ok:
        return no(outcome.code)
    result = outcome.value
    if result.status == "abstained":
        return no("model_abstained")
    report = check_citations(result.answer, result.cited_ids, kept, set(retriever.chunks))
    if report.outside or report.invented:
        return no("unknown_citation")
    if not report.ok:
        return no("unsupported_claim")
    return RagAnswer(result.answer, citations_for(result.cited_ids, kept), False, "ok", tuple(r.chunk.id for r in found))

from evidencedesk.citations import assess, candidates

print(f"{'question':<40}{'top':>6}{'margin':>8}  {'state':<10}top two documents")
for q in [Q, "RD2-24", "Do you sell electric scooters?", "How long are my support chats stored?",
          "What happens after 30 days?"]:
    hits = candidates(retriever, q)
    a = assess(hits)
    print(f"{q:<40}{a.top:>6.3f}{a.margin:>8.3f}  {a.state:<10}{hits[0].chunk.doc_id} / {hits[1].chunk.doc_id}")

question                                   top  margin  state     top two documents
How long do I have to return a bike?     0.555   0.152  ok        kc-returns-v3 / kc-warranty-v2
RD2-24                                   0.295   0.137  weak      kc-recalls-v1 / kc-returns-v3-bn
Do you sell electric scooters?           0.350   0.048  ambiguous kc-battery-v1 / kc-privacy-v1
How long are my support chats stored?    0.397   0.040  ambiguous kc-privacy-v1 / kc-recalls-v1
What happens after 30 days?              0.344   0.020  ambiguous kc-privacy-v1 / kc-battery-v1

from evidencedesk.chunking import chunk_fixed, chunk_sentences, load_documents

sections = [s.text for d in load_documents(Path("corpus")) for s in d.sections]
facts = [(i, a, b) for i, t in enumerate(sections) for a, b in chunk_sentences(t, 1)]  # every sentence is a fact
total, whole = sum(map(len, sections)), {}
print(f"{len(facts)} sentences in {total} characters; windows of at most 120 characters")
print("overlap  chunks  repeated text  sentences whole in one chunk")
for overlap in (0, 40, 80):
    spans = [chunk_fixed(t, 120, overlap) for t in sections]
    covered = sum(len({p for a, b in s for p in range(a, b)}) for s in spans)
    repeated = (sum(b - a for s in spans for a, b in s) - covered) / total
    whole[overlap] = {f for f in facts if any(a <= f[1] and f[2] <= b for a, b in spans[f[0]])}
    print(f"{overlap:>7}{sum(map(len, spans)):>8}{repeated:>14.0%}{len(whole[overlap]):>14} of {len(facts)}")
i, a, b = sorted(whole[40] - whole[0])[0]
print("cut in two at overlap 0, whole at overlap 40:", repr(sections[i][a:b]))

76 sentences in 4087 characters; windows of at most 120 characters
overlap  chunks  repeated text  sentences whole in one chunk
      0      45            0%            67 of 76
     40      46           10%            70 of 76
     80      51           29%            76 of 76
cut in two at overlap 0, whole at overlap 40: 'Inflate the tyres to the pressure printed on the tyre wall.'

# file: evidencedesk/citations.py (append)
# --- 3. Versions: replace, never append ---
@dataclass(frozen=True)
class Stale:
    doc_id: str
    version: int
    problem: str  # superseded (filter it) | outdated_but_current (a bug)


def staleness(chunks: list[Chunk]) -> list[Stale]:
    """One row per document that is not the newest version of its family and language."""
    newest: dict[tuple[str, str], int] = {}
    for c in chunks:
        key = (family(c.doc_id), c.lang)
        newest[key] = max(newest.get(key, 0), c.version)
    rows = {(c.doc_id, c.version, "superseded" if c.status == "superseded" else "outdated_but_current")
            for c in chunks if c.version < newest[(family(c.doc_id), c.lang)]}
    return [Stale(*row) for row in sorted(rows)]


def update_document(chunks: list[Chunk], new: list[Chunk]) -> list[Chunk]:
    """Drop all chunks of the document (not only repeated ids), supersede older versions, add the new chunks."""
    doc_id, version, lang = new[0].doc_id, new[0].version, new[0].lang
    kept = [c for c in chunks if c.doc_id != doc_id]
    kept = [replace(c, status="superseded") if family(c.doc_id) == family(doc_id) and c.lang == lang and c.version < version
            else c for c in kept]
    return kept + new

import re
from dataclasses import replace

from evidencedesk.citations import staleness, update_document

report = lambda cs: [(s.doc_id, s.problem) for s in staleness(cs)]
print("corpus as shipped:", report(chunks))
forgot = [replace(c, status="current") if c.doc_id == "kc-returns-v2" else c for c in chunks]
print("status forgotten:", report(forgot), "->", [r.chunk.id for r in Retriever(forgot).retrieve(Q, k=2)])
v4 = [replace(c, id=c.id.replace("v3", "v4"), doc_id="kc-returns-v4", version=4, text=c.text.replace("30 days", "45 days"))
      for c in chunks if c.doc_id == "kc-returns-v3"]  # the policy changes to 45 days
promises = lambda cs: sorted({d for c in cs if c.status == "current" for d in re.findall(r"within (\d+) days of delivery", c.text)})
for label, state in [("appended", chunks + v4), ("replaced", update_document(chunks, v4))]:
    print(f"{label}: bike return promises {promises(state)}, report {report(state)}")

corpus as shipped: [('kc-returns-v2', 'superseded')]
status forgotten: [('kc-returns-v2', 'outdated_but_current')] -> ['kc-returns-v3:c0', 'kc-returns-v2:c0']
appended: bike return promises ['30', '45'], report [('kc-returns-v2', 'superseded'), ('kc-returns-v3', 'outdated_but_current')]
replaced: bike return promises ['45'], report [('kc-returns-v2', 'superseded'), ('kc-returns-v3', 'superseded')]

# file: evidencedesk/citations.py (append)
# --- 4. Ingestion writes, query time reads ---
class LoadedRetriever(Retriever):
    """A Retriever around a saved index: it embeds no chunk."""

    def __init__(self, index: VectorIndex, chunks: list[Chunk], model: Model | None = None, k: int = 3,
                 min_score: float = MIN_SCORE) -> None:
        self.chunks, self.index, self.skipped = {c.id: c for c in chunks}, index, []
        self.model, self.k, self.min_score = model or Model.toy(), k, min_score


def ingest(folder: Path, out: Path, model: Model | None = None, clock: Callable[[], str] = lambda: "unset") -> bool:
    """Save out/index.npz and out/chunks.json. False = nothing changed, nothing embedded."""
    model, chunks = model or Model.toy(), chunk_corpus(folder)
    digest = hashlib.sha256(json.dumps([asdict(c) for c in chunks], sort_keys=True).encode()).hexdigest()[:12]
    manifest = out / "chunks.json"
    saved = json.loads(manifest.read_text(encoding="utf-8"))["digest"] if manifest.exists() else None
    old = VectorIndex.load(out / "index.npz") if saved == digest else None
    index, rebuilt = ensure_index(old, [c.to_passage() for c in chunks], model, clock)
    if old is not None and not rebuilt:
        return False
    out.mkdir(parents=True, exist_ok=True)
    index.save(out / "index.npz")
    manifest.write_text(json.dumps({"digest": digest, "chunks": [asdict(c) for c in chunks]}, ensure_ascii=False), encoding="utf-8")
    return True


def open_retriever(out: Path, model: Model | None = None, **options: float) -> Retriever:
    """Query time: only load. The question is the only text embedded."""
    data = json.loads((out / "chunks.json").read_text(encoding="utf-8"))
    chunks = [Chunk(**{**d, "location": Location(**d["location"])}) for d in data["chunks"]]
    return LoadedRetriever(VectorIndex.load(out / "index.npz"), chunks, model, **options)

import shutil

from evidencedesk.citations import ingest, open_retriever
from evidencedesk.index import Model, counting

shutil.rmtree("store", ignore_errors=True)
ingest_model, embedded = counting(Model.toy())
print("first ingest rebuilt:", ingest(Path("corpus"), Path("store"), ingest_model), "| texts embedded:", sum(embedded))
print("second ingest rebuilt:", ingest(Path("corpus"), Path("store"), ingest_model), "| texts embedded in total:", sum(embedded))
query_model, embedded_now = counting(Model.toy())
loaded = open_retriever(Path("store"), query_model)  # query time: load only
same = [[r.chunk.id for r in loaded.retrieve(q)] == [r.chunk.id for r in retriever.retrieve(q)] for q in (Q, "When do gift cards expire?")]
print("same answers as the in-memory retriever:", same)
print("texts embedded at query time:", sum(embedded_now), "(the 2 questions);", len(loaded.chunks), "chunks loaded, none embedded")

first ingest rebuilt: True | texts embedded: 36
second ingest rebuilt: False | texts embedded in total: 36
same answers as the in-memory retriever: [True, True]
texts embedded at query time: 2 (the 2 questions); 36 chunks loaded, none embedded

# file: evidencedesk/bm25.py
"""Keyword search baseline: our own BM25 with Lucene-style IDF, k1 = 1.2 and b = 0.75."""
from __future__ import annotations

import math
from collections import Counter
from typing import Iterable

from evidencedesk.chunking import Chunk
from evidencedesk.embeddings import Hit, words

K1, B = 1.2, 0.75


def tokens(text: str) -> list[str]:
    """Lower-cased words without stop words; the Bangla full stop counts as a space."""
    return words(text.replace("\u0964", " "))


class BM25:
    """score = sum over query terms of idf * tf / (tf + k1 * (1 - b + b * len(doc) / avg_len))."""

    def __init__(self, texts: dict[str, str], k1: float = K1, b: float = B) -> None:
        self.k1, self.b = k1, b
        self.tf = {i: Counter(tokens(t)) for i, t in texts.items()}
        self.length = {i: sum(c.values()) for i, c in self.tf.items()}
        self.n = len(texts)
        self.avg_len = sum(self.length.values()) / self.n if self.n else 0.0
        self.df = Counter(t for c in self.tf.values() for t in c)  # documents per term

    @classmethod
    def from_chunks(cls, chunks: Iterable[Chunk], **options: float) -> BM25:
        """Index the text the vector index embeds, so both methods see equal input."""
        return cls({c.id: c.to_passage().embed_text for c in chunks}, **options)

    def idf(self, term: str) -> float:
        n = self.df.get(term, 0)
        return math.log(1 + (self.n - n + 0.5) / (n + 0.5))

    def score(self, query: str, doc_id: str) -> float:
        tf, size = self.tf[doc_id], self.length[doc_id]
        norm = self.k1 * (1 - self.b + self.b * size / self.avg_len) if size else 0.0
        return sum(self.idf(t) * tf[t] / (tf[t] + norm) for t in set(tokens(query)) if tf[t])

    def search(self, query: str, k: int = 3, allowed: set[str] | None = None) -> list[Hit]:
        """Best k with a score above zero; ties by id. `allowed` limits the ids (a status filter)."""
        ids = [i for i in self.tf if allowed is None or i in allowed]
        ranked = sorted(((self.score(query, i), i) for i in ids), key=lambda pair: (-pair[0], pair[1]))
        return [Hit(i, s) for s, i in ranked[:k] if s > 0]

import math

from evidencedesk.bm25 import BM25

bm = BM25({"d1": "red bike", "d2": "blue bike", "d3": "red helmet"})
idf = math.log(1 + (3 - 2 + 0.5) / (2 + 0.5))  # 'red' appears in 2 of the 3 documents
norm = 1.2 * (1 - 0.75 + 0.75 * 2 / 2)  # every document has the average length (2 words), so the length factor is 1
print(f"by hand: idf(red) = {idf:.4f}, score(red, d1) = {idf:.4f} * 1 / (1 + {norm}) = {idf / (1 + norm):.4f}")
print(f"BM25   : idf(red) = {bm.idf('red'):.4f}, score(red, d1) = {bm.score('red', 'd1'):.4f}")
for query in ("red", "helmet"):
    print(f"{query:<11}", [(h.passage_id, round(h.score, 4)) for h in bm.search(query)])

by hand: idf(red) = 0.4700, score(red, d1) = 0.4700 * 1 / (1 + 1.2) = 0.2136
BM25   : idf(red) = 0.4700, score(red, d1) = 0.2136
red         [('d1', 0.2136), ('d3', 0.2136)]
helmet      [('d3', 0.4458)]

# file: evidencedesk/citations.py (append)
# --- 5. Evaluation cases (data/rag_cases.jsonl) ---
Tag = Literal["direct", "exact_term", "paraphrase", "bangla", "stale", "unanswerable"]


class RagCase(BaseModel):
    model_config = ConfigDict(extra="forbid", frozen=True)

    id: str
    question: str = Field(min_length=1)
    expected_doc_ids: list[str]  # documents holding the answer; empty if none
    answerable: bool
    must_cite: bool  # an answer must carry a citation
    tags: list[Tag]

    @model_validator(mode="after")
    def consistent(self) -> RagCase:
        if self.answerable != bool(self.expected_doc_ids):
            raise ValueError("answerable needs expected_doc_ids, and only then")
        if self.must_cite and not self.answerable:
            raise ValueError("an unanswerable question cannot require a citation")
        return self


def load_rag_cases(path: Path, doc_ids: set[str] | None = None) -> list[RagCase]:
    """Validate every line; errors name the line. `doc_ids` (optional) checks expected documents exist."""
    cases: list[RagCase] = []
    for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
        if not line.strip():
            continue
        try:
            case = RagCase.model_validate_json(line)
        except ValidationError as error:
            raise ValueError(f"line {number}: {error.errors()[0]['msg']}") from None
        if doc_ids is not None and not set(case.expected_doc_ids) <= doc_ids:
            raise ValueError(f"line {number}: unknown document in expected_doc_ids")
        cases.append(case)
    return cases

from evidencedesk.citations import load_rag_cases

cases = load_rag_cases(Path("data/rag_cases.jsonl"), {c.doc_id for c in chunks})
print(len(cases), "cases,", sum(c.answerable for c in cases), "answerable")
Path("bad.jsonl").write_text('{"id": "x", "question": "q", "expected_doc_ids": [], "answerable": true, "must_cite": true, "tags": []}\n')
try:
    load_rag_cases(Path("bad.jsonl"))
except ValueError as error:
    print("rejected:", error)

16 cases, 13 answerable
rejected: line 1: Value error, answerable needs expected_doc_ids, and only then

from evidencedesk.citations import assess, candidates

pool = [(c, candidates(retriever, c.question)) for c in cases]
print("min_score  margin  answered unanswerable  turned away (answerable)  wrong top document")
for min_score, min_margin in [(0.2, 0), (0.3, 0), (0.4, 0), (0.5, 0), (0.3, 0.05)]:
    states = [(c, assess(found, min_score, min_margin).state, found) for c, found in pool]
    given = sum(not c.answerable and s == "ok" for c, s, _ in states)
    away = sum(c.answerable and s != "ok" for c, s, _ in states)
    wrong = sum(c.answerable and s == "ok" and f[0].chunk.doc_id not in c.expected_doc_ids for c, s, f in states)
    print(f"{min_score:>9}{min_margin:>8}{given:>13} of 3{away:>19} of 13{wrong:>20}")

min_score  margin  answered unanswerable  turned away (answerable)  wrong top document
      0.2       0            3 of 3                  0 of 13                   3
      0.3       0            1 of 3                  2 of 13                   2
      0.4       0            0 of 3                  4 of 13                   1
      0.5       0            0 of 3                  7 of 13                   0
      0.3    0.05            0 of 3                  4 of 13                   1

from evidencedesk.bm25 import BM25

kw = BM25.from_chunks(chunks)
current = {c.id for c in chunks if c.status == "current"}
loose = Retriever(chunks, min_score=0.0)
ranked = {c.id: ([r.chunk.doc_id for r in loose.retrieve(c.question, k=3)],
                 [h.passage_id.split(":")[0] for h in kw.search(c.question, 3, current)]) for c in cases if c.answerable}
hits = lambda tagged, method, n: sum(any(d in c.expected_doc_ids for d in ranked[c.id][method][:n]) for c in tagged)
print("kind        cases  vector@1  keyword@1  vector@3  keyword@3")
for tag in ("exact_term", "paraphrase", "bangla", "direct", "stale"):
    tagged = [c for c in cases if c.answerable and tag in c.tags]
    print(f"{tag:<12}{len(tagged):>4}{hits(tagged, 0, 1):>10}{hits(tagged, 1, 1):>11}{hits(tagged, 0, 3):>10}{hits(tagged, 1, 3):>11}")

kind        cases  vector@1  keyword@1  vector@3  keyword@3
exact_term     3         2          3         3          3
paraphrase     5         3          3         3          4
bangla         2         2          2         2          2
direct         2         2          2         2          2
stale          1         1          1         1          1

import sys

from evidencedesk.citations import guarded_answer
from evidencedesk.rag import RecordedReplies

sys.path.insert(0, "tests")
from test_citations import REPLIES

client = RecordedReplies({c.question: REPLIES[c.id] for c in cases if c.id in REPLIES})
results = {c.id: guarded_answer(client, retriever, c.question) for c in cases}
print("abstained:", {i: r.reason for i, r in results.items() if r.abstained})
for i in ("rc01", "rc09", "rc10"):
    print(i, "cites", results[i].citations[0].label)
print("answered", sum(not r.abstained for r in results.values()), "of", len(cases), "| model calls:", len(client.calls))

abstained: {'rc03': 'weak', 'rc06': 'weak', 'rc08': 'ambiguous', 'rc13': 'ambiguous', 'rc14': 'weak', 'rc15': 'ambiguous', 'rc16': 'weak'}
rc01 cites Returns policy (Returning a bike, lines 11-12) [kc-returns-v3:c0]
rc09 cites Returns policy (How refunds are paid, line 18) [kc-returns-v3:c2]
rc10 cites রিটার্ন নীতি (বাইক ফেরত, lines 11-12) [kc-returns-v3-bn:c0]
answered 9 of 16 | model calls: 9

python -m pytest -q tests/test_bm25.py tests/test_citations.py | sed 's/ in [0-9.]*s//'

................................                                         [100%]
32 passed

from evidencedesk.citations import RagCase
from evidencedesk.index import recall_at_k

stale = RagCase(id="rc17", question="Can I return a bike after 14 days?", expected_doc_ids=["kc-returns-v3"], answerable=True,
                must_cite=True, tags=["stale"])
labels = [(c.question, c.expected_doc_ids[0]) for c in [*cases, stale] if c.answerable]
for k in (1, 3):
    recall, missed = recall_at_k(retriever.index, retriever.model, labels, k)
    print(f"recall@{k} = {recall:.3f} on {len(labels)} answerable cases; missed: {len(missed)}")

recall@1 = 0.643 on 14 answerable cases; missed: 5
recall@3 = 0.857 on 14 answerable cases; missed: 2

from evidencedesk.citations import check_support

evidence = [retriever.chunks["kc-returns-v3:c0"].text]
for claim in ("You can return a used bike within 30 days of delivery.", "You cannot return a bike within 30 days of delivery.",
              "You can return a bike within 60 days of delivery."):
    s = check_support(claim, evidence)
    print(f"ok={s.ok!s:<5} share={s.share} numbers missing={list(s.numbers_missing)} | {claim}")

ok=True  share=0.833 numbers missing=[] | You can return a used bike within 30 days of delivery.
ok=True  share=0.833 numbers missing=[] | You cannot return a bike within 30 days of delivery.
ok=False share=1.0 numbers missing=['60'] | You can return a bike within 60 days of delivery.