Documentation, Release, and Portfolio Quality

# excerpt of kit/evidencedesk/docgen.py
def demo_md() -> str:
    class Crash:  # a model that fails with text that must never reach the caller
        def complete(self, request):
            raise RuntimeError("password=hunter2 at /srv/app/db.py")

    steps = [("1. Success", create_app(), Q_OK, "demo-1", "The answer cites the passage it used."),
             ("2. Missing evidence", create_app(), Q_NONE, "demo-2", "No document answers this, so the assistant refuses instead of guessing."),
             ("3. Handled error", create_app(client=Crash()), Q_OK, "demo-3",
              "The model crashed with an internal message. The caller gets a plain failure: no path, no password, no stack trace.")]
    from fastapi.testclient import TestClient

    out = ["# Demo", "", "Synthetic data only (the fictional Kestrel Cycles help centre). Offline: a fake model, no key, no network.",
           "Generated by `python -m evidencedesk.docgen` through the in-process test client; do not edit by hand.", ""]
    for title, app, question, rid, note in steps:
        with TestClient(app) as http:
            r = http.post("/ask", json={"question": question}, headers={"x-user-id": "mateo", "x-request-id": rid})
        out += [f"## {title}", "", note, "", "```text", f"POST /ask {_json({'question': question})}", f"-> {r.status_code} {_json(r.json())}", "```", ""]
    return "\n".join(out)

python -m evidencedesk.docgen
sed -n '/^## 1/,$p' docs/DEMO.md

wrote docs/DEMO.md
wrote docs/EVALUATION.md
wrote docs/LIMITATIONS.md
wrote docs/NEXT-STEP.md
wrote docs/CASE-STUDY.md
## 1. Success

The answer cites the passage it used.

```text
POST /ask {"question": "How long do refunds take after inspection?"}
-> 200 {"message": "Refunds go back to the original payment method within 5 business days after the warehouse receives the item. [kc-returns-v3:c2]", "request_id": "demo-1", "status": "answered"}
```

## 2. Missing evidence

No document answers this, so the assistant refuses instead of guessing.

```text
POST /ask {"question": "Can I pay in instalments?"}
-> 200 {"message": "Not enough evidence in the help centre to answer this. Please contact Kestrel Cycles support.", "request_id": "demo-2", "status": "refused"}
```

## 3. Handled error

The model crashed with an internal message. The caller gets a plain failure: no path, no password, no stack
trace.

```text
POST /ask {"question": "How long do refunds take after inspection?"}
-> 500 {"message": "Something went wrong on our side. Please try again later.", "request_id": "demo-3", "status": "failed"}
```

# file: kit/evidencedesk/adapter_example.py
"""A third-party adapter written against evidencedesk.llm only. No core module imports this file."""
from __future__ import annotations

from collections.abc import Callable
from typing import Any

from evidencedesk.errors import from_status
from evidencedesk.llm import LLMClient, LLMRequest, LLMResponse, Message, Usage

Transport = Callable[[dict[str, Any]], dict[str, Any]]  # one JSON object in, {"status", "headers", "body"} out
STOP = {"stop": "end", "max_tokens": "length"}  # their words -> ours; anything else is "other"


class JsonAdapter:
    """For an imaginary provider: system text and input are separate fields, usage and finish reason have their own names."""

    def __init__(self, transport: Transport, model: str = "example-model") -> None:
        self.transport, self.model = transport, model

    def complete(self, request: LLMRequest) -> LLMResponse:
        text = lambda role: "\n".join(m.content for m in request.messages if m.role == role)
        reply = self.transport({"model": self.model, "system": text("system"), "input": text("user"), "max_output": request.max_tokens})
        if reply["status"] != 200:  # job 4: failures become our error classes, so retries and the safe layer work unchanged
            raise from_status(reply["status"], reply.get("headers"), str(reply.get("body", "")))
        body = reply["body"]
        return LLMResponse(body["text"], self.model, Usage(body["tokens"]["in"], body["tokens"]["out"]), STOP.get(body["finish"], "other"))


def pretend_server(inner: LLMClient) -> Transport:
    """Stands in for the network: it speaks the imaginary wire format but answers from any LLMClient (here, a fake)."""
    def transport(payload: dict[str, Any]) -> dict[str, Any]:
        sent = LLMRequest((Message("system", payload["system"]), Message("user", payload["input"])), payload["model"], payload["max_output"])
        got = inner.complete(sent)
        finish = {v: k for k, v in STOP.items()}.get(got.stop_reason, "other")
        return {"status": 200, "body": {"text": got.text, "tokens": {"in": got.usage.input_tokens, "out": got.usage.output_tokens}, "finish": finish}}
    return transport

from fastapi.testclient import TestClient

from evidencedesk.adapter_example import JsonAdapter, pretend_server
from evidencedesk.api import OfflineClient, create_app


def ask(client, question):
    with TestClient(create_app(client=client)) as http:
        return http.post("/ask", json={"question": question}, headers={"x-user-id": "mateo"}).json()


adapted = JsonAdapter(pretend_server(OfflineClient()))
for q in ("How long do refunds take after inspection?", "Can I pay in instalments?"):
    print(ask(adapted, q) == ask(None, q), ask(adapted, q)["status"])

down = JsonAdapter(lambda payload: {"status": 500, "body": "stack trace at /srv/app.py"})
print(ask(down, "How long do refunds take after inspection?"))

True answered
True refused
{'status': 'failed', 'message': 'Something went wrong on our side. Please try again later.', 'request_id': 'req-1'}

# excerpt of kit/tests/test_release.py
def test_core_logic_gives_the_same_answer_through_the_adapter():
    question = "How long do I have to return a new bike?"
    assert answer_question(JsonAdapter(pretend_server(FakeClient())), question) == answer_question(FakeClient(), question)


@pytest.mark.parametrize("question", [Q, "Can I pay in instalments?"])
def test_the_whole_service_answers_identically_through_the_adapter(question):
    def ask(client):
        with TestClient(create_app(client=client)) as http:
            return http.post("/ask", json={"question": question}, **ASK).json()
    assert ask(JsonAdapter(pretend_server(OfflineClient()))) == ask(None)


def test_no_core_module_imports_the_adapter():
    for path in (KIT / "evidencedesk").glob("*.py"):
        if path.name != "adapter_example.py":
            imported = {n.module for n in ast.walk(ast.parse(path.read_text(encoding="utf-8"))) if isinstance(n, ast.ImportFrom)}
            assert "evidencedesk.adapter_example" not in imported, path.name

python -m pytest tests/test_release.py -k "adapter or core or whole_service or stop_reasons or failing_provider" | sed 's/ in [0-9.]*s//'

.......                                                                  [100%]
7 passed, 7 deselected

python -m evidencedesk.gate | sed -n '4,9p;$p'
sed -n '/^## Failures/,$p' docs/EVALUATION.md

  recall_at_k             0.882
  citation_validity       1.0
  abstention_correct      0.773
  tool_pass               1.0
  format_pass             1.0
  pass_rate               0.848
GATE: GO
## Failures

| Case | Question | Expected | What happened |
|---|---|---|---|
| lg03 | কিস্তিতে বাইক কেনা যায় কি? | abstain | a Bangla question no document answers; it was answered, not refused (the unsafe direction) |
| rc03 | RD2-24 | answer | an exact identifier on its own; the toy embedder matches letters and the match was too weak to answer |
| rc06 | How do I get my money back? | answer | a paraphrase ('money back' for 'refund'); the right document was not in the top results |
| rc08 | How long before a voucher lapses? | answer | a paraphrase ('voucher lapses' for 'gift card expires'); the top two results were too close to choose |
| rc13 | How long are my support chats stored? | answer | a paraphrase ('chats stored' for 'retention'); the top two results were too close to choose |

## What these numbers cannot show

- The answers come from recorded, hand-written replies, not from a language model. The suite checks retrieval,
  thresholds, citation checks and the gate, not answer quality of any real model.
- 33 cases is a small sample: one case moves the pass rate by about 3.0 points, so a one-case difference is
  noise.
- There is no held-out set: the same cases are used to judge and to tune, so the scores are optimistic for
  unseen questions.
- A passing citation check shows the cited passage was shown to the model, not that it supports the claim.
- No latency, cost or load was measured here.

cat docs/LIMITATIONS.md

# Limitations

Generated by `python -m evidencedesk.docgen`; the counts come from the kit as it is now.

## Languages

- Supported by tests: English and Bangla only. The corpus has 11 English and 2 Bangla documents; the
  evaluation has 5 Bangla and 3 mixed-script questions against 14 English ones.
- Bangla pass rate is 0.8 on 5 cases; failing: lg03. Any other language is untested.

## Models and retrieval

- The model is a fake with recorded replies. Real providers were never called for this release; their adapters
  are untested sketches.
- The embedder is a toy (character trigrams). It matches letters, not meaning: 4 English questions fail (3
  paraphrases, 1 exact identifier) and one test is a documented expected failure.

## Data

- Synthetic documents only (13 documents, 36 chunks), searched in memory. Large corpora, document updates and
  real customer text are untested.
- 33 evaluation cases and 15 abuse cases: regression sets, not proof of quality or safety.

## Service and security

- `x-user-id` is a stand-in for authentication. Do not expose the service before real authentication exists.
- Prompt injection is limited by design (allowlists, owner filters, screening), not solved; the abuse set
  shows known attacks are blocked, not that new ones are.
- Tested in-process with `TestClient` on Linux with Python 3.12.3. A real server, concurrency, load, macOS and
  Windows were not tested.

## Packaging

- Data folders sit beside the package; a built wheel would not contain them. `gate.hand_written_client`
  imports a test module (a known wart).

# excerpt of kit/evidencedesk/release.py
ALLOWED = {  # (file, kind) -> why a person decided this finding is fine; review it again whenever the file changes
    ("data/abuse/abuse_set.jsonl", "key-shaped"): "made-up bait that the key-shape control must block",
    ("tests/test_security_p17.py", "key-shaped"): "the same made-up bait, asserted by a test",
    ("tests/test_context_quality.py", "hard-coded secret"): "a reserved example address used as a fake secret",
}


def scan_secrets(root: Path = KIT, allowed: dict[tuple[str, str], str] | None = None) -> list[str]:
    """'path:line kind' for every secret-shaped string, secret-like file name or personal-data shape. Values are never returned.
    Findings listed in `allowed` (default ALLOWED) are dropped; pass {} to see everything."""
    allowed = ALLOWED if allowed is None else allowed
    found = []
    for path in project_files(root):
        rel = path.relative_to(root)
        found += [f"{rel} file name" for pattern in BAD_NAMES if path.match(pattern)]
        if path.suffix not in TEXT:
            continue
        for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1):
            found += [f"{rel}:{number} {kind}" for kind, shape in SHAPES.items() if shape.search(line)]
            found += [f"{rel}:{number} key-shaped" for m in KEY_SHAPE.finditer(line) if m[0] != SIMULATED_SECRET]  # the one made-up key is allowed
            found += [f"{rel}:{number} {kind}" for kind in personal_data(line)] if path.suffix in PERSONAL else []
    return [f for f in found if (f.split(":")[0].split(" ")[0], f.split(" ", 1)[1]) not in allowed]

import tempfile
from pathlib import Path

from evidencedesk import release

print("everything the scan sees:", *release.scan_secrets(allowed={}), sep="\n  ")
print("after review:", release.scan_secrets())

with tempfile.TemporaryDirectory() as folder:  # a planted copy; the values are built here so this page holds no key-like text
    (Path(folder) / "notes.md").write_text("key: sk-" + "ab" * 15 + "\npassword = \"" + "q" * 14 + "\"\nask ana@corp.test\n", encoding="utf-8")
    (Path(folder) / ".env").write_text("MODE=demo\n", encoding="utf-8")
    print("planted:", *release.scan_secrets(Path(folder)), sep="\n  ")

everything the scan sees:
  data/abuse/abuse_set.jsonl:6 key-shaped
  tests/test_context_quality.py:123 hard-coded secret
  tests/test_security_p17.py:55 key-shaped
after review: []
planted:
  .env file name
  notes.md:1 provider-style key
  notes.md:2 hard-coded secret
  notes.md:3 email

rm -rf ../try && cp -r . ../try && cd ../try
python -m venv .venv && source .venv/bin/activate
sed -i 's/^numpy==.*/numpy==2.5.3/' requirements.txt
python -m pip install -q --disable-pip-version-check -r requirements.txt
python -m pip freeze | diff - <(grep -v '^#' requirements.lock.txt)
python -m pytest | tail -1 | sed 's/ in [0-9.]*s//'
python -m evidencedesk.gate | tail -1
{ grep '^#' requirements.lock.txt; python -m pip freeze; } > lock.new && mv lock.new requirements.lock.txt
python -m pytest | tail -1 | sed 's/ in [0-9.]*s//'
python -m evidencedesk.release --tests | tail -1

10c10
< numpy==2.5.3
---
> numpy==2.5.4
1 failed, 468 passed, 1 xfailed
GATE: GO
469 passed, 1 xfailed
RESULT: READY (0 failed, 0 skipped)

# excerpt of kit/evidencedesk/release.py
@dataclass(frozen=True)
class Check:
    name: str
    status: str  # "ok", "FAIL" or "skip"
    detail: str


def run_checks(root: Path = KIT, tests: bool = False) -> list[Check]:
    checks: list[Check] = []

    def add(name: str, problem: str | None, ok_detail: str) -> None:
        checks.append(Check(name, "FAIL" if problem else "ok", problem or ok_detail))

    ...
    add("offline by default", None if (s.offline, s.provider, s.api_key) == (True, "fake", None) else "defaults are not offline", "offline, fake provider, no key")
    add("secrets and sensitive fixtures", f"{len(hits)} finding(s): {hits[:3]}" if hits else None, "no findings")
    add("evaluation gate", None if decision.verdict != "block" else "; ".join(decision.blockers), f"{decision.verdict.upper()} against the stored baseline")
    ...
    if tests:
        ...  # run pytest in a subprocess and keep its last line
    else:
        checks.append(Check("test suite", "skip", "run with --tests"))
    return checks

python -m evidencedesk.release

RELEASE CHECK evidencedesk
[ok  ] python version                  3.12 (needs >=3.12)
[ok  ] exact pins                      every line of requirements.txt uses ==
[ok  ] tests configured                testpaths ['tests'], 25 test files
[ok  ] required data                   5 of 5 paths present
[ok  ] offline by default              offline, fake provider, no key
[ok  ] documentation                   9 documents present
[ok  ] generated docs current          match a fresh run
[ok  ] secrets and sensitive fixtures  no findings
[ok  ] evaluation gate                 GO against the stored baseline
[ok  ] abuse set                       no failing case
[skip] test suite                      run with --tests
RESULT: READY (0 failed, 1 skipped)

cat docs/CASE-STUDY.md

# Case study: EvidenceDesk

Numbers below are measured by `python -m evidencedesk.docgen`; nothing is estimated.

## The user problem

Customers of a fictional bike shop ask policy questions. A wrong answer about refunds or recalls is worse than
no answer, so the assistant must cite its source, say when it has none, and never leak private data or obey
text hidden in a document.

## Architecture

A request enters one FastAPI route, passes identity, task and size checks, retrieves evidence from a local
corpus, builds a context in which document text is marked untrusted, asks a provider-neutral client for a
reply, validates the reply, and returns one response shape. The model is replaceable behind a one-method
interface; the safety checks live outside the model.

## Trade-offs

- Offline fake model and toy embedder: every test is repeatable and free, but real quality is unmeasured.
- Refuse when evidence is weak: fewer wrong answers, more refusals. The threshold decides the balance.
- One public response shape with no internal codes: callers learn nothing from errors, and developers rely on
  logs.
- Exact version pins and a lock file: repeatable installs, with the cost of deliberate upgrades.

## Measured results

- Evaluation (33 cases): recall 0.882, citation validity 1.0, abstention 0.773, pass rate 0.848.
- Failures: lg03, rc03, rc06, rc08, rc13; the release gate says GO against the stored baseline.
- Abuse set: 15 cases, checked by the release script.

## What I would do next

See `NEXT-STEP.md`: more Bangla refusal cases before any change to the threshold.

sed -n '/^| min_score/,/^$/p' docs/NEXT-STEP.md

| min_score | Failing cases | Pass rate | Unanswerable answered |
|---|---|---|---|
| 0.25 | lg03, rc06, rc08, rc13 | 0.879 | 1 of 5 |
| 0.3 | lg03, rc03, rc06, rc08, rc13 | 0.848 | 1 of 5 |
| 0.35 | rc03, rc06, rc08, rc13 | 0.879 | 0 of 5 |
| 0.4 | rc02, rc03, rc06, rc08, rc13 | 0.848 | 0 of 5 |

cd project
python3.12 -m venv .venv
source .venv/bin/activate
env | grep -ci api_key
python -m pip install -q --disable-pip-version-check -r requirements.txt
python -m pip freeze | diff - <(grep -v '^#' requirements.lock.txt) && echo "same as the lock file"
python -m evidencedesk.api | diff - <(sed -n '/^GET \/health/,/^POST \/ask -> 401/p' README.md) && echo "demo matches the README"
python - <<'EOF'
import socket


def blocked(*args, **kwargs):
    raise RuntimeError("network call attempted")


socket.socket.connect = blocked  # any outgoing connection now fails loudly
socket.getaddrinfo = blocked
from evidencedesk.api import demo
from evidencedesk.gate import run_gate_suite

print("offline demo lines:", len(demo().splitlines()), "| pass rate:", run_gate_suite()["metrics"]["pass_rate"])
EOF
python -m evidencedesk.release --tests

0
same as the lock file
demo matches the README
offline demo lines: 5 | pass rate: 0.848
RELEASE CHECK evidencedesk
[ok  ] python version                  3.12 (needs >=3.12)
[ok  ] exact pins                      every line of requirements.txt uses ==
[ok  ] tests configured                testpaths ['tests'], 25 test files
[ok  ] required data                   5 of 5 paths present
[ok  ] offline by default              offline, fake provider, no key
[ok  ] documentation                   9 documents present
[ok  ] generated docs current          match a fresh run
[ok  ] secrets and sensitive fixtures  no findings
[ok  ] evaluation gate                 GO against the stored baseline
[ok  ] abuse set                       no failing case
[ok  ] test suite                      469 passed, 1 xfailed
RESULT: READY (0 failed, 0 skipped)

rm -rf ../copy && cp -r . ../copy && cd ../copy && rm docs/LIMITATIONS.md
python -m evidencedesk.release | grep -E 'FAIL|RESULT'

[FAIL] documentation                   docs/LIMITATIONS.md missing or empty
[FAIL] generated docs current          LIMITATIONS.md out of date
RESULT: NOT READY (2 failed, 1 skipped)

python -m evidencedesk.gate --min-score 0.35 | sed -n '/failures/p;$p'

  failures: rc03, rc06, rc08, rc13
GATE: GO

from evidencedesk.gate import contract_problems
from evidencedesk.llm import LLMRequest, LLMResponse, Message, Usage, answer_question


class PartsAdapter:
    """An imaginary provider: the reply is a list of parts and a done reason; it reports no usage at all."""

    def __init__(self, transport):
        self.transport = transport

    def complete(self, request: LLMRequest) -> LLMResponse:
        raw = self.transport({"text": request.messages[-1].content})
        stop = {"complete": "end", "length": "length"}.get(raw["done_reason"], "other")
        return LLMResponse("".join(raw["parts"]), "parts-model", Usage(), stop)  # Usage() = zeros: unknown, and we say so


cut = PartsAdapter(lambda p: {"parts": ["Standard delivery ", "takes 4 to 7 days"], "done_reason": "length"})
print(contract_problems(cut.complete(LLMRequest((Message("user", "How long?"),)))), "|", answer_question(cut, "How long?"))
silent = PartsAdapter(lambda p: {"parts": [""], "done_reason": "complete"})
print(contract_problems(silent.complete(LLMRequest((Message("user", "How long?"),)))))

[] | Standard delivery takes 4 to 7 days [cut off]
['a finished reply needs text']