প্যাকেজিং, কনফিগারেশন ও পুনরুৎপাদনযোগ্যতা

grep -rn 'from tests' evidencedesk

evidencedesk/gate.py:230:    from tests.test_citations import REPLIES  # the hand-written replies of an earlier page

# file: kit/requirements.txt
# Direct dependencies, exact versions, tested with Python 3.12.3. Install: python -m pip install -r requirements.txt
pydantic==2.14.1
fastapi==0.143.0
numpy==2.5.4
pytest==9.1.1
httpx2==2.13.1

python -m pip freeze

annotated-doc==0.0.5
annotated-types==0.8.0
anyio==4.15.1
fastapi==0.143.0
h11==0.16.0
httpcore2==2.13.1
httpx2==2.13.1
idna==3.20
iniconfig==2.3.1
numpy==2.5.4
opentelemetry-api==1.45.1
packaging==26.3
pluggy==1.6.0
pydantic==2.14.1
pydantic_core==2.50.1
Pygments==2.21.0
pytest==9.1.1
starlette==1.7.0
truststore==0.10.5
typing-inspection==0.4.4
typing_extensions==4.16.0

# file: kit/pyproject.toml
[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"

[project]
name = "evidencedesk"
version = "0.19.0"
description = "EvidenceDesk: an offline, tested help-centre assistant (AI Engineering with LLMs, Beginner)."
readme = "README.md"
requires-python = ">=3.12"  # tested on 3.12.3; current numpy needs 3.12 or newer
dependencies = [
    "pydantic>=2.14,<3",
    "fastapi>=0.143,<1",
    "numpy>=2.5,<3",
]

[project.optional-dependencies]
test = [
    "pytest>=9.0.3",  # earlier versions have a published advisory
    "httpx2>=2.13,<3",  # the client behind fastapi.testclient.TestClient
]

[tool.setuptools]
packages = ["evidencedesk"]  # corpus/, data/ and fixtures/ sit beside the package: use a source checkout or an editable install

[tool.pytest.ini_options]
pythonpath = ["."]  # `python -m pytest` and `pytest` both import evidencedesk and tests from the project folder
testpaths = ["tests"]
addopts = "-q"

# file: kit/evidencedesk/api.py
"""EvidenceDesk service layer: validated app settings, a small offline model, /ask, /health and /ready."""
from __future__ import annotations

import itertools
import json
import re
from collections.abc import Mapping
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any, Literal

from fastapi import FastAPI, Header
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict, Field, model_validator

from evidencedesk.chunking import chunk_corpus
from evidencedesk.config import PREFIX, Settings, load_settings
from evidencedesk.llm import LLMRequest, LLMResponse, Usage
from evidencedesk.permissions import Outbox
from evidencedesk.rag import Retriever
from evidencedesk.safefail import KIT, SafeResponse, respond, safe_answer
from evidencedesk.security import OwnedStore, SecretGuard, SIMULATED_SECRET, SafeLog, Session, load_records, session_gate

REQUIRED_DATA = ("corpus", "fixtures/responses.json", "data/rag_cases.jsonl", "data/abuse/abuse_set.jsonl", "data/attacks/attacks.jsonl")
HTTP_OF = {"unauthenticated": 401, "forbidden_task": 403, "input_too_large": 413, "not_ready": 503}  # by code; else by status
HTTP_BY_STATUS = {"answered": 200, "refused": 200, "blocked": 403, "failed": 500}


# --- 1. configuration: the p02 Settings plus two fields; offline is the default --------------------------
class AppSettings(Settings):
    offline: bool = True  # no network, no key: only the fake provider may run
    corpus_dir: Path = KIT / "corpus"

    @model_validator(mode="after")
    def _check_offline(self) -> AppSettings:
        if self.offline and self.provider != "fake":
            raise ValueError(f"{PREFIX}OFFLINE is on, so provider {self.provider!r} is not allowed: set {PREFIX}OFFLINE=0 on purpose")
        return self


def load_app_settings(env: Mapping[str, str]) -> AppSettings:
    """Same variables as load_settings, plus EVIDENCEDESK_OFFLINE (default true) and EVIDENCEDESK_CORPUS_DIR."""
    data = load_settings(env).model_dump()
    for field in ("offline", "corpus_dir"):
        if env.get(PREFIX + field.upper()):
            data[field] = env[PREFIX + field.upper()]
    return AppSettings(**data)


# --- 2. data that travels with the project -----------------------------------------------------------------
def missing_data(root: Path = KIT) -> list[str]:
    """Required data paths that are absent. The paths hang off the project folder, never off the working directory."""
    return [p for p in REQUIRED_DATA if not (root / p).exists()]


# --- 3. a tiny offline model: it answers with the first sentence of the best evidence ------------------------
class OfflineClient:
    """Not a language model. It lets the whole service run with no key and no network."""

    EVIDENCE = re.compile(r'<evidence source="([^"]+)"[^>]*>\n(.*?)\n</evidence>', re.S)

    def complete(self, request: LLMRequest) -> LLMResponse:
        found = self.EVIDENCE.search(request.messages[-1].content)
        text = "I could not read the evidence." if not found else f"{found[2].split('. ')[0].strip().rstrip('.')}. [{found[1]}]"
        return LLMResponse(json.dumps({"answer": text, "tool_requests": []}), "offline-fake", Usage(0, 0))


# --- 4. request and response models --------------------------------------------------------------------------
class AskRequest(BaseModel):
    model_config = ConfigDict(extra="forbid")
    question: str = Field(min_length=1)  # the size limit belongs to safe_answer, so it answers 413 in one shape


class AskResponse(BaseModel):
    status: Literal["answered", "refused", "blocked", "failed"]
    message: str
    request_id: str


class Ready(BaseModel):
    ready: bool
    reason: str = ""
    chunks: int = 0
    offline: bool = True


def load_retriever(settings: AppSettings) -> Retriever:
    if not settings.corpus_dir.is_dir():
        raise FileNotFoundError("corpus folder")
    if not (chunks := chunk_corpus(settings.corpus_dir)):
        raise ValueError("corpus has no documents")
    return Retriever(chunks, clock=lambda: "2026-10-13T09:00:00Z")


# --- 5. the app: liveness answers always; readiness only after the corpus is loaded -------------------------
def create_app(settings: AppSettings | None = None, client: Any = None, loader: Any = load_retriever) -> FastAPI:
    settings = settings or AppSettings()
    guard = SecretGuard({"api_key": SIMULATED_SECRET})
    state: dict[str, Any] = {"retriever": None, "reason": "not_loaded", "log": SafeLog(guard), "ids": itertools.count(1)}
    store, outbox, model = OwnedStore(load_records()), Outbox(), client or OfflineClient()

    @asynccontextmanager
    async def lifespan(_app: FastAPI):
        try:
            state["retriever"], state["reason"] = loader(settings), ""
        except Exception as exc:  # the type is enough to start the investigation; the text may hold paths
            state["reason"] = type(exc).__name__
        yield

    app = FastAPI(title="EvidenceDesk", lifespan=lifespan)
    app.state.service = state

    @app.get("/health")
    def health() -> dict[str, str]:
        return {"status": "alive"}  # the process is running; says nothing about the corpus

    @app.get("/ready", response_model=Ready)
    def ready() -> JSONResponse:
        retriever = state["retriever"]
        body = Ready(ready=retriever is not None, reason=state["reason"], chunks=len(retriever.chunks) if retriever else 0,
                     offline=settings.offline)
        return JSONResponse(body.model_dump(), status_code=200 if body.ready else 503)

    @app.post("/ask", response_model=AskResponse)
    def ask(body: AskRequest, x_user_id: str = Header(default=""), x_request_id: str = Header(default="")) -> JSONResponse:
        request_id = x_request_id[:40] or f"req-{next(state['ids'])}"
        if state["retriever"] is None:
            answer = respond("not_ready", request_id, log=state["log"], reason=state["reason"])
        else:
            session = Session(x_user_id.strip(), frozenset({"faq"}))  # a stand-in for real authentication (Level 0)
            gate = session_gate(session, settings.corpus_dir, store, outbox)
            answer = safe_answer(model, state["retriever"], body.question, session, gate, guard, "faq", request_id, state["log"])
        return JSONResponse(answer.public(), status_code=HTTP_OF.get(answer.code, HTTP_BY_STATUS[answer.status]))

    return app


# --- 6. the offline demo behind `python -m evidencedesk.api` ----------------------------------------------------
def demo() -> str:
    from fastapi.testclient import TestClient  # no server, no network: the app is called in-process

    lines = []
    with TestClient(create_app()) as http:
        for method, path, kw in (("GET", "/health", {}), ("GET", "/ready", {}),
                                 ("POST", "/ask", {"json": {"question": "How long do refunds take after inspection?"}, "headers": {"x-user-id": "mateo"}}),
                                 ("POST", "/ask", {"json": {"question": "Can I pay in instalments?"}, "headers": {"x-user-id": "mateo"}}),
                                 ("POST", "/ask", {"json": {"question": "Can I pay in instalments?"}})):
            r = http.request(method, path, **kw)
            body = r.json()
            shown = json.dumps(body, sort_keys=True) if path != "/ask" else f"{body['status']} {body['request_id']}: {body['message']}"
            lines.append(f"{method} {path} -> {r.status_code} {shown}")
    return "\n".join(lines)


if __name__ == "__main__":
    print(demo())

from pydantic import ValidationError

from evidencedesk.api import load_app_settings

s = load_app_settings({})
print("defaults:", s.offline, s.provider, s.api_key, s.corpus_dir.name)
env = {"EVIDENCEDESK_PROVIDER": "openai", "EVIDENCEDESK_MODEL": "some-model", "EVIDENCEDESK_API_KEY": "sk-demo-1234"}
for label, e in (("provider set, offline on", env), ("offline switched off", {**env, "EVIDENCEDESK_OFFLINE": "0"}),
                 ("timeout -1", {"EVIDENCEDESK_TIMEOUT_S": "-1"})):
    try:
        got = load_app_settings(e)
        print(f"{label:24} -> ok: {got.provider}, key {got.api_key}")
    except ValidationError as exc:
        print(f"{label:24} -> {exc.errors()[0]['msg'].split(':')[0]}")

defaults: True fake None corpus
provider set, offline on -> Value error, EVIDENCEDESK_OFFLINE is on, so provider 'openai' is not allowed
offline switched off     -> ok: openai, key **********
timeout -1               -> Input should be greater than 0

import os
import tempfile
from pathlib import Path

from evidencedesk.api import REQUIRED_DATA, missing_data

here = os.getcwd()
os.chdir(tempfile.mkdtemp())  # a folder that is not the project
print("from another folder, missing:", missing_data())
print("a folder without the data, missing:", len(missing_data(Path("elsewhere"))), "of", len(REQUIRED_DATA))
os.chdir(here)

from another folder, missing: []
a folder without the data, missing: 5 of 5

from contextlib import nullcontext

from fastapi.testclient import TestClient

from evidencedesk.api import AppSettings, create_app


def probe(label, app, started=True):
    client = TestClient(app)
    with (client if started else nullcontext(client)) as http:  # `with` runs the start-up step that loads the corpus
        health, ready = http.get("/health"), http.get("/ready")
        ask = http.post("/ask", json={"question": "How long do refunds take?"}, headers={"x-user-id": "mateo"})
    print(f"{label:16} health {health.status_code} | ready {ready.status_code} {ready.json()['reason'] or 'ok':14} | ask {ask.status_code} {ask.json()['status']}")


probe("loaded", create_app())
probe("corpus missing", create_app(AppSettings(corpus_dir="no-such-folder")))
probe("never started", create_app(), started=False)

loaded           health 200 | ready 200 ok             | ask 200 answered
corpus missing   health 200 | ready 503 FileNotFoundError | ask 503 failed
never started    health 200 | ready 503 not_loaded     | ask 503 failed

python -m pytest | tail -1 | sed 's/ in [0-9.]*s//'

455 passed, 1 xfailed

# file: kit/README.md
# EvidenceDesk

A small help-centre assistant for the fictional company Kestrel Cycles. It retrieves evidence from a local corpus, answers with
citations, refuses when evidence is missing, and fails safe. **It runs fully offline with a fake model: no API key, no network.**
The fake model is not a language model; real models are less predictable, which is why the project is tested.

## Requirements

- Python 3.12 (tested on 3.12.3). Linux, macOS or Windows.
- The packages in `requirements.txt` (exact versions). `requirements.lock.txt` lists every installed package of the tested environment.

## Install

```bash
python3.12 -m venv .venv
source .venv/bin/activate          # Windows: .venv\Scripts\activate
python -m pip install -r requirements.txt
```

## Run the offline demo

The demo calls the API in-process (no server, no network): liveness, readiness, one answer, one missing-evidence refusal, one
request with no signed-in user.

```bash
python -m evidencedesk.api
```

```text
GET /health -> 200 {"status": "alive"}
GET /ready -> 200 {"chunks": 36, "offline": true, "ready": true, "reason": ""}
POST /ask -> 200 answered req-1: Refunds go back to the original payment method within 5 business days after the warehouse receives the item. [kc-returns-v3:c2]
POST /ask -> 200 refused req-2: Not enough evidence in the help centre to answer this. Please contact Kestrel Cycles support.
POST /ask -> 401 refused req-3: I cannot help with that request. Please contact Kestrel Cycles support.
```

## Test

```bash
python -m pytest
```

Expected last line (the time differs): `455 passed, 1 xfailed`. The one expected failure (`xfailed`) is a documented limit of the
toy embedder.

## Configuration

Environment variables, all optional. An empty value counts as not set.

| Variable | Default | Meaning |
|---|---|---|
| `EVIDENCEDESK_OFFLINE` | `true` | When true, only the fake provider may run. Set `0` on purpose to allow another provider. |
| `EVIDENCEDESK_PROVIDER` | `fake` | `fake`, `openai`, `anthropic`, `gemini` or `local`. Only `fake` is tested here. |
| `EVIDENCEDESK_MODEL` | `fake-model` | Required for a non-fake provider. |
| `EVIDENCEDESK_API_KEY` or `_API_KEY_FILE` | none | Needed by hosted providers; never printed. |
| `EVIDENCEDESK_TIMEOUT_S`, `EVIDENCEDESK_MAX_RETRIES` | `30`, `2` | Validated at start-up. |
| `EVIDENCEDESK_CORPUS_DIR` | the project's `corpus/` | Folder of documents to index. |

## API

- `GET /health`: the process is alive. Always 200.
- `GET /ready`: 200 only when the corpus is loaded and indexed; otherwise 503 with a short reason.
- `POST /ask` with `{"question": "..."}` and header `x-user-id` (a stand-in for real authentication). Answers in one shape:
  `status` (`answered`, `refused`, `blocked`, `failed`), `message`, `request_id`. Codes and exception text are never returned.

## Layout

```text
evidencedesk/  llm, config, errors      provider-neutral client, settings, error classes
               context, context_quality  context assembly, trust labels, budgets
               embeddings, index, chunking, bm25, rag, citations   retrieval and cited answers
               tools, permissions       one read-only tool, allowlists, approvals
               results, contracts, streaming   output contracts and streaming
               evals, gate              evaluation datasets and the release gate
               usage, cache, limits     cost, caching, budgets
               security, safefail       injection controls, safe responses, abuse set
               api                      settings, /ask, /health, /ready
corpus/ data/ fixtures/                 synthetic documents, datasets, recorded replies
tests/                                  pytest suite
```

Dependencies point downwards: `api` imports `safefail`, which imports `security`, and so on down to `llm`. Library code does not
import tests, with one known exception (see Limitations).

## Data flow and trust boundaries

```text
 client (untrusted) --POST /ask--> api.py: validate body, read x-user-id
                                      |  safefail: identity, task, size
 ==== boundary 1: input from here on is hostile ================================
 corpus files --> chunking --> index --> Retriever (text is data, not orders)
                                      |  evidence tagged trust="untrusted", secrets screened
 ==== boundary 2: the model is not trusted either ===============================
 model (fake by default) --> reply must be valid JSON --> tool gate (allowlist,
                                      |  limits, approval) --> secret screen
 client <-- {status, message, request_id} -- no codes, no internals, redacted log
```

## Limitations

- The fake model and the toy embedder are stand-ins. Retrieval quality on real documents is not measured.
- Four evaluation cases are known misses (rc03, rc06, rc08, rc13); the release gate compares against a stored baseline.
- The corpus is synthetic: English plus two Bangla pages. Other languages are untested.
- `x-user-id` is not authentication. Add real authentication before any real use.
- Only `fake` was run. Hosted providers are untested adapters; check the provider's current documentation.
- Data folders sit beside the package: use a source checkout or `pip install -e .`. A built wheel contains only `evidencedesk/`.
- `evidencedesk.gate.hand_written_client` imports `tests.test_citations`: a known wart, pinned by a test.
- No real server, concurrency or load was tested; `TestClient` calls the app in-process.

## Hashes

`requirements.lock.txt` pins versions but has no hashes. For installs that must be verified, produce a hashed file with a
dedicated tool and install with `python -m pip install --require-hashes -r <file>`; check pip's current documentation.

 client (untrusted) --POST /ask--> api.py: validate body, read x-user-id
                                      |  safefail: identity, task, size
 ==== boundary 1: input from here on is hostile ================================
 corpus files --> chunking --> index --> Retriever (text is data, not orders)
                                      |  evidence tagged trust="untrusted", secrets screened
 ==== boundary 2: the model is not trusted either ===============================
 model (fake by default) --> reply must be valid JSON --> tool gate (allowlist,
                                      |  limits, approval) --> secret screen
 client <-- {status, message, request_id} -- no codes, no internals, redacted log

cd project
python3.12 -m venv .venv
source .venv/bin/activate
env | grep -ci api_key
python -m pip install -q -r requirements.txt
python -m pip freeze | diff - <(grep -v '^#' requirements.lock.txt) && echo "same as the lock file"
python -m pytest | sed 's/ in [0-9.]*s//'

0
same as the lock file
........................                                                 [100%]
455 passed, 1 xfailed

import tempfile

empty = tempfile.mkdtemp()  # a folder with no documents in it
with TestClient(create_app(AppSettings(corpus_dir=empty))) as http:
    print(http.get("/health").status_code, http.get("/ready").status_code, http.get("/ready").json()["reason"])

200 503 ValueError

with TestClient(create_app()) as http:
    long = http.post("/ask", json={"question": "x" * 2001}, headers={"x-user-id": "mateo"})
    blank = http.post("/ask", json={"question": ""}, headers={"x-user-id": "mateo"})
print(long.status_code, long.json()["status"], "|", blank.status_code, blank.json()["detail"][0]["loc"], blank.json()["detail"][0]["msg"])

413 refused | 422 ['body', 'question'] String should have at least 1 character

rm -rf ../copy && cp -r . ../copy && echo ' ' >> ../copy/data/eval_extra.jsonl && cd ../copy && python -m pytest 2>&1 | tail -3 | cut -c1-118 | sed 's/ in [0-9.]*s//'

ERROR tests/test_gate.py - evidencedesk.evals.DatasetDrift: eval_extra.jsonl ...
!!!!!!!!!!!!!!!!!!! Interrupted: 2 errors during collection !!!!!!!!!!!!!!!!!!!!
2 errors