Chapter 4 · Embeddings and Local Vector Search
Local Indexes and Retrieval Quality
- Page 8 of 22
- 23 min read
The search from the last page lived in memory and forgot everything when the program stopped. A real assistant needs more: an index it can save and reopen, a record of which model made the vectors, a way to skip junk text, filters that keep stale pages out, and a number that says whether retrieval is any good. This page adds all of that to EvidenceDesk, the running project, in one module, evidencedesk/index.py. Everything runs offline with the toy embedder from Embeddings and Similarity.
What you will learn
- How to keep the embedding model's identity with an index, and why you rebuild when the model changes.
- How to embed in batches and what to do with empty or very short text.
- How to measure recall on a small labelled set, and how exact and approximate search trade speed for recall.
- How to filter by metadata before evidence is used, save and reopen an index, and try a local vector database.
- How to finish with a test suite that pins the ranking of the sample corpus.
One module, seven promises
Each job below gets a small, testable piece of index.py. Read the file once as a whole; the next sections run each piece. The module only imports what embeddings.py already gave us.
| Job | Where it lives |
|---|---|
| Model identity | Model, IndexMeta, IndexMismatch |
| Rebuild on change | ensure_index |
| Batching | embed_batches, counting |
| Unusable text | unusable_reason, Skipped |
| Filters and search | VectorIndex.search(..., where=) |
| Saving | VectorIndex.save, VectorIndex.load |
| Quality | recall_at_k |
# file: evidencedesk/index.py
"""A local vector index that remembers its embedding model, skips unusable text, filters and persists."""
from __future__ import annotations
import json
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Callable
import numpy as np
from evidencedesk.embeddings import Hit, Passage, embed_all, top_k, toy_embed, words
BatchEmbedder = Callable[[list[str]], np.ndarray]
MIN_CHARS = 3
class IndexMismatch(ValueError):
"""The index and the model that wants to use it are not compatible."""
@dataclass(frozen=True)
class Model:
"""An embedding function plus the identity that must travel with its vectors."""
name: str
dim: int
embed_batch: BatchEmbedder
normalised: bool = True
@classmethod
def toy(cls, dim: int = 256) -> Model:
return cls(f"toy-trigram-{dim}", dim, lambda texts: embed_all(texts, lambda t: toy_embed(t, dim)))
def counting(model: Model) -> tuple[Model, list[int]]:
"""Wrap a model; the list collects the size of every embed_batch call."""
sizes: list[int] = []
def embed_batch(texts: list[str]) -> np.ndarray:
sizes.append(len(texts))
return model.embed_batch(texts)
return Model(model.name, model.dim, embed_batch, model.normalised), sizes
@dataclass(frozen=True)
class IndexMeta:
model: str
dim: int
normalised: bool
created_at: str # from an injected clock, so tests do not depend on the date
def check(self, model: Model) -> None:
if (self.model, self.dim) != (model.name, model.dim):
raise IndexMismatch(f"index built by {self.model} ({self.dim} dims), query uses {model.name} "
f"({model.dim} dims): rebuild the index")
@dataclass(frozen=True)
class Skipped:
id: str
reason: str # empty | too_short | no_words
def unusable_reason(text: str) -> str | None:
"""Why a text must not be embedded, or None. Never embed whitespace."""
stripped = text.strip()
if not stripped:
return "empty"
if len(stripped) < MIN_CHARS:
return "too_short"
return None if words(stripped) else "no_words" # only stop words: the toy vector would be all zeros
def embed_batches(texts: list[str], model: Model, batch_size: int = 16) -> np.ndarray:
"""Call the model once per batch of batch_size texts, not once per text."""
if batch_size < 1:
raise ValueError("batch_size must be at least 1")
parts = [model.embed_batch(texts[i : i + batch_size]) for i in range(0, len(texts), batch_size)]
return np.vstack(parts) if parts else np.zeros((0, model.dim))
def payload_of(p: Passage) -> dict:
return {"doc_id": p.doc_id, "title": p.title, "version": p.version, "status": p.status, "lang": p.lang}
class VectorIndex:
def __init__(self, meta: IndexMeta, ids: list[str], matrix: np.ndarray, payloads: list[dict]):
if matrix.shape != (len(ids), meta.dim) or len(payloads) != len(ids):
raise IndexMismatch("ids, payloads and vectors do not fit together")
self.meta, self.ids, self.matrix, self.payloads = meta, ids, matrix, payloads
@classmethod
def build(cls, passages: list[Passage], model: Model, clock: Callable[[], str],
batch_size: int = 16) -> tuple[VectorIndex, list[Skipped]]:
keep, skipped = [], []
for p in passages:
reason = unusable_reason(p.text)
if reason:
skipped.append(Skipped(p.id, reason))
else:
keep.append(p)
matrix = embed_batches([p.embed_text for p in keep], model, batch_size)
meta = IndexMeta(model.name, model.dim, model.normalised, clock())
return cls(meta, [p.id for p in keep], matrix, [payload_of(p) for p in keep]), skipped
def __len__(self) -> int:
return len(self.ids)
def search(self, query: str, model: Model, k: int = 3, where: dict | None = None) -> list[Hit]:
"""Refuse a foreign model, filter first, then rank only what is allowed. A blank query finds nothing."""
self.meta.check(model)
if unusable_reason(query):
return []
rows = [i for i, pl in enumerate(self.payloads) if all(pl.get(key) == v for key, v in (where or {}).items())]
if not rows:
return []
return top_k(model.embed_batch([query])[0], self.matrix[rows], [self.ids[i] for i in rows], k)
def save(self, path: Path) -> None:
header = json.dumps({"meta": asdict(self.meta), "ids": self.ids, "payloads": self.payloads})
with open(path, "wb") as f:
np.savez(f, matrix=self.matrix, header=np.array(header))
@classmethod
def load(cls, path: Path) -> VectorIndex:
with np.load(path, allow_pickle=False) as data:
header, matrix = json.loads(str(data["header"])), data["matrix"]
return cls(IndexMeta(**header["meta"]), header["ids"], matrix, header["payloads"])
def ensure_index(index: VectorIndex | None, passages: list[Passage], model: Model, clock: Callable[[], str],
batch_size: int = 16) -> tuple[VectorIndex, bool]:
"""Keep a compatible index; otherwise rebuild from the source passages. Returns (index, rebuilt)."""
if index is not None:
try:
index.meta.check(model)
return index, False
except IndexMismatch:
pass
return VectorIndex.build(passages, model, clock, batch_size)[0], True
def doc_ids(index: VectorIndex, hits: list[Hit]) -> list[str]:
by_id = dict(zip(index.ids, index.payloads))
return [by_id[h.passage_id]["doc_id"] for h in hits]
def recall_at_k(index: VectorIndex, model: Model, labels: list[tuple[str, str]], k: int = 3) -> tuple[float, list[str]]:
"""recall@k = questions whose expected document is among the documents of the top k passages / all questions."""
misses = [q for q, expected in labels if expected not in doc_ids(index, index.search(q, model, k))]
return 1 - len(misses) / len(labels), missesThe file needs only NumPy. The vectors live in one NumPy matrix, so "the index" is a matrix plus a list of IDs plus a list of metadata dictionaries.
Keep the embedding model's identity with the index
A vector is only meaningful together with the model that made it. So the index stores an IndexMeta: the model name, the vector length, whether the vectors have length 1 (are normalised), and when the index was created. The creation time comes from a clock function you pass in, so a test can fix it. Put the model version in the name (for example toy-trigram-256-v2), because a model can change without changing its vector length.
from pathlib import Path
from evidencedesk.embeddings import Passage, load_corpus
from evidencedesk.index import IndexMismatch, Model, VectorIndex, counting, ensure_index, recall_at_k
CLOCK = lambda: "2026-10-12T09:00:00Z" # injected, so the output is the same on every run
show = lambda hits: [(h.passage_id, round(h.score, 3)) for h in hits]
passages = load_corpus(Path("corpus"))
model = Model.toy()
index, skipped = VectorIndex.build(passages, model, CLOCK)
print(len(index), "indexed,", len(skipped), "skipped")
print(index.meta)36 indexed, 0 skipped
IndexMeta(model='toy-trigram-256', dim=256, normalised=True, created_at='2026-10-12T09:00:00Z')All 36 passages of the 13 sample documents are indexed and none was skipped. The show helper turns hits into short pairs for printing.
Rebuild indexes when representations change
Vectors from two different embedders live in different spaces. Comparing them gives numbers, but the numbers mean nothing, and nothing crashes to tell you. So search checks the identity first and refuses. There is no safe way to convert old vectors to a new model: you need the source text and a re-embedding run. ensure_index keeps a compatible index and otherwise rebuilds one from the passages.
other = Model.toy(dim=64) # pretend we switched to a different embedder: new name, new size
try:
index.search("battery", other)
except IndexMismatch as error:
print("refused:", error)
rebuilt, did_rebuild = ensure_index(index, passages, other, CLOCK)
print(did_rebuild, rebuilt.meta.model, rebuilt.matrix.shape)
same, did_rebuild = ensure_index(index, passages, model, CLOCK)
print(did_rebuild, same is index)refused: index built by toy-trigram-256 (256 dims), query uses toy-trigram-64 (64 dims): rebuild the index
True toy-trigram-64 (36, 64)
False TrueThe first line is the error you want to see in production, not a silently wrong answer. The second shows the rebuild with the new 64-number model; the third shows that a compatible index is left alone. For a big corpus, plan the migration: build the new index next to the old one, compare both on your labelled questions (see below), then switch.
Provider note (as of 2026-10-12): Google's embedding documentation states that the embedding spaces of
gemini-embedding-001andgemini-embedding-2are incompatible, so upgrading from one to the other means re-embedding all your data. The OpenAI page we read has no explicit statement of this kind, so treat "never mix models" as our own rule, not as a quote from that page. Model names and limits change often: check the current documentation.
Batch embedding work
A hosted embedding API charges a network round trip per call, and a local model has start-up cost per call. Sending 36 texts one by one is 36 round trips; sending them in groups of 16 is 3. embed_batches cuts the texts into groups of a fixed size. The size is a trade-off: bigger batches mean fewer calls but more memory and more risk of hitting a provider's input limit, so read the limits in the documentation and keep the size a setting. The helper counting wraps a model and records the size of every call, which is how you test batching without a clock.
counted, sizes = counting(model)
VectorIndex.build(passages, counted, CLOCK, batch_size=16)
print("batch sizes:", sizes)
one_by_one, sizes = counting(model)
VectorIndex.build(passages, one_by_one, CLOCK, batch_size=1)
print("calls with batch_size=1:", len(sizes))batch sizes: [16, 16, 4]
calls with batch_size=1: 36The toy embedder does not need batches; the code is shaped like a real API (a list in, a matrix out), so a real adapter drops in later.
Handle empty and very short inputs
Decide once what the indexer does with unusable text. Our rule: skip, report, never crash, never embed whitespace. A passage is unusable when its text is empty or only whitespace (empty), shorter than 3 characters (too_short), or made only of stop words, so the toy embedder would return an all-zero vector (no_words). Skipped passages come back in a list, so the caller can log them or fail a build. A blank query finds nothing instead of returning random-looking hits.
texts = ["", " \n ", "ok", "the of a", "Charge it fully before storing."]
odd = [Passage(f"t#s{n}", "t", "Title", "Head", text, 1, "current", "en") for n, text in enumerate(texts)]
counted, sizes = counting(model)
small, skipped = VectorIndex.build(odd, counted, CLOCK)
print("indexed:", small.ids)
print("skipped:", [(s.id, s.reason) for s in skipped])
print("texts sent to the model:", sum(sizes))
print("empty input:", len(VectorIndex.build([], model, CLOCK)[0]), "| blank query:", index.search(" ", model))indexed: ['t#s4']
skipped: [('t#s0', 'empty'), ('t#s1', 'empty'), ('t#s2', 'too_short'), ('t#s3', 'no_words')]
texts sent to the model: 1
empty input: 0 | blank query: []Only one text reached the model. Note that the title still exists for the skipped passages, but a title alone is not an answer, so the check looks at the body text.
Measure retrieval recall on a small dataset
Hand-label questions with the document that answers each, then ask: does the right document show up in the top results? recall@k = questions whose expected document appears among the top k passages, divided by all questions. The code counts hits and returns the questions it missed, so you can read them.
LABELS = [ # question -> the document that answers it (a smoke test: 8 questions, not a benchmark)
("How long do I have to return a bike?", "kc-returns-v3"),
("How long does the battery charge take?", "kc-battery-v1"),
("What is the RD2-24 recall?", "kc-recalls-v1"),
("How tall must I be for size M?", "kc-sizing-v1"),
("When do gift cards expire?", "kc-giftcards-v1"),
("How much does delivery cost?", "kc-shipping-v1"),
("How do I get my money back?", "kc-returns-v3"), # paraphrase: shares few letters with the page
("Where can I try a bike before buying?", "kc-sizing-v1"), # the page says "test ride"
]
for k in (1, 3, 5):
recall, misses = recall_at_k(index, model, LABELS, k)
print(f"recall@{k} = {recall:.3f} ({len(misses)} of {len(LABELS)} questions missed)")
print("still missed at k=5:", misses)recall@1 = 0.750 (2 of 8 questions missed)
recall@3 = 0.750 (2 of 8 questions missed)
recall@5 = 0.875 (1 of 8 questions missed)
still missed at k=5: ['Where can I try a bike before buying?']Reading it: raising k from 1 to 5 helps one question (the paraphrase "money back", whose answer ranks 4th), but "try a bike before buying" never appears, because the page says "test ride" and the toy embedder matches letters, not meaning. These eight questions are a smoke test, not a benchmark: one question moves recall by 0.125, and if you tune your system on the same questions you measure memory, not quality. Keep a few questions aside that you never tune on, and grow the set from real user questions. Recall also says nothing about whether the passage supports an answer.
Compare exact search and approximate search
Exact search compares the query with every stored vector, so it always finds the true nearest ones, but the work grows with the number of vectors. Approximate search (approximate nearest neighbour, ANN) builds a structure that lets it look at only a fraction of the vectors. It is faster on large sets, but it may miss a true neighbour. That is the speed-recall trade-off. For 36 passages exact search is the right answer; the trade-off matters at millions of vectors.
The block below is a classroom model, not a real ANN library. It cuts 3,000 generated vectors into buckets with random hyperplanes, scores only the vectors in the query's bucket (optionally also in buckets one bit away), and compares with the exact top 10. The seed is fixed. "Examined" is the share of vectors scored, a stand-in for work; the page does not measure or assert wall-clock speed.
import numpy as np
rng = np.random.default_rng(7) # fixed seed: same numbers on every run
centres = rng.normal(size=(30, 32)) # 30 topics, 100 noisy vectors each: 3000 vectors of 32 numbers
unit = lambda m: m / np.linalg.norm(m, axis=-1, keepdims=True)
data = unit(np.repeat(centres, 100, axis=0) + 0.5 * rng.normal(size=(3000, 32)))
queries = unit(centres[rng.integers(0, 30, 50)] + 0.5 * rng.normal(size=(50, 32)))
def exact(q, k=10): # compare with every vector
return set(np.argsort(-(data @ q))[:k])
class Buckets:
"""Classroom model, not a real ANN library: random hyperplanes cut the space into 2**bits buckets."""
def __init__(self, bits, seed=1):
self.planes = np.random.default_rng(seed).normal(size=(bits, 32))
self.weights = 1 << np.arange(bits)
keys = ((data @ self.planes.T) > 0) @ self.weights
self.members = {int(key): np.flatnonzero(keys == key) for key in set(keys)}
def search(self, q, k=10, neighbours=False):
key = int(((self.planes @ q) > 0) @ self.weights)
keys = [key] + ([key ^ w for w in self.weights] if neighbours else []) # own bucket (+ buckets one bit away)
found = np.concatenate([self.members.get(x, np.array([], dtype=int)) for x in keys])
best = found[np.argsort(-(data[found] @ q))[:k]] # exact scoring, but only of the candidates
return set(best), len(found)
print(f"{'method':<22}{'examined':>9}{'recall@10':>11}")
print(f"{'exact (all vectors)':<22}{'100.0%':>9}{1.0:>11.3f}")
for bits, neighbours in [(4, False), (6, False), (6, True), (8, True)]:
index_ = Buckets(bits)
runs = [index_.search(q, neighbours=neighbours) for q in queries]
recall = np.mean([len(found & exact(q)) / 10 for (found, _), q in zip(runs, queries)])
share = np.mean([n for _, n in runs]) / len(data)
name = f"{bits} bits" + (" + neighbours" if neighbours else "")
print(f"{name:<22}{share:>9.1%}{recall:>11.3f}")method examined recall@10
exact (all vectors) 100.0% 1.000
4 bits 7.4% 0.482
6 bits 2.4% 0.338
6 bits + neighbours 13.2% 0.744
8 bits + neighbours 5.2% 0.566More buckets means fewer vectors examined but lower recall; looking in neighbouring buckets buys recall back at the cost of more work. These numbers belong to this toy and this made-up data; they say nothing about any real library.
Provider note (as of 2026-10-12): Qdrant's documentation lists Dot, Cosine, Euclid and Manhattan distances; for Cosine it normalises the vector once when it is added, after which comparison is equivalent to a dot product. Its search parameters include an
exactoption, described as not using approximate search, and warning that a full scan may take long. For dense vectors it uses HNSW, an approximate index; its configuration notes say that a largermgives a more accurate search and needs more space, and a largeref_constructgives a more accurate search and needs more build time. The page gives no recall numbers. Check the current documentation.
Filter results by metadata
Every index row carries a payload: doc_id, title, version, status and lang. search(..., where={...}) keeps only rows whose payload matches every key, then ranks. The same mechanism serves a language filter, a document filter or an access filter ("only documents this user may read"). The filter must run before any text reaches the prompt: a passage the user may not see must never be retrieved at all. Run it before ranking, not after, or you lose results.
Q = "How long do I have to return a bike?"
for where in (None, {"status": "current"}, {"lang": "bn"}, {"doc_id": "kc-returns-v2"}):
print(where, "->", show(index.search(Q, model, 3, where)))
after = [h for h in index.search(Q, model, 2) if index.payloads[index.ids.index(h.passage_id)]["status"] == "current"]
print("filter after top-2 keeps", len(after), "| filter before keeps", len(index.search(Q, model, 2, {"status": "current"})))None -> [('kc-returns-v3#s1', 0.555), ('kc-returns-v2#s1', 0.463), ('kc-warranty-v2#s1', 0.403)]
{'status': 'current'} -> [('kc-returns-v3#s1', 0.555), ('kc-warranty-v2#s1', 0.403), ('kc-returns-v3#s2', 0.383)]
{'lang': 'bn'} -> [('kc-returns-v3-bn#s2', 0.197), ('kc-returns-v3-bn#s1', 0.137), ('kc-shipping-v1-bn#s2', 0.113)]
{'doc_id': 'kc-returns-v2'} -> [('kc-returns-v2#s1', 0.463), ('kc-returns-v2#s2', 0.234)]
filter after top-2 keeps 1 | filter before keeps 2Without a filter the superseded returns policy ranks second. With status set to current it is gone and the list is full again. The last line shows why order matters: dropping stale rows after taking the top 2 leaves only 1; filtering first gives 2. A filter is a rule in code. It does not make the remaining passages correct, and filters on user access belong in code too, never in a prompt (see Context Assembly and Trust Boundaries).
Test index persistence and reopening
An index you cannot reopen has to be rebuilt on every start, which costs time and, for hosted models, money. save writes the matrix, the metadata and the payloads to one NumPy .npz file; load reads them back and checks that the shapes fit. Test it the way you will use it: save, open a fresh object, and compare.
import tempfile
import numpy as np
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / "kestrel.npz"
index.save(path)
reopened = VectorIndex.load(path) # a new object, read from disk
print("same meta:", reopened.meta == index.meta, "| same vectors:", np.array_equal(reopened.matrix, index.matrix))
print("same ranking:", reopened.search(Q, model, 3) == index.search(Q, model, 3))same meta: True | same vectors: True
same ranking: TrueThe vectors are stored exactly (no rounding), so the ranking is identical. The temporary folder is deleted at the end of the with block, so the test leaves nothing behind.
Add a local vector database option
A vector database stores vectors with payloads, filters and searches them for you. qdrant-client can run in a local mode inside your Python process, with no server. We copy our own vectors into a collection, query it with a payload filter, then store a second one on disk with path= and reopen it. We call close() on every client; without that, the folder stays locked, and on Windows the cleanup of the temporary folder can fail.
import tempfile
from qdrant_client import QdrantClient, models
points = [models.PointStruct(id=n, vector=index.matrix[n].tolist(), payload={"pid": pid, **index.payloads[n]})
for n, pid in enumerate(index.ids)]
def fill(client):
client.create_collection("kestrel", vectors_config=models.VectorParams(size=model.dim, distance=models.Distance.COSINE))
client.upsert("kestrel", points=points)
def ask(client, question, status):
only = models.Filter(must=[models.FieldCondition(key="status", match=models.MatchValue(value=status))])
res = client.query_points("kestrel", query=model.embed_batch([question])[0].tolist(), limit=3, query_filter=only)
return [(p.payload["pid"], round(p.score, 3)) for p in res.points]
Q = "How long do I have to return a bike?"
memory = QdrantClient(":memory:")
fill(memory)
print("memory :", ask(memory, Q, "current"))
print("ours :", show(index.search(Q, model, 3, {"status": "current"})))
memory.close()
with tempfile.TemporaryDirectory() as folder:
disk = QdrantClient(path=folder)
fill(disk)
disk.close() # release the folder lock before reopening (and before Windows deletes it)
again = QdrantClient(path=folder)
print("reopened:", ask(again, Q, "superseded"))
again.close()memory : [('kc-returns-v3#s1', 0.555), ('kc-warranty-v2#s1', 0.403), ('kc-returns-v3#s2', 0.383)]
ours : [('kc-returns-v3#s1', 0.555), ('kc-warranty-v2#s1', 0.403), ('kc-returns-v3#s2', 0.383)]
reopened: [('kc-returns-v2#s1', 0.463), ('kc-returns-v2#s2', 0.234)]The in-memory answer equals ours, passage for passage and score for score, because both compute cosine similarity exactly. The reopened disk collection answers from stored data; here we asked it for the superseded status to show that the stored payload filters too. We use the row number as the point ID and keep our own passage ID in the payload.
Tool note (as of 2026-10-12):
qdrant-clientwas at version 1.19.1 (released 2026-09-16, Apache-2.0, Python 3.10 or newer). Its README showsQdrantClient(":memory:")andQdrantClient(path=...)(persisted to disk) and calls local mode useful for development, prototyping and testing. The current search method isquery_points; the oldersearchmethod is not in current source. In local mode the search is exact brute force (sosearch_paramshas no effect), and a warning in the source code, not in a documentation page, says local mode is not recommended above 20,000 points. We do not rely on hybrid or sparse search in local mode: we found it in code only. Separately, Qdrant's documentation says a self-deployed server has no authentication by default and accepts requests from anyone who can reach it; turn on API-key authentication before you expose one. We ran the block above with this version on Python 3.13 and saw no warnings. Check the current documentation.
Mini-lab: Local Semantic Search
Put it together. Index the sample corpus, retrieve top results and pin the behaviour with tests. The test file tests/test_index.py is part of the course kit (it does not fit on this page); read it and run it. It holds the eight labelled questions used above (the LABELS list) and 12 tests:
- the metadata records the model name, vector length and the injected clock;
- unusable text is skipped, reported and never sent to the model;
- embedding runs in batches of 16, 16 and 4;
- a foreign model is refused until the index is rebuilt;
- the filter runs before ranking (status, language), and a blank query finds nothing;
- a saved index reopens with the same ranking;
- four questions keep the same top passage, and recall@1 is 0.75 and recall@5 is 0.875, missing exactly the "try a bike" question.
Run it from the project folder. We cut the run time from pytest's last line with sed, so the output is the same on every run.
python -m pytest -q tests/test_index.py | sed 's/ in [0-9.]*s//'............ [100%]
12 passedTwelve tests pass. They prove the plumbing and the ranking of this corpus with this toy embedder. They do not prove that a real embedding model would rank the same, and a passing recall number does not prove that an answer is supported. The next page puts retrieved passages into a RAG pipeline and checks citations.
Where this shows up at work
Kwame's team in Accra upgraded the embedding model of their help-centre search. Half the nightly index job ran with the new model before someone noticed, and the index held two kinds of vectors with the same length. Nothing crashed; answers just got strange. After that incident they stored the model name in the index, refused to query on a mismatch, and added a recall check of 40 real questions to the release checklist. The same module also gave them a clean way to hide archived pages and to skip pages that arrived empty from a broken export.
Common mistakes
- Checking only the vector length. Two models can share a length and still have different spaces. Compare the model name and version, and rebuild when it changes.
- Embedding blank or stop-word-only text. It adds rows that match nothing or match anything. Skip it and report it.
- Filtering after the top k. You get fewer results than asked for, or none. Filter first, then rank.
- Trusting a small recall number. Eight questions are a smoke test. Do not tune on the questions you report, and do not read recall as proof of a correct answer.
- Treating local mode as a server. It is exact and for small collections. If you later run a server, remember it has no authentication by default.
Try it yourself
- Easy: Four labelled questions have their expected document at rank 1, rank 2, rank 5 and not in the top 10. Compute recall@1, recall@3 and recall@5 by hand.
- Medium: Revisit Embeddings and Similarity: write
confident(index, model, question, min_score=0.3)that returns only current-status hits with a score of at leastmin_score. Try "Can I pay in instalments?", "How much does delivery cost?" and "How do I get my money back?". - Hard: Declare a new model version with the same size and vectors as the old one. Save the index, reopen it, show that the new version is refused, rebuild with
ensure_index, save again and show the new name in the file.
Answers
Easy. recall@1 = 1/4 = 0.25 (only the first question). recall@3 = 2/4 = 0.5 (ranks 1 and 2). recall@5 = 3/4 = 0.75 (ranks 1, 2 and 5). The fourth question counts in no case.
Medium. The function is a filter on the score of hits that already passed the status filter.
def confident(index, model, question, min_score=0.3, k=3):
return [h for h in index.search(question, model, k, {"status": "current"}) if h.score >= min_score]
for question in ("Can I pay in instalments?", "How much does delivery cost?", "How do I get my money back?"):
print(question, "->", show(confident(index, model, question)))Can I pay in instalments? -> []
How much does delivery cost? -> [('kc-shipping-v1#s3', 0.539), ('kc-shipping-v1#s1', 0.532), ('kc-shipping-v1#s2', 0.465)]
How do I get my money back? -> []The threshold abstains on the instalments question, which the documents do not answer: good. It also abstains on "money back", which they do answer (the returns page), because the toy shares few letters with it: a miss. A threshold cuts noise but cannot repair weak retrieval.
Hard. Same vector length is not same identity.
v2 = Model("toy-trigram-256-v2", 256, model.embed_batch) # same size, same vectors, but declared a new version
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / "kestrel.npz"
index.save(path)
old = VectorIndex.load(path)
try:
old.search("battery", v2)
except IndexMismatch as error:
print("refused:", error)
fresh, did_rebuild = ensure_index(old, passages, v2, CLOCK)
fresh.save(path)
print(did_rebuild, VectorIndex.load(path).meta.model)refused: index built by toy-trigram-256 (256 dims), query uses toy-trigram-256-v2 (256 dims): rebuild the index
True toy-trigram-256-v2Pass checklist:
- Easy: you got 0.25, 0.5 and 0.75 and can say why the last question never counts.
- Medium: your abstain list for the instalments question is empty, you can name one answerable question the threshold wrongly dropped, and you did not use a prompt to do the filtering.
- Hard: the error names both model versions,
did_rebuildisTrue, and the reopened file reports the new name.
Summary
- Store the model name, version, vector length and creation time with the index; refuse to query it with another model, and rebuild when the model changes.
- Embed in fixed-size batches, count the calls in tests, and skip empty, very short and stop-word-only text with a report.
- Recall@k on hand-labelled questions is a smoke test with a small set, not a benchmark and not proof of a supported answer.
- Exact search is simple and complete; approximate search trades recall for less work. Our bucket model shows the trade-off but is not a real library, and
qdrant-clientlocal mode is exact and meant for small collections. - Filter by metadata before ranking and before evidence is used; save, reopen and compare to prove persistence.
Next: RAG Ingestion and Citation Flow - turn documents into passages with stable IDs, retrieve them into a prompt and check that every citation points to a passage that supports the answer.