প্রকাশ ও ডকুমেন্টেশন যাচাই: Python দিয়ে সিস্টেম কার্ড পরীক্ষা

CARD = {
    "name": "ShortlistHelper (hypothetical)",
    "card_version": "1.2",
    "versions": {"system": "0.4", "model": "fit-scorer-2026.08", "parser": "1.3"},
    "intended_use": "order the CVs for ONE job so a recruiter knows which to read first",
    "out_of_scope": ["automatic rejection", "ranking current staff"],
    "data_sources": [
        {"name": "past_applicants_2023_2025", "origin": "the agency's own archive",
         "permission": {"basis": "assumed",
                        "assumption": "the 2023 privacy notice covers testing of tools",
                        "checked_by": None}},
    ],
}
from pprint import pprint
for key in ("versions", "data_sources"):
    print(key, "->")
    pprint(CARD[key], width=88, sort_dicts=False)

versions ->
{'system': '0.4', 'model': 'fit-scorer-2026.08', 'parser': '1.3'}
data_sources ->
[{'name': 'past_applicants_2023_2025',
  'origin': "the agency's own archive",
  'permission': {'basis': 'assumed',
                 'assumption': 'the 2023 privacy notice covers testing of tools',
                 'checked_by': None}}]

CARD["output_limitations"] = {
    "score_is": "a number from 0 to 100 computed by a model from the CV text",
    "score_is_not": ["a measure of ability", "a probability of success", "a reason"],
    "highlights": "words that changed the score most; not tested as a human reason",
}
print(CARD["output_limitations"]["score_is_not"])

['a measure of ability', 'a probability of success', 'a reason']

WEAK
Applications may be processed using automated tools in accordance with applicable policies
and procedures, and the outputs of such tools may inform decisions.

BETTER
We use a computer program to suggest the order in which our recruiters read CVs. It reads
only the text of your CV and gives it a score. A recruiter, not the program, decides who is
invited to an interview. A CV low on the list may not be opened unless a recruiter chooses
to. If you want a person to read your whole CV, tell us on the form or by email. The program
has been tested only on CVs written in English. We last checked this notice on 2026-10-01.

import re

def sentence_stats(text):
    sentences = [s for s in re.split(r"(?<=[.!?])\s+", " ".join(text.split())) if s]
    words = sum(len(s.split()) for s in sentences)
    return len(sentences), words, round(words / len(sentences), 1)

weak = ("Applications may be processed using automated tools in accordance with applicable "
        "policies and procedures, and the outputs of such tools may inform decisions.")
better = ("We use a computer program to suggest the order in which our recruiters read CVs. "
          "It reads only the text of your CV and gives it a score. A recruiter, not the program, "
          "decides who is invited to an interview. A CV low on the list may not be opened unless "
          "a recruiter chooses to. If you want a person to read your whole CV, tell us on the "
          "form or by email. The program has been tested only on CVs written in English. "
          "We last checked this notice on 2026-10-01.")
for name, text in (("weak", weak), ("better", better)):
    print(name, "(sentences, words, words per sentence):", sentence_stats(text))

weak (sentences, words, words per sentence): (1, 23, 23.0)
better (sentences, words, words per sentence): (7, 91, 13.0)

CARD.update({
    "tests": [{"name": "group comparison", "result": "done", "evidence": "eval/group_check.csv"},
              {"name": "parser check", "result": "passed", "evidence": None}],
    "evaluation": {"claim": "score order tested on 40 past applicants",
                   "evidence_file": "eval/shortlist_check_01.csv"},
    "limitations": ["uses CV text only", {"status": "UNKNOWN", "why": "no calibration test run"}],
    "language_coverage": {"tested": ["English"], "not_tested": ["Spanish"]},
    "accessibility": {"status": "UNKNOWN", "why": "upload form not reviewed"},
    "human_oversight": {"escalation": ["parse warning -> read by hand"]},
    "failure_modes": [],
    "next_review": "2026-09-30",
})
VISIBILITY = {"name": "public", "card_version": "public", "versions": "public",
              "intended_use": "public", "out_of_scope": "public", "output_limitations": "public",
              "limitations": "public", "language_coverage": "public", "accessibility": "public",
              "human_oversight": "public", "next_review": "public",
              "data_sources": "internal", "tests": "internal", "evaluation": "internal",
              "failure_modes": "public"}

def view(card, audience):
    keep = {k: v for k, v in card.items()
            if audience == "internal" or VISIBILITY[k] == "public"}
    return keep, len(card) - len(keep)

public, withheld = view(CARD, "public")
import textwrap
print(textwrap.fill("public sections: " + ", ".join(public), 90))
print("withheld from the public page:", withheld, "(listed as available to reviewers)")
assert "limitations" in public and "language_coverage" in public

public sections: name, card_version, versions, intended_use, out_of_scope,
output_limitations, limitations, language_coverage, accessibility, human_oversight,
failure_modes, next_review
withheld from the public page: 3 (listed as available to reviewers)

# file: cardcheck.py
import textwrap
from datetime import date

AS_OF = date(2026, 10, 12)   # a fixed "today", so every run gives the same answer
MISSING = object()
REQUIRED = [
    ("name", "Feature"), ("versions.system", "System version"),
    ("versions.model", "Model version"), ("intended_use", "Intended use"),
    ("out_of_scope", "Out of scope"), ("data_sources", "Data sources"),
    ("evaluation.evidence_file", "Evaluation evidence"), ("limitations", "Limitations"),
    ("language_coverage", "Language coverage"), ("accessibility", "Accessibility"),
    ("human_oversight.escalation", "Escalation"), ("failure_modes", "Failure modes"),
    ("next_review", "Next review"),
]

def get(card, path):
    node = card
    for key in path.split("."):
        if not isinstance(node, dict) or key not in node:
            return MISSING
        node = node[key]
    return node

def status(value):
    if value is MISSING or value in ("", [], {}, None):
        return "MISSING"
    if isinstance(value, dict) and value.get("status") == "UNKNOWN":
        return "UNKNOWN"
    return "ok"

def validate(card):
    return [(status(get(card, p)), p, label) for p, label in REQUIRED]

def unknown_paths(x, prefix=""):
    found = []
    if isinstance(x, dict):
        if x.get("status") == "UNKNOWN":
            found.append(prefix)
        for k, v in x.items():
            found += unknown_paths(v, f"{prefix}.{k}" if prefix else k)
    elif isinstance(x, list):
        for i, v in enumerate(x):
            found += unknown_paths(v, f"{prefix}[{i}]")
    return found

def show(value):
    if isinstance(value, str):
        return value
    if isinstance(value, list):
        return "; ".join(show(v) for v in value)
    if isinstance(value, dict) and value.get("status") == "UNKNOWN":
        return "UNKNOWN (" + value["why"] + ")"
    if isinstance(value, dict):
        return ", ".join(f"{k}: {show(v)}" for k, v in value.items())
    return str(value)

def draft(card):
    lines = ["SYSTEM CARD (DRAFT, not reviewed)"]
    for path, label in REQUIRED:
        value, s = get(card, path), status(get(card, path))
        if s == "MISSING":
            text = "MISSING - the owner must supply this"
        elif s == "UNKNOWN":
            text = "UNKNOWN - " + value["why"]
        else:
            text = show(value)
        lines.append(textwrap.fill(f"{label}: {text}", 92, subsequent_indent="    "))
    return "\n".join(lines)

from cardcheck import draft, validate, unknown_paths

print(draft(CARD))
rows = validate(CARD)
print()
print({s: sum(1 for r in rows if r[0] == s) for s in ("ok", "UNKNOWN", "MISSING")})
for s, path, label in rows:
    if s != "ok":
        print(f"{s:<8}{path}")
print("UNKNOWN anywhere in the card:", sorted(unknown_paths(CARD)))

SYSTEM CARD (DRAFT, not reviewed)
Feature: ShortlistHelper (hypothetical)
System version: 0.4
Model version: fit-scorer-2026.08
Intended use: order the CVs for ONE job so a recruiter knows which to read first
Out of scope: automatic rejection; ranking current staff
Data sources: name: past_applicants_2023_2025, origin: the agency's own archive, permission:
    basis: assumed, assumption: the 2023 privacy notice covers testing of tools, checked_by:
    None
Evaluation evidence: eval/shortlist_check_01.csv
Limitations: uses CV text only; UNKNOWN (no calibration test run)
Language coverage: tested: English, not_tested: Spanish
Accessibility: UNKNOWN - upload form not reviewed
Escalation: parse warning -> read by hand
Failure modes: MISSING - the owner must supply this
Next review: 2026-09-30

{'ok': 11, 'UNKNOWN': 1, 'MISSING': 1}
UNKNOWN accessibility
MISSING failure_modes
UNKNOWN anywhere in the card: ['accessibility', 'limitations[1]']

# file: consistency.py
from datetime import date
from pathlib import Path
from cardcheck import AS_OF, get

def evidence_flags(card):
    flags = []
    path = get(card, "evaluation.evidence_file")
    if isinstance(path, str) and not Path(path).exists():
        flags.append(f"evaluation: evidence file {path} not found")
    for t in card.get("tests", []):
        ev = t.get("evidence")
        if not ev:
            flags.append(f"test '{t['name']}' says '{t['result']}' but cites no evidence")
        elif not Path(ev).exists():
            flags.append(f"test '{t['name']}': evidence file {ev} not found")
    for s in card.get("data_sources", []):
        p = s.get("permission", {})
        if p.get("basis") == "assumed" and not p.get("checked_by"):
            flags.append(f"data '{s['name']}': permission is an unchecked assumption")
    return flags

def consistency_flags(card, release, config):
    flags = []
    for part in ("system", "model"):
        said, real = card["versions"][part], release[part]
        if said != real:
            flags.append(f"{part} version: card says {said}, release log says {real}")
    due = date.fromisoformat(card["next_review"])
    if due < AS_OF:
        flags.append(f"review overdue: {due} is {(AS_OF - due).days} days before {AS_OF}")
    if config.get("auto_reject_below") is not None and \
            "automatic rejection" in card.get("out_of_scope", []):
        flags.append("card puts automatic rejection out of scope, but the config sets "
                     f"auto_reject_below={config['auto_reject_below']}")
    tested = set(card["language_coverage"]["tested"])
    for lang in sorted(set(config["languages_enabled"]) - tested):
        flags.append(f"language {lang} is enabled in the config but not tested")
    return flags

from pathlib import Path
from consistency import evidence_flags, consistency_flags

Path("eval").mkdir()
Path("eval/shortlist_check_01.csv").write_text("toy stand-in for the team's results file\n")

RELEASE = {"system": "0.5", "model": "fit-scorer-2026.08"}      # Toy data: the release log
CONFIG = {"auto_reject_below": 15, "languages_enabled": ["English", "Spanish"]}  # Toy data

print("EVIDENCE")
for f in evidence_flags(CARD):
    print(" -", f)
print("CONSISTENCY (as of 2026-10-12)")
for f in consistency_flags(CARD, RELEASE, CONFIG):
    print(" -", f)

EVIDENCE
 - test 'group comparison': evidence file eval/group_check.csv not found
 - test 'parser check' says 'passed' but cites no evidence
 - data 'past_applicants_2023_2025': permission is an unchecked assumption
CONSISTENCY (as of 2026-10-12)
 - system version: card says 0.4, release log says 0.5
 - review overdue: 2026-09-30 is 12 days before 2026-10-12
 - card puts automatic rejection out of scope, but the config sets auto_reject_below=15
 - language Spanish is enabled in the config but not tested

Feature:   ReplyDrafter (hypothetical), version 0.1, model "drafter-2026.09".
Owner:     Kofi, who runs a small shop in Kumasi. Efua, a staff member, reads every draft
           before it is sent. If a customer complains, Kofi handles it.
Use:       drafts English replies to customer emails about orders. It must not answer
           refund or legal questions; those go to Kofi.
Tested:    on 25 past emails (Toy data) by Kofi himself; results in eval/reply_check.csv.
           3 of the 25 drafts stated the refund policy wrongly.
Languages: English tested. Twi (a language widely spoken in Ghana) not tested.
Data:      past customer emails from the shop's inbox; customers have not been asked.
Access:    nobody has checked whether customers can reach the shop by another route.
Review:    next card review 2026-12-01.

from pathlib import Path
from cardcheck import validate

REPLY = {
    "name": "ReplyDrafter (hypothetical)",
    "versions": {"system": "0.1", "model": "drafter-2026.09"},
    "intended_use": "draft English replies to order emails; a staff member reads every draft",
    "out_of_scope": ["answering refund or legal questions", "sending without a person reading"],
    "data_sources": [{"name": "past_customer_emails", "origin": "the shop's inbox",
                      "permission": {"basis": "assumed",
                                     "assumption": "customers were not asked; not yet checked",
                                     "checked_by": None}}],
    "evaluation": {"claim": "25 past emails tested by the owner",
                   "evidence_file": "eval/reply_check.csv"},
    "tests": [{"name": "past-email test", "result": "done", "evidence": "eval/reply_check.csv"}],
    "limitations": ["3 of 25 drafts stated the refund policy wrongly (Toy data)"],
    "language_coverage": {"tested": ["English"], "not_tested": ["Twi"]},
    "accessibility": {"status": "UNKNOWN", "why": "no check of other contact routes"},
    "human_oversight": {"escalation": ["refund or legal question -> Kofi"]},
    "failure_modes": [{"mode": "wrong refund policy", "signal": "Efua spots it or a complaint",
                       "response": "Kofi handles it"}],
    "next_review": "2026-12-01",
}
Path("eval/reply_check.csv").write_text("toy stand-in for Kofi's results file\n")
print([r[:2] for r in validate(REPLY) if r[0] != "ok"])
from consistency import evidence_flags
print(evidence_flags(REPLY))

[('UNKNOWN', 'accessibility')]
["data 'past_customer_emails': permission is an unchecked assumption"]