মডেল কার্ড, সিস্টেম কার্ড ও ডেটাশিট: কী নথিভুক্ত করবেন এবং কার জন্য

# file: card.py
import json

def new_card(name, version, owner, as_of):
    return {"card_type": "system card (draft)", "name": name, "version": version,
            "owner": owner, "as_of": as_of}

def unknown(why):
    """An honest gap: say what is not known and why. Never replace it with a guess."""
    return {"status": "UNKNOWN", "why": why}

def show(card, *keys):
    print(json.dumps({k: card[k] for k in keys}, indent=2, ensure_ascii=False))

from card import new_card, unknown, show

card = new_card("ShortlistHelper (hypothetical)", "0.3",
                "Head of recruitment (a role)", "2026-10-12")
card["intended_use"] = "order the CVs for ONE job, so a recruiter knows which to read first"
card["out_of_scope"] = [
    "rejecting applicants without a person reading the CV",
    "ranking or judging current staff",
    "any decision about pay, promotion or dismissal",
    "jobs other than the one the requirements were written for",
]
card["intended_users"] = {
    "operators": "2 recruiters trained on the tool",
    "affected": "applicants, who do not see their score",
    "not_intended": "managers who see scores without the CV",
}
show(card, "intended_use", "out_of_scope", "intended_users")

{
  "intended_use": "order the CVs for ONE job, so a recruiter knows which to read first",
  "out_of_scope": [
    "rejecting applicants without a person reading the CV",
    "ranking or judging current staff",
    "any decision about pay, promotion or dismissal",
    "jobs other than the one the requirements were written for"
  ],
  "intended_users": {
    "operators": "2 recruiters trained on the tool",
    "affected": "applicants, who do not see their score",
    "not_intended": "managers who see scores without the CV"
  }
}

card["evaluation"] = {
    "question": "does the score order agree with a later full review of each CV?",
    "data": "40 invented past applicants (Toy data), 2 comparison groups, 1 job type",
    "conditions": {"cv_language": "English", "cv_format": "PDF and DOCX with text",
                   "period": "2026-09", "run_by": "internal team, not independent"},
    "evidence_file": "eval/shortlist_check_01.csv",
    "not_covered": ["scanned or image-only CVs", "CVs under 150 words",
                    "more than 1 job type", "groups other than the 2 compared"],
}
card["limitations"] = [
    {"limit": "scores use CV text only; skills not written down are invisible",
     "evidence": "design of the feature"},
    {"limit": "40 people are too few to show that a group difference is absent",
     "evidence": "eval/shortlist_check_01.csv"},
    {"limit": "a score of 80 does not mean 80% chance of being suitable",
     "evidence": unknown("no calibration test has been run")},
]
show(card, "evaluation", "limitations")

{
  "evaluation": {
    "question": "does the score order agree with a later full review of each CV?",
    "data": "40 invented past applicants (Toy data), 2 comparison groups, 1 job type",
    "conditions": {
      "cv_language": "English",
      "cv_format": "PDF and DOCX with text",
      "period": "2026-09",
      "run_by": "internal team, not independent"
    },
    "evidence_file": "eval/shortlist_check_01.csv",
    "not_covered": [
      "scanned or image-only CVs",
      "CVs under 150 words",
      "more than 1 job type",
      "groups other than the 2 compared"
    ]
  },
  "limitations": [
    {
      "limit": "scores use CV text only; skills not written down are invisible",
      "evidence": "design of the feature"
    },
    {
      "limit": "40 people are too few to show that a group difference is absent",
      "evidence": "eval/shortlist_check_01.csv"
    },
    {
      "limit": "a score of 80 does not mean 80% chance of being suitable",
      "evidence": {
        "status": "UNKNOWN",
        "why": "no calibration test has been run"
      }
    }
  ]
}

card["language_regional_coverage"] = {
    "tested": ["English"],
    "not_tested": ["Spanish", "Bangla", "Japanese", "CVs mixing two languages"],
    "regions": unknown("past applicants' locations were not recorded"),
    "rule": "a CV in an untested language is read by a person and not scored",
}
card["accessibility"] = {
    "upload_form": unknown("not reviewed with assistive technology"),
    "alternative_route": "applicants may email a CV; a recruiter reads it without a score",
    "untested_risk": "unusual layouts (some made with accessibility tools) may parse badly",
}
show(card, "language_regional_coverage", "accessibility")

{
  "language_regional_coverage": {
    "tested": [
      "English"
    ],
    "not_tested": [
      "Spanish",
      "Bangla",
      "Japanese",
      "CVs mixing two languages"
    ],
    "regions": {
      "status": "UNKNOWN",
      "why": "past applicants' locations were not recorded"
    },
    "rule": "a CV in an untested language is read by a person and not scored"
  },
  "accessibility": {
    "upload_form": {
      "status": "UNKNOWN",
      "why": "not reviewed with assistive technology"
    },
    "alternative_route": "applicants may email a CV; a recruiter reads it without a score",
    "untested_risk": "unusual layouts (some made with accessibility tools) may parse badly"
  }
}

card["human_oversight"] = {
    "decides": "a recruiter decides whom to interview; the score never decides alone",
    "override": "any recruiter may open any CV, whatever its score",
    "escalation": [
        "parse warning on a CV -> read it by hand",
        "applicant objects to a result -> Head of recruitment reviews it in 5 working days",
        "2 or more suspected scoring errors in a week -> pause the feature",
    ],
    "monitoring": "each month a person reads a sample of 50 low-ranked CVs",
}
card["failure_modes"] = [
    {"mode": "layout not parsed, so a CV scores near 0",
     "signal": "parse warning", "response": "read by hand"},
    {"mode": "recruiters stop opening low-ranked CVs",
     "signal": "monthly sample finds suitable CVs ranked low", "response": "review the workflow"},
    {"mode": "score reused to rank staff",
     "signal": "access log shows non-recruiting use", "response": "block access, report"},
]
show(card, "human_oversight", "failure_modes")

{
  "human_oversight": {
    "decides": "a recruiter decides whom to interview; the score never decides alone",
    "override": "any recruiter may open any CV, whatever its score",
    "escalation": [
      "parse warning on a CV -> read it by hand",
      "applicant objects to a result -> Head of recruitment reviews it in 5 working days",
      "2 or more suspected scoring errors in a week -> pause the feature"
    ],
    "monitoring": "each month a person reads a sample of 50 low-ranked CVs"
  },
  "failure_modes": [
    {
      "mode": "layout not parsed, so a CV scores near 0",
      "signal": "parse warning",
      "response": "read by hand"
    },
    {
      "mode": "recruiters stop opening low-ranked CVs",
      "signal": "monthly sample finds suitable CVs ranked low",
      "response": "review the workflow"
    },
    {
      "mode": "score reused to rank staff",
      "signal": "access log shows non-recruiting use",
      "response": "block access, report"
    }
  ]
}

datasheet = {
    "dataset": "past_applicants_2023_2025 (invented)",
    "motivation": "to test whether the score order matches a later full review",
    "composition": "40 past applicants (Toy data); fields: CV text, label, 2 group tags",
    "collection": "taken from the agency's own archive; group tags added by hand for testing",
    "recommended_uses": ["checking score order for the one job type"],
    "not_recommended": ["training a new model", "any use outside the agency"],
    "permission": unknown("applicants' consent for research use has not been checked"),
}
card["datasets"] = [datasheet]
show(datasheet, "motivation", "recommended_uses", "not_recommended", "permission")

{
  "motivation": "to test whether the score order matches a later full review",
  "recommended_uses": [
    "checking score order for the one job type"
  ],
  "not_recommended": [
    "training a new model",
    "any use outside the agency"
  ],
  "permission": {
    "status": "UNKNOWN",
    "why": "applicants' consent for research use has not been checked"
  }
}

import json, textwrap

def gaps(x):
    if isinstance(x, dict):
        return (1 if x.get("status") == "UNKNOWN" else 0) + sum(gaps(v) for v in x.values())
    if isinstance(x, list):
        return sum(gaps(v) for v in x)
    return 0

with open("card.json", "w", encoding="utf-8") as f:
    json.dump(card, f, indent=2, sort_keys=True, ensure_ascii=False)
print("top-level entries:", len(card))
print(textwrap.fill("sorted: " + ", ".join(sorted(card)), 90))
print("UNKNOWN entries:", gaps(card))

top-level entries: 15
sorted: accessibility, as_of, card_type, datasets, evaluation, failure_modes,
human_oversight, intended_use, intended_users, language_regional_coverage, limitations,
name, out_of_scope, owner, version
UNKNOWN entries: 4

card["fairness_checks"] = {
    "done": "score order compared with a later full review, for 2 groups, on 40 people",
    "result_location": "eval/shortlist_check_01.csv",
    "not_done": ["tests for other groups", "tests with larger samples", "independent review"],
    "claim": "no claim is made that the feature is fair or free of bias",
}
show(card, "fairness_checks")

{
  "fairness_checks": {
    "done": "score order compared with a later full review, for 2 groups, on 40 people",
    "result_location": "eval/shortlist_check_01.csv",
    "not_done": [
      "tests for other groups",
      "tests with larger samples",
      "independent review"
    ],
    "claim": "no claim is made that the feature is fair or free of bias"
  }
}

def paths(x, prefix=""):
    found = []
    if isinstance(x, dict):
        if x.get("status") == "UNKNOWN":
            found.append(prefix)
        for key, value in x.items():
            found += paths(value, f"{prefix}.{key}" if prefix else key)
    elif isinstance(x, list):
        for i, value in enumerate(x):
            found += paths(value, f"{prefix}[{i}]")
    return found

for p in sorted(paths(card)):
    print(p)

accessibility.upload_form
datasets[0].permission
language_regional_coverage.regions
limitations[2].evidence