প্র্যাকটিস ল্যাব: টয় ডেটা ও Python দিয়ে একটা সীমিত AI ফিচারের মূল্যায়ন

# Toy data: six invented risks for FeedbackHelper, a feature that drafts comments on student essays
RISKS = [
    dict(id="R1", title="Comment misreads second-language writing", severity=4, likelihood=3,
         owner="Head of English", mitigation="teacher reads every comment before release"),
    dict(id="R2", title="Essay text and names sent to an outside service", severity=4, likelihood=2,
         owner="", mitigation="remove names before sending"),
    dict(id="R3", title="Suggested grade band believed without question", severity=3, likelihood=4,
         owner="Head of English", mitigation="no grade bands in the draft"),
    dict(id="R4", title="Parents not told AI is used", severity=2, likelihood=5,
         owner="the team", mitigation="letter home before the pilot"),
    dict(id="R5", title="Generic praise lowers trust in feedback", severity=2, likelihood=3,
         owner="Deputy head", mitigation=""),
    dict(id="R6", title="Appeal route unclear", severity=7, likelihood=3,
         owner="Deputy head", mitigation="name a second reader"),
]
RISK_FIELDS = ("id", "title", "severity", "likelihood", "owner", "mitigation")
VAGUE_OWNERS = ("", "the team", "everyone", "tbd")


def check(r):
    """Report problems; never fill a gap."""
    found = [f"{k} is missing" for k in RISK_FIELDS if k not in ("owner",) and not r.get(k)]
    if r.get("owner", "").lower() in VAGUE_OWNERS:
        found.append("owner is missing or not a named role")
    for k in ("severity", "likelihood"):
        if r.get(k) and not 1 <= r[k] <= 5:
            found.append(f"{k} {r[k]} is outside 1-5")
    return found


def summary(risks):
    ok = sorted((r for r in risks if not check(r)), key=lambda r: (-r["severity"] * r["likelihood"], r["id"]))
    lines = ["id  sev lik  pts  owner               title"]
    lines += [f"{r['id']:<4}{r['severity']:>3}{r['likelihood']:>4}{r['severity'] * r['likelihood']:>5}  "
              f"{r['owner']:<20}{r['title']}" for r in ok]
    return "\n".join(lines)


for r in RISKS:
    for p in check(r):
        print(f"{r['id']} problem: {p}")
print(f"\n{sum(not check(r) for r in RISKS)} of {len(RISKS)} records complete\n")
print(summary(RISKS))
print("\nNote: points = severity x likelihood is a method we chose, not a measurement.")

R2 problem: owner is missing or not a named role
R4 problem: owner is missing or not a named role
R5 problem: mitigation is missing
R6 problem: severity 7 is outside 1-5

2 of 6 records complete

id  sev lik  pts  owner               title
R1    4   3   12  Head of English     Comment misreads second-language writing
R3    3   4   12  Head of English     Suggested grade band believed without question

Note: points = severity x likelihood is a method we chose, not a measurement.

import random
from collections import Counter

# Toy data: 160 invented applicants in groups P, Q and R. label = judged suitable in a later review,
# pred = shortlisted by an invented tool. The generator is seeded, so every run gives the same rows.
def make_rows(seed=11, sizes=(("P", 90), ("Q", 60), ("R", 10))):
    rnd, rows = random.Random(seed), []
    for g, n in sizes:
        for _ in range(n):
            label = int(rnd.random() < 0.5)
            hit = (0.80 if g != "Q" else 0.60) if label else 0.15   # Q's suitable people are missed more
            rows.append(dict(group=g, label=label, pred=int(rnd.random() < hit)))
    return rows

ROWS = make_rows()


def rates(rows):
    out = {}
    for g in sorted({r["group"] for r in rows}):
        c = Counter((r["label"], r["pred"]) for r in rows if r["group"] == g)
        n, pos, neg = sum(c.values()), c[1, 1] + c[1, 0], c[0, 1] + c[0, 0]
        out[g] = dict(n=n, pos=pos, neg=neg, sel=(c[1, 1] + c[0, 1]) / n,
                      fpr=c[0, 1] / neg if neg else None, fnr=c[1, 0] / pos if pos else None)
    return out


def small(v, minimum=20):
    return min(v["pos"], v["neg"]) < minimum    # our chosen limit, not a standard


R = rates(ROWS)
f = lambda x: "n/a" if x is None else f"{x:.2f}"
print("group   n  suitable  unsuitable   sel   fpr   fnr  flag")
for g, v in R.items():
    print(f"{g:<6}{v['n']:>4}{v['pos']:>9}{v['neg']:>11}  {v['sel']:.2f}  {f(v['fpr'])}  {f(v['fnr'])}  "
          + (f"SMALL: one person moves fnr by {1 / v['pos']:.2f}" if small(v) else ""))
for g in ("Q",):
    print(f"gap {g} - P: selection {R[g]['sel'] - R['P']['sel']:+.2f}, "
          f"fpr {R[g]['fpr'] - R['P']['fpr']:+.2f}, fnr {R[g]['fnr'] - R['P']['fnr']:+.2f}")
print("Toy data: this illustrates a calculation, not any real system.")

group   n  suitable  unsuitable   sel   fpr   fnr  flag
P       90       50         40  0.48  0.07  0.20
Q       60       27         33  0.32  0.06  0.37
R       10        4          6  0.30  0.17  0.50  SMALL: one person moves fnr by 0.25
gap Q - P: selection -0.16, fpr -0.01, fnr +0.17
Toy data: this illustrates a calculation, not any real system.

from pathlib import Path

MISSING = object()
CARD_FIELDS = [("name", "Feature"), ("versions.system", "System version"), ("intended_use", "Intended use"),
            ("limitations", "Limitations"), ("human_oversight.escalation", "Human oversight"),
            ("evaluation.evidence_file", "Evaluation evidence"), ("accessibility", "Accessibility"),
            ("next_review", "Next review")]


def get(card, path):
    for key in path.split("."):
        if not isinstance(card, dict) or key not in card:
            return MISSING
        card = card[key]
    return card


def status(v):
    if v is MISSING or v in ("", [], {}, None):
        return "MISSING"
    return "UNKNOWN" if isinstance(v, dict) and v.get("status") == "UNKNOWN" else "ok"


def show(v):
    if isinstance(v, list):
        return "; ".join(map(str, v))
    return ", ".join(f"{k}: {x}" for k, x in v.items()) if isinstance(v, dict) else str(v)


def draft(card):
    lines = ["SYSTEM CARD (DRAFT, not reviewed)"]
    for path, label in CARD_FIELDS:
        v, s = get(card, path), status(get(card, path))
        text = ("MISSING - the owner must supply this" if s == "MISSING"
                else "UNKNOWN - " + v["why"] if s == "UNKNOWN" else show(v))
        lines.append(f"{label}: {text}")
    return "\n".join(lines)


def evidence_flags(card):
    p = get(card, "evaluation.evidence_file")
    return [f"evaluation cites {p} but the file does not exist"] if isinstance(p, str) and not Path(p).exists() else []


# Toy data: a partly filled card for LeafletHelper, a tool that rewrites notices in plain language
CARD = {
    "name": "LeafletHelper (hypothetical)",
    "versions": {"system": "0.2"},
    "intended_use": "draft plain-language versions of library notices; a librarian approves each one",
    "limitations": [],
    "evaluation": {"claim": "12 notices tested by the author", "evidence_file": "eval/leaflet_check.csv"},
    "accessibility": {"status": "UNKNOWN", "why": "nobody has tested the notices with a screen reader"},
    "next_review": "2027-01-15",
}
print(draft(CARD))
rows = [(status(get(CARD, p)), label) for p, label in CARD_FIELDS]
print({s: sum(r[0] == s for r in rows) for s in ("ok", "UNKNOWN", "MISSING")})
print("MISSING:", [label for s, label in rows if s == "MISSING"])
print("evidence:", evidence_flags(CARD))

SYSTEM CARD (DRAFT, not reviewed)
Feature: LeafletHelper (hypothetical)
System version: 0.2
Intended use: draft plain-language versions of library notices; a librarian approves each one
Limitations: MISSING - the owner must supply this
Human oversight: MISSING - the owner must supply this
Evaluation evidence: eval/leaflet_check.csv
Accessibility: UNKNOWN - nobody has tested the notices with a screen reader
Next review: 2027-01-15
{'ok': 5, 'UNKNOWN': 1, 'MISSING': 2}
MISSING: ['Limitations', 'Human oversight']
evidence: ['evaluation cites eval/leaflet_check.csv but the file does not exist']

Meridian Bursary Fund (invented) gives 120 awards a year from about 3000 applications. Four
readers take about 12 minutes per form. A vendor offers "MeritScore": it reads each form and
essay and gives a score from 0 to 100, and a three-line summary. The fund's plan: readers open
the top 900 forms; a random 10% of the other 2100 are also read; the rest are not.
Applicants write in several languages. The director wants a decision before the next round.
The vendor keeps essay text for a period not yet stated in writing. There is no appeal route.

# Toy data: Meridian Bursary Fund (invented): 3000 applications, 120 awards, four readers
FORMS = 3000
OPTIONS = {   # forms a person reads in full, minutes per form, and minutes per form for any other check
    "A human-only review": dict(full=3000, other=0),
    "B AI-assisted review": dict(full=900 + 210, other=0),    # top 900 + a 10% sample of the other 2100
    "C non-AI: checklist + draw": dict(full=0, other=3000),   # 4-minute eligibility checklist, draw among eligible
}
print(f"{'option':<28}{'hours':>6}{'per reader':>11}{'read in full':>14}{'never read in full':>24}")
for name, o in OPTIONS.items():
    hours = (o["full"] * 12 + o["other"] * 4) / 60
    never = f"{FORMS - o['full']}" + (" (checklist only)" if o["full"] == 0 else "")
    print(f"{name:<28}{hours:>6.0f}{hours / 4:>11.0f}{o['full']:>14}{never:>24}")

REVIEW = {
    "stakeholders": {   # group: (gains, pays, has a say)
        "applicants": ("a sooner decision", "wrong or unread decisions", "no"),
        "second-language writers": ("none known", "essay scored down for style", "no"),
        "readers": ("less reading", "pressure to follow the score", "some"),
        "donors": ("lower cost", "reputation if awards look unfair", "some"),
    },
    "risks": [  # same fields as the E1 register; check() from E1 runs on them
        dict(id="H1", title="Essay score favours one writing style", severity=4, likelihood=3,
             owner="Fund director", mitigation="group test on last year's forms before live use"),
        dict(id="H2", title="Readers follow the score, skip the sample", severity=4, likelihood=3,
             owner="Head of readers", mitigation="sample drawn by script and logged"),
        dict(id="H3", title="Vendor keeps essay text", severity=3, likelihood=2,
             owner="Data lead", mitigation="written retention limit before the pilot"),
        dict(id="H4", title="No second look for rejected applicants", severity=4, likelihood=4,
             owner="Fund director", mitigation="appeal route below"),
    ],
    "fairness_test": dict(label="awarded after two full reads", groups="region, first language (if policy allows)",
                          metrics="selection and false-negative rate, counts, interval", min_each=20,
                          stop_rule="a gap above 0.10 (chosen) stops live use", owner="Data lead"),
    "appeal": dict(who="any applicant", days=10, reviewer="a reader who did not see the first decision"),
    "recommendation": dict(
        decision="conditional go: shadow pilot only, the score is not shown to readers",
        conditions=[("report group tables on last year's 3000 forms", "tables with counts", "gap > 0.10", "Data lead"),
                    ("readers decide every award", "100% human decisions", "any live use", "Fund director"),
                    ("write the vendor retention limit", "signed text", "none by 2027-01-15", "Data lead")],
        accountable="Fund director", review_date="2027-02-01"),
}
NEED = ("stakeholders", "risks", "fairness_test", "appeal", "recommendation")


def gaps(review):
    found = [f"{k} is missing" for k in NEED if not review.get(k)]
    for r in review.get("risks", []):
        found += [f"{r['id']}: {p}" for p in check(r)]            # check() comes from the E1 solution
    for what, measure, stop, owner in review.get("recommendation", {}).get("conditions", []):
        if not (measure and stop and owner):
            found.append(f"condition '{what[:30]}' lacks a measure, stop rule or owner")
    if review.get("appeal") and not review["appeal"].get("reviewer"):
        found.append("appeal has no named reviewer")
    return found


print("\npay without a say:", [g for g, v in REVIEW["stakeholders"].items() if v[2] == "no"])
print("gaps in the review:", gaps(REVIEW) or "none")
weak = {**REVIEW, "recommendation": {**REVIEW["recommendation"], "conditions": [("add a tool", "", "", "")]}}
print("gaps if a condition is empty:", gaps(weak))
print("recommendation:", REVIEW["recommendation"]["decision"], "| owner:", REVIEW["recommendation"]["accountable"],
      "| review:", REVIEW["recommendation"]["review_date"])

option                       hours per reader  read in full      never read in full
A human-only review            600        150          3000                       0
B AI-assisted review           222         56          1110                    1890
C non-AI: checklist + draw     200         50             0   3000 (checklist only)

pay without a say: ['applicants', 'second-language writers']
gaps in the review: none
gaps if a condition is empty: ["condition 'add a tool' lacks a measure, stop rule or owner"]
recommendation: conditional go: shadow pilot only, the score is not shown to readers | owner: Fund director | review: 2027-02-01

Harbour Water Support (invented) runs a rule-based checker for bill support. In week 3 a
"simplified" rule v3 goes live. By week 4 the Coast group (invented) is rejected far more often
than before. Applications arrive online, giving monthly income, or on paper, giving annual
income. The checker is a plain rule, not a learned model. Staff are asked: stop the harm,
keep evidence, tell people, explain why, and prove the repair works.

import hashlib, json, random, textwrap
from collections import Counter

# Toy data: Harbour Water Support (invented). A rule-based checker approves or rejects bill-support
# applications. Weeks 1-2 use rule v2; on "week 3" a rule change (v3) goes live. Seeded, so repeatable.
LIMIT = 1500   # approve if monthly income is at most this (invented units)


def make_apps(seed=5, n=400):
    rnd, rows = random.Random(seed), []
    for i in range(n):
        group = rnd.choices(["Ridge", "Delta", "Coast"], [5, 3, 2])[0]
        paper = rnd.random() < (0.7 if group == "Coast" else 0.1)   # paper forms give ANNUAL income
        monthly = rnd.randint(500, 2500)
        rows.append(dict(id=f"H{i:03}", week=i * 4 // n + 1, group=group, channel="paper" if paper else "online",
                         period="annual" if paper else "monthly", income=monthly * 12 if paper else monthly))
    return rows


def v2(a):
    return "approve" if (a["income"] / 12 if a["period"] == "annual" else a["income"]) <= LIMIT else "reject"


def v3(a):   # the "simplified" rule: the income period is no longer used
    return "approve" if a["income"] <= LIMIT else "reject"


def v4(a):   # the fix: use the period again, and send unclear cases to a person instead of rejecting
    if a["period"] not in ("annual", "monthly"):
        return "refer"
    return v2(a)


def rej(rows, rule, group=None):
    s = [a for a in rows if group in (None, a["group"])]
    return sum(rule(a) == "reject" for a in s) / len(s) if s else None


APPS = make_apps()
live = lambda a: v2(a) if a["week"] < 3 else v3(a)
print("rejection rate by group (live rule)  weeks 1-2  weeks 3-4")
for g in ("Ridge", "Delta", "Coast"):
    early = [a for a in APPS if a["week"] < 3 and a["group"] == g]
    late = [a for a in APPS if a["week"] >= 3 and a["group"] == g]
    print(f"{g:<37}{sum(live(a) == 'reject' for a in early) / len(early):>8.2f}"
          f"{sum(live(a) == 'reject' for a in late) / len(late):>11.2f}")

# 1 CONTAIN: back to the last known good rule; list decisions that differ, for a person to re-decide
late = [a for a in APPS if a["week"] >= 3]
affected = [a["id"] for a in late if live(a) == "reject" and v2(a) == "approve"]
print("\ncontain: rule v2 restored; decisions to re-open:", len(affected), "of", len(late))

# 2 EVIDENCE: versions, counts and a fingerprint of the decision log; no names or free text
log = [(a["id"], live(a)) for a in APPS]
snapshot = {"rules": "v2, v3", "v3_from": "week 3", "decisions": len(log), "reopen": len(affected),
            "log_sha256": hashlib.sha256(json.dumps(log).encode()).hexdigest()[:16]}
print("evidence:", snapshot)

# 3 TELL THE AFFECTED: only what is known; unknowns are written as UNKNOWN
def notice(ref):
    return textwrap.fill(
        f"Application {ref}: your decision may be wrong because of a rule change. A person is deciding it "
        "again; you need not apply again. Effect on your bill: UNKNOWN. Next update: UNKNOWN date.", 90)


print(notice(affected[0]))

# 4 ROOT CAUSE: compare the two rules on the same applications
c = Counter((a["channel"], a["group"]) for a in late if v2(a) != v3(a))
print("\ndecisions that differ between v2 and v3, by (channel, group):", dict(sorted(c.items())))
print("differences on online forms:", sum(n for k, n in c.items() if k[0] == "online"))

# 5 DID THE FIX WORK? Run the checks on NEW data (another seed) against the broken and the fixed rule.
def tests(rule, rows, slack=0.02, min_n=20):
    out = []
    ok = rule(dict(income=12000, period="annual")) == "approve"             # 12000 a year is 1000 a month
    out.append(("T1 annual income is converted", "PASS" if ok else "FAIL"))
    online = [a for a in rows if a["channel"] == "online"]
    same = all(rule(a) == v2(a) for a in online)
    out.append(("T2 online forms decide as v2 did", "PASS" if same else "FAIL"))
    ok = rule(dict(income=9000, period="")) != "reject"                      # unclear: refer, never reject
    out.append(("T3 unclear period goes to a person", "PASS" if ok else "FAIL"))
    sizes = Counter(a["group"] for a in rows)
    if min(sizes.values()) < min_n:
        out.append(("T4 group gap no worse than v2", "NOT TESTED"))
    else:
        gap = lambda rl: max(rej(rows, rl, g) for g in sizes) - min(rej(rows, rl, g) for g in sizes)
        ok = gap(rule) <= gap(v2) + slack     # compare with the last known good rule, not with "zero"
        out.append((f"T4 group gap {gap(rule):.2f} vs v2 {gap(v2):.2f}", "PASS" if ok else "FAIL"))
    return out


NEW = make_apps(seed=9)
for name, rule in (("v3", v3), ("v4", v4)):
    print(name, [f"{t[:2]} {res}" for t, res in tests(rule, NEW)])
print("T4 on 30 rows only:", tests(v4, NEW[:30])[-1])

rejection rate by group (live rule)  weeks 1-2  weeks 3-4
Ridge                                    0.57       0.59
Delta                                    0.49       0.53
Coast                                    0.49       0.91

contain: rule v2 restored; decisions to re-open: 27 of 200
evidence: {'rules': 'v2, v3', 'v3_from': 'week 3', 'decisions': 400, 'reopen': 27, 'log_sha256': 'd6f3854d61f2b4f4'}
Application H200: your decision may be wrong because of a rule change. A person is
deciding it again; you need not apply again. Effect on your bill: UNKNOWN. Next update:
UNKNOWN date.

decisions that differ between v2 and v3, by (channel, group): {('paper', 'Coast'): 14, ('paper', 'Delta'): 5, ('paper', 'Ridge'): 8}
differences on online forms: 0
v3 ['T1 FAIL', 'T2 PASS', 'T3 FAIL', 'T4 FAIL']
v4 ['T1 PASS', 'T2 PASS', 'T3 PASS', 'T4 PASS']
T4 on 30 rows only: ('T4 group gap no worse than v2', 'NOT TESTED')