#!/usr/bin/env python3
"""Check for the post "What the size of a bank reconciliation discrepancy tells you".

Lab: generates bank statements and a books register that matches each one, injects one error at a
time into the register (row missing, row doubled, sign flipped, two adjacent digits transposed, a
row dated into the next period), 1,000 cases per error type and scenario, computes the discrepancy
(statement total minus books total) and tests the three rules of thumb against it:

  rule "one row"     the discrepancy equals the amount of a statement row
                     (that row is missing from the books, doubled in them, or booked in another period)
  rule "half"        half the discrepancy equals the amount of a statement row (that row's sign is flipped)
  rule "nine"        the discrepancy in cents divides by 9 (two digits are transposed)

For "nine" the candidate rows are the rows where swapping two adjacent digits of the amount gives
exactly the discrepancy. Amounts are compared without their sign, the way a bookkeeper scans a
statement for a figure. Everything uses integer cents and random.Random(SEED), so the numbers are
the same on every machine.

Docs: re-reads two QuickBooks Online help pages and confirms each quoted sentence is still there.

Exit 0: everything in dataset.json still holds. Exit 1: a measurement or a quote differs.
Exit 2: a page could not be reached at all (the lab result is printed first).

  python3 check.py            run the lab and the pages, compare with dataset.json
  python3 check.py --recheck  the same (the lab takes a few seconds, so nothing is skipped)
  python3 check.py --write    run the lab and rewrite dataset.json results and dataset.csv
"""
import csv
import html
import json
import math
import os
import platform
import random
import re
import sys
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
CHECKED_AT = "2026-10-25"
SEED = 20261025
CASES = 1000
UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/130.0 Safari/537.36"
)

END_OF_REC = ("https://quickbooks.intuit.com/learn-support/en-us/help-article/statement-reconciliation/"
              "fix-issues-end-reconciliation-quickbooks-online/L3mZimyAb_US_en_US")
BEGINNING = ("https://quickbooks.intuit.com/learn-support/en-us/help-article/statement-reconciliation/"
             "fix-issues-accounts-reconciled-past-quickbooks/L8lx6PQQ5_US_en_US")
QUOTES = [
    ("q-zero", END_OF_REC, "If everything is in place, the difference between the ending balances should be zero."),
    ("q-missing-duplicate", END_OF_REC, "There are missing or duplicate transactions in QuickBooks."),
    ("q-slightly-off", END_OF_REC, "If you find transactions that are slightly off, reach out to your accountant."),
    ("q-discrepancy", BEGINNING,
     "There's a discrepancy in your beginning balance when something changes on a transaction since a past reconciliation."),
    ("q-report", BEGINNING,
     "This report lists what changed, how it affected your balance, and what you can do to fix each item."),
]

ERRORS = ["missing", "doubled", "sign-flipped", "transposed", "next-period"]
# The rule a bookkeeper would apply for each error, and the rule's name for the cause.
OWN_RULE = {"missing": "one-row", "doubled": "one-row", "next-period": "one-row",
            "sign-flipped": "half", "transposed": "nine"}
RULES = ["one-row", "half", "nine"]

SCENARIOS = {
    # name: (min rows, max rows) per monthly statement
    "small-business": (30, 80),
    "busy-account": (150, 300),
}

FEES = [500, 1000, 1200, 1500, 2500, 3500]  # cents


def lognormal_cents(rng, median_dollars, sigma, whole_dollars=False):
    v = math.exp(rng.gauss(math.log(median_dollars), sigma))
    v = min(max(v, 1.0), 250000.0)
    return int(round(v)) * 100 if whole_dollars else int(round(v * 100))


def one_amount(rng):
    """Signed amount in cents: money in positive, money out negative."""
    r = rng.random()
    if r < 0.45:   # card purchase
        return -lognormal_cents(rng, 38, 1.0)
    if r < 0.60:   # ACH bill payment
        return -lognormal_cents(rng, 250, 0.9)
    if r < 0.70:   # round transfer or withdrawal, $50 steps
        k = min(int(rng.expovariate(1 / 6)) + 1, 40)
        return -k * 5000
    if r < 0.80:   # check, 30% written to whole dollars
        return -lognormal_cents(rng, 400, 1.0, whole_dollars=rng.random() < 0.3)
    if r < 0.95:   # deposit
        return lognormal_cents(rng, 1200, 0.8)
    return -rng.choice(FEES)  # bank fee


def statement(rng, lo, hi):
    n = rng.randint(lo, hi)
    days = sorted(rng.randint(1, 30) for _ in range(n))
    return [(d, one_amount(rng)) for d in days]


def transpose(rng, cents):
    """Swap two adjacent differing digits of |cents|; None when no such pair exists."""
    s = str(abs(cents))
    pairs = [i for i in range(len(s) - 1) if s[i] != s[i + 1] and not (i == 0 and s[1] == "0")]
    if not pairs:
        return None
    i = rng.choice(pairs)
    t = s[:i] + s[i + 1] + s[i] + s[i + 2:]
    return int(t) * (1 if cents > 0 else -1)


def swaps(cents):
    """Every |difference| one adjacent-digit swap of this amount can produce."""
    s = str(abs(cents))
    out = set()
    for i in range(len(s) - 1):
        if s[i] != s[i + 1] and not (i == 0 and s[1] == "0"):
            t = s[:i] + s[i + 1] + s[i] + s[i + 2:]
            out.add(abs(int(s) - int(t)))
    return out


def inject(rng, rows, error):
    """Return (index of the row that went wrong, discrepancy in cents = statement - books)."""
    if error == "next-period":
        late = [i for i, (d, _) in enumerate(rows) if d >= 28]
        i = rng.choice(late) if late else len(rows) - 1
        return i, rows[i][1]
    while True:
        i = rng.randrange(len(rows))
        a = rows[i][1]
        if error == "missing":
            return i, a
        if error == "doubled":
            return i, -a
        if error == "sign-flipped":
            return i, 2 * a
        if error == "transposed":
            t = transpose(rng, a)
            if t is not None:
                return i, a - t


def read(rows, d):
    """What each rule says about discrepancy d: the rows it fits."""
    D = abs(d)
    mags = [abs(a) for _, a in rows]
    return {
        "one-row": [i for i, m in enumerate(mags) if m == D],
        "half": [i for i, m in enumerate(mags) if D % 2 == 0 and m == D // 2],
        "nine": [i for i, m in enumerate(mags) if D in swaps(m)] if D % 9 == 0 else [],
        "nine-fires": D % 9 == 0,
    }


def run_scenario(name, lo, hi):
    rng = random.Random(f"{SEED}-{name}")
    per_error = {}
    fired = {r: {e: 0 for e in ERRORS} for r in RULES}
    for e in ERRORS:
        c = {"cases": CASES, "own_rule_fits": 0, "own_rule_fits_true_row": 0, "own_rule_one_row": 0,
             "own_rule_several_rows": 0, "other_rule_fires": 0, "clean_read": 0,
             "nine_also_fires": 0, "half_also_fires": 0, "one_row_also_fires": 0}
        counts = []
        for _ in range(CASES):
            rows = statement(rng, lo, hi)
            i, d = inject(rng, rows, e)
            r = read(rows, d)
            own = OWN_RULE[e]
            fires = {"one-row": bool(r["one-row"]), "half": bool(r["half"]), "nine": r["nine-fires"]}
            for rule in RULES:
                if fires[rule]:
                    fired[rule][e] += 1
            own_rows = r[own] if own != "nine" else r["nine"]
            counts.append(len(own_rows))
            if fires[own]:
                c["own_rule_fits"] += 1
            if i in own_rows:
                c["own_rule_fits_true_row"] += 1
            if len(own_rows) == 1:
                c["own_rule_one_row"] += 1
            if len(own_rows) > 1:
                c["own_rule_several_rows"] += 1
            others = [rule for rule in RULES if rule != own and fires[rule]]
            if others:
                c["other_rule_fires"] += 1
            for rule, key in (("nine", "nine_also_fires"), ("half", "half_also_fires"),
                              ("one-row", "one_row_also_fires")):
                if rule != own and fires[rule]:
                    c[key] += 1
            if not others and own_rows == [i]:
                c["clean_read"] += 1
        c["own_rule_rows_median"] = sorted(counts)[len(counts) // 2]
        c["own_rule_rows_max"] = max(counts)
        per_error[e] = c
    per_rule = {}
    for rule in RULES:
        own_errs = [e for e in ERRORS if OWN_RULE[e] == rule]
        total = sum(fired[rule].values())
        wrong = sum(v for e, v in fired[rule].items() if e not in own_errs)
        per_rule[rule] = {"fires": total, "fires_on_wrong_cause": wrong,
                          "fires_by_true_error": fired[rule]}
    return {"per_error": per_error, "per_rule": per_rule}


def pct(n, d):
    return round(100.0 * n / d, 1) if d else 0.0


def lab():
    out = {}
    for name, (lo, hi) in SCENARIOS.items():
        out[name] = run_scenario(name, lo, hi)
        for c in out[name]["per_error"].values():
            for k in ("own_rule_one_row", "own_rule_several_rows", "other_rule_fires", "clean_read",
                      "nine_also_fires", "half_also_fires", "one_row_also_fires"):
                c[k + "_pct"] = pct(c[k], c["cases"])
        for r in out[name]["per_rule"].values():
            r["wrong_cause_pct_of_fires"] = pct(r["fires_on_wrong_cause"], r["fires"])
    return out


def summary(res):
    """The headline numbers the post quotes, derived from the lab result."""
    out = {}
    for scen, s in res.items():
        pe, pr = s["per_error"], s["per_rule"]
        out[scen] = {
            "nine rule, share of its fits where the cause was not a transposition":
                f'{pr["nine"]["wrong_cause_pct_of_fires"]}%',
            "half rule, share of its fits where the cause was not a flipped sign":
                f'{pr["half"]["wrong_cause_pct_of_fires"]}%',
            "one-row rule, share of its fits where the cause was a flipped sign or a transposition":
                f'{pr["one-row"]["wrong_cause_pct_of_fires"]}%',
            "missing rows whose amount also fits the nine rule": f'{pe["missing"]["nine_also_fires_pct"]}%',
            "missing rows where more than one statement row has that amount":
                f'{pe["missing"]["own_rule_several_rows_pct"]}%',
            "transposed, cases where exactly one row could produce the discrepancy":
                f'{pe["transposed"]["own_rule_one_row_pct"]}%',
            "transposed, median candidate rows": pe["transposed"]["own_rule_rows_median"],
            "next-period cases that read exactly like a missing row by amount":
                f'{pct(pe["next-period"]["own_rule_fits_true_row"], pe["next-period"]["cases"])}%',
        }
    return out


def csv_rows(res):
    rows = []
    for scen, s in res.items():
        for e, c in s["per_error"].items():
            rows.append({"scenario": scen, "error": e, "own_rule": OWN_RULE[e], "cases": c["cases"],
                         "own_rule_fits_pct": pct(c["own_rule_fits"], c["cases"]),
                         "own_rule_one_row_pct": c["own_rule_one_row_pct"],
                         "own_rule_several_rows_pct": c["own_rule_several_rows_pct"],
                         "own_rule_rows_median": c["own_rule_rows_median"],
                         "one_row_rule_also_fits_pct": c["one_row_also_fires_pct"],
                         "half_rule_also_fits_pct": c["half_also_fires_pct"],
                         "nine_rule_also_fits_pct": c["nine_also_fires_pct"],
                         "another_rule_also_fits_pct": c["other_rule_fires_pct"],
                         "clean_read_pct": c["clean_read_pct"]})
    return rows


class Unreachable(Exception):
    pass


def page_text(url, cache={}, dead={}):
    if url in dead:
        raise Unreachable(dead[url])
    if url not in cache:
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html"})
            with urllib.request.urlopen(req, timeout=30) as r:
                body = r.read().decode("utf-8", "replace")
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            dead[url] = f"{url}: {e}"
            raise Unreachable(dead[url])
        body = re.sub(r"(?is)<script.*?</script>|<style.*?</style>", " ", body)
        text = html.unescape(re.sub(r"<[^>]+>", " ", body)).replace("’", "'")
        cache[url] = re.sub(r"\s+", " ", text)
    return cache[url]


def main():
    write = "--write" in sys.argv
    res = lab()
    path = os.path.join(HERE, "dataset.json")
    if write:
        with open(path) as f:
            ds = json.load(f)
        ds["results"]["lab"] = res
        ds["results"]["summary"] = summary(res)
        ds["tools"] = [{"name": "Python", "version": platform.python_version()},
                       {"name": "random.Random (Mersenne Twister)", "version": "standard library", "seed": SEED}]
        with open(path, "w") as f:
            json.dump(ds, f, indent=2)
            f.write("\n")
        rows = csv_rows(res)
        with open(os.path.join(HERE, "dataset.csv"), "w", newline="") as f:
            w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
            w.writeheader()
            w.writerows(rows)
        print("wrote dataset.json and dataset.csv")
        return 0

    with open(path) as f:
        ds = json.load(f)
    failed = 0
    if ds["results"]["lab"] != res or ds["results"]["summary"] != summary(res):
        print("FAIL lab: the simulation no longer gives the numbers in dataset.json")
        failed += 1
    with open(os.path.join(HERE, "dataset.csv"), newline="") as f:
        saved = [{k: v for k, v in r.items()} for r in csv.DictReader(f)]
    if saved != [{k: str(v) for k, v in r.items()} for r in csv_rows(res)]:
        print("FAIL lab: dataset.csv differs from the simulation")
        failed += 1
    if not failed:
        print("ok   lab: every number in dataset.json and dataset.csv reproduced")

    unreachable = []
    for qid, url, quote in QUOTES:
        try:
            text = page_text(url)
        except Unreachable as e:
            unreachable.append(str(e))
            continue
        if quote.replace("’", "'") in text:
            print(f"ok   {qid}")
        else:
            print(f"FAIL {qid}: quote no longer on {url}")
            failed += 1
    if failed:
        return 1
    if unreachable:
        for u in sorted(set(unreachable)):
            print(f"UNREACHABLE {u}")
        return 2
    return 0


if __name__ == "__main__":
    sys.exit(main())