#!/usr/bin/env python3
"""Check for the post "Catch-up bookkeeping starts from the oldest statement".

Docs (the evidence): re-reads the help page each of five accounting tools publishes on an account's
opening or starting balance and confirms every quoted sentence is still on it.
Illustration (not evidence): builds twelve generated monthly statements, removes one month and
repeats another, and lists every place where a statement's opening balance differs from the closing
balance of the statement before it. Its result is certain by construction.

Exit 0: everything in dataset.json still holds. Exit 1: a quote is gone or the illustration differs.
Exit 2: no quote failed, but at least one page could not be reached at all.

  python3 check.py            run everything and compare with dataset.json
  python3 check.py --recheck  the same (nothing here takes more than a few seconds)
  python3 check.py --write    run everything and rewrite dataset.json and dataset.csv
"""
import csv
import gzip
import html
import json
import os
import random
import re
import sys
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
CHECKED_AT = "2026-10-25"
UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/130.0 Safari/537.36"
)

QBO = "https://quickbooks.intuit.com/learn-support/en-us/help-articles/edit-an-incorrect-opening-balance/00/262087"
QBD = ("https://quickbooks.intuit.com/learn-support/en-us/help-article/product-setup/"
       "enter-opening-balances-accounts-quickbooks-desktop/L4PfiNhRx_US_en_US")
XERO = "https://central.xero.com/s/article/Enter-conversion-balances"
WAVE = "https://support.waveapps.com/hc/en-us/articles/208622796-Enter-a-starting-balance-for-a-new-account"
# The Wave article's HTML page answers 403 to a script; the same article's body is served by the help
# centre's own JSON endpoint, which is what this script reads.
WAVE_READ = "https://support.waveapps.com/api/v2/help_center/en-us/articles/208622796.json"
FRESH = "https://support.freshbooks.com/hc/en-us/articles/360040117691"

READ_FROM = {WAVE: WAVE_READ}

# (tool, url, page title, where, date, against): each cell is a list of quotes, or a "not stated" note.
TOOLS = [
    {
        "tool": "QuickBooks Online",
        "url": QBO,
        "title": "Enter and manage opening balances in QuickBooks Online",
        "where": "Chart of accounts, New account, Opening balance field",
        "date": "The As of date: the day you start tracking the account",
        "against": "Opening Balance Equity",
        "quotes": {
            "where": ["In the Opening balance field, add the balance reflecting your bank or credit card account."],
            "date": ["Select the date you want to start tracking your finances in the As of field."],
            "against": ["QuickBooks tracks all of your opening balances for all of your accounts in an Opening Balance Equity account."],
            "later": ["you can add an opening balance later with a journal entry", "Edit the date, then the amount."],
        },
    },
    {
        "tool": "QuickBooks Desktop",
        "url": QBD,
        "title": "Enter opening balances for accounts in QuickBooks Desktop",
        "where": "Chart of Accounts, New, Enter Opening Balance: Ending date and Ending balance fields",
        "date": "A date older than the oldest transaction you want to track",
        "against": "Not stated for the Enter Opening Balance window; the page's journal entry route uses Opening Balance Equity",
        "quotes": {
            "where": ["Select Enter Opening Balance.",
                      "Enter the date you picked in the Ending date field.",
                      "In the Ending balance field, enter the balance of your real-life account for that date."],
            "date": ["Pick a date that's older than the oldest transaction you want to track in QuickBooks."],
            "against": ["The Opening Balance Equity account shouldn't have a remaining balance."],
        },
    },
    {
        "tool": "Xero",
        "url": XERO,
        "title": "Enter conversion balances",
        "where": "Not read: the page renders by script and returned no article text on 2026-10-25",
        "date": "Not read",
        "against": "Not read",
        "quotes": {},
        "unreadable": "The Xero Central page returned no article text to WebFetch or to a script on 2026-10-25 "
                      "(it is built in the browser by script), so no Xero cell is filled from it.",
    },
    {
        "tool": "Wave",
        "url": WAVE,
        "title": "Enter a starting balance for a new account",
        "where": "Transactions page: an income transaction described Starting balance",
        "date": "The start of the business's fiscal year",
        "against": "An equity account the user picks in the Category field",
        "quotes": {
            "where": ["Create an income transaction.", "In the Description field, type Starting balance."],
            "date": ["When adding a new bank account, enter the starting balance for the account at the start of your business's fiscal year."],
            "against": ["In the Category field, select an appropriate equity account for your business."],
            "later": ["If you enter transactions within the fiscal year with dates before the starting balance transaction, you must adjust the date and amount so it's the first transaction for the account."],
        },
    },
    {
        "tool": "FreshBooks",
        "url": FRESH,
        "title": "How do I set up bank reconciliation?",
        "where": "Accounting, bank reconciliation set-up: Reconciliation Start Date and Opening Balance",
        "date": "The Reconciliation Start Date, suggested as the first day of a recent statement",
        "against": "Opening Balance Adjustments (Equity)",
        "quotes": {
            "where": ["Enter your Opening Balance from the day you chose to start reconciling in the field"],
            "date": ["it's recommended to use the first day of a recent bank statement or the start of a period on your credit card statement",
                     "Enter the Date of your opening balance/balance forward line into FreshBooks' Bank Rec as your Reconciliation Start Date"],
            "against": ["The opening balance you've entered will create a journal entry that affects the Bank (Cash) and Opening Balance Adjustments (Equity) accounts."],
        },
    },
]


class Unreachable(Exception):
    pass


def norm(s):
    s = s.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"')
    s = s.replace(" ", " ")
    s = re.sub(r"\s+", " ", s)
    # Tag stripping leaves a space before punctuation that followed a link; drop it on both sides.
    return re.sub(r" ([.,;:])", r"\1", s).strip()


def page_text(url, cache={}, failed={}):
    if url in failed:
        raise Unreachable(failed[url])
    if url not in cache:
        read = READ_FROM.get(url, url)
        try:
            req = urllib.request.Request(read, headers={"User-Agent": UA, "Accept": "text/html,application/json"})
            with urllib.request.urlopen(req, timeout=30) as r:
                raw = r.read()
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            failed[url] = f"{read}: {e}"
            raise Unreachable(failed[url])
        if raw[:2] == b"\x1f\x8b":
            raw = gzip.decompress(raw)
        body = raw.decode("utf-8", "replace")
        if read.endswith(".json"):
            body = json.loads(body)["article"]["body"]
        body = re.sub(r"(?is)<script.*?</script>|<style.*?</style>", " ", body)
        cache[url] = norm(html.unescape(re.sub(r"<[^>]+>", " ", body)))
    return cache[url]


def check_quotes():
    rows, unreachable = [], []
    for t in TOOLS:
        for cell, quotes in t["quotes"].items():
            for q in quotes:
                try:
                    found = norm(q) in page_text(t["url"])
                    status = "found" if found else "missing"
                except Unreachable as e:
                    status = "unreachable"
                    if str(e) not in unreachable:
                        unreachable.append(str(e))
                rows.append({"tool": t["tool"], "cell": cell, "url": t["url"], "quote": q, "status": status})
    return rows, unreachable


# Illustration only: twelve generated months, one removed, one repeated.
def illustration():
    rng = random.Random(20261025)
    months, opening = [], 1234567  # cents
    for m in range(1, 13):
        movement = rng.randint(-150000, 150000)
        months.append({"month": f"2025-{m:02d}", "opening": opening, "closing": opening + movement})
        opening += movement
    received = [s for s in months if s["month"] != "2025-05"]
    i = next(k for k, s in enumerate(received) if s["month"] == "2025-09")
    received.insert(i + 1, dict(received[i]))
    breaks = []
    for prev, cur in zip(received, received[1:]):
        if cur["opening"] != prev["closing"]:
            breaks.append({"after": prev["month"], "statement": cur["month"],
                           "previousClosing": f"{prev['closing'] / 100:.2f}",
                           "opening": f"{cur['opening'] / 100:.2f}"})
    return {"statementsGenerated": len(months), "statementsReceived": len(received),
            "removed": "2025-05", "repeated": "2025-09",
            "startingBalance": f"{months[0]['opening'] / 100:.2f}", "breaks": breaks}


def main():
    write = "--write" in sys.argv
    quotes, unreachable = check_quotes()
    ill = illustration()
    for u in unreachable:
        print(f"UNREACHABLE {u}")
    for q in quotes:
        if q["status"] != "found":
            print(f"{q['status'].upper():11} {q['tool']} [{q['cell']}] {q['quote']}")
    if write:
        data = {
            "method": "primary-docs",
            "checkedAt": CHECKED_AT,
            "tools": ["WebFetch (Claude Code), 2026-10-25", f"Python {sys.version.split()[0]} urllib, random (illustration)"],
            "inputs": [{"tool": t["tool"], "url": t["url"], "pageTitle": t["title"],
                        **({"readFrom": READ_FROM[t["url"]]} if t["url"] in READ_FROM else {})} for t in TOOLS],
            "results": {
                "table": [{k: t[k] for k in ("tool", "url", "where", "date", "against")} for t in TOOLS],
                "quotes": [{k: q[k] for k in ("tool", "cell", "url", "quote")} for q in quotes],
                "illustration": {"label": "ILLUSTRATION ONLY: generated statements, result certain by construction; not the post's evidence", **ill},
            },
            "notes": [
                "Every quote was read on its page with WebFetch on 2026-10-25. The two Intuit pages time out when fetched "
                "by script from the host this ran on, so a script recheck reports them unreachable (exit 2) rather than failed.",
                "Wave's article HTML answers 403 to a script; its text is read from the help centre's JSON endpoint for the same article id.",
                next(t["unreadable"] for t in TOOLS if "unreadable" in t),
                "QuickBooks Online's page ties the As of date to the day tracking starts and names the statement only as the source of the amount; "
                "it does not tell the user to use a statement's first day.",
                "QuickBooks Desktop's Enter Opening Balance window asks for an Ending date and Ending balance; the page does not say which "
                "account that window posts against, only that Opening Balance Equity should not keep a remaining balance.",
            ],
        }
        with open(os.path.join(HERE, "dataset.json"), "w") as f:
            json.dump(data, f, indent=2, ensure_ascii=False)
            f.write("\n")
        with open(os.path.join(HERE, "dataset.csv"), "w", newline="") as f:
            w = csv.writer(f)
            w.writerow(["tool", "where_the_balance_is_entered", "date_it_applies_to", "posted_against", "source_url", "checked_at"])
            for t in TOOLS:
                w.writerow([t["tool"], t["where"], t["date"], t["against"], t["url"], CHECKED_AT])
        print("wrote dataset.json and dataset.csv")
    else:
        with open(os.path.join(HERE, "dataset.json")) as f:
            data = json.load(f)
        stored = {k: v for k, v in data["results"]["illustration"].items() if k != "label"}
        if stored != ill:
            print("ILLUSTRATION differs from dataset.json")
            return 1
        stored_q = [(q["tool"], q["cell"], q["quote"]) for q in data["results"]["quotes"]]
        if stored_q != [(q["tool"], q["cell"], q["quote"]) for q in quotes]:
            print("QUOTE LIST differs from dataset.json")
            return 1
    if any(q["status"] == "missing" for q in quotes):
        return 1
    if unreachable:
        print(f"{sum(q['status'] == 'found' for q in quotes)} quotes found; "
              f"{sum(q['status'] == 'unreachable' for q in quotes)} on pages that could not be reached")
        return 2
    print(f"all {len(quotes)} quotes found; illustration unchanged")
    return 0


if __name__ == "__main__":
    sys.exit(main())