#!/usr/bin/env python3
"""Which Chase months arrive as a spreadsheet and which only as a PDF statement, per Chase's own pages.

Four chase.com personal-banking pages, read with WebFetch on 2026-10-25 and re-read here with urllib:
each quote below must still be on its page. Two lab rows read Evidilo's own source in this repository
(what its reader knows about Chase), stated as read in code and never as an accuracy figure.

    python3 check.py            re-fetch every page, confirm every quote, re-read the code rows
    python3 check.py --recheck  the same (no step here takes over a minute)
    python3 check.py --write    also rewrite dataset.json and dataset.csv

Exit 0: every quote found and every code row matches. Exit 1: a quote is gone from a page that
loaded, or a code row changed. Exit 2: nothing changed, but at least one page could not be reached.
"""
import csv
import gzip
import html
import json
import os
import platform
import re
import sys
import urllib.error
import urllib.request

CHECKED_AT = "2026-10-25"
NOT_STATED = "not stated on any Chase personal page found"
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/126.0 Safari/537.36")

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.normpath(os.path.join(HERE, "..", "..", "..", "..", ".."))

SOURCES = {
    "paperless": {
        "title": "Paperless Statements (chase.com)",
        "url": "https://www.chase.com/personal/mobile-online-banking/login-paperless",
        "quotes": [
            "Depending on your account type you can access up to 7 years of statements.",
            "Choose the year and account and click the Open or Save icon to the right of each statement.",
            "Click on \"Statements\", select your account and then the year and statement to open as a PDF.",
        ],
    },
    "planner": {
        "title": "What's Chase Spending Planner? (chase.com)",
        "url": "https://www.chase.com/personal/banking/education/budgeting-saving/what-is-chase-spending-planner",
        "quotes": [
            "Spending Planner is a digital tool for Chase checking and credit card customers that can help users monitor their spending.",
            "If you access the tool through your mobile phone, you'll be able to download PDF versions of your transactions for the last calendar year.",
            "If you access the tool via your desktop computer you'll be able to download transactions in a spreadsheet (CSV), Quicken Web Connect (QFX), Quicken or Microsoft Money (QIF), or QuickBooks Web Connect (QBO) format for the last two years.",
            "options to download your spending data (this year and the previous one)",
            "access the tool under the \"Manage Your Spending Plan\" section under the \"Plan, Track, or Manage\" dashboard",
        ],
    },
    "budget": {
        "title": "Spending Budget (chase.com)",
        "url": "https://www.chase.com/personal/financial-tools/build/spending-budget",
        "quotes": [
            "If you access the tool on desktop you can download your data in a spreadsheet (CSV), Quicken Web Connect (QFX), Quicken or Microsoft Money (QIF) or QuickBooks Web Connect (QBO).",
            "Your budget includes posted transactions on your J.P. Morgan or Chase checking accounts, credit cards and linked external accounts.",
            "Transfers and payments between these accounts are not included.",
        ],
    },
    "checks": {
        "title": "Order New Checks Online (chase.com)",
        "url": "https://www.chase.com/personal/checking/view-checks",
        "quotes": [
            "You can see online images of checks you've written for up to three years when you sign in to your account on chase.com or on the Chase Mobile",
        ],
    },
}

# What each page says, one row per item. Every cell is carried by the quotes of the listed source.
DOC_ROWS = [
    {"item": "Spending Planner transaction download, desktop", "format": "CSV, QFX, QIF, QBO",
     "period": "the last two years", "source": "planner"},
    {"item": "Spending Planner transaction download, Chase Mobile app", "format": "PDF",
     "period": "the last calendar year", "source": "planner"},
    {"item": "Spending Summary data download", "format": "not stated",
     "period": "this year and the previous one", "source": "planner"},
    {"item": "Statements online (Statements & documents)", "format": "PDF",
     "period": "up to 7 years, depending on account type; picked by year and account", "source": "paperless"},
    {"item": "Images of checks written", "format": "image",
     "period": "up to three years", "source": "checks"},
    {"item": "Transaction download for personal accounts outside Spending Planner", "format": NOT_STATED,
     "period": NOT_STATED, "source": ""},
]


def norm(text):
    text = re.sub(r"<(script|style)\b.*?</\1>", " ", text, flags=re.S | re.I)
    text = re.sub(r"<[^>]+>", " ", text)
    text = html.unescape(text)
    for a, b in (("’", "'"), ("‘", "'"), ("“", '"'), ("”", '"'), (" ", " ")):
        text = text.replace(a, b)
    return re.sub(r"\s+", " ", text).strip()


def fetch(url):
    req = urllib.request.Request(url, headers={
        "User-Agent": UA,
        "Accept": "text/html,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip",
    })
    with urllib.request.urlopen(req, timeout=25) as resp:
        raw = resp.read()
        if resp.headers.get("Content-Encoding") == "gzip":
            raw = gzip.decompress(raw)
    return norm(raw.decode("utf-8", errors="replace"))


def check_pages():
    status = {}
    for sid, src in SOURCES.items():
        try:
            text = fetch(src["url"])
        except (urllib.error.URLError, TimeoutError, OSError) as exc:
            status[sid] = "unreachable"
            print(f"UNREACHABLE {sid}: {src['url']} ({exc})")
            continue
        missing = [q for q in src["quotes"] if norm(q) not in text]
        status[sid] = "missing" if missing else "ok"
        for q in src["quotes"]:
            print(f"{'GONE ' if q in missing else 'ok   '} {sid}: {q}")
    return status


def read(rel):
    with open(os.path.join(REPO, rel), encoding="utf-8") as fh:
        return fh.read()


def check_code():
    """What Evidilo's reader knows about Chase, read from its source. Returns (rows, ok)."""
    shared = read("packages/shared/src/banks.ts")
    entries = re.findall(r'\{ id: "(\w+)", displayName: "([^"]+)", phase: \d, layoutFamilies: (\w+) \}', shared)
    chase = [e for e in entries if e[0] == "chase"]
    every = re.search(r"const EVERY_LAYOUT[^=]*= \[([^\]]*)\]", shared, flags=re.S)
    families = re.findall(r'"([\w-]+)"', every.group(1)) if every else []
    no_per_bank = "the register claims no per-bank layout" in re.sub(r"\s*//\s*", " ", shared)

    naming = []
    for root, _dirs, files in os.walk(os.path.join(REPO, "packages/parsers/src")):
        for name in files:
            if name.endswith(".ts") and not name.endswith(".test.ts"):
                path = os.path.join(root, name)
                with open(path, encoding="utf-8") as fh:
                    if re.search(r"\bchase\b", fh.read(), flags=re.I):
                        naming.append(os.path.relpath(path, REPO))

    rows = [
        {"item": "Banks in Evidilo's register", "value": str(len(entries)),
         "detail": "; ".join(e[1] for e in entries)},
        {"item": "JPMorgan Chase in the register", "value": "yes" if chase else "no",
         "detail": f"layout families: {', '.join(families)}" if chase and chase[0][2] == "EVERY_LAYOUT" else "entry changed"},
        {"item": "Per-bank layout claimed by the register", "value": "none" if no_per_bank else "comment changed",
         "detail": "comment in packages/shared/src/banks.ts: the register claims no per-bank layout"},
        {"item": "Reader source files (not tests) naming Chase", "value": str(len(naming)),
         "detail": "; ".join(naming) or "packages/parsers/src, *.ts excluding *.test.ts, word match"},
    ]
    expected = {"Banks in Evidilo's register": "12", "JPMorgan Chase in the register": "yes",
                "Per-bank layout claimed by the register": "none",
                "Reader source files (not tests) naming Chase": "0"}
    ok = True
    for r in rows:
        good = expected[r["item"]] == r["value"] and r["detail"] != "entry changed"
        ok = ok and good
        print(f"{'ok   ' if good else 'DIFF '} code: {r['item']} = {r['value']} ({r['detail']})")
    if families != ["csv", "digital-table", "scanned", "multicolumn"]:
        print(f"DIFF  code: layout families now {families}")
        ok = False
    return rows, ok


def write(status, code_rows):
    out = []
    for r in DOC_ROWS:
        src = SOURCES.get(r["source"])
        out.append({"kind": "chase-page", "item": r["item"], "format": r["format"], "period_or_value": r["period"],
                    "source": src["url"] if src else "", "checked_at": CHECKED_AT})
    for r in code_rows:
        out.append({"kind": "evidilo-code", "item": r["item"], "format": "", "period_or_value": r["value"],
                    "source": "repository: " + r["detail"],
                    "checked_at": CHECKED_AT})
    with open(os.path.join(HERE, "dataset.csv"), "w", newline="", encoding="utf-8") as fh:
        w = csv.DictWriter(fh, fieldnames=list(out[0]))
        w.writeheader()
        w.writerows(out)
    data = {
        "method": "primary-docs, plus a read of Evidilo's own source",
        "checkedAt": CHECKED_AT,
        "question": "For a personal Chase account, which months does Chase hand over as a spreadsheet, which only as a PDF statement, and how far back does each reach? And what does Evidilo's reader know about Chase?",
        "tools": [
            "WebFetch (Claude Code), 2026-10-25, to read each page and capture each quote",
            "WebSearch restricted to chase.com, 2026-10-25, to locate Chase's personal help pages",
            f"check.py: Python {platform.python_version()} urllib, re-fetches each page and confirms each quote; reads the repository at the commit checked out",
        ],
        "inputs": [
            {"id": sid, "title": s["title"], "url": s["url"], "fetchedAt": CHECKED_AT,
             "quotes": s["quotes"], "lastCheckStatus": status.get(sid, "not run")}
            for sid, s in SOURCES.items()
        ] + [
            {"id": "code", "title": "Evidilo repository", "paths": ["packages/shared/src/banks.ts", "packages/parsers/src"],
             "commit": "27c7c67"},
        ],
        "results": {"chasePages": DOC_ROWS, "evidiloCode": code_rows},
        "summary": {
            "longestSpreadsheetWindowStated": "the last two years (Spending Planner, desktop: CSV, QFX, QIF, QBO)",
            "longestPdfStatementWindowStated": "up to 7 years, depending on account type",
            "checkImagesWindowStated": "up to three years",
            "monthsOnlyAsPdfStatement": "older than two years and up to 7 years back",
        },
        "notes": [
            "No Chase personal help page found on 2026-10-25 documents a transaction download outside Spending Planner or a date range for one; WebSearch over chase.com returned only commercial and business guides for that feature. The only transaction-download window Chase states for personal accounts is Spending Planner's.",
            "The Chase Connect account-activity guide (https://www.chase.com/content/dam/chase-ux/documents/commercial-banking/chase-connect/cc_quickguide_account_activity.pdf) is for commercial accounts and is kept out of every row. WebFetch could not extract its text on the day.",
            "Chase Business Online and Chase Commercial Online pages state their own statement and history windows; none of them is used here.",
            "Spending Budget says transfers and payments between the customer's accounts are not included in the budget. Neither page says whether the transaction download leaves them out as well, so the post says only that.",
            "The two years and the 7 years are Chase's words; nothing here was measured inside a Chase account. No Chase account was used.",
            "The code rows describe what Evidilo's reader holds about Chase. They are not an accuracy figure; none is published for Chase.",
        ],
    }
    with open(os.path.join(HERE, "dataset.json"), "w", encoding="utf-8") as fh:
        json.dump(data, fh, indent=2, ensure_ascii=False)
        fh.write("\n")


def main():
    status = check_pages()
    code_rows, code_ok = check_code()
    if "--write" in sys.argv:
        write(status, code_rows)
    if "missing" in status.values() or not code_ok:
        print("FAIL: a quote is gone from its page, or a code row changed")
        return 1
    if "unreachable" in status.values():
        print("UNREACHABLE: at least one page could not be read; nothing that loaded has changed")
        return 2
    print("OK: every quote is on its page and every code row matches")
    return 0


if __name__ == "__main__":
    sys.exit(main())
