#!/usr/bin/env python3
"""Check behind /blog/iif-file-format.

1. Re-reads Intuit's three IIF help pages and confirms the quoted sentences are still there and
   that none of the three mentions duplicates.
2. Downloads Intuit's IIF import kit, confirms it is the same zip that was read on 2026-10-25
   (its manual is a PDF; its quotes were read from those exact bytes), re-reads the header help
   for the TRNS/SPL rules, and checks every sample transaction in the kit for a zero sum.
3. Writes one statement's deposit and check with Evidilo's production IIF writer
   (packages/exports/src/iif.ts, through render-iif.harness.ts and vitest), checks the file
   against the kit, then breaks it six ways and records which rule each copy breaks.

Exit 0: everything still holds. Exit 1: a quote is gone or a measurement differs from
dataset.csv. Exit 2: a cited page could not be reached at all (nothing else failed).
`--recheck` runs the same steps (none takes over a minute). `--write` rewrites dataset.csv.
"""

import csv
import hashlib
import html
import io
import os
import re
import subprocess
import sys
import tempfile
import urllib.error
import urllib.request
import zipfile
from datetime import date
from decimal import Decimal
from pathlib import Path

HERE = Path(__file__).resolve().parent
REPO = HERE.parents[4]  # apps/web/public/blog/<slug> -> repository root
UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/130.0 Safari/537.36"
)

OVERVIEW = (
    "https://quickbooks.intuit.com/learn-support/en-us/help-article/list-management/"
    "iif-overview-import-kit-sample-files-headers/L5CZIpJne_US_en_US"
)
EXPORT_IMPORT = (
    "https://quickbooks.intuit.com/learn-support/en-us/help-article/import-export-data-files/"
    "export-import-edit-iif-files/L56LT9Z0Q_US_en_US"
)
IMPROVED_2019 = (
    "https://quickbooks.intuit.com/learn-support/en-us/help-article/import-export-data-files/"
    "improved-iif-import-quickbooks-2019-later/L1K3ZQDX9_US_en_US"
)
KIT = "https://support.quickbooks.intuit.com/support/pages/executable/iif/iif_import_kit.zip"
KIT_SHA256 = "45dde93a8bec83569820a63c50bb2ca0477e10380c358b4838e87c74382f0834"

PAGE_QUOTES = {
    OVERVIEW: [
        "Importing IIF files into QuickBooks provides only limited error checking.",
        "The file headers are in rows that begin with an exclamation point.",
        "that account will be created as a new bank account.",
        "Be sure all accounts in your .iif file are created in QuickBooks before importing.",
        "The correct spelling and placement of these file headers are required to prevent errors during import.",
        "the data will import incorrectly",
    ],
    EXPORT_IMPORT: [
        "When you import IIF files, it doesn't link transactions.",
        "Back up your company file before you import IIF files.",
    ],
    IMPROVED_2019: [
        "It eliminates the possibility of importing bad data by scanning your file for errors.",
        "Provides a new .IIF file of the failed transactions, along with the descriptions of errors found.",
        "Look for ERROR to see what the problem(s) with the file is.",
    ],
}
PAGE_NAMES = {OVERVIEW: "overview", EXPORT_IMPORT: "export-import-edit", IMPROVED_2019: "improved-2019"}

HEADER_HELP_QUOTES = [
    "The keyword ENDTRNS appears after the last distribution line.",
    "The TRNSTYPE, ACCNT, and AMOUNT columns are required.",
    "The date is always in MM/DD/YY format. For example, 1/30/94.",
    "Debit amounts are always positive, credit amounts are always negative.",
    "SPLID (Required) A unique number that identifies the distribution line in the transaction.",
    "The date in this field must match the date in the DATE field for the transaction.",
    "The keyword in this field must match the keyword in the TRNSTYPE field for the transaction.",
]


def norm(text: str) -> str:
    text = html.unescape(text)
    text = text.replace("\u2019", "'").replace("\u2018", "'").replace("\u00a0", " ")
    text = text.replace("\u2013", "-").replace("\u2014", "-")
    return re.sub(r"\s+", " ", text)


def html_text(raw: str) -> str:
    raw = re.sub(r"(?is)<(script|style|xml)\b.*?</\1>", " ", raw)
    return norm(re.sub(r"<[^>]+>", " ", raw))


def fetch(url: str, timeout: int = 25) -> bytes:
    request = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "*/*"})
    with urllib.request.urlopen(request, timeout=timeout) as response:
        return response.read()


# ---------- the rules a file is checked against ----------

DATE_FORM = re.compile(r"^(\d{1,2})/(\d{1,2})/(\d{2}|\d{4})$")


def amount(value: str) -> Decimal:
    value = value.strip().strip('"').replace(",", "")
    return Decimal(value) if value else Decimal(0)


def validate(text: str) -> list:
    """The rules each broken copy is measured against, named after where they come from."""
    broken = set()
    headers, declared, block = {}, set(), None
    blocks = []
    for line in text.replace("\r\n", "\n").split("\n"):
        if line == "":
            continue
        cells = line.split("\t")
        key = cells[0].strip()
        if key.startswith("!"):
            headers[key[1:]] = [c.strip() for c in cells[1:]]
            continue
        if key == "ACCNT":
            declared.add(dict(zip(headers["ACCNT"], cells[1:]))["NAME"])
            continue
        if key == "TRNS":
            if block is not None:
                broken.add("endtrns")  # a new transaction opened before the last one closed
                blocks.append(block)
            block = []
        if key in ("TRNS", "SPL"):
            if block is None:
                broken.add("endtrns")
                block = []
            block.append((key, dict(zip(headers[key], cells[1:]))))
        if key == "ENDTRNS":
            if block is None:
                broken.add("endtrns")
            else:
                blocks.append(block)
                block = None
    if block is not None:
        broken.add("endtrns")
        blocks.append(block)

    for lines in blocks:
        if sum(amount(fields.get("AMOUNT", "")) for _, fields in lines) != 0:
            broken.add("balance")
        trns = lines[0][1] if lines and lines[0][0] == "TRNS" else {}
        for keyword, fields in lines:
            if fields.get("ACCNT", "") not in declared:
                broken.add("declared")
            match = DATE_FORM.match(fields.get("DATE", ""))
            if match is None:
                broken.add("date-form")
            else:
                month, day, year = (int(g) for g in match.groups())
                try:
                    date(year if year > 99 else 2000 + year, month, day)
                except ValueError:
                    broken.add("date-form")
            if keyword == "SPL" and trns:
                if fields.get("DATE") != trns.get("DATE"):
                    broken.add("spl-date")
                if fields.get("TRNSTYPE") != trns.get("TRNSTYPE"):
                    broken.add("spl-type")
    return sorted(broken)


def blocks_of(text: str):
    headers, out, block = {}, [], None
    for line in text.replace("\r\n", "\n").split("\n"):
        cells = line.split("\t")
        key = cells[0].strip()
        if key.startswith("!"):
            headers[key[1:]] = [c.strip() for c in cells[1:]]
        elif key == "TRNS":
            block = [dict(zip(headers["TRNS"], cells[1:]))]
        elif key == "SPL" and block is not None:
            block.append(dict(zip(headers["SPL"], cells[1:])))
        elif key == "ENDTRNS" and block is not None:
            out.append(block)
            block = None
    return headers, out


# ---------- steps ----------


def check_pages(rows, problems, unreachable):
    for url, quotes in PAGE_QUOTES.items():
        name = PAGE_NAMES[url]
        try:
            text = html_text(fetch(url).decode("utf-8", "replace"))
        except (urllib.error.URLError, TimeoutError, OSError) as error:
            print(f"UNREACHABLE {url}: {error!r}")
            unreachable.append(url)
            continue
        for quote in quotes:
            found = norm(quote) in text
            rows.append((f"page:{name}", f"quote: {quote}", "present" if found else "absent"))
            if not found:
                problems.append(f"{name}: quote gone: {quote}")
        dup = re.search(r"duplicat", text, re.I) is not None
        rows.append((f"page:{name}", "mentions duplicates", "yes" if dup else "no"))


def check_kit(rows, problems, unreachable):
    try:
        data = fetch(KIT, timeout=60)
    except (urllib.error.URLError, TimeoutError, OSError) as error:
        print(f"UNREACHABLE {KIT}: {error!r}")
        unreachable.append(KIT)
        return None, None
    digest = hashlib.sha256(data).hexdigest()
    rows.append(("kit", "zip sha256 (the bytes whose manual was read)", digest))
    kit = zipfile.ZipFile(io.BytesIO(data))
    names = kit.namelist()

    help_name = next(n for n in names if n.endswith("IIF Header Help/!TRNS.htm"))
    help_text = html_text(kit.read(help_name).decode("latin-1"))
    for quote in HEADER_HELP_QUOTES:
        found = norm(quote) in help_text
        rows.append(("kit:header-help !TRNS", f"quote: {quote}", "present" if found else "absent"))

    files = transactions = balanced = spl_headers = spl_with_id = 0
    four_digit = two_digit = 0
    kit_columns = set()
    for name in names:
        if not name.lower().endswith(".iif"):
            continue
        files += 1
        text = kit.read(name).decode("latin-1")
        for line in text.splitlines():
            cells = [c.strip() for c in line.split("\t")]
            if cells[0] in ("!TRNS", "!SPL"):
                kit_columns.update(c for c in cells[1:] if c)
            if cells[0] == "!SPL":
                spl_headers += 1
                spl_with_id += "SPLID" in cells
        _, blocks = blocks_of(text)
        for block in blocks:
            transactions += 1
            balanced += sum(amount(f.get("AMOUNT", "")) for f in block) == 0
            for fields in block:
                match = DATE_FORM.match(fields.get("DATE", "").strip())
                if match:
                    four_digit += len(match.group(3)) == 4
                    two_digit += len(match.group(3)) == 2
    rows.append(("kit", "sample .iif files", str(files)))
    rows.append(("kit", "sample transactions (TRNS..ENDTRNS blocks)", str(transactions)))
    rows.append(("kit", "sample transactions whose lines sum to zero", str(balanced)))
    rows.append(("kit", "!SPL header rows that name SPLID", f"{spl_with_id} of {spl_headers}"))
    rows.append(("kit", "sample TRNS/SPL lines dated with a four-digit year", str(four_digit)))
    rows.append(("kit", "sample TRNS/SPL lines dated with a two-digit year", str(two_digit)))

    declarations = next(
        n for n in names if n.endswith("7.0 and later/Import First - List Declarations (Not Version Specific).iif")
    )
    for line in kit.read(declarations).decode("latin-1").splitlines():
        if line.startswith("!ACCNT"):
            return [c.strip() for c in line.split("\t")], kit_columns
    return None, kit_columns


def render_with_evidilo() -> str:
    with tempfile.TemporaryDirectory() as tmp:
        out = os.path.join(tmp, "evidilo.iif")
        result = subprocess.run(
            ["npx", "vitest", "run", "--config", str(HERE / "harness.vitest.config.mjs")],
            cwd=REPO / "packages" / "exports",
            env={**os.environ, "IIF_OUT": out},
            capture_output=True,
            text=True,
            timeout=300,
        )
        if result.returncode != 0 or not os.path.exists(out):
            print(result.stdout[-2000:], result.stderr[-2000:])
            raise SystemExit("the harness did not write a file")
        with open(out, "rb") as handle:
            return handle.read().decode("utf-8")


def check_evidilo(rows, kit_account_header, kit_columns):
    text = render_with_evidilo()
    headers, blocks = blocks_of(text)
    rows.append(("evidilo", "transactions written", str(len(blocks))))
    for block in blocks:
        kind = block[0].get("TRNSTYPE", "")
        amounts = " + ".join(f.get("AMOUNT", "") for f in block)
        total = sum(amount(f.get("AMOUNT", "")) for f in block)
        rows.append(("evidilo", f"{kind} lines and sum", f"{amounts} = {total:.2f}"))
        rows.append(("evidilo", f"{kind} accounts", " / ".join(f.get("ACCNT", "") for f in block)))
        rows.append(("evidilo", f"{kind} date", block[0].get("DATE", "")))
    rows.append(("evidilo", "!TRNS columns", " ".join(headers.get("TRNS", []))))
    rows.append(("evidilo", "!SPL columns", " ".join(headers.get("SPL", []))))
    if kit_columns is not None:
        mine = set(headers.get("TRNS", [])) | set(headers.get("SPL", []))
        rows.append(("evidilo", "!TRNS/!SPL columns that appear in a kit sample header", f"{len(mine & kit_columns)} of {len(mine)}"))
    rows.append(("evidilo", "!SPL names SPLID", "yes" if "SPLID" in headers.get("SPL", []) else "no"))
    account_header = ["!ACCNT", *headers.get("ACCNT", [])]
    same = kit_account_header is not None and account_header == kit_account_header
    rows.append(("evidilo", "!ACCNT header equals the kit's list declarations", "yes" if same else "no"))
    lines = text.split("\r\n")
    first_trns = next(i for i, line in enumerate(lines) if line.startswith("TRNS\t"))
    declared_before = [line.split("\t")[1] for line in lines[:first_trns] if line.startswith("ACCNT\t")]
    rows.append(("evidilo", "accounts declared before the first TRNS", " / ".join(declared_before)))
    rows.append(("evidilo", "rules broken (validator)", ",".join(validate(text)) or "none"))
    return text


def variants(text: str):
    lines = text.split("\r\n")
    check_spl = next(i for i, line in enumerate(lines) if line.startswith("SPL\tCHECK"))
    yield "unbalanced: the check's SPL amount typed as 612.53", lines[:check_spl] + [
        lines[check_spl].replace("\t612.35\t", "\t612.53\t")
    ] + lines[check_spl + 1 :]
    first_end = next(i for i, line in enumerate(lines) if line == "ENDTRNS")
    yield "the deposit's ENDTRNS line deleted", lines[:first_end] + lines[first_end + 1 :]
    yield "the check's SPL posted to Bank Service Charges, never declared", lines[:check_spl] + [
        lines[check_spl].replace("Uncategorized Expense", "Bank Service Charges")
    ] + lines[check_spl + 1 :]
    yield "dates written as 2026-09-03 and 2026-09-08", [
        line.replace("09/03/2026", "2026-09-03").replace("09/08/2026", "2026-09-08") for line in lines
    ]
    yield "dates written day first: 03/09/2026 and 08/09/2026", [
        line.replace("09/03/2026", "03/09/2026").replace("09/08/2026", "08/09/2026") for line in lines
    ]
    body = [line for line in lines if line.startswith(("TRNS", "SPL", "ENDTRNS"))]
    yield "the same two transactions appended a second time", lines[:-1] + body + [""]


def main() -> int:
    write = "--write" in sys.argv
    rows, problems, unreachable = [], [], []

    check_pages(rows, problems, unreachable)
    kit_account_header, kit_columns = check_kit(rows, problems, unreachable)
    text = check_evidilo(rows, kit_account_header, kit_columns)
    for label, lines in variants(text):
        rows.append(("variant", label, ",".join(validate("\r\n".join(lines))) or "none"))

    csv_path = HERE / "dataset.csv"
    header = ("source", "item", "observed")
    if write:
        # A source this host cannot reach keeps the rows it last recorded (see dataset.json notes).
        kept = []
        if csv_path.exists() and unreachable:
            gone = {f"page:{PAGE_NAMES[u]}" for u in unreachable if u in PAGE_NAMES}
            with open(csv_path, newline="", encoding="utf-8") as handle:
                kept = [tuple(r) for r in list(csv.reader(handle))[1:] if r[0] in gone]
        rows = kept + rows
        with open(csv_path, "w", newline="", encoding="utf-8") as handle:
            writer = csv.writer(handle)
            writer.writerow(header)
            writer.writerows(rows)
        print(f"wrote {len(rows)} rows")
    else:
        with open(csv_path, newline="", encoding="utf-8") as handle:
            recorded = [tuple(r) for r in list(csv.reader(handle))[1:]]
        skipped = {f"page:{PAGE_NAMES[u]}" for u in unreachable if u in PAGE_NAMES}
        if KIT in unreachable:
            skipped |= {"kit", "kit:header-help !TRNS"}
        expected = [r for r in recorded if r[0] not in skipped]
        if rows != expected:
            got, want = set(rows), set(expected)
            for row in sorted(want - got):
                problems.append(f"recorded, not observed now: {row}")
            for row in sorted(got - want):
                problems.append(f"observed now, not recorded: {row}")
            if not problems:
                problems.append("rows differ in order")

    for problem in problems:
        print("FAIL", problem)
    if problems:
        return 1
    if unreachable:
        print(f"{len(unreachable)} source(s) unreachable; every other row holds")
        return 2
    print(f"OK: {len(rows)} rows hold")
    return 0


if __name__ == "__main__":
    os.chdir(HERE)
    sys.exit(main())