#!/usr/bin/env python3
"""How far back a bank feed reaches, per each accounting tool's own page.

Six tools, three questions each: how far back the first download reaches, whether the range can be
widened after connecting, and which statement files the tool imports for the months the feed misses.
Every answer is a verbatim quote from the tool's own page, captured with WebFetch on 2026-10-25.
A cell the page does not answer reads "not stated on its help page".

    python3 check.py            re-fetch every page, confirm every quote is still there
    python3 check.py --recheck  the same (there are no lab steps to skip)
    python3 check.py --write    also rewrite dataset.json and dataset.csv from the table below

Exit 0: every quote found. Exit 1: a quote is gone from a page that loaded.
Exit 2: no quote gone, but at least one page could not be reached (403, timeout).
"""
import csv
import gzip
import html
import json
import re
import sys
import urllib.error
import urllib.request

CHECKED_AT = "2026-10-25"
NOT_STATED = "not stated on its help page"
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/120.0 Safari/537.36")

# id -> (tool, page title, URL a reader opens, URL this script reads, quotes)
SOURCES = {
    "qbo-connect": {
        "tool": "QuickBooks Online",
        "title": "Connect bank and credit card accounts to QuickBooks Online",
        "url": "https://quickbooks.intuit.com/learn-support/en-us/help-article/banking/connect-bank-credit-card-accounts-quickbooks/L4yDAHMNH_US_en_US",
        "quotes": [
            "Download timespans vary by bank, ranging from 90 days up to 24 months.",
        ],
    },
    "qbo-upload": {
        "tool": "QuickBooks Online",
        "title": "Manually upload transactions into QuickBooks Online",
        "url": "https://quickbooks.intuit.com/learn-support/en-us/help-article/import-transactions/manually-upload-transactions-quickbooks-online/L0rE9OXBz_US_en_US",
        "quotes": [
            "QuickBooks allows you to upload an account statement (PDF or image) or a list of transactions (QBO, QFX, or CSV file).",
            "If you can't connect to your bank or want to add transactions from before you did so, use this process.",
        ],
    },
    "qbd-feeds": {
        "tool": "QuickBooks Desktop",
        "title": "Download bank feed transactions in QuickBooks Desktop",
        "url": "https://quickbooks.intuit.com/learn-support/en-us/help-article/bank-feeds/download-bank-feed-transactions-quickbooks-desktop/L8o0Aw9VM_US_en_US",
        "quotes": [
            "Some banks offer up to a year of history, while most provide about 90 days.",
            "This may be 30, 60, 90 days, or more depending on the bank.",
            "You can't download transactions into the same QuickBooks Desktop company file more than once.",
            "Web connect only works with .qbo files.",
            "Other types like QFX and QIF Quicken files won't work.",
        ],
    },
    "xero-feeds": {
        "tool": "Xero",
        "title": "Connect Bank Feeds for Easy Financial Admin | Xero US",
        "url": "https://www.xero.com/us/accounting-software/connect-your-bank/",
        "quotes": [
            "Can I import historical transactions when I set up an automated bank feed?",
            "It depends on your bank. When setting up an automated bank feed in Xero, you may be given the option to pull historical data from your bank account.",
            "Import transactions from your online banking in OFX, QIF, QBO, QFX or CSV formats, then upload it to Xero",
            "Upload a PDF bank statement from a supported bank",
        ],
    },
    "wave-connect": {
        "tool": "Wave",
        "title": "Import transactions automatically by connecting a bank account",
        "url": "https://support.waveapps.com/hc/en-us/articles/208624076-Import-transactions-automatically-by-connecting-a-bank-account",
        # The article page sits behind a browser challenge; Wave's help centre serves the same
        # article body as JSON, which is what WebFetch read on the day.
        "fetch": "https://support.waveapps.com/api/v2/help_center/en-us/articles/208624076.json",
        "quotes": [
            "Depending on your bank, you may be able to import up to 18 months of transactions.",
            "To import older transactions, download electronic statements from your bank and upload them to Wave",
            "Automatic bank transaction imports are part of Wave's Pro Plan.",
            "Transactions generally update every 24 hours but can take up to 7 days depending on your bank.",
        ],
    },
    "wave-toggle": {
        "tool": "Wave",
        "title": "Turn automatic transaction imports on or off for individual accounts",
        "url": "https://support.waveapps.com/hc/en-us/articles/31115684544148-Turn-automatic-transaction-imports-on-or-off-for-individual-accounts",
        "fetch": "https://support.waveapps.com/api/v2/help_center/en-us/articles/31115684544148.json",
        "quotes": [
            "To change the date to import transactions from, click Edit date and select a date.",
            "If you select an earlier date, you may import duplicate transactions.",
        ],
    },
    "wave-upload": {
        "tool": "Wave",
        "title": "Upload bank and credit card statements",
        "url": "https://support.waveapps.com/hc/en-us/articles/208621556-Upload-bank-and-credit-card-statements",
        "fetch": "https://support.waveapps.com/api/v2/help_center/en-us/articles/208621556.json",
        "quotes": [
            "Wave accepts the following file types: Microsoft Money (.Ofx) QuickBooks (.Qbo) Quicken (.Qfx) Simply Accounting (.Aso)",
            "Wave also allows you to upload Comma-Separated Variable (.CSV) files.",
        ],
    },
    "freshbooks-plaid": {
        "tool": "FreshBooks",
        "title": "How do I set up a bank connection with Plaid?",
        "url": "https://support.freshbooks.com/hc/en-us/articles/360033417072-How-do-I-set-up-a-bank-connection-with-Plaid",
        "quotes": [
            "Bank Connections can pull up to a maximum of 2 years of activity",
        ],
    },
    "freshbooks-file": {
        "tool": "FreshBooks",
        "title": "How do I import expenses from a file?",
        "url": "https://support.freshbooks.com/hc/en-us/articles/360001651111",
        "quotes": [
            "older expenses than the ones brought in by your bank connection , you can upload expenses in bulk with a CSV file in the same currency",
            "If your bank only allows transactions to be downloaded in other file formats such as PDFs, you'll need to convert the PDF into a CSV manually first.",
        ],
    },
    "sage-feeds": {
        "tool": "Sage Accounting",
        "title": "How to set up Bank Feeds (Sage Knowledgebase, Canada)",
        "url": "https://ca-kb.sage.com/portal/app/portlets/results/viewsolution.jsp?solutionid=240304191719250",
        "quotes": [
            "You can normally download transactions from the last 90 days, but this varies by bank.",
            "If you need transactions more than 90 days old, you can import your bank statement or enter the transactions manually before connecting Bank Feeds.",
        ],
    },
    "sage-formats": {
        "tool": "Sage Accounting",
        "title": "Supported file formats for bank statement imports (Sage Knowledgebase, UK)",
        "url": "https://gb-kb.sage.com/portal/app/portlets/results/view2.jsp?k2dockey=240718093942637",
        "quotes": [
            "OFX or ASO files are an industry-standard format.",
            "If you're uploading an XLS or XLSX file, we only process the first tab in your file.",
        ],
    },
    "sage-import": {
        "tool": "Sage Accounting",
        "title": "Import bank transactions (Sage Knowledgebase, UK)",
        "url": "https://gb-kb.sage.com/portal/app/portlets/results/viewsolution.jsp?solutionid=222001000100789",
        "quotes": [
            "You can upload transactions from a bank statement using a QIF, CSV or other file types.",
            "If your bank statement is only available in PDF format, consider using AutoEntry to convert it into a CSV file",
        ],
    },
}

# One row per tool. Each cell is the answer and the source ids whose quotes carry it.
# range_tied_to_bank: the stated first-download figure is qualified by the bank ("varies by bank",
# "depending on your bank", "most banks"); None where no figure is stated.
ROWS = [
    {
        "tool": "QuickBooks Online",
        "range_tied_to_bank": True,
        "first_download": "90 days up to 24 months, varies by bank",
        "first_download_src": ["qbo-connect"],
        "widen_after_connecting": NOT_STATED,
        "widen_src": [],
        "statement_files_for_earlier_months": "PDF or image statement; QBO, QFX or CSV list",
        "files_src": ["qbo-upload"],
    },
    {
        "tool": "QuickBooks Desktop",
        "range_tied_to_bank": True,
        "first_download": "most banks about 90 days, some up to a year (Direct Connect); 30, 60, 90 days or more (Web Connect)",
        "first_download_src": ["qbd-feeds"],
        "widen_after_connecting": NOT_STATED + "; the same transactions cannot be downloaded into one company file twice",
        "widen_src": ["qbd-feeds"],
        "statement_files_for_earlier_months": ".qbo (Web Connect) only; QFX and QIF will not work",
        "files_src": ["qbd-feeds"],
    },
    {
        "tool": "Xero",
        "range_tied_to_bank": None,
        "first_download": NOT_STATED + " (\"It depends on your bank.\")",
        "first_download_src": ["xero-feeds"],
        "widen_after_connecting": NOT_STATED,
        "widen_src": [],
        "statement_files_for_earlier_months": "OFX, QIF, QBO, QFX or CSV; PDF from a supported bank",
        "files_src": ["xero-feeds"],
    },
    {
        "tool": "Wave",
        "range_tied_to_bank": True,
        "first_download": "up to 18 months, depending on the bank (Pro Plan)",
        "first_download_src": ["wave-connect"],
        "widen_after_connecting": "Edit date can be set to an earlier date; Wave warns this may import duplicates",
        "widen_src": ["wave-toggle"],
        "statement_files_for_earlier_months": "OFX, QBO, QFX, ASO; CSV as the more complex option",
        "files_src": ["wave-connect", "wave-upload"],
    },
    {
        "tool": "FreshBooks",
        "range_tied_to_bank": False,
        "first_download": "up to 2 years (Plaid connection)",
        "first_download_src": ["freshbooks-plaid"],
        "widen_after_connecting": NOT_STATED,
        "widen_src": [],
        "statement_files_for_earlier_months": "CSV; a PDF has to be turned into a CSV first",
        "files_src": ["freshbooks-file"],
    },
    {
        "tool": "Sage Accounting",
        "range_tied_to_bank": True,
        "first_download": "normally the last 90 days, varies by bank",
        "first_download_src": ["sage-feeds"],
        "widen_after_connecting": NOT_STATED + "; older months go in by statement import before connecting",
        "widen_src": ["sage-feeds"],
        "statement_files_for_earlier_months": "OFX/ASO, QIF, CSV, XLS; PDF via AutoEntry",
        "files_src": ["sage-formats", "sage-import"],
    },
]


def norm(text):
    """Page text as a reader sees it: no tags, entities decoded, one space, straight quotes."""
    text = re.sub(r"<(script|style)\b.*?</\1>", " ", text, flags=re.S | re.I)
    text = re.sub(r"<[^>]+>", " ", text)
    text = html.unescape(text)
    text = text.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"')
    text = text.replace(" ", " ")
    return re.sub(r"\s+", " ", text).strip()


def fetch(url):
    req = urllib.request.Request(url, headers={
        "User-Agent": UA,
        "Accept": "text/html,application/json;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip",
    })
    with urllib.request.urlopen(req, timeout=25) as resp:
        raw = resp.read()
        if resp.headers.get("Content-Encoding") == "gzip":
            raw = gzip.decompress(raw)
    body = raw.decode("utf-8", errors="replace")
    if url.endswith(".json"):
        body = json.loads(body)["article"]["body"]
    return norm(body)


def check():
    status = {}
    for sid, src in SOURCES.items():
        try:
            text = fetch(src.get("fetch", src["url"]))
        except (urllib.error.URLError, TimeoutError, OSError, ValueError, KeyError) as exc:
            status[sid] = "unreachable"
            print(f"UNREACHABLE {sid}: {src['url']} ({exc})")
            continue
        missing = [q for q in src["quotes"] if norm(q) not in text]
        status[sid] = "missing" if missing else "ok"
        for q in src["quotes"]:
            print(f"{'GONE ' if q in missing else 'ok   '} {sid}: {q}")
    return status


def write(status):
    rows = []
    for r in ROWS:
        srcs = sorted(set(r["first_download_src"] + r["widen_src"] + r["files_src"]))
        rows.append({
            "tool": r["tool"],
            "first_download": r["first_download"],
            "range_tied_to_bank": "" if r["range_tied_to_bank"] is None else ("yes" if r["range_tied_to_bank"] else "no"),
            "widen_after_connecting": r["widen_after_connecting"],
            "statement_files_for_earlier_months": r["statement_files_for_earlier_months"],
            "sources": " ".join(SOURCES[s]["url"] for s in srcs),
            "checked_at": CHECKED_AT,
        })
    with open("dataset.csv", "w", newline="", encoding="utf-8") as fh:
        w = csv.DictWriter(fh, fieldnames=list(rows[0]))
        w.writeheader()
        w.writerows(rows)
    data = {
        "method": "primary-docs",
        "checkedAt": CHECKED_AT,
        "question": "How far back does each tool's bank feed reach on the first download, can the range be widened after connecting, and which statement files does the tool import for the months before it?",
        "tools": [
            "WebFetch (Claude Code), 2026-10-25, to read every page and capture each quote",
            "check.py: Python 3 urllib, re-fetches each page and confirms each quote",
        ],
        "inputs": [
            {"id": sid, "tool": s["tool"], "title": s["title"], "url": s["url"],
             **({"fetchedVia": s["fetch"]} if "fetch" in s else {}),
             "fetchedAt": CHECKED_AT, "quotes": s["quotes"],
             "lastCheckStatus": status.get(sid, "not run")}
            for sid, s in SOURCES.items()
        ],
        "results": [
            {**{k: v for k, v in r.items() if not k.endswith("_src")},
             "sources": {"first_download": r["first_download_src"], "widen_after_connecting": r["widen_src"],
                         "statement_files_for_earlier_months": r["files_src"]}}
            for r in ROWS
        ],
        "summary": {
            "toolsCompared": len(ROWS),
            "pagesRead": len(SOURCES),
            "toolsTyingTheStatedRangeToTheBank": sum(1 for r in ROWS if r["range_tied_to_bank"]),
            "toolsStatingAFirstDownloadRange": sum(1 for r in ROWS if not r["first_download"].startswith(NOT_STATED)),
            "toolsStatingTheRangeCanBeWidenedAfterConnecting": sum(1 for r in ROWS if not r["widen_after_connecting"].startswith(NOT_STATED)),
            "toolsListingCsvForEarlierMonths": sum(1 for r in ROWS if "CSV" in r["statement_files_for_earlier_months"]),
            "longestStatedFirstDownload": "24 months (QuickBooks Online, for some banks)",
            "shortestStatedFirstDownload": "30 days (QuickBooks Desktop Web Connect, for some banks)",
        },
        "notes": [
            "A cell reads 'not stated on its help page' where the tool's own page does not answer it; no answer was taken from a forum, a reseller or memory.",
            "quickbooks.intuit.com timed out from the host that runs check.py on 2026-10-25, so the four Intuit quotes were captured with WebFetch only; check.py reports those pages as unreachable and exits 2 until the host can reach Intuit.",
            "Xero's help centre (central.xero.com) renders its articles with a script and returned no text to WebFetch or urllib on 2026-10-25, including the OFX import article the plan named. Xero's row comes from Xero's own US bank-feeds page, which answers the question in its FAQ.",
            "Wave's article pages answer a plain request with a browser challenge (403). Wave's help centre serves the same article bodies as JSON at /api/v2/help_center/, which is what was read.",
            "Sage Accounting has no en-us help article for these pages (help.sbc.sage.com/en-us returned 404); the rows come from Sage's Canadian and UK knowledgebases, both tagged Product: Sage Accounting.",
            "FreshBooks offers two connection partners; the Plaid page is used. Its Yodlee page gives a different maximum and is not in the table.",
            "Evidilo, the site publishing this, reads statements and holds no bank connection; it is not one of the rows.",
        ],
    }
    with open("dataset.json", "w", encoding="utf-8") as fh:
        json.dump(data, fh, indent=2, ensure_ascii=False)
        fh.write("\n")


def main():
    status = check()
    if "--write" in sys.argv:
        write(status)
    if "missing" in status.values():
        print("FAIL: a quote is gone from its page")
        return 1
    if "unreachable" in status.values():
        print("UNREACHABLE: at least one page could not be read; nothing that loaded has changed")
        return 2
    print("OK: every quote is still on its page")
    return 0


if __name__ == "__main__":
    sys.exit(main())
