#!/usr/bin/env python3 """Check for the post "What is a QFX file". Lab: writes one five-row statement through Evidilo's production OFX, QFX and QBO writers (packages/exports/src/ofx.ts, qfx.ts and qbo.ts, via write-ofx-qfx.mjs), diffs the OFX and QFX files, compares the QBO with the QFX, and parses both with Python's strict XML parser (xml.etree / expat). Docs: re-reads three Quicken support pages and confirms each quoted sentence is still there, and that none of them names the INTU.BID line. Exit 0: everything in dataset.json still holds. Exit 1: a measurement or a quote differs. Exit 2: a page could not be reached at all. python3 check.py run everything and compare with dataset.json python3 check.py --recheck the same (the lab steps take a few seconds, so none is skipped) python3 check.py --write run everything and rewrite dataset.json and dataset.csv """ import csv import difflib import hashlib import html import json import os import re import subprocess import sys import tempfile import urllib.error import urllib.request import xml.etree.ElementTree as ET import pyexpat HERE = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", "..")) CHECKED_AT = "2026-10-25" UA = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/130.0 Safari/537.36" ) TROUBLESHOOT = "https://www.quicken.com/support/web-connect-troubleshooting/" IMPORT = "https://www.quicken.com/support/how-do-i-import-data-quicken-windows" FI_MESSAGE = "https://www.quicken.com/support/why-am-i-receiving-message-about-my-financial-institution" QUOTES = [ ("q-third-party", TROUBLESHOOT, "Quicken does not support QFX files created by third-party conversion software (such as PDF-to-QFX conversion)."), ("q-unable-to-assist", TROUBLESHOOT, "If you use one of these applications and the converted file does not open, we will be unable to assist."), ("q-validate", TROUBLESHOOT, "Web Connect files require access to online services in order to validate the file and the transactions for security purposes."), ("q-authorized", TROUBLESHOOT, "If the financial institution no longer supports Web Connect in Quicken, the file cannot be authorized when importing."), ("q-qif", TROUBLESHOOT, "QIF files will not work with Web Connect in Quicken."), ("q-qfx-extension", IMPORT, "Your browser will download a Web Connect file that contains your account transactions (you can recognize such a file by the .QFX extension in its file name)."), ("q-no-ofx", IMPORT, "Quicken cannot import Excel (.xls), QuickBooks, PDF, or OFX files."), ("q-ewc", FI_MESSAGE, "Several financial institutions (mostly credit unions and community banks) are discontinuing Express Web Connect"), ] # Strings the thesis would need from Quicken and that its pages do not carry. ABSENT = [("a-intu-bid", u, "INTU.BID") for u in (TROUBLESHOOT, IMPORT, FI_MESSAGE)] class Unreachable(Exception): pass def page_text(url, cache={}): if url not in cache: try: req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html"}) with urllib.request.urlopen(req, timeout=30) as r: body = r.read().decode("utf-8", "replace") except (urllib.error.URLError, TimeoutError, OSError) as e: raise Unreachable(f"{url}: {e}") body = re.sub(r"(?is)|", " ", body) text = html.unescape(re.sub(r"<[^>]+>", " ", body)) cache[url] = text.replace("’", "'") return cache[url] def squash(s): # The page renders "(.xls)" as "(. xls )"; whitespace is compared away on both sides. return re.sub(r"\s+", "", s) def lab(): with tempfile.TemporaryDirectory() as out: subprocess.run( ["node", "--experimental-strip-types", "--no-warnings", os.path.join(HERE, "write-ofx-qfx.mjs"), REPO, out], check=True, cwd=HERE, ) ofx = open(os.path.join(out, "statement.ofx"), "rb").read() qfx = open(os.path.join(out, "statement.qfx"), "rb").read() qbo = open(os.path.join(out, "statement.qbo"), "rb").read() ofx_lines = ofx.decode("ascii").split("\r\n")[:-1] qfx_lines = qfx.decode("ascii").split("\r\n")[:-1] added, removed = [], [] for op in difflib.SequenceMatcher(a=ofx_lines, b=qfx_lines, autojunk=False).get_opcodes(): tag, i1, i2, j1, j2 = op if tag in ("replace", "delete"): removed += ofx_lines[i1:i2] if tag in ("replace", "insert"): added += [(j + 1, qfx_lines[j]) for j in range(j1, j2)] def parse(data): try: ET.fromstring(data) return "parsed" except ET.ParseError as e: return str(e) body = lambda b: b[b.index(b""):] leaf = re.compile(r"^<[A-Z0-9.]+>[^<]") src = open(os.path.join(REPO, "packages/exports/src/ofx.ts")).read() bid = re.search(r'export const INTU_BID = "([^"]+)"', src) tx = sum(1 for line in qfx_lines if line == "") line_before = qfx_lines[added[0][0] - 2] if added else "" return [ ("ofx-lines", "lab", "lines in the OFX file", len(ofx_lines)), ("qfx-lines", "lab", "lines in the QFX file", len(qfx_lines)), ("ofx-bytes", "lab", "bytes in the OFX file", len(ofx)), ("qfx-bytes", "lab", "bytes in the QFX file", len(qfx)), ("transactions", "lab", "transactions in each file", tx), ("lines-added", "lab", "lines the QFX has that the OFX does not", len(added)), ("lines-removed", "lab", "lines the OFX has that the QFX does not", len(removed)), ("added-line", "lab", "the line the QFX adds", added[0][1] if added else ""), ("added-at", "lab", "line number of that line in the QFX", added[0][0] if added else 0), ("added-after", "lab", "the line before it", line_before), ("same-without-bid", "lab", "QFX minus the INTU.BID line equals the OFX byte for byte", "yes" if hashlib.sha256(qfx.replace(b"3000\r\n", b"")).digest() == hashlib.sha256(ofx).digest() else "no"), ("qbo-equals-qfx", "lab", "the QBO file from the same rows is byte for byte the QFX file", "yes" if qbo == qfx else "no"), ("leaves-no-end-tag", "lab", "QFX lines that open a value and never close it", sum(1 for l in qfx_lines if leaf.match(l))), ("end-tags", "lab", "QFX closing tags", sum(1 for l in qfx_lines if l.startswith(" on (header removed)", parse(body(qfx))), ("xml-qfx-body-line", "lab", "the line xml.etree stopped on, counted from ", body(qfx).decode("ascii").split("\r\n")[6]), ("intu-bid-source", "repo", "INTU_BID constant in packages/exports/src/ofx.ts", bid.group(1) if bid else ""), ("intu-bid-placeholder", "repo", "ofx.ts calls it a placeholder", "yes" if "fixes 3000 as a placeholder" in src else "no"), ] def docs(): rows = [] for key, url, quote in QUOTES: rows.append((key, url, quote, "present" if squash(quote) in squash(page_text(url)) else "missing")) for key, url, needle in ABSENT: rows.append((key + "-" + str(ABSENT.index((key, url, needle))), url, f"page names {needle}", "no" if needle not in page_text(url) else "yes")) return rows def main(): write = "--write" in sys.argv try: results = [dict(id=k, source=s, item=i, value=v) for k, s, i, v in lab() + docs()] except Unreachable as e: print(f"UNREACHABLE: {e}") return 2 if write: with open(os.path.join(HERE, "dataset.csv"), "w", newline="") as f: w = csv.DictWriter(f, fieldnames=["id", "source", "item", "value"]) w.writeheader() w.writerows(results) path = os.path.join(HERE, "dataset.json") data = json.load(open(path)) if os.path.exists(path) else {} data["results"] = results data["checkedAt"] = CHECKED_AT data.setdefault("tools", {})["python"] = sys.version.split()[0] data["tools"]["expat"] = pyexpat.EXPAT_VERSION data["tools"]["node"] = subprocess.run(["node", "--version"], capture_output=True, text=True).stdout.strip() with open(path, "w") as f: json.dump(data, f, indent=2) f.write("\n") print("wrote dataset.json and dataset.csv") expected = {r["id"]: str(r["value"]) for r in json.load(open(os.path.join(HERE, "dataset.json")))["results"]} failed = 0 for r in results: want = expected.get(r["id"]) ok = want == str(r["value"]) if r["id"].startswith("q-") and r["value"] != "present": ok = False if not ok: failed += 1 print(f"DIFFERS {r['id']}: expected {want!r}, got {r['value']!r}") print(f"{len(results) - failed} of {len(results)} items hold") return 1 if failed else 0 if __name__ == "__main__": sys.exit(main())