From 3dba054dca3822c0c1272036a69483dfa86ceee4 Mon Sep 17 00:00:00 2001 From: nmemmert <67961292+nmemmert@users.noreply.github.com> Date: Wed, 21 Oct 2026 12:58:29 -0400 Subject: [PATCH] Statement loader web app with Docker image and Gitea Actions build Co-Authored-By: Claude Sonnet 5.5 --- .dockerignore | 9 + .gitea/workflows/build.yml | 47 ++++ .gitignore | 12 + Dockerfile | 18 ++ README.md | 32 +++ Start Statement Loader.command | 3 + app.py | 165 ++++++++++++ app_dashboard.html | 170 ++++++++++++ app_page.html | 104 ++++++++ compose.yaml | 53 ++++ requirements.txt | 2 + statement_loader.py | 466 +++++++++++++++++++++++++++++++++ 12 files changed, 1081 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitea/workflows/build.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100755 Start Statement Loader.command create mode 100644 app.py create mode 100644 app_dashboard.html create mode 100644 app_page.html create mode 100644 compose.yaml create mode 100644 requirements.txt create mode 100644 statement_loader.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c6204df --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.venv +.git +backups +data +*.pdf +*.csv +*.xlsx +loader_rules.json +*.json diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..91dba1e --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,47 @@ +name: build-container + +on: + push: + branches: [main] + tags: ["v*"] + workflow_dispatch: + +env: + REGISTRY: git.necloud.us + IMAGE: git.necloud.us/nate/money + +jobs: + build: + runs-on: ubuntu-latest # runner needs Docker access (act_runner with the docker socket) + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Gitea container registry + run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login "$REGISTRY" -u "${{ gitea.actor }}" --password-stdin + + - name: Build image + run: | + docker build \ + --label org.opencontainers.image.source="${{ gitea.server_url }}/${{ gitea.repository }}" \ + --label org.opencontainers.image.revision="${{ gitea.sha }}" \ + -t "$IMAGE:latest" -t "$IMAGE:${{ gitea.sha }}" . + + - name: Smoke test (starts the app and loads the dashboard page) + run: | + docker run -d --name smoke "$IMAGE:latest" + for i in $(seq 1 20); do + docker exec smoke python -c "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8765/dashboard')" && ok=1 && break + sleep 1 + done + docker logs smoke + docker rm -f smoke + test "$ok" = 1 + + - name: Push image + run: | + docker push "$IMAGE:latest" + docker push "$IMAGE:${{ gitea.sha }}" + if [ "${{ gitea.ref_type }}" = "tag" ]; then + docker tag "$IMAGE:latest" "$IMAGE:${{ gitea.ref_name }}" + docker push "$IMAGE:${{ gitea.ref_name }}" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f3dcb7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# private financial data - never commit +*.pdf +*.csv +*.xlsx +backups/ +data/ +Statement_Coverage.txt +# local tooling +.venv/ +__pycache__/ +.DS_Store +loader_rules.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f6eacb8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 DATA_DIR=/data BIND=0.0.0.0 PORT=8765 +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY statement_loader.py app.py app_page.html app_dashboard.html ./ + +# statements, Bills_Tracker.xlsx and backups live in this volume, never in the image +RUN useradd -m -u 1000 app && mkdir /data && chown app /data +USER app +VOLUME /data +EXPOSE 8765 + +HEALTHCHECK --interval=30s --timeout=5s CMD python -c "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8765/dashboard')" || exit 1 +CMD ["python", "app.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..190abbc --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# money + +Local tool that loads bank/card statements (Chase Visa, Ally, CVFCU PDFs; Capital One CSV) into `Bills_Tracker.xlsx` and shows a web dashboard. +Everything runs locally; statements and the workbook are **never** committed or baked into the image. + +## Run on ZimaOS / CasaOS +Add the app: App Store -> Custom Install -> import `compose.yaml` (it follows the ZimaOS `x-casaos` format). +Copy `Bills_Tracker.xlsx` into `/DATA/AppData/money/data/` (the app's data folder), then open the app's Web UI (port 8765). +The image is built by Gitea Actions (`.gitea/workflows/build.yml`) and pushed to `git.necloud.us/nate/money`. +Add a repository secret `REGISTRY_TOKEN` (a Gitea access token with `write:package`), and on the ZimaOS box run +`docker login git.necloud.us` once so it can pull the private image. + +## Run with plain Docker +```bash +mkdir -p data && cp /path/to/Bills_Tracker.xlsx data/ +AppID=money PUID=1000 PGID=1000 TZ=America/New_York docker compose up -d # then edit the bind path in compose.yaml or symlink +``` + +## Personal rules (not in git) +Your name, card last-4 and similar live in `loader_rules.json` next to the workbook (git-ignored, not in the image): +```json +{"own_names": ["JANE DOE"], "own_bank_hints": ["MY CREDIT UNION"], "csv_accounts": {"1234": "Capital One Card"}} +``` +`own_names` lets the loader recognise transfers between your own accounts. Copy the file into the data folder (`/DATA/AppData/money/data/` on ZimaOS). + +## Run without Docker +```bash +python3 -m venv .venv && .venv/bin/pip install -r requirements.txt && .venv/bin/python app.py +``` +`DATA_DIR` (default: this folder) is where statements, the workbook and `backups/` live. + +The app has no login. Publish it on localhost only, or put it behind your own authentication. diff --git a/Start Statement Loader.command b/Start Statement Loader.command new file mode 100755 index 0000000..82fab8c --- /dev/null +++ b/Start Statement Loader.command @@ -0,0 +1,3 @@ +#!/bin/bash +cd "$(dirname "$0")" +.venv/bin/python app.py diff --git a/app.py b/app.py new file mode 100644 index 0000000..a3ebce4 --- /dev/null +++ b/app.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Local web front end for the statement loader. Run: .venv/bin/python app.py +Opens http://127.0.0.1:8765 - listens on this Mac only; nothing is sent anywhere. +""" +import os, json, secrets, shutil, subprocess, sys, re, calendar, datetime as dt, threading, webbrowser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import urlparse, parse_qs +import statement_loader as L + +PORT = int(os.environ.get("PORT", "8765")) +BIND = os.environ.get("BIND", "127.0.0.1") # in Docker: 0.0.0.0 (publish the port to 127.0.0.1 only) +EXTRA_HOSTS = {h.strip() for h in os.environ.get("ALLOWED_HOSTS", "").split(",") if h.strip()} +TOKEN = secrets.token_urlsafe(24) +LOCK = threading.Lock() +HERE = L.HERE # code and pages +DATA = L.DATA # statements and workbook + +def eom(y, m): return calendar.monthrange(y, m)[1] + +def coverage(sts): + today = dt.date.today() + cfg = {a[0]: a[2] for a in L.ACCOUNTS} + by = {} + for st in sts: by.setdefault(st.account, []).append(st) + if not by: return dict(months=[], accounts=[]) + lo = min(s.start or s.end for s in sts); months = [] + y, m = lo.year, lo.month + while (y, m) <= (today.year, today.month): + months.append(f"{y}-{m:02d}"); m += 1 + if m > 12: y, m = y + 1, 1 + out = [] + for acct, lst in by.items(): + csv_only = all(s.kind == "csv" for s in lst) + cells, first = {}, min((s.start or s.end) for s in lst) + for mo in months: + y, m = int(mo[:4]), int(mo[5:]) + if csv_only: + have = any((s.start.year, s.start.month) <= (y, m) <= (s.end.year, s.end.month) for s in lst) + cells[mo] = "have" if have else ("future" if (y, m) > (max(s.end for s in lst).year, max(s.end for s in lst).month) else ("none" if (y, m) < (first.year, first.month) else "missing")) + continue + have = any((s.end.year, s.end.month) == (y, m) for s in lst) + due = dt.date(y, m, cfg.get(acct) or eom(y, m)) + started = (y, m) >= (min(s.end for s in lst).year, min(s.end for s in lst).month) + cells[mo] = "have" if have else ("none" if not started else ("future" if due > today else "missing")) + last = max(s.end for s in lst) + if csv_only: + nxt = f"Next download: from {last + dt.timedelta(days=1):%b %-d, %Y}" + else: + ny, nm = (last.year, last.month + 1) if last.month < 12 else (last.year + 1, 1) + if cfg.get(acct) is None and last.day != eom(last.year, last.month): ny, nm = last.year, last.month + nd = dt.date(ny, nm, cfg.get(acct) or eom(ny, nm)) + nxt = f"Next statement closes ~{nd:%b %-d, %Y}" + (" (ready to download)" if nd <= today else "") + out.append(dict(account=acct, cells=cells, count=len(lst), through=f"{last:%b %-d, %Y}", next=nxt, kind="csv" if csv_only else "pdf")) + return dict(months=months, accounts=out) + +def state(): + res = L.run_preview(DATA) + new = res["new"] + rows = [dict(date=f"{r.txn.date:%Y-%m-%d}", account=r.txn.account, desc=r.txn.desc[:60], amount=r.txn.amount, kind=r.kind, + payee=r.payee, what=r.what, reason=r.reason, new_payee=r.is_new_payee) for r in new] + sm = L.summarize(new) + return dict(coverage=coverage(res["statements"]), skipped=[dict(file=f, why=w) for f, w in res["skipped"]], + duplicates=[dict(file=a, same_as=b) for a, b in res["dups"]], + bad=[dict(file=f, parsed=p, stated=s) for f, p, s, ok in res["checks"] if not ok], + already=res["already"], new=rows, summary=sm, statements=len(res["statements"]), + excel_open=(DATA / "~$Bills_Tracker.xlsx").exists() or (DATA / ".~lock.Bills_Tracker.xlsx#").exists(), + backups=sorted(p.name for p in L.BACKUPS.glob("*.xlsx"))[-5:] if L.BACKUPS.exists() else []) + +_DATA = {} +def workbook_data(): + """Read-only snapshot of the Bill Log for the web dashboard (computed from raw cells, not cached formulas).""" + from openpyxl import load_workbook + key = L.WORKBOOK.stat().st_mtime_ns + if _DATA.get("key") == key: return _DATA["val"] + wb = load_workbook(L.WORKBOOK, read_only=True) + pay = {} + for r in wb["Lists"].iter_rows(min_row=4, max_row=153, max_col=3, values_only=True): + if r[0]: pay[r[0]] = (r[1] or "Other / Unsorted", r[2] or "Spending") + cats = [r[0] for r in wb["Lists"].iter_rows(min_row=4, max_row=40, min_col=5, max_col=5, values_only=True) if r[0] and not str(r[0]).startswith("Slots")] + tx = [] + for r in wb["Bill Log"].iter_rows(min_row=6, max_col=9, values_only=True): + if isinstance(r[0], dt.datetime) and r[1] and r[5] is not None: + c, t = pay.get(r[1], ("Other / Unsorted", "Spending")) + tx.append([f"{r[0]:%Y-%m-%d}", r[1], c, t, r[4] or "", round(float(r[5]), 2), r[6] or "", r[8] or ""]) + ex = sum(1 for r in wb["Excluded Transfers"].iter_rows(min_row=5, max_col=1, values_only=True) if isinstance(r[0], dt.datetime)) + wb.close() + _DATA["key"], _DATA["val"] = key, dict(tx=tx, categories=cats, excluded=ex, updated=f"{dt.datetime.fromtimestamp(L.WORKBOOK.stat().st_mtime):%b %-d, %Y %-I:%M %p}") + return _DATA["val"] + +def safe_name(name): + name = re.sub(r"[^\w.\- ]", "_", Path(name).name) + if Path(name).suffix.lower() not in (".pdf", ".csv"): raise ValueError("Only PDF or CSV files can be added.") + p = DATA / name; i = 1 + while p.exists(): + p = DATA / f"{Path(name).stem}_{i}{Path(name).suffix}"; i += 1 + return p + +class H(BaseHTTPRequestHandler): + def log_message(self, *a): pass + def _send(self, code, body, ctype="application/json"): + b = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(code); self.send_header("Content-Type", ctype); self.send_header("Content-Length", str(len(b))) + self.send_header("Cache-Control", "no-store"); self.end_headers(); self.wfile.write(b) + def _ok_host(self): + if "*" in EXTRA_HOSTS: return True # LAN/NAS use: any hostname or IP (the app has no login, so keep it on a trusted network) + h = self.headers.get("Host", ""); return h.rsplit(":", 1)[0] in {"127.0.0.1", "localhost"} | EXTRA_HOSTS or h in EXTRA_HOSTS + def do_GET(self): + if not self._ok_host(): return self._send(403, {"error": "forbidden"}) + u = urlparse(self.path) + if u.path == "/": return self._send(200, PAGE.replace("__TOKEN__", TOKEN).encode(), "text/html; charset=utf-8") + if u.path == "/dashboard": + f = HERE / "app_dashboard.html" + return self._send(200, f.read_bytes() if f.exists() else b"app_dashboard.html missing", "text/html; charset=utf-8") + if u.path == "/api/data": + with LOCK: + try: return self._send(200, workbook_data()) + except Exception as e: return self._send(500, {"error": str(e)}) + if u.path == "/api/state": + with LOCK: + try: return self._send(200, state()) + except Exception as e: return self._send(500, {"error": str(e)}) + self._send(404, {"error": "not found"}) + def do_POST(self): + if not self._ok_host() or self.headers.get("X-Token") != TOKEN: return self._send(403, {"error": "forbidden"}) + u = urlparse(self.path); n = int(self.headers.get("Content-Length") or 0) + try: + if u.path == "/api/upload": + name = parse_qs(u.query).get("name", [""])[0]; data = self.rfile.read(n) + with LOCK: + p = safe_name(name); p.write_bytes(data) + return self._send(200, {"saved": p.name}) + self.rfile.read(n) + if u.path == "/api/import": + with LOCK: + res = L.run_preview(DATA) + if not res["new"]: return self._send(200, {"message": "Nothing new to import."}) + out = L.apply(res["new"]) + return self._send(200, {"message": f"Added {out['log_rows']} rows to the Bill Log and {out['excluded_rows']} to Excluded Transfers.", + "backup": Path(out["backup"]).name, "new_payees": out["new_payees"]}) + if u.path == "/api/undo": + with LOCK: + b = sorted(L.BACKUPS.glob("*.xlsx")) if L.BACKUPS.exists() else [] + if not b: return self._send(200, {"message": "No backups to restore."}) + keep = L.BACKUPS / f"before_undo_{dt.datetime.now():%Y%m%d_%H%M%S}.xlsx" + shutil.copy2(L.WORKBOOK, keep); shutil.copy2(b[-1], L.WORKBOOK) + return self._send(200, {"message": f"Restored the workbook from {b[-1].name}."}) + if u.path == "/api/open": + if sys.platform != "darwin": return self._send(200, {"message": "Open Bills_Tracker.xlsx from the data folder on the host."}) + subprocess.run(["open", str(L.WORKBOOK)], check=False) + return self._send(200, {"message": "Opening the workbook…"}) + except Exception as e: + return self._send(400, {"error": str(e)}) + self._send(404, {"error": "not found"}) + +PAGE = (HERE / "app_page.html").read_text() if (HERE / "app_page.html").exists() else "

app_page.html missing

" + +if __name__ == "__main__": + if not L.WORKBOOK.exists(): print(f"WARNING: {L.WORKBOOK} not found - put Bills_Tracker.xlsx in the data folder.") + srv = ThreadingHTTPServer((BIND, PORT), H) + url = f"http://127.0.0.1:{PORT}/" + print(f"Statement loader running at {url} (Ctrl+C to stop)") + if "--no-browser" not in sys.argv and not os.environ.get("DATA_DIR"): threading.Timer(0.8, lambda: webbrowser.open(url)).start() + try: srv.serve_forever() + except KeyboardInterrupt: print("\nStopped.") diff --git a/app_dashboard.html b/app_dashboard.html new file mode 100644 index 0000000..fc597b5 --- /dev/null +++ b/app_dashboard.html @@ -0,0 +1,170 @@ + + + + + +Bills Dashboard + + + +
+

Bills Dashboard

+ ← Statement Loader
+
+
+

Monthly total

+

Monthly spending by category

+

Year over year by month

+

Year over year trend

+

Category totals by year

+

Share by category

+
+
+

Top payees

+

Category detail · year over year

+
+
+

Transactions

+ +
+
+
+

Read-only view of Bills_Tracker.xlsx. Card payments and transfers are not included (see Excluded Transfers in the workbook). Refunds are negative amounts.

+
+
+ + + diff --git a/app_page.html b/app_page.html new file mode 100644 index 0000000..4bed13d --- /dev/null +++ b/app_page.html @@ -0,0 +1,104 @@ + + + + + +Statement Loader + + + +
+

Statement Loader

Dashboard →
+
Runs only on this Mac. Drop in statements, check what’s covered, then add the new transactions to Bills_Tracker.xlsx. A backup is saved first, every time.
+ +
+ +
+

1 · Add statements

+
Drop PDF or CSV statements here, or click to choose files
+
Supported: Chase Visa PDF · Ally Bank PDF · CVFCU PDF · Capital One CSV download. + Adding a different account?
+
Open statement_loader.py, add one line to ACCOUNTS (a name, a word printed on page 1 of its statement, and the day it closes), then ask Claude to write the parser for that bank’s layout. Any file the app can’t read is listed under “Needs attention”.
+
+ +
+

2 · Coverage

+
+
✓ have it! missing· not yet due / before first statementColumns = month the statement closes.
+
+ +
+

3 · Review & import

+
+
+
+
+
+ + + diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..26f313d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,53 @@ +name: money +services: + money: + image: git.necloud.us/nate/money:latest + container_name: money + restart: unless-stopped + user: $PUID:$PGID + ports: + - target: 8765 + published: "8765" + protocol: tcp + volumes: + - type: bind + source: /DATA/AppData/$AppID/data + target: /data + environment: + - DATA_DIR=/data + - ALLOWED_HOSTS=* + - TZ=$TZ + x-casaos: + envs: + - container: ALLOWED_HOSTS + description: + en_us: "Hostnames the web page accepts. * = any (the app has no login, so only use it on a trusted network)." + - container: TZ + description: + en_us: Time zone + ports: + - container: "8765" + description: + en_us: Web UI port + volumes: + - container: /data + description: + en_us: "Bills_Tracker.xlsx, your statements (PDF/CSV) and backups. Put Bills_Tracker.xlsx here before first use." +x-casaos: + architectures: + - amd64 + - arm64 + main: money + author: nate + developer: nate + category: Finance + icon: https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/png/actual-budget.png + tagline: + en_us: Load bank statements into your bills workbook + title: + en_us: Money + description: + en_us: "Drop in Chase, Ally, CVFCU and Capital One statements, see which months are missing, add new transactions to Bills_Tracker.xlsx (with a backup first), and view a bills dashboard. Runs entirely on your server." + port_map: "8765" + index: / + scheme: http diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..48f3b48 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +openpyxl==3.1.5 +pdfplumber==0.11.10 diff --git a/statement_loader.py b/statement_loader.py new file mode 100644 index 0000000..16be842 --- /dev/null +++ b/statement_loader.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 +"""Statement loader: parses bank/card statements and adds new transactions to Bills_Tracker.xlsx. + +Sources understood: Chase Visa PDF, Ally Bank PDF, CVFCU PDF, Capital One CSV download. +Everything runs locally. The workbook is backed up before every write. + +CLI: .venv/bin/python statement_loader.py preview # what would be added (writes nothing) + .venv/bin/python statement_loader.py import # back up, then add new rows +""" +import re, csv, sys, shutil, zipfile, datetime as dt, calendar, collections +from dataclasses import dataclass, field +from pathlib import Path +import pdfplumber + +import os +HERE = Path(__file__).parent # code lives here +DATA = Path(os.environ.get("DATA_DIR", HERE)) # statements, workbook and backups live here (a mounted volume in Docker) +WORKBOOK = DATA / "Bills_Tracker.xlsx" +BACKUPS = DATA / "backups" + +# ---------------------------------------------------------------- accounts +# name, text on page 1 that identifies it, day-of-month the statement closes (None = month end) +ACCOUNTS = [ + ("Chase Visa", ["chase.com", "Prime Visa"], 15), + ("Ally Checking", ["COMBINED CUSTOMER STATEMENT", "ally.com"], 10), + ("CVFCU Checking", ["cvfcu.com"], None), +] +CSV_ACCOUNTS = {} # optional: last 4 of card no. in the CSV -> account name (set in loader_rules.json) +CSV_COLUMNS = ("Transaction Date", "Debit", "Credit", "Description") + +# Personal matching rules live in DATA/loader_rules.json (git-ignored, never baked into the image), e.g. +# {"own_names": ["JANE DOE"], "own_bank_hints": ["MY CREDIT UNION"], "csv_accounts": {"1234": "Capital One Card"}} +# own_names: your name as it appears in transfer descriptions, so transfers between your own accounts are not counted as spending. +import json +def _rules(): + try: return json.loads((DATA / "loader_rules.json").read_text()) + except (OSError, ValueError): return {} +def own(d): return any(n.upper() in d for n in _rules().get("own_names", [])) +def bank_hint(d): return any(n.upper() in d for n in _rules().get("own_bank_hints", [])) + +def money(s): + s = s.replace(",", "").replace("$", "") + return float(s) +def pdate(s): + for f in ("%m/%d/%Y", "%m/%d/%y", "%Y-%m-%d"): + try: return dt.datetime.strptime(s, f).date() + except ValueError: pass + +@dataclass +class Txn: + account: str + date: dt.date + desc: str # statement description as shown + detail: str # continuation lines (merchant detail on Ally etc.) + amount: float # positive = money out / charge, negative = money in / refund + source: str = "" # file name + extra: str = "" # e.g. mortgage principal/interest note + category_hint: str = "" + +@dataclass +class Statement: + file: str + account: str + start: dt.date | None + end: dt.date + txns: list = field(default_factory=list) + stated_out: float | None = None # total "money out" the statement itself reports (for checking) + kind: str = "pdf" + +# ---------------------------------------------------------------- PDF parsers +def _pages(path): + with pdfplumber.open(path) as pdf: + return [(pg.extract_text() or "") for pg in pdf.pages] + +def identify_pdf(pages): + first = pages[0].lower() + for name, keys, _ in ACCOUNTS: + if any(k.lower() in first for k in keys): return name + return None + +def parse_chase(path, pages): + full = "\n".join(pages) + m = re.search(r"Opening/Closing Date\s+(\S+) - (\S+)", full) + start, end = pdate(m.group(1)), pdate(m.group(2)) + out = {} + for k in ("Purchases", "Fees Charged", "Interest Charged"): + mm = re.search(rf"^{k}\s+\+?\$([\d,]+\.\d\d)", full, re.M) + out[k] = money(mm.group(1)) if mm else 0.0 + st = Statement(Path(path).name, "Chase Visa", start, end, stated_out=sum(out.values())) + in_rewards = False + for line in full.splitlines(): + if re.search(r"\$ Amount Rewards", line): in_rewards = True; continue + if re.match(r"ACCOUNT ACTIVITY|PAYMENTS AND OTHER|PURCHASE$|FEES CHARGED|INTEREST CHARGED", line): in_rewards = False + if in_rewards: continue + mm = re.match(r"^(\d\d)/(\d\d) (.+?) (-?(?:[\d,]+)?\.\d\d)$", line) + if not mm: continue + mo, dy = int(mm.group(1)), int(mm.group(2)) + yr = end.year if mo <= end.month or (mo, dy) <= (end.month, end.day) else end.year - 1 + if dt.date(yr, mo, dy) > end + dt.timedelta(days=31): yr -= 1 + st.txns.append(Txn("Chase Visa", dt.date(yr, mo, dy), mm.group(3).strip(), "", money(mm.group(4)), st.file)) + return st + +ALLY_TX = re.compile(r"^(\d\d/\d\d/\d{4}) (.+?) \$([\d,]+\.\d\d) -\$([\d,]+\.\d\d) \$([\d,]+\.\d\d)$") +def parse_ally(path, pages): + full = "\n".join(pages) + m = re.search(r"Statement Date.{0,120}?(\d\d/\d\d/\d{4})", pages[0], re.S) + end = pdate(m.group(1)) + mb = re.search(r"Beginning Balance, as of (\d\d/\d\d/\d{4})", full) + start = pdate(mb.group(1)) + dt.timedelta(days=1) if mb else None + md = re.search(r"Withdrawals and Other Debits\s+-?\$([\d,]+\.\d\d)", full) + st = Statement(Path(path).name, "Ally Checking", start, end, stated_out=money(md.group(1)) if md else None) + acct, cur = "Ally Checking", None + for line in full.splitlines(): + if re.match(r"^(Spending|Savings) Account\b", line) and "xxxx" not in line.lower() and "www" in line: + acct = "Ally Checking" if line.startswith("Spending") else "Ally Savings" + mm = ALLY_TX.match(line) + if mm: + if mm.group(2) in ("Beginning Balance", "Ending Balance"): cur = None; continue + cr, db = money(mm.group(3)), money(mm.group(4)) + cur = Txn(acct, pdate(mm.group(1)), re.sub(r"^(\w) (\w)", r"\1\2", mm.group(2)).strip(), "", round(db - cr, 2), st.file) + st.txns.append(cur); continue + if re.match(r"^\d\d/\d\d/\d{4} (Beginning|Ending) Balance", line): cur = None; continue + if cur and not re.match(r"^(\d{6}/|Ally Bank Member|Page \d|Statement Date|COMBINED|\d{6}-\d\d)", line): + cur.detail = (cur.detail + " | " + line.strip()).strip(" |") + if re.match(r"^\d{6}/\d+//", line): cur = None + st.txns = [t for t in st.txns if t.account == "Ally Checking"] # savings has no spending + return st + +CV_TX = re.compile(r"^(\d\d/\d\d/\d{4}) (.+?) \$([\d,]+\.\d\d) \$([\d,]+\.\d\d)$") +def parse_cvfcu(path, pages): + full = "\n".join(pages) + end = pdate(re.search(r"Statement Ending (\d\d/\d\d/\d{4})", pages[0]).group(1)) + st = Statement(Path(path).name, "CVFCU Checking", dt.date(end.year, end.month, 1), end) + block, prev, cur = None, None, None + pr = fin = None + for line in full.splitlines(): + h = re.match(r"^([A-Z0-9][A-Z0-9 &/\.\-]+?) - XXXXXX\d{4}$", line) + if h: + if h.group(1) != block: prev = None + block, cur = h.group(1), None; continue + if re.match(r"^30 YR MORTGAGE|^Statement Year to Date", line): block = None + if block is None: + mp = re.match(r"^Principal Amount -?\$([\d,.]+)", line); mf = re.match(r"^Finance Charge \$([\d,.]+)", line) + if mp: pr = money(mp.group(1)) + if mf: fin = money(mf.group(1)) + continue + if block != "PREFERRED CHECKING": continue + md = re.match(r"^([\d,]+) Debit\(s\) This Period -\$([\d,]+\.\d\d)", line) + if md: st.stated_out = money(md.group(2)); continue + mb = re.match(r"^\d\d/\d\d/\d{4} Beginning Balance \$([\d,]+\.\d\d)$", line) + if mb: prev = money(mb.group(1)); continue + mm = CV_TX.match(line) + if mm and prev is not None: + amt, bal = money(mm.group(3)), money(mm.group(4)) + sign = 1 if abs(prev - amt - bal) < 0.005 else -1 if abs(prev + amt - bal) < 0.005 else 1 + cur = Txn("CVFCU Checking", pdate(mm.group(1)), mm.group(2).strip(), "", round(sign * amt, 2), st.file) + st.txns.append(cur); prev = bal; continue + if re.match(r"^\d\d/\d\d/\d{4} Ending Balance", line): cur = None + for t in st.txns: + if "SCHEDULED TRANSFER TO LOAN" in t.desc.upper() and pr and fin: + t.extra = f"Principal ${pr:,.2f} / Interest ${fin:,.2f}" + return st + +def parse_capone_csv(path): + rows = list(csv.DictReader(open(path, newline="", encoding="utf-8-sig"))) + rows = [r for r in rows if r.get("Transaction Date") and pdate(r["Transaction Date"])] + cards = {r.get("Card No.", "") for r in rows} + mapped = {**CSV_ACCOUNTS, **_rules().get("csv_accounts", {})} + unknown = sorted(c for c in cards if c not in mapped) + acct = next((mapped[c] for c in cards if c in mapped), None) or ("Capital One Card" if len(cards) == 1 else f"Capital One Card ****{unknown[0]}") + ds = [pdate(r["Transaction Date"]) for r in rows] + st = Statement(Path(path).name, acct, min(ds), max(ds), kind="csv") + st.stated_out = round(sum(money(r["Debit"]) for r in rows if r.get("Debit")), 2) + for r in rows: + amt = money(r["Debit"]) if r.get("Debit") else -money(r["Credit"]) + st.txns.append(Txn(acct, pdate(r["Transaction Date"]), r["Description"].strip(), "", round(amt, 2), st.file, category_hint=r.get("Category", ""))) + return st + +_CACHE = {} +def load_statements(folder=DATA): + """Parse every statement file in folder. Returns (statements, skipped, duplicates).""" + sts, skipped, dups, seen = [], [], [], {} + for p in sorted(Path(folder).glob("*")): + if p.suffix.lower() not in (".pdf", ".csv"): continue + ck = (str(p), p.stat().st_mtime_ns, p.stat().st_size) + if ck in _CACHE: + st, err = _CACHE[ck] + if err: skipped.append((p.name, err)) + elif st: + key = (st.account, st.end, st.kind) + if key in seen and st.kind == "pdf": dups.append((st.file, seen[key])) + else: seen[key] = st.file; sts.append(st) + continue + st, err = None, None + try: + if p.suffix.lower() == ".pdf": + pages = _pages(p); name = identify_pdf(pages) + if not name: err = "not recognized (new account? see ACCOUNTS in statement_loader.py)" + else: st = {"Chase Visa": parse_chase, "Ally Checking": parse_ally, "CVFCU Checking": parse_cvfcu}[name](p, pages) + elif p.suffix.lower() == ".csv": + head = open(p, encoding="utf-8-sig").readline() + if not all(c in head for c in CSV_COLUMNS): _CACHE[ck] = (None, None); continue + st = parse_capone_csv(p) + except Exception as e: + err = f"could not read: {e}" + _CACHE[ck] = (st, err) + if err: skipped.append((p.name, err)); continue + if st is None: continue + key = (st.account, st.end, st.kind) + if key in seen and st.kind == "pdf": dups.append((st.file, seen[key])); continue + seen[key] = st.file; sts.append(st) + return sts, skipped, dups + +# ---------------------------------------------------------------- classification +def norm(desc): + toks = [] + for t in re.sub(r"[*,;#\-]", " ", desc.upper()).split(): + if re.search(r"\d|#", t): break + toks.append(t) + if len(toks) == 3: break + return " ".join(toks) + +def _is_chase_payment(t): return t.account == "Chase Visa" and re.match(r"(AUTOMATIC )?PAYMENT", t.desc.upper()) and t.amount < 0 + +def exclusion(t): + """Return (reason, direction) if this is NOT spending (card payment, transfer, income, interest), else None.""" + d = (t.desc + " " + t.detail).upper(); out = "Money out" if t.amount > 0 else "Money in" + CARD = "Card payment (spending counted when purchased)" + if _is_chase_payment(t): return CARD, out + if t.account == "Capital One Card" and t.category_hint.startswith("Payment") and t.amount < 0: return CARD, out + if t.account == "Ally Checking": + if t.desc.upper().startswith("INTEREST PAID"): return "Interest earned", out + if "CASH APP*" in d and own(d): return "Cash App transfer to your own Cash App (confirm destination)", out + if "CASH APP" in d and t.amount < 0: return "Deposit (Cash App / Amazon) – not spending", out + if "AMAZON.COM SERVI" in d and t.amount < 0: return "Deposit (Cash App / Amazon) – not spending", out + if "SAVINGS" in d and "TRANSFER" in d: return "Transfer to savings account", out + if ("TRANSFER TO" in d and own(d)) or bank_hint(d): return "Transfer to CVFCU checking", out + if ("TRANSFER FROM" in d and own(d)) or ("ALLY BANK TRANSFER" in d and t.amount < 0): return "Transfer from CVFCU checking", out + if t.account == "CVFCU Checking": + if "PAYROLL" in d: return "Income (payroll deposit)", out + if "CHASE CREDIT CRD" in d or "DISCOVER" in d: return CARD, out + if "ALLY BANK" in d: return "Transfer to Ally account", out + if "P2P" in d and own(d): return "Transfer from Ally account", out + if d.startswith("DIVIDEND EARNED"): return "Interest earned", out + return None + +SEED_RULES = [ # (text to find in desc+detail, payee, what for) for sources whose log description is generic + ("T-MOBILE", "T-Mobile", "Cell phone service"), + ("SCHEDULED TRANSFER TO LOAN", "CVFCU Mortgage", "Monthly mortgage payment (principal + interest)"), +] + +@dataclass +class Row: + txn: Txn + kind: str # "log" or "excluded" + payee: str = "" + what: str = "" + reason: str = "" + direction: str = "" + note: str = "" + is_new_payee: bool = False + needs_review: bool = False + +def learn_rules(wb): + """payee/what-for per normalized description, learned from rows already in the Bill Log.""" + c = collections.defaultdict(collections.Counter) + for r in wb["Bill Log"].iter_rows(min_row=6, values_only=True): + if r[0] and r[7] and r[1]: + c[norm(r[7])][(r[1], r[4])] += 1 + return {k: v.most_common(1)[0][0] for k, v in c.items() if k} + +def classify(t, rules, known_payees): + ex = exclusion(t) + if ex: return Row(t, "excluded", reason=ex[0], direction=ex[1]) + d = (t.desc + " " + t.detail).upper() + payee = what = None + for needle, p, w in SEED_RULES: + if needle in d: payee, what = p, w; break + if not payee and t.account == "Ally Checking" and "ATM" in d: + payee, what = "ATM Cash", "Cash withdrawal (net of ATM-fee reimbursement)" + if not payee: + k = norm(t.desc) + hit = rules.get(k) + if not hit and k: # fall back to the first two words + two = " ".join(k.split()[:2]) + cands = {v for kk, v in rules.items() if " ".join(kk.split()[:2]) == two} + if len(cands) == 1: hit = next(iter(cands)) + if hit: payee, what = hit + row = Row(t, "log") + if payee: row.payee, row.what = payee, what + else: + row.payee = " ".join(w.capitalize() for w in (norm(t.desc) or t.desc.upper()).split())[:40] or "Unknown" + row.what = "NEW – please categorize"; row.is_new_payee = True; row.needs_review = True + row.note = "NEW – review payee/category" + if payee and payee not in known_payees: row.is_new_payee = True + if t.amount < 0: row.note = (row.note + "; " if row.note else "") + "Refund / credit" + if t.extra: row.note = (row.note + "; " if row.note else "") + t.extra + return row + +# ---------------------------------------------------------------- workbook access +def existing_keys(wb): + """Multiset of (account, date, |amount|) already in Bill Log + Excluded Transfers.""" + keys = collections.Counter() + for r in wb["Bill Log"].iter_rows(min_row=6, values_only=True): + if r[0]: keys[(r[6], r[0].date(), round(abs(r[5]), 2))] += 1 + for r in wb["Excluded Transfers"].iter_rows(min_row=5, values_only=True): + if r[0] and isinstance(r[0], dt.datetime): keys[(r[1], r[0].date(), round(abs(r[4]), 2))] += 1 + return keys + +def plan(wb, statements): + """Work out which transactions are new. ATM withdrawal + same-day fee line are merged like the existing log does.""" + rules = learn_rules(wb) + lists = wb["Lists"] + known = {r[0] for r in lists.iter_rows(min_row=4, max_row=153, max_col=1, values_only=True) if r[0]} + have = existing_keys(wb) + rows = [] + for st in statements: + for t in st.txns: + rows.append(classify(t, rules, known)) + new, already = [], 0 + for r in rows: + t = r.txn; k = (t.account, t.date, round(abs(t.amount), 2)) + if have[k] > 0: have[k] -= 1; already += 1; continue + new.append(r) + return new, already + +def check_statements(statements): + """Compare what we parsed to the totals each statement prints. Returns [(file, parsed, stated, ok)].""" + res = [] + for st in statements: + if st.stated_out is None: continue + parsed = round(sum(t.amount for t in st.txns if t.amount > 0), 2) + res.append((st.file, parsed, st.stated_out, abs(parsed - st.stated_out) < 0.005)) + return res + +# ---------------------------------------------------------------- writing +def _chart_sig(xml): + refs = re.findall(rb"<(?:c:)?f>(.*?)", xml) + kinds = tuple(re.findall(rb"<(?:c:)?(bar|line|pie|doughnut|area)Chart>", xml)) + return kinds + tuple(sorted(re.sub(rb"'|'|\$| ", b"", r) for r in refs)) + +def _restore_charts(original, saved): + """openpyxl rewrites charts and drops some styling; put the original chart XML back (matched by plotted ranges).""" + with zipfile.ZipFile(original) as zo, zipfile.ZipFile(saved) as zs: + orig = {_chart_sig(zo.read(n)): zo.read(n) for n in zo.namelist() if re.match(r"xl/charts/chart\d+\.xml$", n)} + out = {} + for n in zs.namelist(): + data = zs.read(n) + if re.match(r"xl/charts/chart\d+\.xml$", n): + data = orig.get(_chart_sig(data), data) + out[n] = data + infos = {i.filename: i for i in zs.infolist()} + tmp = Path(str(saved) + ".tmp") + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) as z: + for n, data in out.items(): z.writestr(n, data) + tmp.replace(saved) + +def _copy_style(src, dst): + from copy import copy + dst.font, dst.fill, dst.border, dst.alignment, dst.number_format = copy(src.font), copy(src.fill), copy(src.border), copy(src.alignment), src.number_format + +def apply(new_rows, workbook=WORKBOOK): + """Back up, then append new rows. Returns a summary dict.""" + from openpyxl import load_workbook + BACKUPS.mkdir(exist_ok=True) + backup = BACKUPS / f"Bills_Tracker_{dt.datetime.now():%Y%m%d_%H%M%S}.xlsx" + shutil.copy2(workbook, backup) + wb = load_workbook(workbook) + bl, ex, ls, rc, rv = wb["Bill Log"], wb["Excluded Transfers"], wb["Lists"], wb["Reconciliation"], wb["Review Items"] + log_rows = sorted([r for r in new_rows if r.kind == "log"], key=lambda r: (r.txn.date, r.txn.account)) + ex_rows = sorted([r for r in new_rows if r.kind == "excluded"], key=lambda r: (r.txn.date, r.txn.account)) + # Bill Log: fill the next empty rows (formulas in C/D are already there) + r = 6 + while bl.cell(r, 1).value is not None: r += 1 + if r + len(log_rows) - 1 > 2505: raise RuntimeError("Bill Log is full (2,500 rows). Archive a year first.") + for row in log_rows: + t = row.txn + for c, v in ((1, dt.datetime.combine(t.date, dt.time())), (2, row.payee), (5, row.what), (6, t.amount), (7, t.account), (8, t.desc), (9, row.note or None)): + bl.cell(r, c).value = v + r += 1 + # Excluded Transfers + er = 5 + while ex.cell(er, 1).value is not None: er += 1 + for row in ex_rows: + t = row.txn + for c, v in enumerate((dt.datetime.combine(t.date, dt.time()), t.account, t.desc, t.detail or None, abs(t.amount), row.direction, row.reason), 1): + ex.cell(er, c).value = v + _copy_style(ex.cell(5, c), ex.cell(er, c)) + er += 1 + # reconciliation formulas read 'Excluded Transfers'!E5:E111; widen them + for rr in rc.iter_rows(): + for c in rr: + if isinstance(c.value, str) and "$111" in c.value: c.value = c.value.replace("$111", "$2000") + ex.auto_filter.ref = f"A4:G{max(er - 1, 111)}" + # Lists: new payees + lr = 4 + while ls.cell(lr, 1).value is not None and lr <= 153: lr += 1 + added_payees = [] + for row in log_rows: + if row.is_new_payee and row.payee not in added_payees and row.payee not in {ls.cell(i, 1).value for i in range(4, lr)}: + if lr > 153: break + ls.cell(lr, 1).value, ls.cell(lr, 2).value, ls.cell(lr, 3).value = row.payee, "Other / Unsorted", "Spending" + for c in (1, 2, 3): _copy_style(ls.cell(lr - 1, c), ls.cell(lr, c)) + added_payees.append(row.payee); lr += 1 + # Reconciliation: statement money-out added per account (statements are verified against their printed totals before import) + by_acct = collections.Counter() + for row in new_rows: + if row.txn.amount > 0: by_acct[row.txn.account] += row.txn.amount + for rr in range(5, 9): + a = rc.cell(rr, 1).value + if by_acct.get(a): rc.cell(rr, 2).value = round(rc.cell(rr, 2).value + by_acct[a], 2) + # coverage lines + lo, hi = {}, {} + for sheet, a_col, d_col, first in ((bl, 7, 1, 6), (ex, 2, 1, 5)): + for rr in sheet.iter_rows(min_row=first, values_only=True): + d, a = rr[d_col - 1], rr[a_col - 1] + if isinstance(d, dt.datetime) and a: + lo[a] = min(lo.get(a, d), d); hi[a] = max(hi.get(a, d), d) + for rr in range(1, rc.max_row + 1): + a = rc.cell(rr, 1).value + if a in lo and rc.cell(rr, 2).value and "–" in str(rc.cell(rr, 2).value): + rc.cell(rr, 2).value = f"{lo[a]:%-m/%-d/%Y} – {hi[a]:%-m/%-d/%Y}" + # Review Items + if added_payees or any(r.needs_review for r in log_rows): + n = max([x[0] for x in rv.iter_rows(min_row=5, max_col=1, values_only=True) if isinstance(x[0], int)] + [0]) + 1 + rr = rv.max_row + 1 + for c, v in enumerate((n, f"{len(added_payees)} new payee(s) from imported statements: " + ", ".join(added_payees[:12]) + ("…" if len(added_payees) > 12 else ""), + "Added to the Lists tab as 'Other / Unsorted' (Spending) so they appear in every summary.", + "Set the right Category/Type on the Lists tab (and rename the payee if you like)."), 1): + rv.cell(rr, c).value = v; _copy_style(rv.cell(rr - 1, c), rv.cell(rr, c)) + wb.calculation.fullCalcOnLoad = True + tmp = Path(str(workbook) + ".new") + wb.save(tmp) + _restore_charts(workbook, tmp) + tmp.replace(workbook) + return dict(backup=str(backup), log_rows=len(log_rows), excluded_rows=len(ex_rows), new_payees=added_payees) + +def summarize(rows): + return dict(log=sum(r.kind == "log" for r in rows), excluded=sum(r.kind == "excluded" for r in rows), + new_payees=sorted({r.payee for r in rows if r.is_new_payee})) + +def run_preview(folder=DATA): + from openpyxl import load_workbook + sts, skipped, dups = load_statements(folder) + checks = check_statements(sts) + bad = {f for f, p, s, ok in checks if not ok} + good = [s for s in sts if s.file not in bad] + wb = load_workbook(WORKBOOK) + new, already = plan(wb, good) + return dict(statements=sts, skipped=skipped, dups=dups, checks=checks, bad=bad, new=new, already=already) + +if __name__ == "__main__": + cmd = sys.argv[1] if len(sys.argv) > 1 else "preview" + res = run_preview() + for f, p, st, ok in res["checks"]: + if not ok: print(f"!! {f}: parsed {p:,.2f} but statement says {st:,.2f} - NOT imported") + for f, why in res["skipped"]: print(f"skipped {f}: {why}") + new = res["new"]; sm = summarize(new) + print(f"{len(res['statements'])} statements read, {res['already']} transactions already in workbook, {len(new)} new " + f"({sm['log']} to Bill Log, {sm['excluded']} to Excluded Transfers).") + for r in new[:200]: + print(f" {r.txn.date} {r.txn.account:16} {r.txn.amount:>10,.2f} {r.kind:8} {r.payee or r.reason} <- {r.txn.desc[:40]}") + if sm["new_payees"]: print("New payees to categorize:", ", ".join(sm["new_payees"])) + if cmd == "import" and new: + print(apply(new)) + elif cmd == "import": print("Nothing new to import.")