diff --git a/Dockerfile b/Dockerfile index d761826..c6db74d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY statement_loader.py app.py app_page.html app_dashboard.html ./ +COPY statement_loader.py seed_rules.py app.py app_page.html app_dashboard.html ./ COPY template/ ./template/ # statements, Bills_Tracker.xlsx and backups live in this volume, never in the image diff --git a/README.md b/README.md index 598d3f8..77dfafd 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,10 @@ AppID=money PUID=1000 PGID=1000 TZ=America/New_York docker compose up -d # the "Create a blank workbook" button that copies it into the data folder. Then drop statements in; new payees land on the Lists tab as "Other / Unsorted" for you to categorize. `loader_rules.example.json` shows the private rules file format. +## Categories +New payees are categorized by built-in generic merchant rules (`seed_rules.py`); anything unknown lands in "Other / Unsorted". +Use the **Categorize payees** card in the app (or the Lists tab) to set the rest. Your choices are saved in the workbook. + ## Personal rules (not in git) Your name, card last-4 and similar live in `loader_rules.json` next to the workbook (git-ignored, not in the image): ```json diff --git a/app.py b/app.py index 33aded9..1e57516 100644 --- a/app.py +++ b/app.py @@ -74,7 +74,7 @@ _DATA = {} def workbook_data(): """Read-only snapshot of the Bill Log for the web dashboard (computed from raw cells, not cached formulas).""" from openpyxl import load_workbook - if not L.WORKBOOK.exists(): return dict(tx=[], categories=[], excluded=0, updated="", setup=True, data_dir=str(DATA)) + if not L.WORKBOOK.exists(): return dict(tx=[], income=[], categories=[], excluded=0, updated="", setup=True, data_dir=str(DATA)) key = L.WORKBOOK.stat().st_mtime_ns if _DATA.get("key") == key: return _DATA["val"] wb = load_workbook(L.WORKBOOK, read_only=True) @@ -87,9 +87,16 @@ def workbook_data(): if isinstance(r[0], dt.datetime) and r[1] and r[5] is not None: c, t = pay.get(r[1], ("Other / Unsorted", "Spending")) tx.append([f"{r[0]:%Y-%m-%d}", r[1], c, t, r[4] or "", round(float(r[5]), 2), r[6] or "", r[8] or ""]) - ex = sum(1 for r in wb["Excluded Transfers"].iter_rows(min_row=5, max_col=1, values_only=True) if isinstance(r[0], dt.datetime)) + ex, income = 0, [] + for r in wb["Excluded Transfers"].iter_rows(min_row=5, max_col=7, values_only=True): + if not isinstance(r[0], dt.datetime): continue + ex += 1 + if r[5] == "Money in" and r[6] in ("Income (payroll deposit)", "Interest earned"): + m = re.search(r"EFT (.+?)\s*[;,]", r[2] or "") + src = (m.group(1).strip().title() if m else (r[2] or "Other").title()) if r[6].startswith("Income") else "Interest" + income.append([f"{r[0]:%Y-%m-%d}", src, "Interest" if r[6].startswith("Interest") else "Paycheck", round(float(r[4]), 2), r[1] or ""]) wb.close() - _DATA["key"], _DATA["val"] = key, dict(tx=tx, categories=cats, excluded=ex, updated=f"{dt.datetime.fromtimestamp(L.WORKBOOK.stat().st_mtime):%b %-d, %Y %-I:%M %p}") + _DATA["key"], _DATA["val"] = key, dict(tx=tx, income=income, categories=cats, excluded=ex, updated=f"{dt.datetime.fromtimestamp(L.WORKBOOK.stat().st_mtime):%b %-d, %Y %-I:%M %p}") return _DATA["val"] def safe_name(name): @@ -124,6 +131,10 @@ class H(BaseHTTPRequestHandler): with LOCK: try: return self._send(200, workbook_data()) except Exception as e: return self._send(500, {"error": str(e)}) + if u.path == "/api/payees": + with LOCK: + try: return self._send(200, L.uncategorized()) + except Exception as e: return self._send(500, {"error": str(e)}) if u.path == "/api/state": with LOCK: try: return self._send(200, state()) @@ -140,7 +151,13 @@ class H(BaseHTTPRequestHandler): if p.name == "loader_rules.json": json.loads(data) # must be valid JSON DATA.mkdir(parents=True, exist_ok=True); p.write_bytes(data) return self._send(200, {"saved": p.name}) - self.rfile.read(n) + body = self.rfile.read(n) + if u.path == "/api/set_categories": + with LOCK: out = L.set_categories(json.loads(body or b"[]")) + return self._send(200, {"message": f"Updated {out['updated']} payee(s). Backup: {out['backup']}."}) + if u.path == "/api/autocategorize": + with LOCK: out = L.autocategorize() + return self._send(200, {"message": f"Auto-categorized {out['updated']} payee(s); {out['remaining']} still need you. Backup: {out['backup']}."}) if u.path == "/api/import": with LOCK: res = L.run_preview(DATA) diff --git a/app_dashboard.html b/app_dashboard.html index ad704b2..848e4c9 100644 --- a/app_dashboard.html +++ b/app_dashboard.html @@ -33,6 +33,8 @@ td.n,th.n{text-align:right;font-variant-numeric:tabular-nums}.scroll{max-height: ← Statement Loader
+

Income vs spending

+

Income sources

Monthly total

Monthly spending by category

Year over year by month

@@ -50,7 +52,7 @@ td.n,th.n{text-align:right;font-variant-numeric:tabular-nums}.scroll{max-height:
-

Read-only view of Bills_Tracker.xlsx. Card payments and transfers are not included (see Excluded Transfers in the workbook). Refunds are negative amounts.

+

Read-only view of Bills_Tracker.xlsx. Income = paychecks and interest. Card payments and transfers between your accounts are in neither income nor spending (see Excluded Transfers in the workbook). Refunds are negative amounts.

diff --git a/seed_rules.py b/seed_rules.py new file mode 100644 index 0000000..edf1d18 --- /dev/null +++ b/seed_rules.py @@ -0,0 +1,54 @@ +"""Generic merchant rules: statement text -> (payee, category, type, what it was for). +Keywords are matched as whole words (case-insensitive) in the statement description, in order: first match wins, +so put specific entries before generic ones. payee=None keeps the cleaned-up statement name. +Categories must exist on the workbook's Lists tab (otherwise the payee falls back to "Other / Unsorted"). +No personal data belongs in this file; private rules go in loader_rules.json. +""" +import re + +B, S = "Bill", "Spending" +# (keywords, payee, category, type, what) +SEEDS = [ + # --- housing / insurance / taxes + (["SCHEDULED TRANSFER TO LOAN", "MORTGAGE"], "Mortgage", "Housing", B, "Monthly mortgage payment"), + (["RENT", "HOA", "PROPERTY MGMT"], None, "Housing", B, "Housing payment"), + (["GEICO", "PROGRESSIVE", "STATE FARM", "ALLSTATE", "LIBERTY MUTUAL", "FARMERS INS", "NATIONWIDE", "HOMESITE", "TRUSTAGE", "INSUR", "INS PREM"], None, "Insurance", B, "Insurance premium"), + (["DMV", "DEPARTMENT OF MOTOR", "DEPARTMENT MOTOR", "DEPT OF MOTOR", "TREASURER", "COUNTY TAX", "TAXFINESFEES", "IRS", "USPS", "POST OFFICE", "POSTAL"], None, "Taxes & Gov Fees", B, "Tax / government fee / postage"), + # --- phone, internet, utilities + (["T-MOBILE", "TMOBILE", "VERIZON", "AT&T", "ATT", "SPRINT", "XFINITY", "COMCAST", "SPECTRUM", "VONAGE", "SHENTEL", "COX", "FRONTIER", "MINT MOBILE", "GOOGLE FI", "STARLINK", "SIMPLEFIN"], None, "Phone & Internet", B, "Phone / internet service"), + (["ELECTRIC", "POWER", "ENERGY", "DOMINION", "DUKE", "WATER", "SEWER", "PROPANE", "NATURAL GAS", "UTILIT", "TRASH", "WASTE"], None, "Utilities", B, "Utility bill"), + # --- subscriptions & software + (["NETFLIX", "SPOTIFY", "HULU", "DISNEY", "HBO", "PEACOCK", "PARAMOUNT", "PRIME VIDEO", "YOUTUBE", "PURE FLIX", "EMBY", "AUDIBLE", "KINDLE", "AMZN DIGITAL", "AMAZON DIGITAL", "AMAZON PRIME", "APPLE.COM/BILL", "ITUNES", "GOOGLE", "MICROSOFT", "MSFT", "ADOBE", "DROPBOX", "ICLOUD", "ANTHROPIC", "CLAUDE.AI", "OPENAI", "CLOUDFLARE", "GITHUB", "ELEVENLABS", "1PASSWORD", "DISCORD", "NINTENDO", "PLAYSTATION", "XBOX", "STEAM", "LOGOS"], None, "Subscriptions & Software", B, "Subscription / software"), + # --- groceries & household + (["ALDI", "FOOD LION", "KROGER", "PUBLIX", "SAFEWAY", "HARRIS TEETER", "WEGMANS", "GIANT", "TRADER JOE", "WHOLE FOODS", "SAMS CLUB", "SAMSCLUB", "COSTCO", "WM SUPERCENTER", "WAL MART", "WALMART", "WAL-MART", "DOLLAR TREE", "DOLLARTREE", "DOLLAR GENERAL", "FAMILY DOLLAR", "OLLIES", "GROCERY"], None, "Groceries & Household", S, "Groceries / household"), + # --- dining + (["MCDONALD", "WENDY", "BURGER KING", "TACO BELL", "CHICK FIL", "SUBWAY", "DOMINO", "PIZZA", "LITTLE CAESARS", "PANDA EXPRESS", "STARBUCKS", "DUNKIN", "HARDEE", "CHIPOTLE", "SONIC", "ARBY", "KFC", "WAFFLE HOUSE", "APPLEBEE", "LONGHORN", "OLIVE GARDEN", "PANERA", "CICIS", "CAFE", "RESTAURANT", "GRILL", "DINER", "DOORDASH", "UBER EATS", "GRUBHUB", "CANTEEN", "VENDING", "MEXICAN", "WASABI", "SUSHI", "BBQ"], None, "Dining Out", S, "Restaurant / food"), + # --- fuel & auto + (["SHEETZ", "WAWA", "EXXON", "SHELL", "BP", "CHEVRON", "MARATHON", "SPEEDWAY", "CIRCLE K", "MURPHY", "QUIKTRIP", "RACETRAC", "7-ELEVEN", "CITGO", "GAS STATION", "FUEL", "AUTOZONE", "OREILLY", "O'REILLY", "ADVANCE AUTO", "NAPA", "JIFFY LUBE", "TIRE", "PEP BOYS", "AUTO"], None, "Fuel & Auto", S, "Fuel / auto"), + # --- health + (["CVS", "WALGREENS", "RITE AID", "PHARMACY", "DENTAL", "DENTIST", "ORTHODONT", "MYEYEDR", "OPTICAL", "EYE", "HOSPITAL", "MEDICAL", "CLINIC", "URGENT CARE", "PHYSICIAN", "HEALTH", "PLANET FITNESS", "GYM"], None, "Health & Medical", S, "Health / medical"), + # --- pets + (["PETCO", "PETSMART", "VETERINAR", "ANIMAL HOSPITAL", "PET &", "PET SUPPLIES", "CHEWY"], None, "Pets", S, "Pet care / supplies"), + # --- home & garden + (["LOWES", "HOME DEPOT", "TRACTOR SUPPLY", "ACE HARDWARE", "MENARDS", "HARBOR FREIGHT", "HARDWARE", "NURSERY", "GARDEN"], None, "Home & Garden", S, "Home / garden / hardware"), + # --- travel & outings + (["EXPEDIA", "AIRBNB", "HOTEL", "MARRIOTT", "HILTON", "DELTA", "SOUTHWEST", "UNITED AIR", "AMTRAK", "UBER", "LYFT", "CINEMA", "REGAL", "AMC", "THEATER", "THEATRE", "MUSEUM", "RESORT", "SETTLEMENT", "PARK", "BOWLING"], None, "Travel & Outings", S, "Travel / outing"), + # --- education, giving + (["SCHOOL", "TUITION", "ACADEMY", "UDEMY", "COURSERA", "UNIVERSITY", "COLLEGE", "ABEKA"], None, "Education", S, "Education"), + (["DONATION", "CHURCH", "CHARITY", "RED CROSS", "GOFUNDME", "BAPTIST", "MINISTR"], None, "Giving & Charity", S, "Giving / donation"), + # --- shopping (broad, keep last) + (["AMAZON", "AMZN", "EBAY", "ETSY", "BEST BUY", "TJMAXX", "TJ MAXX", "ROSS", "KOHL", "MACY", "STAPLES", "HOBBY LOBBY", "HOBBYLOBBY", "MICHAELS", "GOODWILL", "THRIFT", "ZENNI", "TARGET", "BOOKSHOP", "MARDEL", "CHRISTIANBOOK"], None, "Shopping", S, "Shopping"), +] + +def _kw(k): + # match at the start of a word; short keywords (BP, ATT, IRS...) must also end a word + return re.escape(k) + (r"(?![A-Z0-9])" if len(k) <= 4 else "") + +_COMPILED = [(re.compile(r"(? 153: break - ls.cell(lr, 1).value, ls.cell(lr, 2).value, ls.cell(lr, 3).value = row.payee, "Other / Unsorted", "Spending" + ls.cell(lr, 1).value, ls.cell(lr, 2).value, ls.cell(lr, 3).value = row.payee, row.category or "Other / Unsorted", row.type or "Spending" for c in (1, 2, 3): _copy_style(ls.cell(lr - 1, c), ls.cell(lr, c)) added_payees.append(row.payee); lr += 1 # Reconciliation: statement money-out added per account (statements are verified against their printed totals before import) @@ -425,7 +439,7 @@ def apply(new_rows, workbook=WORKBOOK): n = max([x[0] for x in rv.iter_rows(min_row=5, max_col=1, values_only=True) if isinstance(x[0], int)] + [0]) + 1 rr = rv.max_row + 1 for c, v in enumerate((n, f"{len(added_payees)} new payee(s) from imported statements: " + ", ".join(added_payees[:12]) + ("…" if len(added_payees) > 12 else ""), - "Added to the Lists tab as 'Other / Unsorted' (Spending) so they appear in every summary.", + "Added to the Lists tab (category from the built-in merchant rules, or 'Other / Unsorted' when unknown) so they appear in every summary.", "Set the right Category/Type on the Lists tab (and rename the payee if you like)."), 1): rv.cell(rr, c).value = v; _copy_style(rv.cell(rr - 1, c), rv.cell(rr, c)) wb.calculation.fullCalcOnLoad = True @@ -435,6 +449,61 @@ def apply(new_rows, workbook=WORKBOOK): tmp.replace(workbook) return dict(backup=str(backup), log_rows=len(log_rows), excluded_rows=len(ex_rows), new_payees=added_payees) + +# ---------------------------------------------------------------- categorizing existing payees +def _backup(workbook): + BACKUPS.mkdir(exist_ok=True) + b = BACKUPS / f"Bills_Tracker_{dt.datetime.now():%Y%m%d_%H%M%S_%f}.xlsx" + shutil.copy2(workbook, b); return b + +def _save_workbook(wb, workbook): + wb.calculation.fullCalcOnLoad = True + tmp = Path(str(workbook) + ".new"); wb.save(tmp); _restore_charts(workbook, tmp); tmp.replace(workbook) + +def uncategorized(workbook=WORKBOOK): + """Payees on the Lists tab still set to Other / Unsorted, with how much they account for.""" + from openpyxl import load_workbook + wb = load_workbook(workbook, read_only=True) + tot, cnt = collections.Counter(), collections.Counter() + for r in wb["Bill Log"].iter_rows(min_row=6, max_col=6, values_only=True): + if r[0] and r[1]: tot[r[1]] += r[5] or 0; cnt[r[1]] += 1 + cats = _categories(wb) + out = [dict(payee=r[0], category=r[1] or "Other / Unsorted", type=r[2] or "Spending", n=cnt[r[0]], total=round(tot[r[0]], 2)) + for r in wb["Lists"].iter_rows(min_row=4, max_row=153, max_col=3, values_only=True) + if r[0] and (not r[1] or r[1] == "Other / Unsorted")] + wb.close() + return dict(payees=sorted(out, key=lambda x: -x["total"]), categories=cats, types=["Bill", "Spending"]) + +def set_categories(changes, workbook=WORKBOOK): + """changes: [{payee, category, type}] -> written to the Lists tab (backup first).""" + from openpyxl import load_workbook + b = _backup(workbook); wb = load_workbook(workbook); ls = wb["Lists"] + cats = set(_categories(wb)); rows = {ls.cell(r, 1).value: r for r in range(4, 154) if ls.cell(r, 1).value} + n = 0 + for ch in changes: + r = rows.get(ch.get("payee")) + if r and ch.get("category") in cats and ch.get("type") in ("Bill", "Spending"): + ls.cell(r, 2).value, ls.cell(r, 3).value = ch["category"], ch["type"]; n += 1 + _save_workbook(wb, workbook) + return dict(updated=n, backup=b.name) + +def autocategorize(workbook=WORKBOOK): + """Apply the built-in merchant rules to payees still set to Other / Unsorted.""" + from openpyxl import load_workbook + b = _backup(workbook); wb = load_workbook(workbook); ls = wb["Lists"]; cats = set(_categories(wb)) + descs = collections.defaultdict(list) + for r in wb["Bill Log"].iter_rows(min_row=6, max_col=8, values_only=True): + if r[0] and r[1] and r[7] and len(descs[r[1]]) < 3: descs[r[1]].append(str(r[7])) + n = left = 0 + for r in range(4, 154): + p = ls.cell(r, 1).value + if not p or (ls.cell(r, 2).value and ls.cell(r, 2).value != "Other / Unsorted"): continue + sm = seed_match(" ".join([p] + descs[p])) + if sm and sm[1] in cats: ls.cell(r, 2).value, ls.cell(r, 3).value = sm[1], sm[2]; n += 1 + else: left += 1 + _save_workbook(wb, workbook) + return dict(updated=n, remaining=left, backup=b.name) + def summarize(rows): return dict(log=sum(r.kind == "log" for r in rows), excluded=sum(r.kind == "excluded" for r in rows), new_payees=sorted({r.payee for r in rows if r.is_new_payee}))