"""Cleanup engine for the Xero + asset-tracking sample. Reads data/ only; writes results/. 1 contacts: duplicate detection and merge map 2 invoice lines: category rules -> account + tracking, review queue, tax-code check 3 recode journals (monthly) 4 asset log: clean, correct, rebuild the ledger; fleet reconciliation 5 hire billing check (billable vs billed unit-weeks) 6 Make S2 billing run simulation for July 2026 (with an idempotency re-run) 7 regression test against truth/ (the generator's answer key), written to results/engine_test.json""" import csv, os, re, json, difflib, datetime as dt from collections import defaultdict, Counter HERE = os.path.dirname(os.path.abspath(__file__)); os.makedirs(os.path.join(HERE, "results"), exist_ok=True) rd = lambda p: list(csv.DictReader(open(os.path.join(HERE, p)))) def W(name, rows, head=None): with open(os.path.join(HERE, "results", name), "w", newline="") as f: w = csv.DictWriter(f, fieldnames=head or list(rows[0].keys())); w.writeheader(); w.writerows(rows) D = dt.date.fromisoformat; FY0, FY1 = dt.date(2025, 7, 1), dt.date(2026, 6, 30) contacts = rd("data/xero_contacts.csv"); lines = rd("data/xero_invoice_lines_fy26.csv"); sites = {s["site_id"]: s for s in rd("data/sites.csv")} log = rd("data/asset_movement_log_fy26.csv"); stock = {r["asset"]: int(r["counted_in_yard"]) for r in rd("data/yard_stocktake_2026-06-30.csv")} fleet = {r["asset"]: int(r["owned"]) for r in rd("data/fleet_register_2025-06-30.csv")}; purch = rd("data/asset_purchases_fy26.csv") RATE = dict(FP=4.50, BR=3.20, ST=95.00, TK=35.00); LOSS_RATE = 145.0 # ------------------------------------------------------------------ 1 contacts SFX = r"\b(pty\.? ?ltd\.?|p/l|pty|ltd|limited)\b|\(accounts\)|- ?accounts\b" def norm(s): s = re.sub(SFX, " ", s.lower()); s = s.replace("&", " and "); s = re.sub(r"[^a-z0-9 ]", " ", s); return re.sub(r"\s+", " ", s).strip() dom = lambda e: e.split("@")[-1].lower() if "@" in e else "" cs = sorted(contacts, key=lambda c: (c["created"], c["contact_id"])); master = {}; cmap = [] for c in cs: best = None for m in [x for x in cs if x["contact_id"] in master and master[x["contact_id"]] == x["contact_id"]]: a, b = norm(c["name"]), norm(m["name"]); sc = difflib.SequenceMatcher(None, a, b).ratio() tn_c, tn_m = c["tax_number"].replace(" ", ""), m["tax_number"].replace(" ", "") if tn_c and tn_c == tn_m: cand = ("M1", 1.0, "Same tax number") elif a == b and (dom(c["email"]) == dom(m["email"]) or not (tn_c and tn_m)): cand = ("M2", 1.0, "Same name after removing suffixes and punctuation") elif sc >= .9 and dom(c["email"]) == dom(m["email"]) and not (tn_c and tn_m and tn_c != tn_m): cand = ("M3", round(sc, 3), "Near-identical name and same email domain") elif sc >= .85 and tn_c and tn_m and tn_c != tn_m: cand = ("N1", round(sc, 3), "Similar name but different tax numbers: kept separate") elif sc >= .85: cand = ("V1", round(sc, 3), "Similar name, nothing else in common: review") else: continue rank = {"M1": 5, "M2": 4, "M3": 3, "V1": 2, "N1": 1}[cand[0]] if not best or (rank, cand[1]) > (best[0], best[2][1]): best = (rank, m, cand) if best and best[2][0].startswith("M"): master[c["contact_id"]] = best[1]["contact_id"]; r, s_, note = best[2] else: master[c["contact_id"]] = c["contact_id"]; r, s_, note = (best[2] if best else ("", "", "")) cmap.append(dict(contact_id=c["contact_id"], name=c["name"], tax_number=c["tax_number"], email=c["email"], created=c["created"], master_id=master[c["contact_id"]], master_name=next(x["name"] for x in contacts if x["contact_id"] == master[c["contact_id"]]), rule=r, score=s_, note=note, related_id=(best[1]["contact_id"] if best else ""), action="Merge into master" if master[c["contact_id"]] != c["contact_id"] else ("Keep separate" if r == "N1" else "Review" if r == "V1" else "Keep"))) cmap.sort(key=lambda r: r["contact_id"]); W("contact_map.csv", cmap) # ------------------------------------------------------------------ 2 categorisation ACCTS = {"4010": "Equipment hire income", "4020": "Labour and services income", "4030": "Materials and disposal income", "4040": "Equipment loss and damage recoveries", "835": "Customer deposits held"} RULES = [("R00", r"package|\bfence\b.*make safe|make safe.*\bfence\b", "MIXED", "", "", "Two service lines in one line: needs a person"), ("R01", r"security deposit", "DEP", "835", "Fencing", "Refundable deposits are a liability, not income"), ("R02", r"lost/damaged|lost or damaged|damaged panels", "LOSS", "4040", "Fencing", "Charges for equipment not returned"), ("R03", r"fence hire", "HIRE-FP", "4010", "Fencing", "Weekly panel hire"), ("R04", r"barrier hire", "HIRE-BR", "4010", "Fencing", "Weekly barrier hire"), ("R05", r"scaffold tower hire", "HIRE-ST", "4010", "Scaffolding", "Weekly tower hire"), ("R06", r"tarp kit hire", "HIRE-TK", "4010", "Make-safe", "Weekly tarp kit hire"), ("R07", r"delivery and installation|installation labour|collection and dismantle", "DEL-FP", "4020", "Fencing", "Delivery, install and collection"), ("R08", r"scaffold (erect|dismantle)", "LAB-ST", "4020", "Scaffolding", "Erect and dismantle labour"), ("R09", r"make safe|tarp supply", "MS", "4020", "Make-safe", "Call-outs and tarping"), ("R10", r"roof repair labour", "LAB-RF", "4020", "Roofing", "Roof repair labour"), ("R11", r"^materials -|waste removal|tip fees", "MAT-RF", "4030", "Roofing", "Materials, waste and tip fees")] SVC_OF_JOB = {"Fencing": "Fencing", "Fencing (event)": "Fencing", "Scaffolding": "Scaffolding", "Make-safe": "Make-safe", "Roof repair": "Roofing"} out = []; review = []; tax = [] for l in lines: d = l["description"].lower(); hit = next((r for r in RULES if re.search(r[1], d)), None) amt = float(l["line_amount"]) if hit and hit[2] != "MIXED": rule, cat, acct, svc, conf = hit[0], hit[2], hit[3], hit[4], "Rule" elif l["reference"] in sites: rule, cat, acct, svc, conf = ("R12", "CONTEXT", "", SVC_OF_JOB[sites[l["reference"]]["job_type"]], "Context: service line from the site, account needs a person") if hit: rule = "R00" else: rule, cat, acct, svc, conf = (hit[0] if hit else "--"), "REVIEW", "", "", "No rule and no site reference" exp_tax = "BAS Excluded" if cat == "DEP" else "GST on Income" void = l["status"] == "VOIDED" o = dict(line_id=l["line_id"], invoice_no=l["invoice_no"], date=l["date"], month=l["date"][:7], contact_id=l["contact_id"], master_id=master[l["contact_id"]], reference=l["reference"], description=l["description"], amount=amt, status=l["status"], old_account=l["account_code"], tax_type=l["tax_type"], rule=rule, category=cat, new_account=acct, service_line=svc, expected_tax=exp_tax if cat not in ("CONTEXT", "REVIEW") else "GST on Income", counted=0 if void else 1) out.append(o) if not void and cat in ("CONTEXT", "REVIEW"): review.append(dict(line_id=o["line_id"], invoice_no=o["invoice_no"], date=o["date"], contact=next(c["name"] for c in contacts if c["contact_id"] == o["master_id"]), reference=o["reference"], description=o["description"], amount=amt, proposed_service=svc, proposed_account="", reason=conf, owner="Owner" if cat == "REVIEW" else "Bookkeeper")) if not void and l["tax_type"] != o["expected_tax"]: if o["expected_tax"] == "GST on Income": imp = round(amt * .10, 2); kind = "GST not charged on a taxable sale" else: imp = round(-amt * .10, 2); kind = "GST charged on a refundable deposit" tax.append(dict(line_id=o["line_id"], invoice_no=o["invoice_no"], date=o["date"], description=o["description"], amount=amt, coded_tax=l["tax_type"], expected_tax=o["expected_tax"], gst_effect=imp, issue=kind, bas_quarter=f"FY26 Q{(int(o['month'][5:]) - 7) % 12 // 3 + 1}")) W("lines_coded.csv", out); W("review_queue.csv", review); W("tax_exceptions.csv", tax) W("rules.csv", [dict(rule=r[0], pattern=r[1], category=r[2], account=r[3], service_line=r[4], why=r[5]) for r in RULES] + [dict(rule="R12", pattern="(no rule) and the invoice Reference is a known site", category="CONTEXT", account="", service_line="from site job type", why="Service line known, account confirmed by a person")]) # ------------------------------------------------------------------ 3 recode journals (one manual journal per month, dated month end) jr = defaultdict(float) for o in out: if not o["counted"] or o["category"] in ("CONTEXT", "REVIEW"): continue jr[(o["month"], o["old_account"], o["new_account"], o["service_line"])] += o["amount"] journ = [] for (m, a, b, s), v in sorted(jr.items()): y, mo = map(int, m.split("-")); me = (dt.date(y + (mo == 12), mo % 12 + 1, 1) - dt.timedelta(1)).isoformat() jn = f"MJ-{m.replace('-', '')}" journ += [dict(journal=jn, date=me, account=a, tracking="", debit=round(v, 2) if v > 0 else 0, credit=round(-v, 2) if v < 0 else 0, narration=f"Reclass {m} income from {a} to {b} ({s})"), dict(journal=jn, date=me, account=b, tracking=s if b != "835" else "", debit=round(-v, 2) if v < 0 else 0, credit=round(v, 2) if v > 0 else 0, narration=f"Reclass {m} income from {a} to {b} ({s})")] W("recode_journals.csv", journ) # ------------------------------------------------------------------ 4 asset log CLS = {"fp": "FP", "fence panel": "FP", "panels": "FP", "fb": "FB", "base": "FB", "fence bases": "FB", "br": "BR", "barrier": "BR", "st": "ST", "scaffold tower": "ST", "tk": "TK", "tarp kit": "TK"} def site_of(s): m = re.match(r"\s*[sS]?0*(\d{1,4})\b", s); return f"S{int(m.group(1)):04d}" if m else "" seen = set(); clean = []; issues = [] for i, r in enumerate(log): sid = site_of(r["site"]); cls = CLS[r["asset"].strip().lower()]; key = (r["docket"], cls, r["qty"], r["direction"]) fix = "" if r["site"] == sid else "Site text normalised" if key in seen: issues.append(dict(docket=r["docket"], date=r["date"], site=sid, asset=cls, qty=int(r["qty"]), direction=r["direction"], issue="DUPLICATE", action="Removed: same docket entered twice", status="Fixed")); continue seen.add(key); clean.append(dict(docket=r["docket"], date=r["date"], site_text=r["site"], site=sid, asset=cls, qty=int(r["qty"]), direction=r["direction"], fix=fix, entered_by=r["entered_by"])) # collection invoices tell us when a site was cleared coll = defaultdict(list) for l in lines: if l["status"] != "VOIDED" and re.search(r"collection and dismantle|scaffold dismantle", l["description"].lower()) and l["reference"] in sites: coll[l["reference"]].append(D(l["date"])) def balances(rows): b = defaultdict(int) for r in rows: if D(r["date"]) <= FY1: b[(r["site"], r["asset"])] += r["qty"] if r["direction"] == "OUT" else -r["qty"] return b bal = balances(clean) def transposed(a, b): a, b = a[1:], b[1:]; d = [i for i in range(4) if a[i] != b[i]] return len(d) == 2 and d[1] == d[0] + 1 and a[d[0]] == b[d[1]] and a[d[1]] == b[d[0]] # pass 0: a return dated before the site's job even started cannot belong to that site for r in clean: if r["direction"] != "IN" or D(r["date"]) >= D(sites[r["site"]]["start"]): continue w, c, q = r["site"], r["asset"], r["qty"] cands = [s for s in sites if transposed(s, w) and bal.get((s, c), 0) >= q and D(sites[s]["start"]) <= D(r["date"])] if len(cands) == 1: s = cands[0]; issues.append(dict(docket=r["docket"], date=r["date"], site=w, asset=c, qty=q, direction="IN", issue="WRONG SITE", action=f"Moved to {s}: return dated before {w} started; {s} (digits swapped) had units on hire", status="Fixed")) bal[(w, c)] += q; bal[(s, c)] -= q; r["site"] = s; r["fix"] = f"Site corrected from {w} (return before the job started)" else: issues.append(dict(docket=r["docket"], date=r["date"], site=w, asset=c, qty=q, direction="IN", issue="RETURN BEFORE START", action="Check the docket with the yard", status="Open")) # a return written against the wrong site: the real site was collected (collection invoiced within 2 days) but still shows units, # and exactly one return docket of that size, dated that day, sits on another site for s in sorted(sites): for c in ("FP", "FB", "BR", "ST", "TK"): q = bal.get((s, c), 0) if q <= 0 or not coll.get(s): continue cands = [r for r in clean if r["site"] != s and r["asset"] == c and r["direction"] == "IN" and r["qty"] == q and any(0 <= (cd - D(r["date"])).days <= 2 for cd in coll[s])] if len(cands) == 1: r = cands[0]; w = r["site"] issues.append(dict(docket=r["docket"], date=r["date"], site=w, asset=c, qty=q, direction="IN", issue="WRONG SITE", action=f"Moved to {s}: {s} was collected that day with exactly {q} units still on the ledger", status="Fixed")) bal[(w, c)] += q; bal[(s, c)] -= q; r["site"] = s; r["fix"] = f"Site corrected from {w} (return written against the wrong site)" # second pass: a site below zero and a site with exactly that many units left over, whose numbers differ by two swapped digits for (w, c), q in sorted(bal.items()): if q >= 0: continue cands = [s for (s, c2), v in bal.items() if c2 == c and v == -q and transposed(s, w)] if len(cands) == 1: s = cands[0]; moved = [r for r in clean if r["site"] == w and r["asset"] == c and r["direction"] == "IN"] need = -q for r in sorted(moved, key=lambda r: r["date"], reverse=True): if need <= 0: break if r["qty"] <= need: issues.append(dict(docket=r["docket"], date=r["date"], site=w, asset=c, qty=r["qty"], direction="IN", issue="WRONG SITE", action=f"Moved to {s}: {w} fell below zero by {-q} and {s} (digits swapped) had exactly {-q} left over; confirm with the yard", status="Fixed")) r["site"] = s; r["fix"] = f"Site corrected from {w} (digits swapped)"; need -= r["qty"] bal[(w, c)] += -q - need; bal[(s, c)] -= -q - need for (s, c), q in sorted(bal.items()): if q < 0: issues.append(dict(docket="", date="", site=s, asset=c, qty=q, direction="", issue="NEGATIVE BALANCE", action="Check dockets with the yard", status="Open")) for r in clean: if r["direction"] == "LOST": pass closed = {s for s in sites if coll.get(s)} pos = [] for (s, c), q in sorted(bal.items()): if q == 0: continue st = "Closed site: units not returned" if s in closed and c != "FB" and q > 0 else "On hire" if s in closed and q > 0: issues.append(dict(docket="", date="", site=s, asset=c, qty=q, direction="", issue="NOT RETURNED", action=f"Site collected but {q} {c} still on the ledger: confirm with the customer, then charge as lost ({LOSS_RATE:.2f} each) or write off", status="Open")) pos.append(dict(site=s, customer=sites[s]["customer_name"], job_type=sites[s]["job_type"], asset=c, units=q, status=st)) W("asset_ledger_clean.csv", clean); W("asset_log_issues.csv", issues); W("asset_position.csv", pos) lost = Counter(); lost_pre = Counter() for r in clean: if r["direction"] == "LOST" and FY0 <= D(r["date"]) <= FY1: lost[r["asset"]] += r["qty"] if r["direction"] == "LOST" and D(r["date"]) < FY0: lost_pre[r["asset"]] += r["qty"] fr = [] for c in ["FP", "FB", "BR", "ST", "TK"]: bought = sum(int(p["qty"]) for p in purch if p["asset"] == c) onh = sum(p["units"] for p in pos if p["asset"] == c and p["status"] == "On hire"); nr = sum(p["units"] for p in pos if p["asset"] == c and p["status"] != "On hire") var = fleet[c] - lost_pre[c] + bought - lost[c] - stock[c] - onh - nr fr.append(dict(asset=c, register_start=fleet[c], lost_before_fy_not_written_off=lost_pre[c], bought=bought, lost_billed=lost[c], owned_end=fleet[c] - lost_pre[c] + bought - lost[c], yard_count=stock[c], on_hire=onh, not_returned=nr, variance=var)) if lost_pre[c]: issues.append(dict(docket="", date="2025-06-30", site="REGISTER", asset=c, qty=lost_pre[c], direction="", issue="REGISTER NOT UPDATED", action=f"{lost_pre[c]} {c} billed as lost before 1 Jul 2025 are still on the fleet register: write them off the register", status="Fixed")) if var: issues.append(dict(docket="", date="2026-06-30", site="YARD", asset=c, qty=var, direction="", issue="STOCKTAKE VARIANCE", action="Recount the yard; if confirmed, write off", status="Open")) W("fleet_reconciliation.csv", fr); W("asset_log_issues.csv", issues) # ------------------------------------------------------------------ 5 hire billing check moves = defaultdict(list) for r in clean: if r["direction"] in ("OUT", "IN") and D(r["date"]) <= dt.date(2026, 12, 31): moves[(r["site"], r["asset"])].append((D(r["date"]), r["qty"] if r["direction"] == "OUT" else -r["qty"])) billed_u = defaultdict(int); billed_v = defaultdict(float); credits = defaultdict(float) for o in out: if not o["counted"] or not o["category"].startswith("HIRE-"): continue k = (o["reference"], o["category"][5:]) if o["amount"] < 0: credits[k] += o["amount"] else: billed_u[k] += int(next(l["quantity"] for l in lines if l["line_id"] == o["line_id"])); billed_v[k] += o["amount"] hb = []; cyc = []; open_hire = {} for (s, c), evs in sorted(moves.items()): if c not in RATE: continue evs.sort(); start = D(sites[s]["start"]); fin = s in closed or sum(q for _, q in evs) <= 0 end = max(d for d, q in evs if q < 0) if fin and any(q < 0 for _, q in evs) else None weeks = []; k = 0 while True: ws = start + dt.timedelta(7 * k); we = ws + dt.timedelta(6) if end and ws > end: break if not end and ws > dt.date(2026, 12, 31): break b = sum(q for d, q in evs if d < ws); mx = b for d, q in evs: if ws <= d <= we: b += q; mx = max(mx, b) if mx <= 0 and ws > evs[0][0]: break weeks.append((ws, mx)); k += 1 if end and we >= end: break due = 0; acc = 0 for i in range(0, len(weeks), 4): cy = weeks[i:i + 4]; cend = min(cy[-1][0] + dt.timedelta(6), end) if end else cy[-1][0] + dt.timedelta(6); idate = cend + dt.timedelta(1) u = sum(q for _, q in cy) if FY0 <= idate <= FY1: due += u; cyc.append(dict(site=s, asset=c, cycle_start=cy[0][0].isoformat(), invoice_due=idate.isoformat(), weeks=len(cy), billable_units=u)) elif idate > FY1 and cy[0][0] <= FY1: acc += sum(q for ws, q in cy if ws <= FY1); open_hire[(s, c)] = (cy[0][0], idate, cy[-1][1]) bu = billed_u[(s, c)] if due == 0 and bu == 0 and acc == 0: continue gap = due - bu hb.append(dict(site=s, customer=sites[s]["customer_name"], asset=c, rate=RATE[c], billable_units=due, billed_units=bu, unit_gap=gap, gap_value=round(gap * RATE[c], 2), credits=round(credits[(s, c)], 2), accrued_units_at_30jun=acc, finding="Under-billed" if gap > 0 else "Over-billed" if gap < 0 else "Agrees")) W("hire_billing_check.csv", hb); W("hire_cycles.csv", cyc) # ------------------------------------------------------------------ 6 Make S2 simulation: billing run on 1-31 Jul 2026 from the Airtable on-hire table def billing_run(existing): drafts = [] for (s, c), (cs_, idate, units) in sorted(open_hire.items()): key = f"{s}|{c}|{cs_.isoformat()}" if key in existing or not (dt.date(2026, 7, 1) <= idate <= dt.date(2026, 7, 31)): continue u = bal.get((s, c), 0) if u <= 0: continue existing.add(key); drafts.append(dict(key=key, site=s, customer=sites[s]["customer_name"], asset=c, invoice_date=idate.isoformat(), weeks=4, units_on_hire=u, unit_weeks=u * 4, rate=RATE[c], amount=round(u * 4 * RATE[c], 2), status="DRAFT")) return drafts keys = set(); run1 = billing_run(keys); run2 = billing_run(keys) W("make_s2_july_run.csv", run1, head=["key", "site", "customer", "asset", "invoice_date", "weeks", "units_on_hire", "unit_weeks", "rate", "amount", "status"]) # ------------------------------------------------------------------ 7 regression test against the generator's answer key T = {r["line_id"]: r for r in rd("truth/truth_lines.csv")}; tests = {} ok = sum(1 for o in out if (o["category"] == T[o["line_id"]]["category"]) or (T[o["line_id"]]["category"] == "UNCLEAR" and o["category"] in ("CONTEXT", "REVIEW"))) tests["lines"] = dict(items=len(out), agree=ok) dt_ = {r["contact_id"]: r["master"] for r in rd("truth/dup_truth.csv")}; ents = rd("truth/entities.csv") ent_of = {c: m for c, m in dt_.items()}; ent_of.update({m: m for m in dt_.values()}) tests["dups"] = dict(items=len(dt_), agree=sum(1 for c, m in dt_.items() if master[c] == master[m]), false_merges=sum(1 for c in contacts if master[c["contact_id"]] != c["contact_id"] and ent_of.get(c["contact_id"], c["contact_id"]) != ent_of.get(master[c["contact_id"]], master[c["contact_id"]]))) tt = {o["line_id"]: o for o in out} tests["tax"] = dict(items=sum(1 for r in T.values() if r["tax_ok"] == "0" and tt[r["line_id"]]["counted"]), agree=sum(1 for r in T.values() if r["tax_ok"] == "0" and tt[r["line_id"]]["counted"] and any(x["line_id"] == r["line_id"] for x in tax)), false_flags=sum(1 for x in tax if T[x["line_id"]]["tax_ok"] == "1")) tl = rd("truth/truth_log.csv"); seeded = [r for r in tl if r["issue"]] fixed_d = {(i["docket"], i["issue"]) for i in issues} written = {r["docket"]: site_of(r["site"]) for r in log}; neg = {(i["site"], i["asset"]) for i in issues if i["issue"] == "NEGATIVE BALANCE"}; nret = {(i["site"], i["asset"]) for i in issues if i["issue"] == "NOT RETURNED"} tests["log"] = dict(items=len(seeded), fixed=sum(1 for r in seeded if (r["docket"], {"DUP": "DUPLICATE", "WRONGSITE": "WRONG SITE"}[r["issue"]]) in fixed_d), agree=sum(1 for r in seeded if (r["docket"], {"DUP": "DUPLICATE", "WRONGSITE": "WRONG SITE"}[r["issue"]]) in fixed_d or (r["issue"] == "WRONGSITE" and ((written[r["docket"]], r["cls"]) in neg or (r["site"], r["cls"]) in nret)))) tc = defaultdict(lambda: [0, 0]) voided = {l["line_id"] for l in lines if l["status"] == "VOIDED"} for r in rd("truth/cycles_truth.csv"): if r["in_fy"] == "1" and r["weeks"] != "0": tc[(r["site"], r["cls"])][0] += int(r["true_units"]) hbk = {(h["site"], h["asset"]): h for h in hb} openx = {(i["site"], i["asset"]) for i in issues if i["status"] == "Open"} tests["billing"] = dict(items=sum(1 for k in tc if k not in openx), agree=sum(1 for k, v in tc.items() if k not in openx and hbk.get(k, {}).get("billable_units", 0) == v[0]), excluded_open_exception=sum(1 for k in tc if k in openx)) TF = json.load(open(os.path.join(HERE, "truth/fleet.json"))) tests["fleet"] = dict(items=len(fr), agree=sum(1 for f in fr if f["variance"] == -TF["count_err"].get(f["asset"], 0) and f["owned_end"] == TF["fleet_owned"][f["asset"]] - TF["lost_before_fy"].get(f["asset"], 0) - f["lost_billed"])) tests["make_idempotent"] = dict(items=len(run1), second_run_new=len(run2)) json.dump(tests, open(os.path.join(HERE, "results/engine_test.json"), "w"), indent=1) print(json.dumps(tests)); print("review", len(review), "tax", len(tax), "issues", Counter(i["issue"] for i in issues), "hb", Counter(h["finding"] for h in hb), "drafts", len(run1)) print("merges", sum(1 for c in cmap if c["action"] == "Merge into master"), Counter(c["rule"] for c in cmap)) print(fr)