Files
erp/.claude/skills/arcodange-bank-reco/scripts/bank-match.sh
Gabriel Radureau 4b17e5f22c feat(bank-match): exact tx-id match pass (consumes payment transaction_id)
The consumer side of erp#26/#27: now that a règlement stores its originating bank
transaction id (transaction_id -> llx_bank.num_chq), bank-match uses it.

New PASS 0 (exact), highest priority, before wire-ref and amt+date:
- carry each feed movement's own id (Qonto transaction id; Wise activity + transfer
  resource id) as feed_ids, and each Dolibarr payment's num.
- match when a payment's num equals a feed id. Tagged [tx-id].
- DATE-WINDOW-INDEPENDENT — the id is proof, so it pairs movements whose bank
  settlement and Dolibarr saisie are weeks apart (which amt+date would miss).
Pass 0 runs before the ref index is built, so its matches are excluded from the
later passes (no double-match).

Fixture-proven: a payment dated 15d off the bank movement (outside the ±7d window)
matches via [tx-id] when num carries the Qonto id, and correctly does NOT match
when num is empty.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-06-30 00:39:21 +02:00

387 lines
19 KiB
Bash
Executable File

#!/usr/bin/env bash
# Match bank movements (Qonto + Wise) against Dolibarr payments.
#
# Usage:
# bank-match.sh [--month YYYY-MM | --since YYYY-MM-DD --until YYYY-MM-DD]
# [--window-days N] # date tolerance, default 7
# [--include-fees] # include Wise cashback / charges (default off)
#
# Output: three buckets
# - MATCHED bank movement ↔ Dolibarr payment
# - BANK-ONLY bank movement with no Dolibarr counterpart (potential
# missing supplier invoice or unrecorded incoming payment)
# - DOLIBARR-ONLY Dolibarr payment with no bank movement (timing or error)
#
# Internal Wise↔Qonto consolidations (e.g. 5000 € moved Wise→Qonto same day)
# are auto-detected and excluded from matching against Dolibarr.
#
# Exit 0 if everything in the window matches cleanly, 1 if there's any bank-only
# or dolibarr-only entry.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BANK_CURL="${SCRIPT_DIR}/bank-curl.sh"
DOL_CURL="${SCRIPT_DIR}/../../dolibarr/scripts/dol-curl.sh"
SINCE=""; UNTIL=""; MONTH=""; WINDOW=7; INCLUDE_FEES=0; ENRICH=0
while [[ $# -gt 0 ]]; do
case "$1" in
--since) SINCE="$2"; shift 2 ;;
--until) UNTIL="$2"; shift 2 ;;
--month) MONTH="$2"; shift 2 ;;
--window-days) WINDOW="$2"; shift 2 ;;
--include-fees) INCLUDE_FEES=1; shift ;;
--enrich) ENRICH=1; shift ;;
-h|--help) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
*) echo "bank-match.sh: unknown arg: $1" >&2; exit 2 ;;
esac
done
if [[ -n "${MONTH}" ]]; then
SINCE="${MONTH}-01"
UNTIL="$(python3 -c "import calendar; y,m=map(int,'${MONTH}'.split('-')); print(f'{y:04d}-{m:02d}-{calendar.monthrange(y,m)[1]:02d}')")"
fi
[[ -z "${SINCE}" ]] && SINCE="$(python3 -c "import datetime; print((datetime.date.today()-datetime.timedelta(days=365)).strftime('%Y-%m-%d'))")"
[[ -z "${UNTIL}" ]] && UNTIL="$(python3 -c "import datetime; print(datetime.date.today().strftime('%Y-%m-%d'))")"
set -a; source "${SCRIPT_DIR}/../../dolibarr/.env"; set +a
: "${WISE_PROFILE_ID:?bank-match.sh: WISE_PROFILE_ID not set}"
WORK="$(mktemp -d -t bankmatch.XXXXXX)"
trap 'rm -rf "${WORK}"' EXIT
# --- 1. Pull Qonto transactions ---
TMP_ORG=$(mktemp -t qontoorg.XXXXXX.json)
"${BANK_CURL}" qonto /v2/organization > "${TMP_ORG}"
QONTO_ACCT=$(python3 -c "
import json, sys
d = json.load(open(sys.argv[1]))
accs = (d.get('organization') or {}).get('bank_accounts') or []
print([a for a in accs if a.get('status')=='active'][0]['id'])" "${TMP_ORG}")
rm -f "${TMP_ORG}"
QURL="/v2/transactions?bank_account_id=${QONTO_ACCT}&settled_at_from=${SINCE}T00:00:00Z&settled_at_to=${UNTIL}T23:59:59Z&per_page=100"
"${BANK_CURL}" qonto "${QURL}" > "${WORK}/qonto.json"
# --- 2. Pull Wise activities ---
"${BANK_CURL}" wise "/v1/profiles/${WISE_PROFILE_ID}/activities?size=100&since=${SINCE}T00:00:00.000Z&until=${UNTIL}T23:59:59.999Z" > "${WORK}/wise.json"
# --- 3. Pull Dolibarr customer + supplier invoices, payments, and bank accounts ---
"${DOL_CURL}" '/invoices?limit=500&sortfield=t.datef&sortorder=ASC' > "${WORK}/dol_inv.json"
"${DOL_CURL}" '/supplierinvoices?limit=500' > "${WORK}/dol_sup.json"
"${DOL_CURL}" '/bankaccounts' > "${WORK}/dol_acct.json"
mkdir -p "${WORK}/dol_pay" "${WORK}/dol_supay"
for id in $(python3 -c "import json,sys; print(' '.join(str(r['id']) for r in json.load(open(sys.argv[1])) if r.get('id')))" "${WORK}/dol_inv.json"); do
"${DOL_CURL}" "/invoices/${id}/payments" > "${WORK}/dol_pay/${id}.json" 2>/dev/null || echo "[]" > "${WORK}/dol_pay/${id}.json"
done
for id in $(python3 -c "import json,sys; print(' '.join(str(r['id']) for r in json.load(open(sys.argv[1])) if r.get('id')))" "${WORK}/dol_sup.json"); do
"${DOL_CURL}" "/supplierinvoices/${id}/payments" > "${WORK}/dol_supay/${id}.json" 2>/dev/null || echo "[]" > "${WORK}/dol_supay/${id}.json"
done
# --- 3b. Optional: enrich Wise TRANSFER activities with wire references ---
if [[ "${ENRICH}" == "1" ]]; then
mkdir -p "${WORK}/wise_refs"
for tid in $(python3 -c "
import json, sys
acts = json.load(open(sys.argv[1])).get('activities') or []
for a in acts:
r = a.get('resource') or {}
if r.get('type')=='TRANSFER' and r.get('id'): print(r['id'])
" "${WORK}/wise.json"); do
"${BANK_CURL}" wise "/v1/transfers/${tid}" > "${WORK}/wise_refs/${tid}.json" 2>/dev/null || true
done
fi
# --- 4. Match in python ---
PATTERNS_FILE="${SCRIPT_DIR}/../known-patterns.json"
python3 - "${WORK}" "${SINCE}" "${UNTIL}" "${WINDOW}" "${INCLUDE_FEES}" "${PATTERNS_FILE}" "${ENRICH}" <<'PY'
import json, sys, os, re, datetime, collections
work, since, until, window_days, include_fees, patterns_file, enrich = sys.argv[1:8]
window = int(window_days); include_fees = include_fees == "1"; enrich = enrich == "1"
since_d = datetime.date.fromisoformat(since); until_d = datetime.date.fromisoformat(until)
def strip(s): return re.sub(r'<[^>]+>', '', s or '').strip()
# 4a. Normalize Qonto
qonto_movs = []
for t in (json.load(open(os.path.join(work,"qonto.json"))).get("transactions") or []):
dt = datetime.date.fromisoformat((t.get("settled_at") or "")[:10])
if dt < since_d or dt > until_d: continue
amt = float(t.get("amount") or 0)
sign = "+" if t.get("side") == "credit" else "-"
label = t.get("label") or t.get("operation_type") or "-"
# feed_ids: the Qonto transaction's own id — exact-match key against a Dolibarr
# payment whose stored num (num_chq) is that id (set via payment transaction_id).
feed_ids = [str(t["id"])] if t.get("id") else []
qonto_movs.append({"bank":"Qonto", "date":dt, "sign":sign, "amount":amt, "label":label[:40], "op":t.get("operation_type",""), "feed_ids":feed_ids, "matched_dol":None, "matched_internal":False})
# 4b. Normalize Wise
wise_movs = []
for a in (json.load(open(os.path.join(work,"wise.json"))).get("activities") or []):
dt = datetime.date.fromisoformat((a.get("createdOn") or "")[:10])
if dt < since_d or dt > until_d: continue
typ = a.get("type","-")
if not include_fees and typ in ("BALANCE_CASHBACK", "BALANCE_INTEREST"):
continue
pa = strip(a.get("primaryAmount") or "")
sign = "+" if pa.startswith("+") else "-"
m = re.search(r'([\d,.]+)\s*([A-Z]{3})', pa)
amt = float(m.group(1).replace(",", "")) if m else 0.0
title = strip(a.get("title") or "")[:40]
res = a.get("resource") or {}
resource_id = str(res.get("id")) if res.get("type") == "TRANSFER" else None
# feed_ids: the Wise activity id and (for transfers) the resource id — either
# may have been stored as the payment's transaction_id; match against both.
feed_ids = []
if a.get("id"): feed_ids.append(str(a["id"]))
if resource_id: feed_ids.append(resource_id)
wise_movs.append({"bank":"Wise", "date":dt, "sign":sign, "amount":amt, "label":title, "op":typ, "feed_ids":feed_ids, "matched_dol":None, "matched_internal":False, "wise_resource_id":resource_id, "wire_ref":""})
# 4b'. If --enrich, load per-transfer wire references and attach to Wise movs
if enrich:
ref_dir = os.path.join(work, "wise_refs")
if os.path.isdir(ref_dir):
for m in wise_movs:
if not m["wise_resource_id"]: continue
p = os.path.join(ref_dir, f"{m['wise_resource_id']}.json")
if not os.path.isfile(p): continue
try:
t = json.load(open(p))
m["wire_ref"] = (t.get("reference") or "")
except Exception: pass
bank_movs = qonto_movs + wise_movs
# 4c. Detect internal Wise<->Qonto consolidations: same date, equal amount, opposite signs, one Wise + one Qonto
for w in [m for m in bank_movs if m["bank"]=="Wise" and m["sign"]=="-"]:
for q in [m for m in bank_movs if m["bank"]=="Qonto" and m["sign"]=="+" and not m["matched_internal"]]:
if abs(w["amount"] - q["amount"]) < 0.01 and abs((w["date"] - q["date"]).days) <= 3:
w["matched_internal"] = q; q["matched_internal"] = w
break
# 4d. Normalize Dolibarr payments — carry socid too for avoir netting
dol_pays = []
inv_by_id = {str(r["id"]): r for r in json.load(open(os.path.join(work,"dol_inv.json")))}
for fn in os.listdir(os.path.join(work,"dol_pay")):
iid = fn[:-5]; inv = inv_by_id.get(iid)
if not inv: continue
for p in json.load(open(os.path.join(work,"dol_pay",fn))):
d = datetime.datetime.strptime(p["date"], "%Y-%m-%d %H:%M:%S").date()
if d < since_d or d > until_d: continue
amt = float(p.get("amount") or 0)
dol_pays.append({"side":"customer", "ref":inv["ref"], "date":d, "amount":amt,
"fk_account":inv.get("fk_account"), "socid":inv.get("socid"),
"num": (p.get("num") or ""),
"matched_bank":None, "netted_against":None})
sup_by_id = {str(r["id"]): r for r in json.load(open(os.path.join(work,"dol_sup.json")))}
for fn in os.listdir(os.path.join(work,"dol_supay")):
iid = fn[:-5]; sup = sup_by_id.get(iid)
if not sup: continue
for p in json.load(open(os.path.join(work,"dol_supay",fn))):
d = datetime.datetime.strptime(p["date"], "%Y-%m-%d %H:%M:%S").date()
if d < since_d or d > until_d: continue
amt = float(p.get("amount") or 0)
dol_pays.append({"side":"supplier", "ref":sup["ref"], "date":d, "amount":amt,
"fk_account":sup.get("fk_account"), "socid":sup.get("socid"),
"num": (p.get("num") or ""),
"matched_bank":None, "netted_against":None})
# 4d.1. AVOIR cycle netting: an AVC (credit note) for -X on socid S cancels out
# a FAC for +X on the same socid, within a small date window. Bank sees the NET
# of the cycle (typically +X for the reissued FAC with the new ref scheme).
# Pair an AVC with a FAC of opposite sign + equal abs(amount) + same socid +
# within ±5d. Mark both as "netted" so they're excluded from matching and
# excluded from the dolibarr-only failure count.
avcs = [p for p in dol_pays if p["side"]=="customer" and p["ref"].startswith("AVC") and p["amount"] < 0]
for avc in avcs:
candidates = [p for p in dol_pays
if p is not avc
and p["side"]=="customer"
and p["socid"] == avc["socid"]
and abs(p["amount"] + avc["amount"]) < 0.01 # opposite signs equal magnitude
and abs((p["date"] - avc["date"]).days) <= 5
and p["netted_against"] is None
and p["matched_bank"] is None]
if candidates:
# Prefer the OLDEST (the original cancelled FAC), not the reissue.
# Heuristic: refs with shorter / older numbering scheme. If multiple,
# pick smallest date delta.
candidates.sort(key=lambda p: (abs((p["date"] - avc["date"]).days), p["ref"]))
partner = candidates[0]
avc["netted_against"] = partner["ref"]
partner["netted_against"] = avc["ref"]
# 4e. Match — three-pass, highest confidence first:
# PASS 0 (exact) : a Dolibarr payment whose stored num (num_chq = the
# transaction_id recorded with the règlement) equals the feed
# transaction's own id. Date-window-independent — the id is proof.
# PASS 1 (strong) : Wise transfers with an --enrich'd wire reference containing
# a "FAC***" pattern match the Dolibarr invoice with that ref.
# PASS 2 (loose) : remaining bank movements use the date+amount heuristic.
# Netted Dolibarr entries (avoir cycle) are excluded from all passes.
# Pass 0: exact match on the feed transaction id stored as the payment's num
# (num_chq = the transaction_id recorded with the règlement). Date-independent.
for m in [x for x in bank_movs if not x["matched_internal"] and not x["matched_dol"] and x.get("feed_ids")]:
fids = set(m["feed_ids"])
for p in dol_pays:
if (p["matched_bank"] is None and p["netted_against"] is None
and p.get("num") and str(p["num"]) in fids):
m["matched_dol"] = p; m["match_kind"] = "tx-id"
p["matched_bank"] = m
break
# Build customer ref -> dol payment index (only un-netted, un-matched entries)
ref_index = collections.defaultdict(list)
for p in dol_pays:
if p["matched_bank"] is None and p["netted_against"] is None:
# Strip trailing dash/suffix variants — FAC002CL0001002 vs FAC002-CL0001002 are equivalent
normalized = re.sub(r'[^A-Z0-9]', '', p["ref"].upper())
ref_index[normalized].append(p)
# Pass 1: strong match on wire references
for m in [x for x in bank_movs if not x["matched_internal"] and x.get("wire_ref")]:
refs_in_wire = re.findall(r'FAC\d+(?:CL\d+)?', (m["wire_ref"] or "").upper().replace("-",""))
for r in refs_in_wire:
if r in ref_index and ref_index[r]:
p = ref_index[r].pop(0)
m["matched_dol"] = p; m["match_kind"] = "wire-ref"
p["matched_bank"] = m
break
# Pass 2: loose date+amount match for remaining bank movements
for m in [x for x in bank_movs if not x["matched_internal"] and not x["matched_dol"]]:
candidates = [p for p in dol_pays
if p["matched_bank"] is None and p["netted_against"] is None
and abs(p["amount"] - m["amount"]) < 0.01
and abs((p["date"] - m["date"]).days) <= window]
if candidates:
candidates.sort(key=lambda p: abs((p["date"] - m["date"]).days))
p = candidates[0]
m["matched_dol"] = p; m["match_kind"] = "amt+date"
p["matched_bank"] = m
# 4f. Annotate non-matched movements with known-patterns catalog
patterns = []
if os.path.isfile(patterns_file):
try: patterns = json.load(open(patterns_file)).get("patterns", [])
except Exception as e: print(f" /!\\ failed to load {patterns_file}: {e}", file=sys.stderr)
def match_pattern(mov):
# Match against both the bank label AND the operation type — different
# banks surface useful info in different fields (Qonto puts the operation
# type in `op`, e.g. "qonto_fee"; Wise puts the activity type in `op`,
# e.g. "BALANCE_DEPOSIT", and the human title in `label`).
haystack = (mov.get("label") or "") + " | " + (mov.get("op") or "")
for pat in patterns:
if pat.get("bank") and pat["bank"] != mov["bank"].lower(): continue
if pat.get("side") and pat["side"] != ("credit" if mov["sign"]=="+" else "debit"): continue
amin = pat.get("amount_min"); amax = pat.get("amount_max")
if amin is not None and mov["amount"] < amin: continue
if amax is not None and mov["amount"] > amax: continue
if re.search(pat["pattern"], haystack, re.IGNORECASE):
return pat
return None
for m in bank_movs:
if m["matched_dol"] or m["matched_internal"]: continue
m["known"] = match_pattern(m)
# --- 5. Render ---
# Load Dolibarr bank accounts (for fk_account context on dolibarr-only)
dol_accts = {}
try:
for a in json.load(open(os.path.join(work, "dol_acct.json"))):
dol_accts[str(a["id"])] = {"ref": a.get("ref","-"), "label": a.get("label","-"), "country": a.get("country_code","")}
except Exception: pass
# Heuristic: which Dolibarr accounts are NOT covered by Qonto/Wise API today?
# Convention: CCA = Compte Courant d'Associé (personal). Anything not QON*/WIS*
# is treated as "API-invisible" and tagged as such.
def account_kind(fk_account):
if not fk_account: return ("unknown", "fk_account=None")
a = dol_accts.get(str(fk_account))
if not a: return ("unknown", f"fk_account={fk_account} (not in /bankaccounts)")
ref = (a["ref"] or "").upper()
if ref.startswith(("QON", "WIS")):
return ("api_tracked", f"{a['ref']} ({a['label']})")
return ("personal_or_other", f"{a['ref']} ({a['label']})")
def fmt_bank(m):
return f" {m['bank']:<5} {m['date']} {m['sign']:<2}{m['amount']:>9.2f} {m['op'][:18]:<18} {m['label']}"
print(f"# Bank reconciliation: {since} → {until} (window ±{window}d, fees: {'on' if include_fees else 'off'}, enrich: {'on' if enrich else 'off'})")
print()
matched = [m for m in bank_movs if m["matched_dol"]]
internal = [m for m in bank_movs if m["matched_internal"] and m["sign"]=="-"]
bank_only = [m for m in bank_movs if not m["matched_dol"] and not m["matched_internal"]]
netted_dol_pairs = [p for p in dol_pays if p["netted_against"]]
dol_only = [p for p in dol_pays if p["matched_bank"] is None and p["netted_against"] is None]
print(f"=== MATCHED ({len(matched)} bank ↔ Dolibarr) ===")
for m in sorted(matched, key=lambda m: m["date"]):
p = m["matched_dol"]
delta = (p["date"] - m["date"]).days
kind = m.get("match_kind", "?")
print(fmt_bank(m) + f" ↔[{kind}] {p['side']:<8} {p['ref']:<24} ({p['date']}, Δ{delta:+d}d)")
print()
print(f"=== INTERNAL (Wise↔Qonto consolidations, {len(internal)}) ===")
for m in sorted(internal, key=lambda m: m["date"]):
other = m["matched_internal"]
print(fmt_bank(m) + f" ↔ {other['bank']} {other['date']} {other['sign']}{other['amount']:.2f}")
print()
# Avoir cycles netted out (informational; bank correctly sees only the net)
if netted_dol_pairs:
print(f"=== AVOIR-NETTED ({len(netted_dol_pairs)} Dolibarr entries pairing AVC↔FAC cancellation cycles) ===")
for p in sorted(netted_dol_pairs, key=lambda p: (p["date"], p["ref"])):
print(f" {p['side']:<8} {p['date']} {p['amount']:>9.2f} {p['ref']:<24} ↔ netted against {p['netted_against']}")
print()
bank_known = [m for m in bank_only if m.get("known")]
bank_unknown = [m for m in bank_only if not m.get("known")]
print(f"=== BANK-ONLY — known patterns ({len(bank_known)}, intentional gaps documented in known-patterns.json) ===")
for m in sorted(bank_known, key=lambda m: m["date"]):
k = m["known"]
cls = k.get("classification","?")
print(fmt_bank(m) + f" [{cls}]")
print(f" └─ {k.get('note','')}")
print()
print(f"=== BANK-ONLY — unknown ({len(bank_unknown)}, NEEDS attention: missing supplier invoice / unrecorded payment / new pattern) ===")
for m in sorted(bank_unknown, key=lambda m: m["date"]):
print(fmt_bank(m))
print()
# Split dolibarr-only by whether the fk_account is API-tracked (real gap)
# or personal_or_other (expected gap — we have no API on those accounts)
dol_only_api = [p for p in dol_only if account_kind(p["fk_account"])[0] == "api_tracked"]
dol_only_personal = [p for p in dol_only if account_kind(p["fk_account"])[0] != "api_tracked"]
print(f"=== DOLIBARR-ONLY — on API-tracked accounts ({len(dol_only_api)}, REAL GAP: bank should have shown this) ===")
for p in sorted(dol_only_api, key=lambda p: p["date"]):
_, ctx = account_kind(p["fk_account"])
print(f" {p['side']:<8} {p['date']} {p['amount']:>9.2f} {p['ref']:<24} ({ctx})")
print()
print(f"=== DOLIBARR-ONLY — on accounts NOT in API scope ({len(dol_only_personal)}, expected gap: CCA1 perso etc.) ===")
for p in sorted(dol_only_personal, key=lambda p: p["date"]):
_, ctx = account_kind(p["fk_account"])
print(f" {p['side']:<8} {p['date']} {p['amount']:>9.2f} {p['ref']:<24} ({ctx})")
print()
# Verdict: only UNKNOWN bank-only AND dol-only-on-API-tracked count as failures.
# Avoir-netted pairs and personal-account dolibarr entries are intentional/expected.
fails = len(bank_unknown) + len(dol_only_api)
print("-" * 100)
print(f"# {len(matched)} matched, {len(internal)} internal, {len(netted_dol_pairs)} avoir-netted, {len(bank_known)} bank-known, {len(bank_unknown)} bank-UNKNOWN, {len(dol_only_api)} dol-only-API, {len(dol_only_personal)} dol-only-personal")
print(f"# patterns loaded from {patterns_file}: {len(patterns)} pattern(s)")
sys.exit(0 if fails == 0 else 1)
PY