|
|
|
|
@@ -24,7 +24,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
|
BANK_CURL="${SCRIPT_DIR}/bank-curl.sh"
|
|
|
|
|
DOL_CURL="${SCRIPT_DIR}/../../dolibarr/scripts/dol-curl.sh"
|
|
|
|
|
|
|
|
|
|
SINCE=""; UNTIL=""; MONTH=""; WINDOW=7; INCLUDE_FEES=0
|
|
|
|
|
SINCE=""; UNTIL=""; MONTH=""; WINDOW=7; INCLUDE_FEES=0; ENRICH=0
|
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
|
|
|
case "$1" in
|
|
|
|
|
--since) SINCE="$2"; shift 2 ;;
|
|
|
|
|
@@ -32,6 +32,7 @@ while [[ $# -gt 0 ]]; do
|
|
|
|
|
--month) MONTH="$2"; shift 2 ;;
|
|
|
|
|
--window-days) WINDOW="$2"; shift 2 ;;
|
|
|
|
|
--include-fees) INCLUDE_FEES=1; shift ;;
|
|
|
|
|
--enrich) ENRICH=1; shift ;;
|
|
|
|
|
-h|--help) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
|
|
|
|
*) echo "bank-match.sh: unknown arg: $1" >&2; exit 2 ;;
|
|
|
|
|
esac
|
|
|
|
|
@@ -66,9 +67,10 @@ QURL="/v2/transactions?bank_account_id=${QONTO_ACCT}&settled_at_from=${SINCE}T00
|
|
|
|
|
# --- 2. Pull Wise activities ---
|
|
|
|
|
"${BANK_CURL}" wise "/v1/profiles/${WISE_PROFILE_ID}/activities?size=100&since=${SINCE}T00:00:00.000Z&until=${UNTIL}T23:59:59.999Z" > "${WORK}/wise.json"
|
|
|
|
|
|
|
|
|
|
# --- 3. Pull Dolibarr customer + supplier invoices and their payments ---
|
|
|
|
|
# --- 3. Pull Dolibarr customer + supplier invoices, payments, and bank accounts ---
|
|
|
|
|
"${DOL_CURL}" '/invoices?limit=500&sortfield=t.datef&sortorder=ASC' > "${WORK}/dol_inv.json"
|
|
|
|
|
"${DOL_CURL}" '/supplierinvoices?limit=500' > "${WORK}/dol_sup.json"
|
|
|
|
|
"${DOL_CURL}" '/bankaccounts' > "${WORK}/dol_acct.json"
|
|
|
|
|
|
|
|
|
|
mkdir -p "${WORK}/dol_pay" "${WORK}/dol_supay"
|
|
|
|
|
for id in $(python3 -c "import json,sys; print(' '.join(str(r['id']) for r in json.load(open(sys.argv[1])) if r.get('id')))" "${WORK}/dol_inv.json"); do
|
|
|
|
|
@@ -78,12 +80,26 @@ for id in $(python3 -c "import json,sys; print(' '.join(str(r['id']) for r in js
|
|
|
|
|
"${DOL_CURL}" "/supplierinvoices/${id}/payments" > "${WORK}/dol_supay/${id}.json" 2>/dev/null || echo "[]" > "${WORK}/dol_supay/${id}.json"
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
# --- 3b. Optional: enrich Wise TRANSFER activities with wire references ---
|
|
|
|
|
if [[ "${ENRICH}" == "1" ]]; then
|
|
|
|
|
mkdir -p "${WORK}/wise_refs"
|
|
|
|
|
for tid in $(python3 -c "
|
|
|
|
|
import json, sys
|
|
|
|
|
acts = json.load(open(sys.argv[1])).get('activities') or []
|
|
|
|
|
for a in acts:
|
|
|
|
|
r = a.get('resource') or {}
|
|
|
|
|
if r.get('type')=='TRANSFER' and r.get('id'): print(r['id'])
|
|
|
|
|
" "${WORK}/wise.json"); do
|
|
|
|
|
"${BANK_CURL}" wise "/v1/transfers/${tid}" > "${WORK}/wise_refs/${tid}.json" 2>/dev/null || true
|
|
|
|
|
done
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# --- 4. Match in python ---
|
|
|
|
|
PATTERNS_FILE="${SCRIPT_DIR}/../known-patterns.json"
|
|
|
|
|
python3 - "${WORK}" "${SINCE}" "${UNTIL}" "${WINDOW}" "${INCLUDE_FEES}" "${PATTERNS_FILE}" <<'PY'
|
|
|
|
|
python3 - "${WORK}" "${SINCE}" "${UNTIL}" "${WINDOW}" "${INCLUDE_FEES}" "${PATTERNS_FILE}" "${ENRICH}" <<'PY'
|
|
|
|
|
import json, sys, os, re, datetime, collections
|
|
|
|
|
work, since, until, window_days, include_fees, patterns_file = sys.argv[1:7]
|
|
|
|
|
window = int(window_days); include_fees = include_fees == "1"
|
|
|
|
|
work, since, until, window_days, include_fees, patterns_file, enrich = sys.argv[1:8]
|
|
|
|
|
window = int(window_days); include_fees = include_fees == "1"; enrich = enrich == "1"
|
|
|
|
|
since_d = datetime.date.fromisoformat(since); until_d = datetime.date.fromisoformat(until)
|
|
|
|
|
|
|
|
|
|
def strip(s): return re.sub(r'<[^>]+>', '', s or '').strip()
|
|
|
|
|
@@ -111,7 +127,22 @@ for a in (json.load(open(os.path.join(work,"wise.json"))).get("activities") or [
|
|
|
|
|
m = re.search(r'([\d,.]+)\s*([A-Z]{3})', pa)
|
|
|
|
|
amt = float(m.group(1).replace(",", "")) if m else 0.0
|
|
|
|
|
title = strip(a.get("title") or "")[:40]
|
|
|
|
|
wise_movs.append({"bank":"Wise", "date":dt, "sign":sign, "amount":amt, "label":title, "op":typ, "matched_dol":None, "matched_internal":False})
|
|
|
|
|
res = a.get("resource") or {}
|
|
|
|
|
resource_id = str(res.get("id")) if res.get("type") == "TRANSFER" else None
|
|
|
|
|
wise_movs.append({"bank":"Wise", "date":dt, "sign":sign, "amount":amt, "label":title, "op":typ, "matched_dol":None, "matched_internal":False, "wise_resource_id":resource_id, "wire_ref":""})
|
|
|
|
|
|
|
|
|
|
# 4b'. If --enrich, load per-transfer wire references and attach to Wise movs
|
|
|
|
|
if enrich:
|
|
|
|
|
ref_dir = os.path.join(work, "wise_refs")
|
|
|
|
|
if os.path.isdir(ref_dir):
|
|
|
|
|
for m in wise_movs:
|
|
|
|
|
if not m["wise_resource_id"]: continue
|
|
|
|
|
p = os.path.join(ref_dir, f"{m['wise_resource_id']}.json")
|
|
|
|
|
if not os.path.isfile(p): continue
|
|
|
|
|
try:
|
|
|
|
|
t = json.load(open(p))
|
|
|
|
|
m["wire_ref"] = (t.get("reference") or "")
|
|
|
|
|
except Exception: pass
|
|
|
|
|
|
|
|
|
|
bank_movs = qonto_movs + wise_movs
|
|
|
|
|
|
|
|
|
|
@@ -122,7 +153,7 @@ for w in [m for m in bank_movs if m["bank"]=="Wise" and m["sign"]=="-"]:
|
|
|
|
|
w["matched_internal"] = q; q["matched_internal"] = w
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
# 4d. Normalize Dolibarr payments
|
|
|
|
|
# 4d. Normalize Dolibarr payments — carry socid too for avoir netting
|
|
|
|
|
dol_pays = []
|
|
|
|
|
inv_by_id = {str(r["id"]): r for r in json.load(open(os.path.join(work,"dol_inv.json")))}
|
|
|
|
|
for fn in os.listdir(os.path.join(work,"dol_pay")):
|
|
|
|
|
@@ -132,7 +163,9 @@ for fn in os.listdir(os.path.join(work,"dol_pay")):
|
|
|
|
|
d = datetime.datetime.strptime(p["date"], "%Y-%m-%d %H:%M:%S").date()
|
|
|
|
|
if d < since_d or d > until_d: continue
|
|
|
|
|
amt = float(p.get("amount") or 0)
|
|
|
|
|
dol_pays.append({"side":"customer", "ref":inv["ref"], "date":d, "amount":amt, "fk_account":inv.get("fk_account"), "matched_bank":None})
|
|
|
|
|
dol_pays.append({"side":"customer", "ref":inv["ref"], "date":d, "amount":amt,
|
|
|
|
|
"fk_account":inv.get("fk_account"), "socid":inv.get("socid"),
|
|
|
|
|
"matched_bank":None, "netted_against":None})
|
|
|
|
|
|
|
|
|
|
sup_by_id = {str(r["id"]): r for r in json.load(open(os.path.join(work,"dol_sup.json")))}
|
|
|
|
|
for fn in os.listdir(os.path.join(work,"dol_supay")):
|
|
|
|
|
@@ -142,20 +175,71 @@ for fn in os.listdir(os.path.join(work,"dol_supay")):
|
|
|
|
|
d = datetime.datetime.strptime(p["date"], "%Y-%m-%d %H:%M:%S").date()
|
|
|
|
|
if d < since_d or d > until_d: continue
|
|
|
|
|
amt = float(p.get("amount") or 0)
|
|
|
|
|
dol_pays.append({"side":"supplier", "ref":sup["ref"], "date":d, "amount":amt, "fk_account":sup.get("fk_account"), "matched_bank":None})
|
|
|
|
|
dol_pays.append({"side":"supplier", "ref":sup["ref"], "date":d, "amount":amt,
|
|
|
|
|
"fk_account":sup.get("fk_account"), "socid":sup.get("socid"),
|
|
|
|
|
"matched_bank":None, "netted_against":None})
|
|
|
|
|
|
|
|
|
|
# 4e. Match: each bank movement (non-internal) tries to find a Dolibarr counterpart
|
|
|
|
|
for m in [x for x in bank_movs if not x["matched_internal"]]:
|
|
|
|
|
bank_signed = m["amount"] if m["sign"]=="+" else -m["amount"]
|
|
|
|
|
# For customer payments (Dol records them as positive amounts): +bank credit matches +dol customer payment
|
|
|
|
|
# For supplier payments: -bank debit matches +dol supplier payment (positive in Dol since it's the amount paid out)
|
|
|
|
|
# Heuristic: match abs(amount) within 0.01 and date within window.
|
|
|
|
|
candidates = [p for p in dol_pays if p["matched_bank"] is None and abs(p["amount"] - m["amount"]) < 0.01 and abs((p["date"] - m["date"]).days) <= window]
|
|
|
|
|
# 4d.1. AVOIR cycle netting: an AVC (credit note) for -X on socid S cancels out
|
|
|
|
|
# a FAC for +X on the same socid, within a small date window. Bank sees the NET
|
|
|
|
|
# of the cycle (typically +X for the reissued FAC with the new ref scheme).
|
|
|
|
|
# Pair an AVC with a FAC of opposite sign + equal abs(amount) + same socid +
|
|
|
|
|
# within ±5d. Mark both as "netted" so they're excluded from matching and
|
|
|
|
|
# excluded from the dolibarr-only failure count.
|
|
|
|
|
avcs = [p for p in dol_pays if p["side"]=="customer" and p["ref"].startswith("AVC") and p["amount"] < 0]
|
|
|
|
|
for avc in avcs:
|
|
|
|
|
candidates = [p for p in dol_pays
|
|
|
|
|
if p is not avc
|
|
|
|
|
and p["side"]=="customer"
|
|
|
|
|
and p["socid"] == avc["socid"]
|
|
|
|
|
and abs(p["amount"] + avc["amount"]) < 0.01 # opposite signs equal magnitude
|
|
|
|
|
and abs((p["date"] - avc["date"]).days) <= 5
|
|
|
|
|
and p["netted_against"] is None
|
|
|
|
|
and p["matched_bank"] is None]
|
|
|
|
|
if candidates:
|
|
|
|
|
# Prefer the OLDEST (the original cancelled FAC), not the reissue.
|
|
|
|
|
# Heuristic: refs with shorter / older numbering scheme. If multiple,
|
|
|
|
|
# pick smallest date delta.
|
|
|
|
|
candidates.sort(key=lambda p: (abs((p["date"] - avc["date"]).days), p["ref"]))
|
|
|
|
|
partner = candidates[0]
|
|
|
|
|
avc["netted_against"] = partner["ref"]
|
|
|
|
|
partner["netted_against"] = avc["ref"]
|
|
|
|
|
|
|
|
|
|
# 4e. Match — two-pass:
|
|
|
|
|
# PASS 1 (strong) : Wise transfers with an --enrich'd wire reference containing
|
|
|
|
|
# a "FAC***" pattern try to match the Dolibarr invoice with
|
|
|
|
|
# that exact ref. This is the highest-confidence match.
|
|
|
|
|
# PASS 2 (loose) : remaining bank movements use the date+amount heuristic.
|
|
|
|
|
# Netted Dolibarr entries (avoir cycle) are excluded from both passes.
|
|
|
|
|
|
|
|
|
|
# Build customer ref -> dol payment index (only un-netted, un-matched entries)
|
|
|
|
|
ref_index = collections.defaultdict(list)
|
|
|
|
|
for p in dol_pays:
|
|
|
|
|
if p["matched_bank"] is None and p["netted_against"] is None:
|
|
|
|
|
# Strip trailing dash/suffix variants — FAC002CL0001002 vs FAC002-CL0001002 are equivalent
|
|
|
|
|
normalized = re.sub(r'[^A-Z0-9]', '', p["ref"].upper())
|
|
|
|
|
ref_index[normalized].append(p)
|
|
|
|
|
|
|
|
|
|
# Pass 1: strong match on wire references
|
|
|
|
|
for m in [x for x in bank_movs if not x["matched_internal"] and x.get("wire_ref")]:
|
|
|
|
|
refs_in_wire = re.findall(r'FAC\d+(?:CL\d+)?', (m["wire_ref"] or "").upper().replace("-",""))
|
|
|
|
|
for r in refs_in_wire:
|
|
|
|
|
if r in ref_index and ref_index[r]:
|
|
|
|
|
p = ref_index[r].pop(0)
|
|
|
|
|
m["matched_dol"] = p; m["match_kind"] = "wire-ref"
|
|
|
|
|
p["matched_bank"] = m
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
# Pass 2: loose date+amount match for remaining bank movements
|
|
|
|
|
for m in [x for x in bank_movs if not x["matched_internal"] and not x["matched_dol"]]:
|
|
|
|
|
candidates = [p for p in dol_pays
|
|
|
|
|
if p["matched_bank"] is None and p["netted_against"] is None
|
|
|
|
|
and abs(p["amount"] - m["amount"]) < 0.01
|
|
|
|
|
and abs((p["date"] - m["date"]).days) <= window]
|
|
|
|
|
if candidates:
|
|
|
|
|
# Pick smallest date delta
|
|
|
|
|
candidates.sort(key=lambda p: abs((p["date"] - m["date"]).days))
|
|
|
|
|
p = candidates[0]
|
|
|
|
|
m["matched_dol"] = p; p["matched_bank"] = m
|
|
|
|
|
m["matched_dol"] = p; m["match_kind"] = "amt+date"
|
|
|
|
|
p["matched_bank"] = m
|
|
|
|
|
|
|
|
|
|
# 4f. Annotate non-matched movements with known-patterns catalog
|
|
|
|
|
patterns = []
|
|
|
|
|
@@ -184,21 +268,43 @@ for m in bank_movs:
|
|
|
|
|
m["known"] = match_pattern(m)
|
|
|
|
|
|
|
|
|
|
# --- 5. Render ---
|
|
|
|
|
|
|
|
|
|
# Load Dolibarr bank accounts (for fk_account context on dolibarr-only)
|
|
|
|
|
dol_accts = {}
|
|
|
|
|
try:
|
|
|
|
|
for a in json.load(open(os.path.join(work, "dol_acct.json"))):
|
|
|
|
|
dol_accts[str(a["id"])] = {"ref": a.get("ref","-"), "label": a.get("label","-"), "country": a.get("country_code","")}
|
|
|
|
|
except Exception: pass
|
|
|
|
|
|
|
|
|
|
# Heuristic: which Dolibarr accounts are NOT covered by Qonto/Wise API today?
|
|
|
|
|
# Convention: CCA = Compte Courant d'Associé (personal). Anything not QON*/WIS*
|
|
|
|
|
# is treated as "API-invisible" and tagged as such.
|
|
|
|
|
def account_kind(fk_account):
|
|
|
|
|
if not fk_account: return ("unknown", "fk_account=None")
|
|
|
|
|
a = dol_accts.get(str(fk_account))
|
|
|
|
|
if not a: return ("unknown", f"fk_account={fk_account} (not in /bankaccounts)")
|
|
|
|
|
ref = (a["ref"] or "").upper()
|
|
|
|
|
if ref.startswith(("QON", "WIS")):
|
|
|
|
|
return ("api_tracked", f"{a['ref']} ({a['label']})")
|
|
|
|
|
return ("personal_or_other", f"{a['ref']} ({a['label']})")
|
|
|
|
|
|
|
|
|
|
def fmt_bank(m):
|
|
|
|
|
return f" {m['bank']:<5} {m['date']} {m['sign']:<2}{m['amount']:>9.2f} {m['op'][:18]:<18} {m['label']}"
|
|
|
|
|
|
|
|
|
|
print(f"# Bank reconciliation: {since} → {until} (window ±{window}d, fees: {'on' if include_fees else 'off'})")
|
|
|
|
|
print(f"# Bank reconciliation: {since} → {until} (window ±{window}d, fees: {'on' if include_fees else 'off'}, enrich: {'on' if enrich else 'off'})")
|
|
|
|
|
print()
|
|
|
|
|
matched = [m for m in bank_movs if m["matched_dol"]]
|
|
|
|
|
internal = [m for m in bank_movs if m["matched_internal"] and m["sign"]=="-"]
|
|
|
|
|
bank_only = [m for m in bank_movs if not m["matched_dol"] and not m["matched_internal"]]
|
|
|
|
|
dol_only = [p for p in dol_pays if p["matched_bank"] is None]
|
|
|
|
|
netted_dol_pairs = [p for p in dol_pays if p["netted_against"]]
|
|
|
|
|
dol_only = [p for p in dol_pays if p["matched_bank"] is None and p["netted_against"] is None]
|
|
|
|
|
|
|
|
|
|
print(f"=== MATCHED ({len(matched)} bank ↔ Dolibarr) ===")
|
|
|
|
|
for m in sorted(matched, key=lambda m: m["date"]):
|
|
|
|
|
p = m["matched_dol"]
|
|
|
|
|
delta = (p["date"] - m["date"]).days
|
|
|
|
|
print(fmt_bank(m) + f" ↔ {p['side']:<8} {p['ref']:<24} ({p['date']}, Δ{delta:+d}d)")
|
|
|
|
|
kind = m.get("match_kind", "?")
|
|
|
|
|
print(fmt_bank(m) + f" ↔[{kind}] {p['side']:<8} {p['ref']:<24} ({p['date']}, Δ{delta:+d}d)")
|
|
|
|
|
print()
|
|
|
|
|
|
|
|
|
|
print(f"=== INTERNAL (Wise↔Qonto consolidations, {len(internal)}) ===")
|
|
|
|
|
@@ -207,6 +313,13 @@ for m in sorted(internal, key=lambda m: m["date"]):
|
|
|
|
|
print(fmt_bank(m) + f" ↔ {other['bank']} {other['date']} {other['sign']}{other['amount']:.2f}")
|
|
|
|
|
print()
|
|
|
|
|
|
|
|
|
|
# Avoir cycles netted out (informational; bank correctly sees only the net)
|
|
|
|
|
if netted_dol_pairs:
|
|
|
|
|
print(f"=== AVOIR-NETTED ({len(netted_dol_pairs)} Dolibarr entries pairing AVC↔FAC cancellation cycles) ===")
|
|
|
|
|
for p in sorted(netted_dol_pairs, key=lambda p: (p["date"], p["ref"])):
|
|
|
|
|
print(f" {p['side']:<8} {p['date']} {p['amount']:>9.2f} {p['ref']:<24} ↔ netted against {p['netted_against']}")
|
|
|
|
|
print()
|
|
|
|
|
|
|
|
|
|
bank_known = [m for m in bank_only if m.get("known")]
|
|
|
|
|
bank_unknown = [m for m in bank_only if not m.get("known")]
|
|
|
|
|
|
|
|
|
|
@@ -223,15 +336,28 @@ for m in sorted(bank_unknown, key=lambda m: m["date"]):
|
|
|
|
|
print(fmt_bank(m))
|
|
|
|
|
print()
|
|
|
|
|
|
|
|
|
|
print(f"=== DOLIBARR-ONLY ({len(dol_only)} Dolibarr payments without bank movement) ===")
|
|
|
|
|
for p in sorted(dol_only, key=lambda p: p["date"]):
|
|
|
|
|
print(f" {p['side']:<8} {p['date']} {p['amount']:>9.2f} {p['ref']} (fk_account={p['fk_account']})")
|
|
|
|
|
# Split dolibarr-only by whether the fk_account is API-tracked (real gap)
|
|
|
|
|
# or personal_or_other (expected gap — we have no API on those accounts)
|
|
|
|
|
dol_only_api = [p for p in dol_only if account_kind(p["fk_account"])[0] == "api_tracked"]
|
|
|
|
|
dol_only_personal = [p for p in dol_only if account_kind(p["fk_account"])[0] != "api_tracked"]
|
|
|
|
|
|
|
|
|
|
print(f"=== DOLIBARR-ONLY — on API-tracked accounts ({len(dol_only_api)}, REAL GAP: bank should have shown this) ===")
|
|
|
|
|
for p in sorted(dol_only_api, key=lambda p: p["date"]):
|
|
|
|
|
_, ctx = account_kind(p["fk_account"])
|
|
|
|
|
print(f" {p['side']:<8} {p['date']} {p['amount']:>9.2f} {p['ref']:<24} ({ctx})")
|
|
|
|
|
print()
|
|
|
|
|
|
|
|
|
|
# Verdict: only UNKNOWN bank-only and dolibarr-only count as "needs attention"
|
|
|
|
|
fails = len(bank_unknown) + len(dol_only)
|
|
|
|
|
print("-" * 80)
|
|
|
|
|
print(f"# {len(matched)} matched, {len(internal)} internal, {len(bank_known)} bank-known, {len(bank_unknown)} bank-UNKNOWN, {len(dol_only)} dolibarr-only")
|
|
|
|
|
print(f"=== DOLIBARR-ONLY — on accounts NOT in API scope ({len(dol_only_personal)}, expected gap: CCA1 perso etc.) ===")
|
|
|
|
|
for p in sorted(dol_only_personal, key=lambda p: p["date"]):
|
|
|
|
|
_, ctx = account_kind(p["fk_account"])
|
|
|
|
|
print(f" {p['side']:<8} {p['date']} {p['amount']:>9.2f} {p['ref']:<24} ({ctx})")
|
|
|
|
|
print()
|
|
|
|
|
|
|
|
|
|
# Verdict: only UNKNOWN bank-only AND dol-only-on-API-tracked count as failures.
|
|
|
|
|
# Avoir-netted pairs and personal-account dolibarr entries are intentional/expected.
|
|
|
|
|
fails = len(bank_unknown) + len(dol_only_api)
|
|
|
|
|
print("-" * 100)
|
|
|
|
|
print(f"# {len(matched)} matched, {len(internal)} internal, {len(netted_dol_pairs)} avoir-netted, {len(bank_known)} bank-known, {len(bank_unknown)} bank-UNKNOWN, {len(dol_only_api)} dol-only-API, {len(dol_only_personal)} dol-only-personal")
|
|
|
|
|
print(f"# patterns loaded from {patterns_file}: {len(patterns)} pattern(s)")
|
|
|
|
|
sys.exit(0 if fails == 0 else 1)
|
|
|
|
|
PY
|
|
|
|
|
|