fix(txid): normalize bank tx ids to fit Dolibarr's num_payment varchar(50)
Qonto transaction ids run ~67 chars (<org>-<n>-<n>-transaction-<uuid>) but Dolibarr stores num_payment in varchar(50) (llx_paiement.num_paiement, llx_paiementfourn.num_paiement) — POSTing a payment with the raw id fails HTTP 400 "value too long for type character varying(50)". Parade proven live on the sandbox (2026-07-11): store the UUID suffix (globally unique, ~37 chars). Wise ids (short numerics) are unaffected. Writer side — payment-record.sh strips everything through "transaction-" before POST, announces the normalization on stderr, REFUSES (never truncates) ids still >50 chars after normalization, and emits the normalized num in the output JSON. Reader side — bank-match.sh PASS 0 (exact tx-id, erp#28) now compares BOTH sides in raw AND canonical short form: Qonto feed ids are carried long+short, payment nums are normalized on compare — so nums stored short (the varchar(50) form) and historical long-form nums both keep matching. Wise ids untouched. Proven offline (no credentials, no network, no sandbox/prod writes): - arcodange-bank-reco/tests/run-tests.sh — new bank-match --fixtures offline mode: long feed id ↔ short num, long ↔ long (back-compat), Wise numeric, each Δ+19d outside the ±7d window so only PASS 0 can pair them (exit 0, 3×[tx-id]); plus the empty-num negative (exit 1, 0 matched). - dolibarr-sandbox-write/tests/run-tests.sh — payment-record via a stubbed dol-write.sh (DOL_WRITE hook): long→short in POST body + output JSON, Wise untouched, >50-after-normalization refused BEFORE any POST, citing varchar(50). Both SKILL.md document the canonical short form + the varchar(50) constraint. Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
Claude Fable 5
@@ -5,6 +5,7 @@
|
||||
# bank-match.sh [--month YYYY-MM | --since YYYY-MM-DD --until YYYY-MM-DD]
|
||||
# [--window-days N] # date tolerance, default 7
|
||||
# [--include-fees] # include Wise cashback / charges (default off)
|
||||
# [--fixtures DIR] # offline: match pre-pulled JSON from DIR (tests)
|
||||
#
|
||||
# Output: three buckets
|
||||
# - MATCHED bank movement ↔ Dolibarr payment
|
||||
@@ -24,7 +25,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BANK_CURL="${SCRIPT_DIR}/bank-curl.sh"
|
||||
DOL_CURL="${SCRIPT_DIR}/../../dolibarr/scripts/dol-curl.sh"
|
||||
|
||||
SINCE=""; UNTIL=""; MONTH=""; WINDOW=7; INCLUDE_FEES=0; ENRICH=0
|
||||
SINCE=""; UNTIL=""; MONTH=""; WINDOW=7; INCLUDE_FEES=0; ENRICH=0; FIXTURES=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--since) SINCE="$2"; shift 2 ;;
|
||||
@@ -33,7 +34,8 @@ while [[ $# -gt 0 ]]; do
|
||||
--window-days) WINDOW="$2"; shift 2 ;;
|
||||
--include-fees) INCLUDE_FEES=1; shift ;;
|
||||
--enrich) ENRICH=1; shift ;;
|
||||
-h|--help) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
--fixtures) FIXTURES="$2"; shift 2 ;;
|
||||
-h|--help) sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "bank-match.sh: unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
@@ -45,53 +47,62 @@ fi
|
||||
[[ -z "${SINCE}" ]] && SINCE="$(python3 -c "import datetime; print((datetime.date.today()-datetime.timedelta(days=365)).strftime('%Y-%m-%d'))")"
|
||||
[[ -z "${UNTIL}" ]] && UNTIL="$(python3 -c "import datetime; print(datetime.date.today().strftime('%Y-%m-%d'))")"
|
||||
|
||||
set -a; source "${SCRIPT_DIR}/../../dolibarr/.env"; set +a
|
||||
: "${WISE_PROFILE_ID:?bank-match.sh: WISE_PROFILE_ID not set}"
|
||||
|
||||
WORK="$(mktemp -d -t bankmatch.XXXXXX)"
|
||||
trap 'rm -rf "${WORK}"' EXIT
|
||||
|
||||
# --- 1. Pull Qonto transactions ---
|
||||
TMP_ORG=$(mktemp -t qontoorg.XXXXXX.json)
|
||||
"${BANK_CURL}" qonto /v2/organization > "${TMP_ORG}"
|
||||
QONTO_ACCT=$(python3 -c "
|
||||
if [[ -n "${FIXTURES}" ]]; then
|
||||
# --- 1-3 (offline): fixture mode for tests. DIR mirrors the layout the live
|
||||
# pulls produce (qonto.json, wise.json, dol_inv.json, dol_sup.json,
|
||||
# dol_acct.json, dol_pay/<id>.json, dol_supay/<id>.json[, wise_refs/]).
|
||||
# No credentials, no network, nothing written anywhere.
|
||||
cp -R "${FIXTURES}/." "${WORK}/"
|
||||
mkdir -p "${WORK}/dol_pay" "${WORK}/dol_supay"
|
||||
else
|
||||
set -a; source "${SCRIPT_DIR}/../../dolibarr/.env"; set +a
|
||||
: "${WISE_PROFILE_ID:?bank-match.sh: WISE_PROFILE_ID not set}"
|
||||
|
||||
# --- 1. Pull Qonto transactions ---
|
||||
TMP_ORG=$(mktemp -t qontoorg.XXXXXX.json)
|
||||
"${BANK_CURL}" qonto /v2/organization > "${TMP_ORG}"
|
||||
QONTO_ACCT=$(python3 -c "
|
||||
import json, sys
|
||||
d = json.load(open(sys.argv[1]))
|
||||
accs = (d.get('organization') or {}).get('bank_accounts') or []
|
||||
print([a for a in accs if a.get('status')=='active'][0]['id'])" "${TMP_ORG}")
|
||||
rm -f "${TMP_ORG}"
|
||||
rm -f "${TMP_ORG}"
|
||||
|
||||
QURL="/v2/transactions?bank_account_id=${QONTO_ACCT}&settled_at_from=${SINCE}T00:00:00Z&settled_at_to=${UNTIL}T23:59:59Z&per_page=100"
|
||||
"${BANK_CURL}" qonto "${QURL}" > "${WORK}/qonto.json"
|
||||
QURL="/v2/transactions?bank_account_id=${QONTO_ACCT}&settled_at_from=${SINCE}T00:00:00Z&settled_at_to=${UNTIL}T23:59:59Z&per_page=100"
|
||||
"${BANK_CURL}" qonto "${QURL}" > "${WORK}/qonto.json"
|
||||
|
||||
# --- 2. Pull Wise activities ---
|
||||
"${BANK_CURL}" wise "/v1/profiles/${WISE_PROFILE_ID}/activities?size=100&since=${SINCE}T00:00:00.000Z&until=${UNTIL}T23:59:59.999Z" > "${WORK}/wise.json"
|
||||
# --- 2. Pull Wise activities ---
|
||||
"${BANK_CURL}" wise "/v1/profiles/${WISE_PROFILE_ID}/activities?size=100&since=${SINCE}T00:00:00.000Z&until=${UNTIL}T23:59:59.999Z" > "${WORK}/wise.json"
|
||||
|
||||
# --- 3. Pull Dolibarr customer + supplier invoices, payments, and bank accounts ---
|
||||
"${DOL_CURL}" '/invoices?limit=500&sortfield=t.datef&sortorder=ASC' > "${WORK}/dol_inv.json"
|
||||
"${DOL_CURL}" '/supplierinvoices?limit=500' > "${WORK}/dol_sup.json"
|
||||
"${DOL_CURL}" '/bankaccounts' > "${WORK}/dol_acct.json"
|
||||
# --- 3. Pull Dolibarr customer + supplier invoices, payments, and bank accounts ---
|
||||
"${DOL_CURL}" '/invoices?limit=500&sortfield=t.datef&sortorder=ASC' > "${WORK}/dol_inv.json"
|
||||
"${DOL_CURL}" '/supplierinvoices?limit=500' > "${WORK}/dol_sup.json"
|
||||
"${DOL_CURL}" '/bankaccounts' > "${WORK}/dol_acct.json"
|
||||
|
||||
mkdir -p "${WORK}/dol_pay" "${WORK}/dol_supay"
|
||||
for id in $(python3 -c "import json,sys; print(' '.join(str(r['id']) for r in json.load(open(sys.argv[1])) if r.get('id')))" "${WORK}/dol_inv.json"); do
|
||||
"${DOL_CURL}" "/invoices/${id}/payments" > "${WORK}/dol_pay/${id}.json" 2>/dev/null || echo "[]" > "${WORK}/dol_pay/${id}.json"
|
||||
done
|
||||
for id in $(python3 -c "import json,sys; print(' '.join(str(r['id']) for r in json.load(open(sys.argv[1])) if r.get('id')))" "${WORK}/dol_sup.json"); do
|
||||
"${DOL_CURL}" "/supplierinvoices/${id}/payments" > "${WORK}/dol_supay/${id}.json" 2>/dev/null || echo "[]" > "${WORK}/dol_supay/${id}.json"
|
||||
done
|
||||
mkdir -p "${WORK}/dol_pay" "${WORK}/dol_supay"
|
||||
for id in $(python3 -c "import json,sys; print(' '.join(str(r['id']) for r in json.load(open(sys.argv[1])) if r.get('id')))" "${WORK}/dol_inv.json"); do
|
||||
"${DOL_CURL}" "/invoices/${id}/payments" > "${WORK}/dol_pay/${id}.json" 2>/dev/null || echo "[]" > "${WORK}/dol_pay/${id}.json"
|
||||
done
|
||||
for id in $(python3 -c "import json,sys; print(' '.join(str(r['id']) for r in json.load(open(sys.argv[1])) if r.get('id')))" "${WORK}/dol_sup.json"); do
|
||||
"${DOL_CURL}" "/supplierinvoices/${id}/payments" > "${WORK}/dol_supay/${id}.json" 2>/dev/null || echo "[]" > "${WORK}/dol_supay/${id}.json"
|
||||
done
|
||||
|
||||
# --- 3b. Optional: enrich Wise TRANSFER activities with wire references ---
|
||||
if [[ "${ENRICH}" == "1" ]]; then
|
||||
mkdir -p "${WORK}/wise_refs"
|
||||
for tid in $(python3 -c "
|
||||
# --- 3b. Optional: enrich Wise TRANSFER activities with wire references ---
|
||||
if [[ "${ENRICH}" == "1" ]]; then
|
||||
mkdir -p "${WORK}/wise_refs"
|
||||
for tid in $(python3 -c "
|
||||
import json, sys
|
||||
acts = json.load(open(sys.argv[1])).get('activities') or []
|
||||
for a in acts:
|
||||
r = a.get('resource') or {}
|
||||
if r.get('type')=='TRANSFER' and r.get('id'): print(r['id'])
|
||||
" "${WORK}/wise.json"); do
|
||||
"${BANK_CURL}" wise "/v1/transfers/${tid}" > "${WORK}/wise_refs/${tid}.json" 2>/dev/null || true
|
||||
done
|
||||
"${BANK_CURL}" wise "/v1/transfers/${tid}" > "${WORK}/wise_refs/${tid}.json" 2>/dev/null || true
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- 4. Match in python ---
|
||||
@@ -104,6 +115,15 @@ since_d = datetime.date.fromisoformat(since); until_d = datetime.date.fromisofor
|
||||
|
||||
def strip(s): return re.sub(r'<[^>]+>', '', s or '').strip()
|
||||
|
||||
def norm_txid(s):
|
||||
# Canonical short form of a bank-feed tx id. Qonto ids run ~67 chars
|
||||
# (<org>-<n>-<n>-transaction-<uuid>) but Dolibarr stores num_payment in
|
||||
# varchar(50), so règlements keep only the UUID suffix (globally unique —
|
||||
# see dolibarr-sandbox-write's payment-record.sh, which strips the same
|
||||
# prefix before POST). Wise ids (short numerics, no "transaction-") are
|
||||
# returned unchanged.
|
||||
return re.sub(r'^.*transaction-', '', s or '')
|
||||
|
||||
# 4a. Normalize Qonto
|
||||
qonto_movs = []
|
||||
for t in (json.load(open(os.path.join(work,"qonto.json"))).get("transactions") or []):
|
||||
@@ -114,7 +134,13 @@ for t in (json.load(open(os.path.join(work,"qonto.json"))).get("transactions") o
|
||||
label = t.get("label") or t.get("operation_type") or "-"
|
||||
# feed_ids: the Qonto transaction's own id — exact-match key against a Dolibarr
|
||||
# payment whose stored num (num_chq) is that id (set via payment transaction_id).
|
||||
feed_ids = [str(t["id"])] if t.get("id") else []
|
||||
# Carried in BOTH forms — raw (~67 chars) and canonical short (the UUID suffix,
|
||||
# what payment-record.sh stores under the varchar(50) limit) — so nums recorded
|
||||
# either way keep matching.
|
||||
feed_ids = []
|
||||
if t.get("id"):
|
||||
tid = str(t["id"])
|
||||
feed_ids = list(dict.fromkeys([tid, norm_txid(tid)]))
|
||||
qonto_movs.append({"bank":"Qonto", "date":dt, "sign":sign, "amount":amt, "label":label[:40], "op":t.get("operation_type",""), "feed_ids":feed_ids, "matched_dol":None, "matched_internal":False})
|
||||
|
||||
# 4b. Normalize Wise
|
||||
@@ -225,11 +251,14 @@ for avc in avcs:
|
||||
|
||||
# Pass 0: exact match on the feed transaction id stored as the payment's num
|
||||
# (num_chq = the transaction_id recorded with the règlement). Date-independent.
|
||||
# Both sides are compared in raw AND canonical short form (norm_txid), so a num
|
||||
# stored short (the varchar(50) form) or long (historical) matches either way.
|
||||
for m in [x for x in bank_movs if not x["matched_internal"] and not x["matched_dol"] and x.get("feed_ids")]:
|
||||
fids = set(m["feed_ids"])
|
||||
for p in dol_pays:
|
||||
if (p["matched_bank"] is None and p["netted_against"] is None
|
||||
and p.get("num") and str(p["num"]) in fids):
|
||||
and p.get("num")
|
||||
and {str(p["num"]), norm_txid(str(p["num"]))} & fids):
|
||||
m["matched_dol"] = p; m["match_kind"] = "tx-id"
|
||||
p["matched_bank"] = m
|
||||
break
|
||||
|
||||
Reference in New Issue
Block a user