Merge pull request 'fix(txid): normalize bank tx ids to fit Dolibarr's num_payment varchar(50)' (#37) from arcodange/txid-varchar50-normalize into main
Reviewed-on: #37
This commit was merged in pull request #37.
This commit is contained in:
@@ -197,6 +197,19 @@ Matching runs in three passes, highest confidence first:
|
|||||||
2. **`[wire-ref]` (strong)** — via `--enrich`, below.
|
2. **`[wire-ref]` (strong)** — via `--enrich`, below.
|
||||||
3. **`[amt+date]` (loose)** — the fallback heuristic.
|
3. **`[amt+date]` (loose)** — the fallback heuristic.
|
||||||
|
|
||||||
|
**Canonical short form — the varchar(50) constraint.** Qonto feed ids run ~67 chars
|
||||||
|
(`<org>-<n>-<n>-transaction-<uuid>`) but Dolibarr stores `num_payment` in
|
||||||
|
**varchar(50)** columns, so règlements store the **UUID suffix** — the canonical
|
||||||
|
short form `payment-record.sh` normalizes to (strip everything through
|
||||||
|
`transaction-`). PASS 0 therefore compares **both sides in raw AND short form**:
|
||||||
|
a num stored short (the varchar(50) form) or long (historical) matches either way.
|
||||||
|
Wise ids (short numerics, no `transaction-`) are untouched.
|
||||||
|
|
||||||
|
Offline proof: `tests/run-tests.sh` runs `bank-match.sh --fixtures` on
|
||||||
|
`tests/fixtures/` (no credentials, no network, nothing written) — a long Qonto
|
||||||
|
feed id matches one règlement stored short and one stored long, each ~19d outside
|
||||||
|
the ±7d window (so only PASS 0 can pair them), plus the empty-num negative case.
|
||||||
|
|
||||||
### `--enrich` — wire-reference strong matching
|
### `--enrich` — wire-reference strong matching
|
||||||
|
|
||||||
`bank-match.sh --enrich` fetches `/v1/transfers/{id}` for each Wise TRANSFER and reads the `reference` field (the wire memo from the sender, e.g. `FROM KISSMETRICS HOLDINGS INC FOR INVOICE FAC002CL0001002/ VENDOR:DEV`). When the reference contains a `FAC\d+(CL\d+)?` pattern matching a Dolibarr customer invoice, that pairing takes precedence over the loose date+amount match. Only the strong-matched ones get `[wire-ref]`; the rest fall through to `[amt+date]`. Cost: 1 extra HTTP call per Wise transfer.
|
`bank-match.sh --enrich` fetches `/v1/transfers/{id}` for each Wise TRANSFER and reads the `reference` field (the wire memo from the sender, e.g. `FROM KISSMETRICS HOLDINGS INC FOR INVOICE FAC002CL0001002/ VENDOR:DEV`). When the reference contains a `FAC\d+(CL\d+)?` pattern matching a Dolibarr customer invoice, that pairing takes precedence over the loose date+amount match. Only the strong-matched ones get `[wire-ref]`; the rest fall through to `[amt+date]`. Cost: 1 extra HTTP call per Wise transfer.
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
# bank-match.sh [--month YYYY-MM | --since YYYY-MM-DD --until YYYY-MM-DD]
|
# bank-match.sh [--month YYYY-MM | --since YYYY-MM-DD --until YYYY-MM-DD]
|
||||||
# [--window-days N] # date tolerance, default 7
|
# [--window-days N] # date tolerance, default 7
|
||||||
# [--include-fees] # include Wise cashback / charges (default off)
|
# [--include-fees] # include Wise cashback / charges (default off)
|
||||||
|
# [--fixtures DIR] # offline: match pre-pulled JSON from DIR (tests)
|
||||||
#
|
#
|
||||||
# Output: three buckets
|
# Output: three buckets
|
||||||
# - MATCHED bank movement ↔ Dolibarr payment
|
# - MATCHED bank movement ↔ Dolibarr payment
|
||||||
@@ -24,7 +25,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||||||
BANK_CURL="${SCRIPT_DIR}/bank-curl.sh"
|
BANK_CURL="${SCRIPT_DIR}/bank-curl.sh"
|
||||||
DOL_CURL="${SCRIPT_DIR}/../../dolibarr/scripts/dol-curl.sh"
|
DOL_CURL="${SCRIPT_DIR}/../../dolibarr/scripts/dol-curl.sh"
|
||||||
|
|
||||||
SINCE=""; UNTIL=""; MONTH=""; WINDOW=7; INCLUDE_FEES=0; ENRICH=0
|
SINCE=""; UNTIL=""; MONTH=""; WINDOW=7; INCLUDE_FEES=0; ENRICH=0; FIXTURES=""
|
||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--since) SINCE="$2"; shift 2 ;;
|
--since) SINCE="$2"; shift 2 ;;
|
||||||
@@ -33,7 +34,8 @@ while [[ $# -gt 0 ]]; do
|
|||||||
--window-days) WINDOW="$2"; shift 2 ;;
|
--window-days) WINDOW="$2"; shift 2 ;;
|
||||||
--include-fees) INCLUDE_FEES=1; shift ;;
|
--include-fees) INCLUDE_FEES=1; shift ;;
|
||||||
--enrich) ENRICH=1; shift ;;
|
--enrich) ENRICH=1; shift ;;
|
||||||
-h|--help) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
--fixtures) FIXTURES="$2"; shift 2 ;;
|
||||||
|
-h|--help) sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
*) echo "bank-match.sh: unknown arg: $1" >&2; exit 2 ;;
|
*) echo "bank-match.sh: unknown arg: $1" >&2; exit 2 ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
@@ -45,12 +47,20 @@ fi
|
|||||||
[[ -z "${SINCE}" ]] && SINCE="$(python3 -c "import datetime; print((datetime.date.today()-datetime.timedelta(days=365)).strftime('%Y-%m-%d'))")"
|
[[ -z "${SINCE}" ]] && SINCE="$(python3 -c "import datetime; print((datetime.date.today()-datetime.timedelta(days=365)).strftime('%Y-%m-%d'))")"
|
||||||
[[ -z "${UNTIL}" ]] && UNTIL="$(python3 -c "import datetime; print(datetime.date.today().strftime('%Y-%m-%d'))")"
|
[[ -z "${UNTIL}" ]] && UNTIL="$(python3 -c "import datetime; print(datetime.date.today().strftime('%Y-%m-%d'))")"
|
||||||
|
|
||||||
set -a; source "${SCRIPT_DIR}/../../dolibarr/.env"; set +a
|
|
||||||
: "${WISE_PROFILE_ID:?bank-match.sh: WISE_PROFILE_ID not set}"
|
|
||||||
|
|
||||||
WORK="$(mktemp -d -t bankmatch.XXXXXX)"
|
WORK="$(mktemp -d -t bankmatch.XXXXXX)"
|
||||||
trap 'rm -rf "${WORK}"' EXIT
|
trap 'rm -rf "${WORK}"' EXIT
|
||||||
|
|
||||||
|
if [[ -n "${FIXTURES}" ]]; then
|
||||||
|
# --- 1-3 (offline): fixture mode for tests. DIR mirrors the layout the live
|
||||||
|
# pulls produce (qonto.json, wise.json, dol_inv.json, dol_sup.json,
|
||||||
|
# dol_acct.json, dol_pay/<id>.json, dol_supay/<id>.json[, wise_refs/]).
|
||||||
|
# No credentials, no network, nothing written anywhere.
|
||||||
|
cp -R "${FIXTURES}/." "${WORK}/"
|
||||||
|
mkdir -p "${WORK}/dol_pay" "${WORK}/dol_supay"
|
||||||
|
else
|
||||||
|
set -a; source "${SCRIPT_DIR}/../../dolibarr/.env"; set +a
|
||||||
|
: "${WISE_PROFILE_ID:?bank-match.sh: WISE_PROFILE_ID not set}"
|
||||||
|
|
||||||
# --- 1. Pull Qonto transactions ---
|
# --- 1. Pull Qonto transactions ---
|
||||||
TMP_ORG=$(mktemp -t qontoorg.XXXXXX.json)
|
TMP_ORG=$(mktemp -t qontoorg.XXXXXX.json)
|
||||||
"${BANK_CURL}" qonto /v2/organization > "${TMP_ORG}"
|
"${BANK_CURL}" qonto /v2/organization > "${TMP_ORG}"
|
||||||
@@ -93,6 +103,7 @@ for a in acts:
|
|||||||
"${BANK_CURL}" wise "/v1/transfers/${tid}" > "${WORK}/wise_refs/${tid}.json" 2>/dev/null || true
|
"${BANK_CURL}" wise "/v1/transfers/${tid}" > "${WORK}/wise_refs/${tid}.json" 2>/dev/null || true
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# --- 4. Match in python ---
|
# --- 4. Match in python ---
|
||||||
PATTERNS_FILE="${SCRIPT_DIR}/../known-patterns.json"
|
PATTERNS_FILE="${SCRIPT_DIR}/../known-patterns.json"
|
||||||
@@ -104,6 +115,15 @@ since_d = datetime.date.fromisoformat(since); until_d = datetime.date.fromisofor
|
|||||||
|
|
||||||
def strip(s): return re.sub(r'<[^>]+>', '', s or '').strip()
|
def strip(s): return re.sub(r'<[^>]+>', '', s or '').strip()
|
||||||
|
|
||||||
|
def norm_txid(s):
|
||||||
|
# Canonical short form of a bank-feed tx id. Qonto ids run ~67 chars
|
||||||
|
# (<org>-<n>-<n>-transaction-<uuid>) but Dolibarr stores num_payment in
|
||||||
|
# varchar(50), so règlements keep only the UUID suffix (globally unique —
|
||||||
|
# see dolibarr-sandbox-write's payment-record.sh, which strips the same
|
||||||
|
# prefix before POST). Wise ids (short numerics, no "transaction-") are
|
||||||
|
# returned unchanged.
|
||||||
|
return re.sub(r'^.*transaction-', '', s or '')
|
||||||
|
|
||||||
# 4a. Normalize Qonto
|
# 4a. Normalize Qonto
|
||||||
qonto_movs = []
|
qonto_movs = []
|
||||||
for t in (json.load(open(os.path.join(work,"qonto.json"))).get("transactions") or []):
|
for t in (json.load(open(os.path.join(work,"qonto.json"))).get("transactions") or []):
|
||||||
@@ -114,7 +134,13 @@ for t in (json.load(open(os.path.join(work,"qonto.json"))).get("transactions") o
|
|||||||
label = t.get("label") or t.get("operation_type") or "-"
|
label = t.get("label") or t.get("operation_type") or "-"
|
||||||
# feed_ids: the Qonto transaction's own id — exact-match key against a Dolibarr
|
# feed_ids: the Qonto transaction's own id — exact-match key against a Dolibarr
|
||||||
# payment whose stored num (num_chq) is that id (set via payment transaction_id).
|
# payment whose stored num (num_chq) is that id (set via payment transaction_id).
|
||||||
feed_ids = [str(t["id"])] if t.get("id") else []
|
# Carried in BOTH forms — raw (~67 chars) and canonical short (the UUID suffix,
|
||||||
|
# what payment-record.sh stores under the varchar(50) limit) — so nums recorded
|
||||||
|
# either way keep matching.
|
||||||
|
feed_ids = []
|
||||||
|
if t.get("id"):
|
||||||
|
tid = str(t["id"])
|
||||||
|
feed_ids = list(dict.fromkeys([tid, norm_txid(tid)]))
|
||||||
qonto_movs.append({"bank":"Qonto", "date":dt, "sign":sign, "amount":amt, "label":label[:40], "op":t.get("operation_type",""), "feed_ids":feed_ids, "matched_dol":None, "matched_internal":False})
|
qonto_movs.append({"bank":"Qonto", "date":dt, "sign":sign, "amount":amt, "label":label[:40], "op":t.get("operation_type",""), "feed_ids":feed_ids, "matched_dol":None, "matched_internal":False})
|
||||||
|
|
||||||
# 4b. Normalize Wise
|
# 4b. Normalize Wise
|
||||||
@@ -225,11 +251,14 @@ for avc in avcs:
|
|||||||
|
|
||||||
# Pass 0: exact match on the feed transaction id stored as the payment's num
|
# Pass 0: exact match on the feed transaction id stored as the payment's num
|
||||||
# (num_chq = the transaction_id recorded with the règlement). Date-independent.
|
# (num_chq = the transaction_id recorded with the règlement). Date-independent.
|
||||||
|
# Both sides are compared in raw AND canonical short form (norm_txid), so a num
|
||||||
|
# stored short (the varchar(50) form) or long (historical) matches either way.
|
||||||
for m in [x for x in bank_movs if not x["matched_internal"] and not x["matched_dol"] and x.get("feed_ids")]:
|
for m in [x for x in bank_movs if not x["matched_internal"] and not x["matched_dol"] and x.get("feed_ids")]:
|
||||||
fids = set(m["feed_ids"])
|
fids = set(m["feed_ids"])
|
||||||
for p in dol_pays:
|
for p in dol_pays:
|
||||||
if (p["matched_bank"] is None and p["netted_against"] is None
|
if (p["matched_bank"] is None and p["netted_against"] is None
|
||||||
and p.get("num") and str(p["num"]) in fids):
|
and p.get("num")
|
||||||
|
and {str(p["num"]), norm_txid(str(p["num"]))} & fids):
|
||||||
m["matched_dol"] = p; m["match_kind"] = "tx-id"
|
m["matched_dol"] = p; m["match_kind"] = "tx-id"
|
||||||
p["matched_bank"] = m
|
p["matched_bank"] = m
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[
|
||||||
|
{ "id": 1, "ref": "QON1", "label": "Qonto principal", "country_code": "FR" }
|
||||||
|
]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
[]
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[
|
||||||
|
{ "id": 201, "ref": "FS-OVH-2606", "fk_account": 1, "socid": 12 }
|
||||||
|
]
|
||||||
+8
@@ -0,0 +1,8 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"date": "2026-06-20 12:00:00",
|
||||||
|
"amount": "96.00",
|
||||||
|
"num": "",
|
||||||
|
"fk_bank_line": "556"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"transactions": [
|
||||||
|
{
|
||||||
|
"id": "arcodange-1246-1-transaction-019f14c5-e254-7ac9-9e9f-307ed9-d55f44",
|
||||||
|
"settled_at": "2026-06-01T09:00:00Z",
|
||||||
|
"amount": "96.0",
|
||||||
|
"side": "debit",
|
||||||
|
"label": "OVH SAS",
|
||||||
|
"operation_type": "transfer"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{ "activities": [] }
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
[
|
||||||
|
{ "id": 1, "ref": "QON1", "label": "Qonto principal", "country_code": "FR" },
|
||||||
|
{ "id": 2, "ref": "WIS1", "label": "Wise EUR", "country_code": "BE" }
|
||||||
|
]
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[
|
||||||
|
{ "id": 101, "ref": "FAC003-CL0001003", "fk_account": 2, "socid": 7 }
|
||||||
|
]
|
||||||
+8
@@ -0,0 +1,8 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"date": "2026-06-24 12:00:00",
|
||||||
|
"amount": "2147.00",
|
||||||
|
"num": "2159468139",
|
||||||
|
"fk_bank_line": "555"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
[
|
||||||
|
{ "id": 201, "ref": "FS-OVH-2606", "fk_account": 1, "socid": 12 },
|
||||||
|
{ "id": 202, "ref": "FS-SCW-2606", "fk_account": 1, "socid": 13 }
|
||||||
|
]
|
||||||
+8
@@ -0,0 +1,8 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"date": "2026-06-20 12:00:00",
|
||||||
|
"amount": "96.00",
|
||||||
|
"num": "019f14c5-e254-7ac9-9e9f-307ed9-d55f44",
|
||||||
|
"fk_bank_line": "556"
|
||||||
|
}
|
||||||
|
]
|
||||||
+8
@@ -0,0 +1,8 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"date": "2026-06-22 12:00:00",
|
||||||
|
"amount": "42.50",
|
||||||
|
"num": "arcodange-1246-1-transaction-019f22aa-4b31-7c02-8d5e-11aa22-bb33cc",
|
||||||
|
"fk_bank_line": "557"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"transactions": [
|
||||||
|
{
|
||||||
|
"id": "arcodange-1246-1-transaction-019f14c5-e254-7ac9-9e9f-307ed9-d55f44",
|
||||||
|
"settled_at": "2026-06-01T09:00:00Z",
|
||||||
|
"amount": "96.0",
|
||||||
|
"side": "debit",
|
||||||
|
"label": "OVH SAS",
|
||||||
|
"operation_type": "transfer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "arcodange-1246-1-transaction-019f22aa-4b31-7c02-8d5e-11aa22-bb33cc",
|
||||||
|
"settled_at": "2026-06-03T09:00:00Z",
|
||||||
|
"amount": "42.5",
|
||||||
|
"side": "debit",
|
||||||
|
"label": "SCALEWAY",
|
||||||
|
"operation_type": "card"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"activities": [
|
||||||
|
{
|
||||||
|
"id": "act-9f0e77aa",
|
||||||
|
"type": "TRANSFER",
|
||||||
|
"createdOn": "2026-06-05T10:00:00Z",
|
||||||
|
"primaryAmount": "<positive>+ 2,147.00 EUR</positive>",
|
||||||
|
"title": "Kissmetrics Holdings Inc",
|
||||||
|
"resource": { "type": "TRANSFER", "id": 2159468139 }
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
+42
@@ -0,0 +1,42 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Offline fixture tests for bank-match.sh — no credentials, no network, no writes.
|
||||||
|
# Proves the PASS 0 tx-id normalization (varchar(50) canonical short form):
|
||||||
|
# 1. txid-normalize — a long Qonto feed id (~67 chars) matches a règlement whose
|
||||||
|
# num was stored SHORT (UUID suffix, what payment-record.sh stores) AND one
|
||||||
|
# stored LONG (historical); a Wise numeric id matches unchanged. Every pair
|
||||||
|
# is ~19 days apart — far outside the ±7d window — so only the id-based
|
||||||
|
# PASS 0 can pair them. Expect exit 0, 3 × [tx-id].
|
||||||
|
# 2. txid-no-num — same bank movement but the payment has num="" → must NOT
|
||||||
|
# match (the id is proof; its absence isn't). Expect exit 1, 0 matched.
|
||||||
|
set -euo pipefail
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
BM="${SCRIPT_DIR}/../scripts/bank-match.sh"
|
||||||
|
|
||||||
|
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||||
|
count() { grep -c "$1" <<<"$2" || true; }
|
||||||
|
|
||||||
|
bash -n "${BM}" || fail "bash -n bank-match.sh"
|
||||||
|
|
||||||
|
# --- Case 1: long-Qonto-id ↔ short-num (+ long-num back-compat + Wise) ---
|
||||||
|
OUT="$(bash "${BM}" --fixtures "${SCRIPT_DIR}/fixtures/txid-normalize" \
|
||||||
|
--since 2026-06-01 --until 2026-06-30)" \
|
||||||
|
|| fail "txid-normalize: expected exit 0, got $?"
|
||||||
|
[[ "$(count '↔\[tx-id\]' "${OUT}")" == 3 ]] || fail "txid-normalize: expected 3 [tx-id] matches
|
||||||
|
${OUT}"
|
||||||
|
grep -q 'FS-OVH-2606' <<<"${OUT}" || fail "txid-normalize: long feed id ↔ SHORT num (the varchar(50) form) did not match"
|
||||||
|
grep -q 'FS-SCW-2606' <<<"${OUT}" || fail "txid-normalize: long feed id ↔ LONG num (historical form) did not match"
|
||||||
|
grep -q 'FAC003-CL0001003' <<<"${OUT}" || fail "txid-normalize: Wise numeric id match broken"
|
||||||
|
grep -q '# 3 matched, 0 internal, 0 avoir-netted, 0 bank-known, 0 bank-UNKNOWN, 0 dol-only-API' <<<"${OUT}" \
|
||||||
|
|| fail "txid-normalize: unexpected bucket counts
|
||||||
|
${OUT}"
|
||||||
|
|
||||||
|
# --- Case 2: payment without num must not tx-id-match ---
|
||||||
|
rc=0
|
||||||
|
OUT2="$(bash "${BM}" --fixtures "${SCRIPT_DIR}/fixtures/txid-no-num" \
|
||||||
|
--since 2026-06-01 --until 2026-06-30)" || rc=$?
|
||||||
|
[[ "${rc}" == 1 ]] || fail "txid-no-num: expected exit 1, got ${rc}"
|
||||||
|
grep -q '# 0 matched' <<<"${OUT2}" || fail "txid-no-num: nothing should match
|
||||||
|
${OUT2}"
|
||||||
|
[[ "$(count '↔\[tx-id\]' "${OUT2}")" == 0 ]] || fail "txid-no-num: empty num must not produce a [tx-id] match"
|
||||||
|
|
||||||
|
echo "OK: bank-match fixture tests passed (3 tx-id matches incl. long↔short + long↔long + Wise; empty-num negative)"
|
||||||
@@ -96,9 +96,10 @@ to leave a draft. Emits `{id, ref, ref_supplier, total_ht, total_ttc, statut}`.
|
|||||||
### 3 · Payment (règlement) — `scripts/payment-record.sh`
|
### 3 · Payment (règlement) — `scripts/payment-record.sh`
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
echo '{"invoice_id":19,"mode":"VIR","account_id":1,"transaction_id":"QONTO-TX-1234"}' | scripts/payment-record.sh
|
echo '{"invoice_id":19,"mode":"VIR","account_id":1,"transaction_id":"2159468139"}' | scripts/payment-record.sh
|
||||||
echo '{"invoice_id":13,"kind":"supplier","mode":"VIR","account_id":1,"amount":96,"transaction_id":"WISE-TX-5678"}' \
|
echo '{"invoice_id":13,"kind":"supplier","mode":"VIR","account_id":1,"amount":96,
|
||||||
| scripts/payment-record.sh
|
"transaction_id":"arcodange-1246-1-transaction-019f14c5-e254-7ac9-9e9f-307ed9-d55f44"}' \
|
||||||
|
| scripts/payment-record.sh # stores num 019f14c5-e254-7ac9-9e9f-307ed9-d55f44
|
||||||
```
|
```
|
||||||
The invoice must be **validated** first. `mode`: `VIR|CB|CHQ|LIQ`. Customer
|
The invoice must be **validated** first. `mode`: `VIR|CB|CHQ|LIQ`. Customer
|
||||||
payments settle the full remaining amount and mark the invoice paid; **supplier**
|
payments settle the full remaining amount and mark the invoice paid; **supplier**
|
||||||
@@ -112,9 +113,22 @@ reconciliation matches **by id** rather than by fuzzy amount/date. (`num` is a
|
|||||||
back-compat alias for the same field.) Recording without it prints a warning — the
|
back-compat alias for the same field.) Recording without it prints a warning — the
|
||||||
payment still posts, but it won't auto-reconcile.
|
payment still posts, but it won't auto-reconcile.
|
||||||
|
|
||||||
|
**Canonical short form — the varchar(50) constraint.** Dolibarr stores
|
||||||
|
`num_payment` in **varchar(50)** columns (`llx_paiement.num_paiement`,
|
||||||
|
`llx_paiementfourn.num_paiement`) while Qonto transaction ids run ~67 chars
|
||||||
|
(`<org>-<n>-<n>-transaction-<uuid>`) — POSTing one raw fails with HTTP 400
|
||||||
|
"value too long for type character varying(50)". The script therefore normalizes
|
||||||
|
before POST: everything through `transaction-` is stripped and the **UUID suffix**
|
||||||
|
(globally unique, ~37 chars) is what gets stored. Wise ids (short numerics) pass
|
||||||
|
through unchanged. An id still >50 chars after normalization is **refused** with
|
||||||
|
an explicit error — never truncated silently. `arcodange-bank-reco`'s bank-match
|
||||||
|
normalizes feed ids the same way, so short-form nums keep reconciling by id
|
||||||
|
(historical long-form nums too). Pass the raw feed id; the script does the rest.
|
||||||
|
|
||||||
Emits **`{id, bank_transaction_id, transaction_id}`**. `bank_transaction_id` is the
|
Emits **`{id, bank_transaction_id, transaction_id}`**. `bank_transaction_id` is the
|
||||||
Dolibarr bank line (`llx_bank.fk_bank_line`) the payment created — the id the
|
Dolibarr bank line (`llx_bank.fk_bank_line`) the payment created — the id the
|
||||||
reconciliation (`arcodange-bank-reco`) keys on. Both ends are captured at write time.
|
reconciliation (`arcodange-bank-reco`) keys on. `transaction_id` echoes the
|
||||||
|
**normalized** num actually stored. Both ends are captured at write time.
|
||||||
|
|
||||||
### 4 · Credit note (avoir) — `scripts/creditnote-create.sh`
|
### 4 · Credit note (avoir) — `scripts/creditnote-create.sh`
|
||||||
|
|
||||||
@@ -180,6 +194,10 @@ sandbox KissMetrics on `--target sandbox` and the prod one on `--target prod`.
|
|||||||
REST create needs `code_client`/`code_fournisseur = "-1"` to trigger it — the
|
REST create needs `code_client`/`code_fournisseur = "-1"` to trigger it — the
|
||||||
script does this; without it the API errors `ErrorCustomerCodeRequired`.
|
script does this; without it the API errors `ErrorCustomerCodeRequired`.
|
||||||
- **Dates** are sent as Unix epochs; pass `date:"YYYY-MM-DD"` or omit for today.
|
- **Dates** are sent as Unix epochs; pass `date:"YYYY-MM-DD"` or omit for today.
|
||||||
|
- **Qonto tx ids don't fit varchar(50).** `payment-record.sh` strips the
|
||||||
|
`<org>-<n>-<n>-transaction-` prefix and stores the UUID suffix (see workflow 3);
|
||||||
|
ids still >50 chars after that are refused. Offline proof: `tests/run-tests.sh`
|
||||||
|
(stubbed `dol-write.sh` via the `DOL_WRITE` hook — writes nothing anywhere).
|
||||||
- **`banque lire`** (rights id 111) is granted → `scripts/bank-accounts.sh` lists
|
- **`banque lire`** (rights id 111) is granted → `scripts/bank-accounts.sh` lists
|
||||||
accounts (id/label/bank) so a payment can pick its `account_id`. It's in the
|
accounts (id/label/bank) so a payment can pick its `account_id`. It's in the
|
||||||
provisioner's `WRITE_IDS`, so a fresh `provisionSandbox.ts` run includes it.
|
provisioner's `WRITE_IDS`, so a fresh `provisionSandbox.ts` run includes it.
|
||||||
|
|||||||
@@ -12,10 +12,18 @@
|
|||||||
# tx id from the feed). Stored on the payment's bank line
|
# tx id from the feed). Stored on the payment's bank line
|
||||||
# (llx_bank.num_chq) so the règlement reconciles to the feed by id.
|
# (llx_bank.num_chq) so the règlement reconciles to the feed by id.
|
||||||
# `num` is a back-compat alias for the same field.
|
# `num` is a back-compat alias for the same field.
|
||||||
|
# NORMALIZED before POST: Dolibarr stores num_payment in varchar(50)
|
||||||
|
# (llx_paiement.num_paiement / llx_paiementfourn.num_paiement) while
|
||||||
|
# Qonto ids run ~67 chars (<org>-<n>-<n>-transaction-<uuid>), so
|
||||||
|
# everything through "transaction-" is stripped and the UUID suffix
|
||||||
|
# (globally unique, ~37 chars) is the canonical stored form. Wise ids
|
||||||
|
# (short numerics) pass through unchanged. An id still >50 chars after
|
||||||
|
# normalization is REFUSED with an error — never truncated silently.
|
||||||
# comment (optional)
|
# comment (optional)
|
||||||
#
|
#
|
||||||
# The invoice must be VALIDATED first (invoice-create.sh ... "validate":true).
|
# The invoice must be VALIDATED first (invoice-create.sh ... "validate":true).
|
||||||
# Emits {id, bank_transaction_id, transaction_id} on stdout. `bank_transaction_id`
|
# Emits {id, bank_transaction_id, transaction_id} on stdout — `transaction_id` is
|
||||||
|
# the NORMALIZED num actually stored (what bank-match keys on). `bank_transaction_id`
|
||||||
# is the Dolibarr bank line (llx_bank.fk_bank_line) the payment created — the id
|
# is the Dolibarr bank line (llx_bank.fk_bank_line) the payment created — the id
|
||||||
# bank reconciliation (arcodange-bank-reco) keys on to link this règlement to a
|
# bank reconciliation (arcodange-bank-reco) keys on to link this règlement to a
|
||||||
# statement line. Recording without a transaction_id warns (it won't auto-reconcile).
|
# statement line. Recording without a transaction_id warns (it won't auto-reconcile).
|
||||||
@@ -29,7 +37,7 @@ if [[ -n "${SRC}" && "${SRC}" != "-" ]]; then INPUT="$(cat "${SRC}")"; else INPU
|
|||||||
PYF="$(mktemp -t dolpy.XXXXXX)"; PYF2="$(mktemp -t dolpy2.XXXXXX)"
|
PYF="$(mktemp -t dolpy.XXXXXX)"; PYF2="$(mktemp -t dolpy2.XXXXXX)"
|
||||||
trap 'rm -f "${PYF}" "${PYF2}"' EXIT
|
trap 'rm -f "${PYF}" "${PYF2}"' EXIT
|
||||||
cat > "${PYF}" <<'PY'
|
cat > "${PYF}" <<'PY'
|
||||||
import json, sys, datetime
|
import json, sys, datetime, re
|
||||||
d = json.loads(sys.stdin.read())
|
d = json.loads(sys.stdin.read())
|
||||||
if not d.get("invoice_id"):
|
if not d.get("invoice_id"):
|
||||||
sys.exit("payment-record.sh: 'invoice_id' is required")
|
sys.exit("payment-record.sh: 'invoice_id' is required")
|
||||||
@@ -46,7 +54,23 @@ epoch = int((datetime.datetime.strptime(ds, "%Y-%m-%d") if ds
|
|||||||
else datetime.datetime.now()).timestamp())
|
else datetime.datetime.now()).timestamp())
|
||||||
inv = d["invoice_id"]
|
inv = d["invoice_id"]
|
||||||
# transaction_id is the first-class bank-feed tx id; num is the back-compat alias.
|
# transaction_id is the first-class bank-feed tx id; num is the back-compat alias.
|
||||||
tx = str(d.get("transaction_id") or d.get("num") or "")
|
raw_tx = str(d.get("transaction_id") or d.get("num") or "")
|
||||||
|
# Normalize to the canonical short form. Dolibarr stores num_payment in
|
||||||
|
# varchar(50) (llx_paiement.num_paiement / llx_paiementfourn.num_paiement) and
|
||||||
|
# a Qonto id (~67 chars, <org>-<n>-<n>-transaction-<uuid>) blows past it —
|
||||||
|
# HTTP 400 "value too long for type character varying(50)". Strip everything
|
||||||
|
# through "transaction-" and store the UUID suffix (globally unique). Wise ids
|
||||||
|
# (short numerics, no "transaction-") pass through unchanged. bank-match.sh
|
||||||
|
# normalizes feed ids the same way, so the short form still reconciles by id.
|
||||||
|
tx = re.sub(r'^.*transaction-', '', raw_tx)
|
||||||
|
if tx != raw_tx:
|
||||||
|
sys.stderr.write("payment-record.sh: transaction_id normalized to %r "
|
||||||
|
"(Qonto prefix stripped — num_payment is varchar(50))\n" % tx)
|
||||||
|
if len(tx) > 50:
|
||||||
|
sys.exit("payment-record.sh: transaction_id %r is %d chars even after "
|
||||||
|
"normalization — Dolibarr's num_payment is varchar(50) and silent "
|
||||||
|
"truncation would break bank reconciliation; pass a shorter id"
|
||||||
|
% (tx, len(tx)))
|
||||||
if not tx:
|
if not tx:
|
||||||
sys.stderr.write("payment-record.sh: WARNING — no transaction_id given; this "
|
sys.stderr.write("payment-record.sh: WARNING — no transaction_id given; this "
|
||||||
"règlement won't auto-reconcile to the bank feed\n")
|
"règlement won't auto-reconcile to the bank feed\n")
|
||||||
@@ -83,9 +107,11 @@ if tx:
|
|||||||
if pick is None and rows:
|
if pick is None and rows:
|
||||||
pick = max(rows, key=lambda r: r.get("date", ""))
|
pick = max(rows, key=lambda r: r.get("date", ""))
|
||||||
btx = (pick or {}).get("fk_bank_line")
|
btx = (pick or {}).get("fk_bank_line")
|
||||||
|
# transaction_id: the num as stored (normalized) — fall back to the sent TX so
|
||||||
|
# the JSON always reports the canonical short form even on the recency fallback.
|
||||||
print(json.dumps({"id": pid,
|
print(json.dumps({"id": pid,
|
||||||
"bank_transaction_id": int(btx) if btx and str(btx).isdigit() else btx,
|
"bank_transaction_id": int(btx) if btx and str(btx).isdigit() else btx,
|
||||||
"transaction_id": (pick or {}).get("num", "")}))
|
"transaction_id": (pick or {}).get("num", "") or tx}))
|
||||||
PY
|
PY
|
||||||
|
|
||||||
MAPPED="$(printf '%s' "${INPUT}" | python3 "${PYF}")"
|
MAPPED="$(printf '%s' "${INPUT}" | python3 "${PYF}")"
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Offline tests for payment-record.sh's transaction_id normalization — the writer
|
||||||
|
# side of the varchar(50) fix (Dolibarr's num_payment columns truncate at 50 chars,
|
||||||
|
# Qonto ids run ~67). Uses tests/stub-dol-write.sh via the DOL_WRITE env hook, so
|
||||||
|
# NOTHING is written to the sandbox or prod.
|
||||||
|
# 1. Long Qonto id → POST carries the UUID suffix; JSON reports it; stderr says so.
|
||||||
|
# 2. Wise numeric id → passes through untouched, no normalization notice.
|
||||||
|
# 3. Id still >50 chars after normalization → refused BEFORE any POST, error
|
||||||
|
# cites varchar(50) (never a silent truncation).
|
||||||
|
set -euo pipefail
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PR="${SCRIPT_DIR}/../scripts/payment-record.sh"
|
||||||
|
STUB="${SCRIPT_DIR}/stub-dol-write.sh"
|
||||||
|
|
||||||
|
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
bash -n "${PR}" || fail "bash -n payment-record.sh"
|
||||||
|
bash -n "${STUB}" || fail "bash -n stub-dol-write.sh"
|
||||||
|
|
||||||
|
STATE="$(mktemp -d -t prtest.XXXXXX)"
|
||||||
|
trap 'rm -rf "${STATE}"' EXIT
|
||||||
|
|
||||||
|
LONG="arcodange-1246-1-transaction-019f14c5-e254-7ac9-9e9f-307ed9-d55f44"
|
||||||
|
SHORT="019f14c5-e254-7ac9-9e9f-307ed9-d55f44"
|
||||||
|
|
||||||
|
# --- Case 1: long Qonto id is normalized to the UUID suffix ---
|
||||||
|
OUT="$(printf '{"invoice_id":13,"kind":"supplier","account_id":1,"amount":96,"transaction_id":"%s"}' "${LONG}" \
|
||||||
|
| DOL_WRITE="${STUB}" STUB_STATE="${STATE}" bash "${PR}" 2>"${STATE}/stderr1")" \
|
||||||
|
|| fail "long-qonto-id: expected success, got $?"
|
||||||
|
grep -q "\"num_payment\": \"${SHORT}\"" "${STATE}/post_body.json" \
|
||||||
|
|| fail "long-qonto-id: POST body must carry the SHORT num, got: $(cat "${STATE}/post_body.json")"
|
||||||
|
python3 -c "
|
||||||
|
import json, sys
|
||||||
|
o = json.loads('''${OUT}''')
|
||||||
|
assert o['transaction_id'] == '${SHORT}', o
|
||||||
|
assert o['id'] == 77 and o['bank_transaction_id'] == 556, o
|
||||||
|
" || fail "long-qonto-id: output JSON must report the normalized num, got: ${OUT}"
|
||||||
|
grep -q 'normalized' "${STATE}/stderr1" || fail "long-qonto-id: normalization must be announced on stderr"
|
||||||
|
|
||||||
|
# --- Case 2: Wise numeric id passes through unchanged ---
|
||||||
|
OUT="$(printf '{"invoice_id":19,"account_id":2,"transaction_id":"2159468139"}' \
|
||||||
|
| DOL_WRITE="${STUB}" STUB_STATE="${STATE}" bash "${PR}" 2>"${STATE}/stderr2")" \
|
||||||
|
|| fail "wise-id: expected success, got $?"
|
||||||
|
grep -q '"num_payment": "2159468139"' "${STATE}/post_body.json" \
|
||||||
|
|| fail "wise-id: POST body must carry the id untouched"
|
||||||
|
grep -q 'normalized' "${STATE}/stderr2" && fail "wise-id: must NOT announce a normalization"
|
||||||
|
|
||||||
|
# --- Case 3: >50 chars after normalization is refused before any POST ---
|
||||||
|
rm -f "${STATE}/post_body.json" "${STATE}/post_endpoint"
|
||||||
|
BAD="qonto-migration-batch-7-payment-reference-0123456789-0123456789" # 63 chars, no "transaction-"
|
||||||
|
rc=0
|
||||||
|
printf '{"invoice_id":13,"kind":"supplier","account_id":1,"amount":96,"transaction_id":"%s"}' "${BAD}" \
|
||||||
|
| DOL_WRITE="${STUB}" STUB_STATE="${STATE}" bash "${PR}" >/dev/null 2>"${STATE}/stderr3" || rc=$?
|
||||||
|
[[ "${rc}" -ne 0 ]] || fail "overlong-id: must exit non-zero"
|
||||||
|
[[ ! -f "${STATE}/post_body.json" ]] || fail "overlong-id: must refuse BEFORE any POST"
|
||||||
|
grep -q 'varchar(50)' "${STATE}/stderr3" || fail "overlong-id: error must cite the varchar(50) constraint"
|
||||||
|
|
||||||
|
echo "OK: payment-record normalization tests passed (long→short, wise untouched, >50 refused pre-POST)"
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Offline stand-in for dol-write.sh, used ONLY by tests/run-tests.sh (injected via
|
||||||
|
# the DOL_WRITE env hook). Records the POST body under $STUB_STATE, then serves it
|
||||||
|
# back as the payments list on GET so payment-record.sh can correlate. Never talks
|
||||||
|
# to any host — sandbox and prod are both out of reach by construction.
|
||||||
|
set -euo pipefail
|
||||||
|
STATE="${STUB_STATE:?stub-dol-write.sh: STUB_STATE not set}"
|
||||||
|
METHOD="$1"; ENDPOINT="$2"; BODY="${3:-}"
|
||||||
|
case "${METHOD}" in
|
||||||
|
POST)
|
||||||
|
printf '%s' "${BODY}" > "${STATE}/post_body.json"
|
||||||
|
printf '%s\n' "${ENDPOINT}" > "${STATE}/post_endpoint"
|
||||||
|
echo "77"
|
||||||
|
;;
|
||||||
|
GET)
|
||||||
|
python3 - "${STATE}/post_body.json" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
body = json.load(open(sys.argv[1]))
|
||||||
|
print(json.dumps([{"num": body.get("num_payment", ""),
|
||||||
|
"date": "2026-06-20 12:00:00",
|
||||||
|
"fk_bank_line": "556"}]))
|
||||||
|
PY
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "stub-dol-write.sh: unexpected method ${METHOD}" >&2; exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
Reference in New Issue
Block a user