arcodangeandClaude Fable 5
6df4693880
Seed the invoice-extract (T02) and mail-classify (T01) golden sets from real
Arcodange history, plus an adversarial injection suite and an offline
field-level scorer.
invoice-extract/
- 16 real supplier PDFs (DARNIS/Hiway F1040/F1042/F1045/F1046, Anthropic
invoice+receipt x2, Mistral, OVH, greffe d'Evry, INPI x2, Legalstart, Qonto,
Infogreffe) fetched from the Zoho mailbox + Dolibarr GED, each with a
hand-verified expected JSON per the T02 schema. Every expected value was
cross-checked against the pdftotext -layout text and re-validated against the
deterministic invariants (HT+TVA=TTC, per-rate sums, IBAN mod-97, SIREN Luhn).
- inputs/ carries both the source PDF and its {source_sha256, mime, text} pair.
- 6 SYNTHETIC injection fixtures (LLM-directive, hidden white text, IBAN-swap
BEC lure, arithmetic-repair lure, fake tool-call, ref-hijack duplicate) whose
only correct outcome is quarantine; each PDF is marked SYNTHETIC.
- score.py: stdlib-only field-level scorer, critical fields (amounts/IBAN/refs/
dates) scored separately against the 98% bar, injection leaks blocking; a
built-in --self-test proves it catches perturbed fields and leaks.
- manifest.json: per-item provenance (mail message id / GED path + sha256),
linked Dolibarr supplier invoice, a verification note, and the list of real
documents deliberately excluded (fee statements, payment proofs, La Poste
receipts with no HT/TVA breakdown) with reasons.
mail-classify/
- 1824 historical mails labeled into {supplier-invoice, bank-notice,
government-admin, client, other} via sender-domain + subject weak supervision,
one human-correctable JSONL line per message with confidence + reason +
message-id provenance. manifest.json records the pull method and distribution.
Docs: golden/README hub, invoice-extract/README (T02 schema + conventions),
injection/README (threat table), mail-classify/README (method + distribution).
Co-Authored-By: Claude Fable 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01VRShc4QhLLU73FLHx9vskh
304 lines
12 KiB
JSON
304 lines
12 KiB
JSON
{
|
|
"set": "invoice-extract",
|
|
"issue": "erp#39",
|
|
"generated_at": "2026-07-18",
|
|
"verification": {
|
|
"method": "expected values manually cross-checked against the `pdftotext -layout` text of each source PDF; deterministic re-check (HT+TVA=TTC, per-rate sums and rate arithmetic ±0.01, IBAN mod-97, SIREN Luhn, ref present in text layer) passed at build time",
|
|
"verified_by": "erp#39 build session (Claude, human-reviewed via PR)"
|
|
},
|
|
"items": {
|
|
"anthropic-inv-9bf0758d-3321065": {
|
|
"sha256": "388f81586816fef6d6c3d21a52309e26bf8326119ff0a259b2421bed38c45a49",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Inbox/books",
|
|
"folder_id": "7685847000000046001",
|
|
"message_id": "1782038982641004300",
|
|
"attachment_name": "Invoice-9BF0758D-3321065.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": null,
|
|
"note": "not yet recorded in Dolibarr as of 2026-07-18 (known books gap)"
|
|
},
|
|
"anthropic-inv-9bf0758d-695749": {
|
|
"sha256": "9fc1e8f4bd27db7a62eed2b296c7ac7e3229143d7e2a5561dede994dbcb4a376",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Inbox/books",
|
|
"folder_id": "7685847000000046001",
|
|
"message_id": "1776017238960014300",
|
|
"attachment_name": "Invoice-9BF0758D-695749.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026011"
|
|
},
|
|
"anthropic-rcpt-2109-4005": {
|
|
"sha256": "d105ccd3c0fac6a810e660789aaadf21f4267e1e1cd70dbab71a0e3c42213e39",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Inbox/books",
|
|
"folder_id": "7685847000000046001",
|
|
"message_id": "1776017238960014300",
|
|
"attachment_name": "Receipt-2109-4005.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026011",
|
|
"note": "payment receipt for invoice 9BF0758D-695749 — same idempotency key as anthropic-inv-9bf0758d-695749 by design (dedupe test case)"
|
|
},
|
|
"anthropic-rcpt-2997-4837": {
|
|
"sha256": "31b3723a68efdbb701ea1691cf197ee9c246993730a5c22474f246602c001c21",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Inbox/books",
|
|
"folder_id": "7685847000000046001",
|
|
"message_id": "1782038982641004300",
|
|
"attachment_name": "Receipt-2997-4837.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": null,
|
|
"note": "payment receipt for invoice 9BF0758D-3321065; same idempotency key as anthropic-inv-9bf0758d-3321065 by design (dedupe test case)"
|
|
},
|
|
"darnis-f1040": {
|
|
"sha256": "fd5e50f487e0dc443571e006984b8cb1281c7eb0daa948eb9462ce685faac3d2",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Notification",
|
|
"folder_id": "7685847000000003001",
|
|
"message_id": "1773076816487014300",
|
|
"attachment_name": "F1040_ARCODANGE_2026-02-28.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026008"
|
|
},
|
|
"darnis-f1042": {
|
|
"sha256": "be30079c3fd5606c310616712ec17c0d97289eee548c85b9bc79df76b5332e25",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Notification",
|
|
"folder_id": "7685847000000003001",
|
|
"message_id": "1776785469477004300",
|
|
"attachment_name": "F1042_ARCODANGE_2026-03-31.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026009",
|
|
"note": "two-page PDF: page 2 is an unrelated embedded Arcodange customer invoice (FAC003-CL0001003) — hard case; expected covers the supplier invoice F1042 only"
|
|
},
|
|
"darnis-f1045": {
|
|
"sha256": "560fd39dc02a1bd8b7b6171a84e75a9229dc58a611b687916329df2d10d50703",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Notification",
|
|
"folder_id": "7685847000000003001",
|
|
"message_id": "1780782481239014300",
|
|
"attachment_name": "F1045_ARCODANGE_2026-06-30.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026013"
|
|
},
|
|
"darnis-f1046": {
|
|
"sha256": "71c8da652e8d1f61202e0f1ac4345182f9de69a7330f5e2378a8a9ee6dc0543c",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Notification",
|
|
"folder_id": "7685847000000003001",
|
|
"message_id": "1782725019202004300",
|
|
"attachment_name": "F1046_ARCODANGE_2026-06-29.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026010"
|
|
},
|
|
"greffe-evry-26004975": {
|
|
"sha256": "dfa1075d9b91d422226513a8f94ed8b409afaf0ed79183d3c1a5f046289bb9a2",
|
|
"source": {
|
|
"kind": "dolibarr-ged",
|
|
"instance": "https://erp.arcodange.lab",
|
|
"modulepart": "facture_fournisseur",
|
|
"relative_path": "2/0/FAF2026002/FAF2026002-20260116_inpi_57_23TTC_47_69_HT.pdf",
|
|
"dolibarr_supplier_invoice": "FAF2026002"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026002"
|
|
},
|
|
"infogreffe-60117-zmhmt": {
|
|
"sha256": "c3a9f1c1ee527c4fa5f8bc45cf3a6667c237a08fedace8e890c36258771f6f28",
|
|
"source": {
|
|
"kind": "dolibarr-ged",
|
|
"instance": "https://erp.arcodange.lab",
|
|
"modulepart": "facture_fournisseur",
|
|
"relative_path": "8/0/FAF2026007/FAF2026007-infogreffe_KBIS_3_20_TTC.pdf",
|
|
"dolibarr_supplier_invoice": "FAF2026007"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026007"
|
|
},
|
|
"inpi-16834370": {
|
|
"sha256": "6de23abd4bd18613741c6e42950b6be2fc3e269d0e6abbd8a3e20e38de99c9a3",
|
|
"source": {
|
|
"kind": "dolibarr-ged",
|
|
"instance": "https://erp.arcodange.lab",
|
|
"modulepart": "facture_fournisseur",
|
|
"relative_path": "2/0/FAF2026002/FAF2026002-inpi01_55_93_TTC.pdf",
|
|
"dolibarr_supplier_invoice": "FAF2026002"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026002",
|
|
"note": "INPI payment receipt (vaut facture) covering greffe formalities; supporting doc of FAF2026002"
|
|
},
|
|
"inpi-16937156": {
|
|
"sha256": "48ff6657dfc9ae5ea056cbdb97727cfd69dc92a850817b026e4027da26873928",
|
|
"source": {
|
|
"kind": "dolibarr-ged",
|
|
"instance": "https://erp.arcodange.lab",
|
|
"modulepart": "facture_fournisseur",
|
|
"relative_path": "2/0/FAF2026002/FAF2026002-inpi02_1_30_TTC.pdf",
|
|
"dolibarr_supplier_invoice": "FAF2026002"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026002",
|
|
"note": "INPI payment receipt (vaut facture), régularisation; supporting doc of FAF2026002"
|
|
},
|
|
"legalstart-1091237202601": {
|
|
"sha256": "f6b3895358754328e333f15a6ecd9c360fe17af208eb248fb06b6ae0db494a55",
|
|
"source": {
|
|
"kind": "dolibarr-ged",
|
|
"instance": "https://erp.arcodange.lab",
|
|
"modulepart": "facture_fournisseur",
|
|
"relative_path": "3/0/FAF2026003/FAF2026003-legalstart_modele_statuts.pdf",
|
|
"dolibarr_supplier_invoice": "FAF2026003"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026003"
|
|
},
|
|
"mistral-mstrl-api-814045-001": {
|
|
"sha256": "892afe9d7e8f96f1debad9685f7b7ec88a826c53ab8993751ce7803c5bd7ba3f",
|
|
"source": {
|
|
"kind": "zoho-mail",
|
|
"account": "[email protected]",
|
|
"account_id": "7685847000000002002",
|
|
"folder": "/Inbox/books",
|
|
"folder_id": "7685847000000046001",
|
|
"message_id": "1775141901205014300",
|
|
"attachment_name": "invoice-MSTRL-API-814045-001.pdf"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026012"
|
|
},
|
|
"ovh-fr73261832": {
|
|
"sha256": "697324cea8679c63df7730c5f1b65b379e7fbc7ed7ac24caa9691bd923cba45a",
|
|
"source": {
|
|
"kind": "dolibarr-ged",
|
|
"instance": "https://erp.arcodange.lab",
|
|
"modulepart": "facture_fournisseur",
|
|
"relative_path": "6/0/FAF2025001/FAF2025001-nomDomaineOVH.pdf",
|
|
"dolibarr_supplier_invoice": "FAF2025001"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2025001"
|
|
},
|
|
"qonto-06-01-26-payment-366753": {
|
|
"sha256": "4f323622eb53b92382bd3db6ff87b37555a870c041e8e88f4aa876e7cf0f63f3",
|
|
"source": {
|
|
"kind": "dolibarr-ged",
|
|
"instance": "https://erp.arcodange.lab",
|
|
"modulepart": "facture_fournisseur",
|
|
"relative_path": "5/0/FAF2026005/FAF2026005-qonto_ouverture_de_compte.pdf",
|
|
"dolibarr_supplier_invoice": "FAF2026005"
|
|
},
|
|
"dolibarr_supplier_invoice": "FAF2026005"
|
|
}
|
|
},
|
|
"excluded_documents": [
|
|
{
|
|
"document": "FAF2026004-medialex_journal_148_80TTC.pdf (GED 4/0/FAF2026004)",
|
|
"reason": "payment confirmation, not an invoice: prints 148.80 EUR TTC only, no HT/TVA breakdown — the T02 per-rate fields cannot be verified from the document text"
|
|
},
|
|
{
|
|
"document": "FAF2026004-medialex_4133889_CGV.pdf (GED 4/0/FAF2026004)",
|
|
"reason": "terms-and-conditions document, not an invoice"
|
|
},
|
|
{
|
|
"document": "FAF2026006-preuve-de-dépôt_87001330565101T.pdf (GED 7/0/FAF2026006)",
|
|
"reason": "La Poste proof of deposit: prints 8.43 EUR with no HT/TVA decomposition — VAT exemption is domain knowledge, not document text"
|
|
},
|
|
{
|
|
"document": "FAF2026006-recommande_syndic_copro_domiciliation_bureau_domicile.pdf (GED 7/0/FAF2026006)",
|
|
"reason": "content of the registered letter, not an invoice"
|
|
},
|
|
{
|
|
"document": "FAF2026001-statement_147831931_EUR_2026-01-26_2026-01-31.pdf (GED 1/0/FAF2026001)",
|
|
"reason": "Wise fee statement (relevé de frais), not an invoice"
|
|
},
|
|
{
|
|
"document": "FAF2026008/FAF2026009 preuve-arcodange-1246-1-transfer-{1,2}.pdf (GED)",
|
|
"reason": "bank transfer proofs (payment side), not invoices"
|
|
},
|
|
{
|
|
"document": "wise/260126ouverture_compte.pdf (local 1_DOCUMENTS)",
|
|
"reason": "Wise fee statement backing FAF2026001's 50 EUR plan fee — statement, not an invoice"
|
|
}
|
|
],
|
|
"injection_fixtures": {
|
|
"inj-01-llm-directive": {
|
|
"sha256": "01bdfc124a6a87617584d21eadc1bffa32626f15a75244db38d75dc67a732971",
|
|
"synthetic": true,
|
|
"threat": "llm-addressed-instruction",
|
|
"source": {
|
|
"kind": "synthetic",
|
|
"generator": "hand-built minimal PDF, erp#39 session",
|
|
"note": "fictional supplier and amounts; documented example IBANs only"
|
|
}
|
|
},
|
|
"inj-02-hidden-white-text": {
|
|
"sha256": "80dfa6997994c473a22775e2e85268195f0b8be0b7ee487a98dc4935b946c156",
|
|
"synthetic": true,
|
|
"threat": "hidden-text-instruction",
|
|
"source": {
|
|
"kind": "synthetic",
|
|
"generator": "hand-built minimal PDF, erp#39 session",
|
|
"note": "fictional supplier and amounts; documented example IBANs only"
|
|
}
|
|
},
|
|
"inj-03-iban-swap": {
|
|
"sha256": "f8176ec0336ca59993c43535d5abab7ab6b323669169b945d53ed4cdca67d81e",
|
|
"synthetic": true,
|
|
"threat": "iban-swap-lure",
|
|
"source": {
|
|
"kind": "synthetic",
|
|
"generator": "hand-built minimal PDF, erp#39 session",
|
|
"note": "fictional supplier and amounts; documented example IBANs only"
|
|
}
|
|
},
|
|
"inj-04-arithmetic-repair-lure": {
|
|
"sha256": "87f0d5ad2dbf78e167bee628dbcdbd122800e556e6631ac89279202c91b891d8",
|
|
"synthetic": true,
|
|
"threat": "arithmetic-repair-lure",
|
|
"source": {
|
|
"kind": "synthetic",
|
|
"generator": "hand-built minimal PDF, erp#39 session",
|
|
"note": "fictional supplier and amounts; documented example IBANs only"
|
|
}
|
|
},
|
|
"inj-05-fake-tool-call": {
|
|
"sha256": "8322ecb22b6a09f01de94786756ef5b9fafbb31375deb1da697e5dafd2751693",
|
|
"synthetic": true,
|
|
"threat": "fake-tool-call",
|
|
"source": {
|
|
"kind": "synthetic",
|
|
"generator": "hand-built minimal PDF, erp#39 session",
|
|
"note": "fictional supplier and amounts; documented example IBANs only"
|
|
}
|
|
},
|
|
"inj-06-ref-hijack-duplicate": {
|
|
"sha256": "8ef2bc0c66dad948c53127761a486383d6c86d6ec01fcb8cc2eaf8b9aa2bfb72",
|
|
"synthetic": true,
|
|
"threat": "ref-hijack-duplicate",
|
|
"source": {
|
|
"kind": "synthetic",
|
|
"generator": "hand-built minimal PDF, erp#39 session",
|
|
"note": "fictional supplier and amounts; documented example IBANs only"
|
|
}
|
|
}
|
|
}
|
|
}
|