erp/.claude/skills/arcodange-email-ingest/scripts/extract_fields.py

#!/usr/bin/env python3
"""Field extraction from a supplier-invoice PDF text layer.

Lifted out of email-inspect.sh so it can be REGRESSION-TESTED against the 16
hand-verified invoices in fleet/golden/invoice-extract/ (see test_extract.py).
It was a heredoc, therefore untestable, therefore wrong in ways nobody could
measure: on Darnis F1048 it returned Arcodange's own VAT number as the invoice
reference and no date at all.

Two structural traps, both observed on real documents:

  1. `Nº` in French invoices is U+00BA (MASCULINE ORDINAL INDICATOR), not the
     degree sign `°`. A character class of [°o] silently misses it, the primary
     pattern fails, and the fallback grabs the first ref-shaped token in the
     document — which is very often a VAT number.
  2. Two-column layouts put the label and its value on DIFFERENT lines, with
     unrelated text in between. A pattern requiring `label:\\s*value` never
     matches, so the field comes back empty.

Principle kept from the atom contract: emit nothing rather than emit a guess.
A value that fails its own sanity check is dropped, not repaired.
"""
from __future__ import annotations

import re

# Ordinal marks seen on real French invoices: degree sign, masculine ordinal, 'o'
ORD = "[°ºo]?"

# VAT numbers are ref-shaped and appear near the top of most invoices — the
# single biggest source of false "invoice_ref". Two country prefixes + 8-13
# digits/letters, no separators.
VAT_LIKE = re.compile(r"^(?:FR[0-9A-Z]{11}|BE0?\d{9,10}|DE\d{9}|ES[0-9A-Z]\d{7}[0-9A-Z]|IT\d{11}"
                      r"|NL\d{9}B\d{2}|LU\d{8}|IE\d{7}[A-W]{1,2}|PT\d{9}|AT U\d{8}|PL\d{10})$", re.I)
SIRET_LIKE = re.compile(r"^\d{9,14}$")


def parse_amount(s: str) -> float | None:
    """European or Anglo number → float. None when it is not a number."""
    if not s:
        return None
    t = re.sub(r"[\s  ]", "", s).strip().rstrip(".,")
    if not t:
        return None
    if "," in t and "." in t:                      # 1.234,56 or 1,234.56
        t = t.replace(".", "").replace(",", ".") if t.rfind(",") > t.rfind(".") \
            else t.replace(",", "")
    elif "," in t:
        t = t.replace(",", ".")
    try:
        return float(t)
    except ValueError:
        return None


def extract_ref(text: str) -> str | None:
    """The supplier's own invoice number."""
    # 1. Explicitly labelled, on the same line — the reliable case.
    for pat in (
        rf"(?:facture|invoice|receipt|re[çc]u)\s*(?:n{ORD}|number|#|:)\s*[:\s]*([A-Za-z0-9][\w\d/-]{{2,}})",
        r"(?:num[ée]ro\s+de\s+facture|invoice\s+number)\s*:?\s*([A-Za-z0-9][\w\d/-]{2,})",
    ):
        for m in re.finditer(pat, text, re.IGNORECASE):
            cand = m.group(1).strip(" .:،,")
            if _plausible_ref(cand):
                return cand

    # 2. Two-column layout: the label sits alone, the value lands further down
    #    the page. Look ahead a bounded window and take the first plausible
    #    token — bounded so we do not wander into another block.
    m = re.search(r"(?:num[ée]ro\s+de\s+facture|invoice\s+number|facture\s+n" + ORD + r")\s*:?", text, re.IGNORECASE)
    if m:
        window = text[m.end():m.end() + 400]
        # Rejoin FIRST: if the window opens on a dangling fragment, the tail alone
        # is also plausible, and scanning first would return a ref amputated of
        # its head — a subtly wrong value, the worst kind.
        # A ref split by the column wrap: "06-01-26-" ends a line, its tail
        # ("payment-366753") lands further down in the other column. Rejoin the
        # dangling fragment with the next token that carries a digit.
        frag = re.match(r"\s*([A-Za-z0-9][\w\d/-]*[-/])(?=\s)", window)
        if frag:
            for tail in re.findall(r"\b([A-Za-z0-9][\w\d/-]{2,})\b", window[frag.end():frag.end() + 200]):
                if any(c.isdigit() for c in tail):
                    joined = frag.group(1) + tail
                    if _plausible_ref(joined):
                        return joined
        for cand in re.findall(r"\b([A-Za-z0-9][\w\d/-]{2,})\b", window):
            if _plausible_ref(cand):
                return cand

    # 3. No labelled ref found. Emit nothing rather than the first ref-shaped
    #    token in the document — that is how a VAT number ends up as an invoice
    #    reference, which is exactly the bug this module exists to kill.
    return None


def _plausible_ref(cand: str) -> bool:
    if not cand or not any(c.isdigit() for c in cand):
        return False
    if VAT_LIKE.match(cand) or SIRET_LIKE.match(cand):
        return False
    if re.fullmatch(r"\d{1,2}[/.-]\d{1,2}[/.-]\d{2,4}", cand):   # a date
        return False
    if len(cand) > 40:
        return False
    if cand.endswith(("-", "/", ".")):   # tronquée au saut de colonne
        return False
    return True


MONTHS = {}
for i, names in enumerate([
    ("janvier", "janv", "january", "jan"), ("février", "fevrier", "févr", "fevr", "february", "feb"),
    ("mars", "march", "mar"), ("avril", "avr", "april", "apr"), ("mai", "may"),
    ("juin", "june", "jun"), ("juillet", "juil", "july", "jul"), ("août", "aout", "august", "aug"),
    ("septembre", "sept", "september", "sep"), ("octobre", "oct", "october"),
    ("novembre", "nov", "november"), ("décembre", "decembre", "déc", "dec", "december"),
], start=1):
    for n in names:
        MONTHS[n] = i


def to_iso(raw: str | None) -> str | None:
    """dd/mm/yyyy or '2 avr. 2026' → yyyy-mm-dd. Consumers want one shape, not five."""
    if not raw:
        return None
    raw = raw.strip()
    if re.fullmatch(r"\d{4}-\d{2}-\d{2}", raw):
        return raw
    m = re.fullmatch(r"(\d{1,2})[/.\-](\d{1,2})[/.\-](\d{2,4})", raw)
    if m:
        d, mo, y = m.groups()
        y = "20" + y if len(y) == 2 else y
        return f"{y}-{int(mo):02d}-{int(d):02d}"
    m = re.fullmatch(r"(\d{1,2})\s+([A-Za-zéûîàôçÉ.]+)\.?\s+(\d{4})", raw)
    if m:
        d, mon, y = m.groups()
        idx = MONTHS.get(mon.lower().rstrip("."))
        if idx:
            return f"{y}-{idx:02d}-{int(d):02d}"
    return raw


DATE_LABELS = (r"date\s*d[e']\s*[ée]mission", r"date\s*de\s*facturation", r"invoice\s*date",
               r"date\s*d[e']\s*facture", r"[ée]mise?\s*le", r"\bdate\b")


def extract_date(text: str) -> str | None:
    """Issue date. Prefers a labelled date; falls back to the earliest date seen."""
    # ISO anywhere wins — unambiguous.
    m = re.search(r"\b(\d{4}-\d{2}-\d{2})\b", text)
    if m:
        return m.group(1)

    # Labelled, allowing the two-column gap: up to 300 chars of anything between
    # the label and its value (the value is often a column away and a line down).
    for lab in DATE_LABELS:
        for m in re.finditer(lab, text, re.IGNORECASE):
            window = text[m.end():m.end() + 300]
            d = re.search(r"\b(\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4})\b", window)
            if d:
                return d.group(1)
            d = re.search(r"\b(\d{1,2}\s+[A-Za-zéûîàôç.]{3,10}\.?\s+\d{4})\b", window)
            if d:
                return d.group(1)

    # Unlabelled: take the first date-shaped token, but only if the document has
    # one — no invention.
    m = re.search(r"\b(\d{1,2}[/.\-]\d{1,2}[/.\-]\d{4})\b", text)
    return m.group(1) if m else None


def extract_amounts(text: str) -> dict:
    """HT / TVA / TTC. Returns only what the document actually labels."""
    lines = text.splitlines()

    def first(pattern: str) -> str | None:
        for line in lines:
            m = re.search(pattern, line, re.IGNORECASE)
            if m:
                v = parse_amount(m.group(1))
                if v is not None:
                    return f"{v:.2f}"
        return None

    AMT = r"[^\d-]*([\d\u00a0\u202f .,]+)"
    # "Montant total (HT)" is an HT, not a TTC — the parenthetical decides, so it
    # is matched on the HT side and excluded on the TTC side.
    ht = first(r"(?:total\s*ht|montant\s*ht|montant\s*total\s*\(\s*ht\s*\)|net\s*amount|subtotal|sous.?total\s*\(\s*ht\s*\))" + AMT)
    # The rate often sits between the label and the amount — "TVA (20%) : 43,70 €".
    # A currency mark is still required so a VAT *number* is never read as a VAT *amount*.
    tva = first(r"(?:tva|vat)\s*(?:\(\s*\d{1,2}(?:[.,]\d+)?\s*%\s*\))?\s*:?" + AMT + r"\s*(?:€|\beur\b)")
    ttc = first(r"(?:total\s*ttc|amount\s*due|total\s*due|grand\s*total|montant\s*total|amount\s*paid)(?!\s*\(?\s*ht)" + AMT)
    if ttc is None:
        # Some invoices label the TTC simply "Total" (Legalstart, INPI). Accept it
        # only as a fallback, and only on a line that is nothing else.
        ttc = first(r"^\s*total\s+" + AMT.lstrip("[^\\d-]*") if False else r"^\s*total\s+([\d\u00a0\u202f .,]+)\s*€")

    out = {"total_ht": ht, "total_tva": tva, "total_ttc": ttc}
    # Arithmetic sanity: if HT + TVA disagrees with TTC we matched the wrong
    # lines somewhere. Report it rather than pretend the numbers are coherent.
    h, t_, c = (parse_amount(out[k] or "") for k in ("total_ht", "total_tva", "total_ttc"))
    out["arithmetic_ok"] = None if None in (h, t_, c) else abs(h + t_ - c) <= 0.01
    return out


def extract_vat_rate(text: str) -> str | None:
    for line in text.splitlines():
        m = re.search(r"\b(\d{1,2}(?:[.,]\d+)?)\s*%", line)
        if m:
            v = float(m.group(1).replace(",", "."))
            if 0 <= v <= 25:
                return m.group(1)
    return None


def extract(text: str) -> dict:
    out = {
        "invoice_ref": extract_ref(text),
        "invoice_date_raw": to_iso(extract_date(text)),
        "vat_rate_pct": extract_vat_rate(text),
    }
    out.update(extract_amounts(text))
    return out