#!/usr/bin/env python3
"""trims.py: what did this version DROP?

The merge seat's question when a profile is rewritten to fit the 16,000-byte
cap: a diff that adds one paragraph and shortens fourteen others is a diff
whose additions everyone reads and whose removals nobody does. Three wakes
running (#181, #185, #187) the interesting finding was in the removals, and
the one that mattered was a VENDOR's sentence, not the researcher's narration:
#181 dropped "We recommend that you start with the supervised mode before
using the autonomous mode." while its body said it had dropped four asides.

So this asks one question per changed file: which PROTECTED ATOMS were in the
base and are not in the head? An atom is something no trim is allowed to lose
silently -- a quoted span (the vendor's words), a link, a date, a money figure,
a backticked identifier. Everything else is narration, and narration is the
author's to cut.

It is a reading aid, not a gate: a lost atom can be perfectly correct (a
retired quotation, a page that died). It names them so the reading happens.

WHY NOT THE OBVIOUS ONE-LINER. The quick version of this (set of `"..."`
matches in base minus head) RE-PAIRS EVERY QUOTE AFTER THE FIRST CHANGE: drop
one quoted span and the opening quote of the next pairs with the closing quote
of the one after it, so the tool reports losses and gains that are artifacts of
its own scanner. I ran exactly that one-liner at wake 22dc036f and it printed
six "gains" that were not in the diff at all. Quotes here are paired WITHIN A
LINE only, which is what a paragraph-per-line markdown corpus actually holds.

  python3 tools/trims.py --clone /tmp/reg --base origin/main --head pr187
  python3 tools/trims.py --clone /tmp/reg --ref <merge sha>      # one commit
  python3 tools/trims.py --selftest                              # no network, no clone
"""

import argparse
import os
import re
import subprocess
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from corpus import mask_code  # noqa: E402  (one shared masker, never a copy)

CAP = 16000  # the registry's profile byte cap


def nbytes(text):
    """UTF-8 bytes, which is what the gate counts.

    src/lib/profile.ts refuses PROFILE_TOO_LARGE on
    `Buffer.byteLength(markdown, "utf8") > 16000`. Until wake 6add8abb this
    file reported `len(text)` — CHARACTERS — so every non-ASCII character in a
    profile bought one byte of headroom that does not exist. On #338's github
    profile (two U+2013 en dashes) it printed "28 under the cap" where the gate
    sees 24, in the one direction that matters: an entry at the cap, blessed
    here, refused by CI. A dash, a curly quote or an accent is where the error
    lives, and tight profiles are exactly the ones that acquire them.
    """
    return len(text.encode("utf-8"))

# --- atoms ------------------------------------------------------------------
# Every pattern is line-local on purpose (see the docstring).

MIN_QUOTE = 4  # shorter than this is punctuation, not a quotation worth tracking
CURLY_QUOTE = re.compile(r'“([^”\n]{%d,})”' % MIN_QUOTE)


def straight_quotes(line):
    """Quoted spans of one line, PAIRED BY POSITION and only then filtered.

    The obvious regex -- `"([^"\\n]{4,})"` -- de-synchronizes on a SHORT
    quotation. Found on shopify-ucp v7 (#275): the tier row the head quotes as
    `"Yes"` is three characters, so the pattern failed at its opening mark,
    re-used its CLOSING mark as an opening one, and mis-paired every span after
    it in that paragraph. The vendor's own exclusion sentence (`complete_checkout
    and order tools aren't available at this tier`) was byte-identical in base
    and head and came out LOST, with a fake atom ('under Checkout tools, while
    ...') reported as a gain beside it.

    This is the THIRD generation of the same parity bug: the one-liner that
    re-paired across the whole document (this file's own docstring), the same
    across-paragraphs bug in profiles.py, and now a filter that silently
    consumed a delimiter. The lesson each time: pair the delimiters FIRST,
    filter the contents AFTER -- never let a content test decide what a
    delimiter is.

    An unterminated final segment (an odd number of marks on the line) is
    dropped rather than run to the end of the paragraph.
    """
    # A `"` inside backticks is a character being named, not a delimiter:
    # github and hubspot both write ``Every `"`-delimited span`` and each
    # leaves its file with an odd number of marks. Mask first, then pair on
    # the masked copy, then take the slice from the ORIGINAL (same offsets,
    # because mask_code preserves length).
    masked = mask_code(line)
    segs = masked.split('"')
    at = 0
    for i, seg in enumerate(segs):
        if i % 2 == 1 and i < len(segs) - 1 and len(seg) >= MIN_QUOTE:
            yield line[at:at + len(seg)]
        at += len(seg) + 1


LINK = re.compile(r'\[([^\]\n]+)\]\((https?://[^)\s]+)\)')
BARE_URL = re.compile(r'(?<![(\]])\bhttps?://[^\s)\]}>"\'`,;]+')
DATE = re.compile(r'\b(?:19|20)\d\d-\d\d-\d\d\b')
MONEY = re.compile(r'\$\s?\d[\d,]*(?:\.\d+)?')
# `%` is not a word character, so a trailing \b after it never matches before a
# space: "12% of list" found nothing until the boundary was moved inside the
# spelled-out alternative. Caught by rule "a dropped percent is reported".
PERCENT = re.compile(r'\b\d+(?:\.\d+)?\s?(?:%|percent\b)')
CODE = re.compile(r'`([^`\n]{2,})`')

# --- the seventh class: a bare count is a measurement too --------------------
#
# WHY IT EXISTS, and it took two misses in two days to write it. MONEY is
# `$`-prefixed and PERCENT needs a `%`, so a measurement written as a plain
# numeral matched NO class above and this tool printed "no protected atom
# lost" over the top of it. Twice:
#
#   wake c475269b, #316 (marco v2): the removed bullet carried `99 measurable`
#     (1 -> 0) and `6 broken targets` (1 -> 0).
#   wake f46aeece, #285 (mintlify-index-mcp): the rewritten revision-log
#     bullet dropped `12,000-byte working target`, the figure the compression
#     it describes was aiming at -- in the same commit that moved the file 413
#     bytes further from it.
#
# THE DESIGN PROBLEM is that bare integers are everywhere, so "every digit"
# floods and a flooding report is one nobody reads -- the exact defect this
# tool exists to avoid. The rule is therefore not a number, it is a number
# that MEASURES SOMETHING: a numeral followed, within one word, by a word
# that names what was counted. Three shapes, and nothing else:
#
#   N <noun>          99 measurable, 6 broken, 15 calls, 104.0 ms
#   N-<unit>          12,000-byte, 118-byte
#   N of M            0 of 8, 7 of 8        (the cross-reference shape)
#
# A one- or two-letter word after the numeral is a connective, not a noun
# ("version 3 of the schema", "section 6 is exploratory"), so the following
# word must be at least three letters. That one condition is what keeps the
# class from matching ordinary prose.
NUM = r'\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?'
# Spans another class already owns. Masked FIRST, with spaces of equal length
# so nothing bridges the gap, because a count must never be a second reading
# of a date ("2026-10-28 restores" is not "28 restores"), a price, a
# percentage or a backticked identifier.
COUNT_MASK = re.compile(
    r'`[^`\n]*`'
    # An issue or pull-request reference is a NAME, not a measurement: on the
    # #285 restoration "Detail on #285 and #317" produced the atom "285 and".
    # Harmless as a gain, noise as a loss, and this class lives or dies on
    # whether its rows are worth reading.
    r'|#\d+'
    r'|\$\s?\d[\d,]*(?:\.\d+)?'
    r'|\b(?:19|20)\d\d-\d\d-\d\d\b'
    r'|\b\d+(?:\.\d+)?\s?(?:%|percent\b)'
)
COUNT_OF = re.compile(rf'\b({NUM})\s+of\s+({NUM})\b')
COUNT_UNIT = re.compile(rf'\b({NUM})-([A-Za-z]{{2,}})\b')
COUNT_NOUN = re.compile(rf'\b({NUM})\s+([A-Za-z]{{3,}})\b')
# The three-letter floor costs the short UNITS, and `1007.2 ms` is as much a
# measurement as `99 measurable`. They come back as an explicit list rather
# than by relaxing the floor, because the floor is the only thing standing
# between this class and every numeral in the corpus. Single letters stay out:
# `s`, `m` and `h` after a numeral are as often a stray initial as a unit, and
# this corpus writes milliseconds.
COUNT_UNITS = "ms|kb|mb|gb|tb|kib|mib|gib|px|hz|khz|mhz|bps|kbps|mbps|rpm|rps"
COUNT_SHORT_UNIT = re.compile(rf'\b({NUM})\s+({COUNT_UNITS})\b', re.IGNORECASE)

KINDS = ("quote", "link", "url", "date", "money", "percent", "code", "count")


def atoms(text):
    """kind -> {atom: count}. Counts matter: losing one of three copies is a
    change too, and a set would hide it."""
    out = {k: {} for k in KINDS}

    def add(kind, value):
        value = re.sub(r"\s+", " ", value).strip()
        if value:
            out[kind][value] = out[kind].get(value, 0) + 1

    for line in text.split("\n"):
        for span in straight_quotes(line):
            add("quote", span)
        for m in CURLY_QUOTE.finditer(line):
            add("quote", m.group(1))
        for m in LINK.finditer(line):
            add("link", m.group(2))
        # A markdown link's URL is not also a bare URL: blank the links first.
        bare_line = LINK.sub(lambda m: "[" + m.group(1) + "]()", line)
        for m in BARE_URL.finditer(bare_line):
            add("url", m.group(0).rstrip(".)"))
        for m in DATE.finditer(line):
            add("date", m.group(0))
        for m in MONEY.finditer(line):
            add("money", m.group(0))
        for m in PERCENT.finditer(line):
            add("percent", m.group(0))
        for m in CODE.finditer(line):
            add("code", m.group(1))
        # Counts last, on a line whose date/money/percent/code spans are
        # blanked to spaces of the same width.
        count_line = COUNT_MASK.sub(lambda m: " " * len(m.group(0)), line)
        for m in COUNT_OF.finditer(count_line):
            add("count", m.group(0))
        for m in COUNT_UNIT.finditer(count_line):
            add("count", m.group(0))
        for m in COUNT_NOUN.finditer(count_line):
            add("count", m.group(0))
        for m in COUNT_SHORT_UNIT.finditer(count_line):
            add("count", m.group(0))
    return out


def widened(atom, head_atoms):
    """Did this lost QUOTE survive inside a LONGER quotation in the head?

    A profile at the byte cap that re-attributes a span routinely quotes MORE
    of the vendor's sentence than before: shopify-ucp v7 (#275) replaced the
    unattributed `always remains the merchant of record` with the whole
    sentence `The merchant always remains the merchant of record`, and
    `aren't available at this tier` with the row it came from. Both printed
    LOST, both were standing verbatim in the head, and a true loss delivered
    beside two artifacts is a true loss I will average away -- the same
    failure the `moved` and `thinned` classifications already fixed.

    Two deliberate narrownesses, each a selftest rule:

    1. QUOTES ONLY. For code, urls and links, containment means a DIFFERENT
       thing: `get_order` inside `get_orders` is another identifier and
       `.../api` inside `.../api/mcp` is another page. Widening is normal
       only for a quoted span, which is also the kind this report tells the
       reader to read first, so a false positive there costs the most.
    2. INSIDE A HEAD *ATOM*, never merely inside the head TEXT. A span the
       author kept but stopped quoting has lost its attribution: it is now
       the author's sentence, which is exactly the change worth reading.

    Returns the containing head atom, or None.
    """
    for cand in head_atoms:
        if len(cand) <= len(atom):
            continue
        i = cand.find(atom)
        while i != -1:
            before_ok = i == 0 or not (cand[i - 1].isalnum() or cand[i - 1] == "_")
            j = i + len(atom)
            after_ok = j == len(cand) or not (cand[j].isalnum() or cand[j] == "_")
            if before_ok and after_ok:
                return cand
            i = cand.find(atom, i + 1)
    return None


def delta(before, after):
    """kind -> (lost, gained, thinned), each a list of (atom, count).

    LOST means the head holds the atom ZERO times. THINNED means the head
    holds fewer copies but still holds one, which is not a loss: the
    vendor's sentence still stands in the file. Until wake 27836fa9 this
    function counted a 2->1 drop as a loss, and on the statsig v6 rewrite
    (f0a9c34) that made it print three LOST quotations of which two were
    plainly still in the head, one of them the vendor's own "This installs
    the Statsig MCP Server with OAuth." A duplicate falls whenever a
    profile at the byte cap folds its own correction history into the
    paragraph that already carried the quotation, which is a normal and
    good edit. An instrument that cries loss there is the same failure as
    the one the 'moved' classification fixed: a true positive delivered
    inside false ones is one I will average away.
    """
    result = {}
    for kind in KINDS:
        b, a = before[kind], after[kind]
        lost = [(k, b[k]) for k in b if a.get(k, 0) == 0]
        thinned = [(k, b[k] - a[k]) for k in b if 0 < a.get(k, 0) < b[k]]
        gained = [(k, a[k] - b.get(k, 0)) for k in a if a[k] > b.get(k, 0)]
        result[kind] = (sorted(lost), sorted(gained), sorted(thinned))
    return result


# --- git --------------------------------------------------------------------

def git(clone, *args):
    r = subprocess.run(["git", "-C", clone, *args], capture_output=True, text=True)
    if r.returncode != 0:
        raise SystemExit("git " + " ".join(args) + ": " + r.stderr.strip())
    return r.stdout


def show(clone, ref, path):
    r = subprocess.run(["git", "-C", clone, "show", f"{ref}:{path}"],
                       capture_output=True, text=True)
    return r.stdout if r.returncode == 0 else None


def changed_files(clone, base, head):
    out = git(clone, "diff", "--name-only", f"{base}...{head}")
    return [p for p in out.split("\n") if p.strip()]


# --- report -----------------------------------------------------------------

def sibling_text(clone, head, files, exclude):
    """Everything the head says in the OTHER files this commit touched, flattened.

    An atom that leaves profile.md and lands in a probe record was MOVED, not
    lost, and the first calibration of this tool reported six such moves as
    losses on the notion entry alone (the Bearer challenge's `invalid_token`,
    `error_description` and the protected-resource URL moved out of the profile
    into the evidence JSON filed in the same commit). A tool that cries wolf on
    a restructuring is one whose real findings get skimmed."""
    out = []
    for p in files:
        if p == exclude:
            continue
        t = show(clone, head, p)
        if t:
            out.append(t)
    return re.sub(r"\s+", " ", "\n".join(out))


def report(clone, base, head, only_md=True, quiet_gains=False):
    all_changed = changed_files(clone, base, head)
    files = [f for f in all_changed
             if f.endswith(".md") or (not only_md and f.endswith(".json"))]
    if not files:
        print("no markdown files changed between these two trees")
        return 0
    total_lost = {k: 0 for k in KINDS}
    total_moved = 0
    total_thinned = 0
    total_widened = 0
    for path in files:
        old, new = show(clone, base, path), show(clone, head, path)
        if old is None:
            print(f"\n=== {path}: new file ({nbytes(new or '')} bytes), nothing can be lost")
            continue
        if new is None:
            print(f"\n=== {path}: DELETED. every atom in it is lost; read the reason on the thread")
            continue
        d = delta(atoms(old), atoms(new))
        head_quotes = list(atoms(new)["quote"])
        elsewhere = sibling_text(clone, head, all_changed, path)
        nlost = sum(1 for k in KINDS for a, _ in d[k][0]
                    if a not in elsewhere
                    and not (k == "quote" and widened(a, head_quotes)))
        old_b, new_b = nbytes(old), nbytes(new)
        headroom = CAP - new_b
        print(f"\n=== {path}: {old_b} -> {new_b} bytes "
              f"({new_b - old_b:+d}, {headroom} under the {CAP} cap)")
        if nlost == 0:
            print("    no protected atom lost: every quotation, link, date, figure "
                  "and identifier in the base survives here or in a file this "
                  "commit moved it to")
        for kind in KINDS:
            lost, gained, thinned = d[kind]
            for atom, n in thinned:
                total_thinned += 1
                print(f"  thin  {kind:<7} (-{n}, still in the file) {atom[:120]}")
            for atom, n in lost:
                times = "" if n == 1 else f" (x{n})"
                if atom in elsewhere:
                    total_moved += 1
                    print(f"  moved {kind:<7}{times} {atom[:120]}"
                          f"   -> another file of this commit")
                    continue
                wider = widened(atom, head_quotes) if kind == "quote" else None
                if wider:
                    total_widened += 1
                    print(f"  wider {kind:<7}{times} {atom[:110]}")
                    print(f"        still quoted, inside: \"{wider[:110]}\"")
                    continue
                total_lost[kind] += n
                print(f"  LOST  {kind:<7}{times} {atom[:150]}")
            if not quiet_gains:
                for atom, n in gained:
                    times = "" if n == 1 else f" (x{n})"
                    print(f"  new   {kind:<7}{times} {atom[:150]}")
    print()
    total = sum(total_lost.values())
    if total_moved:
        print(f"{total_moved} atom(s) left the file and are in another file this "
              f"commit changed: moved, not lost.")
    if total_thinned:
        print(f"{total_thinned} atom(s) lost a duplicate and still stand in the "
              f"file: thinned, not lost.")
    if total_widened:
        print(f"{total_widened} quotation(s) are quoted inside a LONGER span in "
              f"the head: widened, not lost (usually a re-attribution).")
    if total == 0:
        print("0 protected atoms lost across every changed markdown file.")
    else:
        parts = ", ".join(f"{total_lost[k]} {k}" for k in KINDS if total_lost[k])
        print(f"{total} protected atoms lost: {parts}.")
        print("A LOST QUOTE IS THE ONE TO READ FIRST: it is the vendor's sentence, "
              "not the author's. Ask the thread whether the removal is deliberate.")
    return total


# --- selftest ---------------------------------------------------------------

def selftest():
    """Falsify the rules with no clone and no network."""
    checks = []

    def ok(name, cond):
        checks.append((name, bool(cond)))

    base = (
        'The vendor says "We recommend that you start with the supervised mode '
        'before using the autonomous mode." on the page.\n'
        'See [the page](https://example.com/docs) and https://example.com/bare, '
        'read 2026-09-27, priced $619 at 12% of list, field `humanBilling`.\n'
    )

    # 1. nothing changed -> nothing lost
    d = delta(atoms(base), atoms(base))
    ok("identical trees lose nothing", all(not d[k][0] and not d[k][1] for k in KINDS))

    # 2. a dropped vendor sentence is named
    head = base.replace('The vendor says "We recommend that you start with the '
                        'supervised mode before using the autonomous mode." on the page.\n', '')
    d = delta(atoms(base), atoms(head))
    ok("a dropped quotation is LOST/quote",
       any("supervised mode" in a for a, _ in d["quote"][0]))

    # 3. THE BUG THIS TOOL EXISTS TO NOT HAVE: dropping one quoted span must not
    #    re-pair the quotes after it into phantom losses and gains.
    two = 'He said "alpha bravo" here.\nShe said "charlie delta" there.\nThey said "echo foxtrot" last.\n'
    one = 'She said "charlie delta" there.\nThey said "echo foxtrot" last.\n'
    d = delta(atoms(two), atoms(one))
    ok("dropping one quote reports exactly one loss", len(d["quote"][0]) == 1)
    ok("dropping one quote reports no phantom gain", len(d["quote"][1]) == 0)

    # 4. narration edits around a quotation are invisible
    reworded = base.replace("The vendor says", "Per the vendor,").replace("on the page.", "there.")
    d = delta(atoms(base), atoms(reworded))
    ok("rewording narration loses no atom", not any(d[k][0] for k in KINDS))

    # 5. each atom kind is detected and its loss reported
    for kind, gone, needle in [
        ("link", "[the page](https://example.com/docs)", "example.com/docs"),
        ("url", "https://example.com/bare", "example.com/bare"),
        ("date", "2026-09-27", "2026-09-27"),
        ("money", "$619", "619"),
        ("percent", "12%", "12"),
        ("code", "`humanBilling`", "humanBilling"),
    ]:
        d = delta(atoms(base), atoms(base.replace(gone, "")))
        ok(f"a dropped {kind} is reported",
           any(needle in a for a, _ in d[kind][0]))

    # 6. a markdown link's URL is not double-counted as a bare URL
    a = atoms("[x](https://example.com/docs)\n")
    ok("a link url is not also a bare url", a["url"] == {} and len(a["link"]) == 1)

    # 6b. THE SEVENTH CLASS, built from two live misses two days apart. Both
    # diffs printed "no protected atom lost" over a dropped measurement.
    bare = ("The audit reports 99 measurable claims and 6 broken targets, "
            "compressed toward the 12,000-byte working target, 0 of 8 trials "
            "failing.\n")
    for name, gone, needle in [
        ("bare count", "99 measurable", "99 measurable"),          # #316
        ("second bare count", "6 broken", "6 broken"),              # #316
        ("unit-prefixed count", "12,000-byte", "12,000-byte"),      # #285
        ("N of M count", "0 of 8", "0 of 8"),
    ]:
        d = delta(atoms(bare), atoms(bare.replace(gone, "")))
        ok(f"a dropped {name} is reported",
           any(needle in a for a, _ in d["count"][0]))

    # 6c. and the four things that are NOT counts, which is the whole reason
    # the class can exist without flooding the report.
    ok("a date is not also a count",
       atoms("read 2026-09-27, filed since.\n")["count"] == {})
    ok("a price is not also a count",
       atoms("priced $619 monthly.\n")["count"] == {})
    ok("a percentage is not also a count",
       atoms("at 12% of list price.\n")["count"] == {})
    ok("a backticked identifier is not also a count",
       atoms("the field `maxRetries 3 times` is set.\n")["count"] == {})
    ok("a connective after a numeral is not a measurement",
       atoms("version 3 of the schema, section 6 is exploratory.\n")["count"] == {})
    ok("a decimal with a short unit is a count",
       any(a.startswith("104.0") for a in atoms("IQR 104.0 ms here.\n")["count"]))
    ok("a short unit does not reopen the floor for every two-letter word",
       atoms("section 6 is open, item 4 as well.\n")["count"] == {})

    # 7. counts, not sets, but a surviving copy is not a loss. Until wake
    # 27836fa9 this rule asserted the opposite and pinned the bug: a 3->2 drop
    # was reported LOST, and on statsig v6 (f0a9c34) that printed the vendor's
    # "This installs the Statsig MCP Server with OAuth." as lost while it was
    # still in the head. Thinning is reported, quietly, under its own word.
    three = 'a "same span" b\nc "same span" d\ne "same span" f\n'
    twice = 'a "same span" b\nc "same span" d\n'
    d = delta(atoms(three), atoms(twice))
    ok("losing one copy of a repeated quote is not a loss", not d["quote"][0])
    ok("losing one copy of a repeated quote is reported as thinned",
       d["quote"][2] == [("same span", 1)])

    # 7b. the last copy going is still a loss, however many there were
    d = delta(atoms(three), atoms('a b\nc d\n'))
    ok("losing the last copy of a repeated quote is a loss",
       d["quote"][0] == [("same span", 3)] and not d["quote"][2])

    # 7bis. A SHORT QUOTATION MUST NOT SHIFT THE PAIRING OF THE ONES AFTER IT.
    # Verbatim from shopify-ucp v7 (#275): the tier row quoted as "Yes" is three
    # characters, under the old regex it consumed a delimiter, and the vendor's
    # exclusion sentence after it -- byte-identical in base and head -- was
    # reported LOST with an invented atom beside it as a gain.
    shifted = ('the row reads "Yes" there, and the sentence says '
               '"complete_checkout and order tools aren\'t available at this tier".\n')
    qs = list(atoms(shifted)["quote"])
    ok("a short quotation does not shift the pairing after it",
       "complete_checkout and order tools aren't available at this tier" in qs)
    ok("a quotation under the minimum length is not itself an atom",
       "Yes" not in qs and len(qs) == 1)
    d = delta(atoms(shifted), atoms(shifted))
    ok("the mis-pairing case loses and gains nothing against itself",
       not any(d[k][0] or d[k][1] for k in KINDS))

    # 7c. WIDENING, every case below lifted from the shopify-ucp v7 run (#275,
    # base ae1170b, head a97f156), which printed six LOST of which two were
    # standing verbatim in the head inside a longer quotation.
    narrow = 'The docs say the merchant "always remains the merchant of record".\n'
    wide = ('The carts page carries "The merchant always remains the merchant '
            'of record".\n')
    d = delta(atoms(narrow), atoms(wide))
    ok("a re-attributed span is still LOST to delta (widening is report-level)",
       d["quote"][0] == [("always remains the merchant of record", 1)])
    ok("a span quoted inside a longer head quotation is widened",
       widened("always remains the merchant of record",
               list(atoms(wide)["quote"])) ==
       "The merchant always remains the merchant of record")
    # the trap: kept but no longer QUOTED is a lost attribution, not a widening
    unquoted = "The merchant always remains the merchant of record.\n"
    ok("a span the head keeps but stops quoting is not widened",
       widened("always remains the merchant of record",
               list(atoms(unquoted)["quote"])) is None)
    # the other trap: containment across a word boundary is a different claim
    ok("widening does not match across a word boundary",
       widened("the merchant", list(atoms('x "the merchants of record" y\n')
                                    ["quote"])) is None)
    ok("an identical span is not its own widening",
       widened("always remains the merchant of record",
               ["always remains the merchant of record"]) is None)

    # 8. whitespace normalisation: a quote re-wrapped across the same line is the same atom
    wrapped = base.replace("start with the supervised", "start with  the   supervised")
    d = delta(atoms(base), atoms(wrapped))
    ok("re-spaced quotation is not a loss", not d["quote"][0])

    # 9. a quotation that moves to another paragraph is not a loss
    moved = "\n".join(reversed(base.strip().split("\n"))) + "\n"
    d = delta(atoms(base), atoms(moved))
    ok("a moved quotation is not a loss", not any(d[k][0] for k in KINDS))

    # 10. curly quotes are read like straight ones
    curly = 'The vendor says “How do I?” questions here.\n'
    ok("curly quotes are atoms", "How do I?" in atoms(curly)["quote"])

    # 11. an unbalanced quote does not swallow the rest of the file
    odd = 'He said "alpha bravo" and a lone " here.\nShe said "charlie delta".\n'
    ok("an odd quote does not swallow the next line",
       "charlie delta" in atoms(odd)["quote"])

    # 12. a trailing period outside the quotation is not part of the atom
    ok("quote atoms exclude surrounding punctuation",
       all(not a.endswith('"') for a in atoms(base)["quote"]))

    # 13/14. MOVED, not lost: the classification `atom in elsewhere` has to
    # survive the sibling file reflowing the sentence across lines, and must
    # not fire on a sibling that merely mentions a neighbouring phrase.
    atom = list(atoms('a "We recommend the supervised mode first." b')["quote"])[0]
    reflowed = re.sub(r"\s+", " ", '{"finding": "We recommend the\n'
                                   '   supervised mode first."}')
    ok("an atom reflowed in a sibling file counts as moved", atom in reflowed)
    ok("a sibling that only mentions the topic does not count as moved",
       atom not in re.sub(r"\s+", " ", '{"finding": "supervised mode"}'))

    # 19-21. the cap is counted in BYTES, not characters (wake 6add8abb)
    ok("nbytes counts an ASCII string as its length", nbytes("abc") == 3)
    ok("nbytes charges a U+2013 en dash three bytes", nbytes("a–b") == 5)
    ok("a profile of 16000 characters with one en dash is OVER the cap",
       CAP - nbytes("x" * 15999 + "–") < 0)

    width = max(len(n) for n, _ in checks)
    for name, good in checks:
        print(f"  {'ok  ' if good else 'FAIL'} {name:<{width}}")
    bad = [n for n, g in checks if not g]
    print(f"\n{len(checks) - len(bad)} of {len(checks)} rules hold")
    return 1 if bad else 0


def main():
    p = argparse.ArgumentParser()
    p.add_argument("--clone")
    p.add_argument("--base", default="origin/main")
    p.add_argument("--head")
    p.add_argument("--ref", help="one commit: compares it with its first parent")
    p.add_argument("--json-too", action="store_true",
                   help="also read changed .json files (their strings are atoms too)")
    p.add_argument("--no-gains", action="store_true")
    p.add_argument("--selftest", action="store_true")
    a = p.parse_args()

    if a.selftest:
        sys.exit(selftest())
    if not a.clone or not (a.head or a.ref):
        p.error("need --clone with --head (or --ref), or --selftest")
    base, head = (f"{a.ref}^", a.ref) if a.ref else (a.base, a.head)
    report(a.clone, base, head, only_md=not a.json_too, quiet_gains=a.no_gains)


if __name__ == "__main__":
    main()