#!/usr/bin/env python3
"""claims.py: does the page a job claim cites still say what the entry quotes?

Every job claim in this registry names a `source` URL, and the good ones quote
the vendor verbatim. `surfaces.py` proves those pages still answer. This asks
the harder question: are the words still there?

A vendor rewrites a product page every quarter. A claim filed in September
quoting a sentence deleted in October still reads, in this registry, as the
vendor's own words. Nothing in CI re-checks it, and neither did I until now.

The check is deliberately literal. It pulls every verbatim quotation out of a
claim's `summary` (anything in double quotes, 15 characters or more), fetches
the cited page, strips tags and normalises whitespace and curly quotes, and
looks for the phrase.

Read a miss carefully. A page that renders its text in the browser serves an
HTML shell to this script, so NOT FOUND means "not in the served HTML", which
is evidence of nothing on its own. That is why the output distinguishes a page
that clearly carries prose from one that served a shell.

    python3 tools/claims.py [--clone /tmp/reg] [--json out.json]

Written at wake 7e8954b9 (2026-09-13).
"""

import sys as _sys

# tools/queue.py shadows the stdlib `queue`; keep it off the path.
_here = __file__.rsplit("/", 1)[0]
_sys.path[:] = [p for p in _sys.path if p not in ("", ".", _here)]

import argparse
import concurrent.futures
import datetime
import glob
import html
import json
import os
import re
import subprocess
import urllib.error
import urllib.request

# corpus.py refuses to score an empty corpus. Loaded by path because this
# script keeps its own directory off sys.path (see the note above).
import importlib.util as _ilu
_cspec = _ilu.spec_from_file_location("corpus", os.path.join(_here, "corpus.py"))
corpus = _ilu.module_from_spec(_cspec)
_cspec.loader.exec_module(corpus)

import importlib.util as _flilu
import os as _flos
# One disclosure for every sweep (tools/fetchlib.py). Loaded by path
# because this file drops its own directory from sys.path above, which is
# this directory's convention for importing a sibling.
_flspec = _flilu.spec_from_file_location(
    "fetchlib", _flos.path.join(_flos.path.dirname(_flos.path.abspath(__file__)), "fetchlib.py"))
fetchlib = _flilu.module_from_spec(_flspec)
_flspec.loader.exec_module(fetchlib)
UA = fetchlib.agent("claims/1")


def normalise(s):
    s = html.unescape(s)
    s = s.replace("’", "'").replace("‘", "'")
    s = s.replace("“", '"').replace("”", '"')
    s = s.replace("—", "-").replace("–", "-").replace(" ", " ")
    s = re.sub(r"\s+", " ", s).strip().lower()
    # Wake c15fd999: a stripped inline tag leaves a space before the
    # punctuation that follows it ("api post , patch" for `API <code>POST</code>,`),
    # so a verbatim phrase read as missing. Close that gap; found first in
    # profiles.py, the same class here.
    return re.sub(r"\s+([,.;:!?)])", r"\1", s)


def page_text(body):
    # Wake c8b1f644: comments go FIRST and to NOTHING, not to a space. React
    # and friends emit `<!-- -->` between adjacent text nodes of one paragraph
    # (`Effective date<!-- -->: <!-- -->June 09, 2026` on linear.app/terms), so
    # a stripper that spaces them out breaks a sentence in the middle. A
    # browser ignores comments; so does this. Found in quoted.py and the same
    # class here, which is the fifth defect of this shape across my tools.
    body = re.sub(r"<!--.*?-->", "", body, flags=re.S)
    body = re.sub(r"(?is)<(script|style|noscript)[^>]*>.*?</\1>", " ", body)
    body = re.sub(r"(?s)<[^>]+>", " ", body)
    return normalise(body)


def fetch(url):
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html,*/*"})
    try:
        with urllib.request.urlopen(req, timeout=40) as r:
            # decode by Content-Encoding (wake 51fa3bb7): docs.datadoghq.com and
            # legal.ironcladapp.com gzip a body nobody asked them to compress
            return r.status, corpus.decode_body(r.headers, r.read())[0].decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        try:
            return e.code, corpus.decode_body(e.headers, e.read())[0].decode("utf-8", "replace")
        except Exception:
            return e.code, ""
    except Exception as e:
        return None, type(e).__name__ + ": " + str(e)[:80]


def quoted(summary):
    """Every quotation in a claim summary, paired from the left.

    Wake 9112e04d, found on HubSpot's version 2 (#88). This used to be
    `re.findall(r'"([^"]{15,})"')`, which does not pair quotation marks: it
    pairs whatever it can reach. A summary quoting four fragments, one of
    them SHORTER than the 15-character floor ("for 72 hours"), made the
    regex skip that pair and then open a new one at its CLOSING mark, so
    the sweep reported the AUTHOR'S OWN PROSE between two quotations
    (" or the agent qualifying ") as a phrase missing from the vendor's
    page. A false fault, and the same instrument would swallow a real
    miss on the next line for the same reason.

    So: split on the quotation marks, take the odd-indexed pieces (the
    insides), and only then apply the length floor.
    """
    parts = summary.split('"')
    if len(parts) % 2 == 0:  # an unbalanced summary: say so rather than guess
        return []
    found = [p for p in parts[1::2] if len(p) >= 15]
    # Wake ab4e216b, found on wix v1 (#283): eight spans on main (exa-mcp,
    # grafana-cloud-mcp, mintlify-docs-mcp) and the three in wix's summary
    # are quoted in SINGLE quotes, and this read only double ones, so the
    # sweep had nothing to say about any of them. An apostrophe is the
    # hazard: "Wix's" and "don't" must not open or close a span, so the
    # opening mark may not follow a letter and the closing mark may not
    # precede one. A curly apostrophe inside the span is fine.
    for m in SINGLE_QUOTED.finditer(summary):
        q = m.group(1)
        if len(q) >= 15 and q not in found:
            found.append(q)
    return found


SINGLE_QUOTED = re.compile(r"(?<![A-Za-z0-9])'([^'\n]{1,400}?)'(?![A-Za-z0-9])")


URL_IN_PROSE = re.compile(
    r"(?<![\w/@.-])((?:https?://)?(?:[a-z0-9-]+\.)+[a-z]{2,}(?:/[\w./#?=&%+-]*[\w/#=&+-])?)")

# "its README:", "(agent management page)", "the MEIO page:", "on the disputes
# page" -- a summary saying, in prose, that these words live somewhere other
# than `source`. Cue words only; a cue is never a pass.
ATTRIBUTION_CUE = re.compile(
    r"((?:its|the|on|in|at|from)\s+(?:[\w'./-]+\s+){0,4}"
    r"(?:README|readme|page|pages|docs|documentation|changelog|appendix|file|table|post)"
    r"(?:'s)?)\s*[:,(]?\s*$")


def summary_urls(summary):
    """Pages a summary names in prose, as candidate sources.

    Wake 3dbf9edb: six of eight misses in one run were phrases the summary
    itself attributes to another document ("the disputes page
    (docs.stripe.com/disputes) adds ...", "On fin.ai/pricing: ...", "its
    README: ..."). The `sources` array added at #103 is the structured way to
    say that and almost no claim fills it, so read the prose too: a URL a
    summary names is a page a reader of that summary would open.
    """
    out = []
    for m in URL_IN_PROSE.finditer(summary or ""):
        u = m.group(1).rstrip(".,;:)\"'")
        # the dot test is about the HOST, so look past any scheme: "https:" has
        # no dot in it and silently dropped every full URL on the first run.
        if "." not in u.split("://")[-1].split("/")[0]:
            continue
        if not u.startswith("http"):
            u = "https://" + u
        if u not in out:
            out.append(u)
    return out


TRAILING_CUE = re.compile(
    r"^\s*\(([^)]*\b(?:README|readme|page|docs|documentation|changelog|appendix)\b[^)]*)\)")


def attribution(summary, quote):
    """The prose cue, if any, that hands THIS quotation to another document.

    A cue sits on either side of the quotation: github's job summary writes
    `"available for all paid Copilot plans" (agent management page)` AFTER
    the closing mark, and reading only what came before missed it.
    """
    i = (summary or "").find('"' + quote + '"')
    if i < 0:
        i = (summary or "").find("'" + quote + "'")
    if i < 0:
        return None
    m = ATTRIBUTION_CUE.search(summary[:i])
    if m:
        return m.group(1).strip()
    m = TRAILING_CUE.match(summary[i + len(quote) + 2:])
    return m.group(1).strip() if m else None


def _selftest():
    cases = [
        ('the disputes page (docs.stripe.com/disputes) adds "x"',
         ["https://docs.stripe.com/disputes"]),
        ('On fin.ai/pricing: "$0.99 per Fin outcome."', ["https://fin.ai/pricing"]),
        ('no page here, version 5 of it, 2026-10-12, 1,000 items', []),
        ('see https://docs.github.com/en/copilot/x.', ["https://docs.github.com/en/copilot/x"]),
    ]
    bad = 0
    for text, want in cases:
        got = summary_urls(text)
        if got != want:
            bad += 1
            print("FAIL urls %r -> %r, wanted %r" % (text, got, want))
    cues = [
        ('its README: "This public documentation server"', "This public documentation server", True),
        ('the vendor\'s words on the servers page: "Get up to date"', "Get up to date", True),
        ('the MEIO page: "Reduce excess inventory"', "Reduce excess inventory", True),
        ('route execution that lets a dispatcher "manage exceptions"', "manage exceptions", False),
        ('it is "available for all paid Copilot plans" (agent management page). Not measured.',
         "available for all paid Copilot plans", True),
        ('the vendor says "x is y" (2026-10-12) and nothing else', "x is y", False),
    ]
    for text, q, want in cues:
        got = attribution(text, q) is not None
        if got != want:
            bad += 1
            print("FAIL cue %r -> %r, wanted %r" % (text, got, want))
    # Wake c8b1f644. The page text a stripper produces, not the cues: React's
    # comment separators sit INSIDE a paragraph, so the sentence must come out
    # contiguous. The second row is the control that matters -- deleting
    # comments must not also delete the gap a real tag leaves between words.
    texts = [
        ("<p>Effective date<!-- -->: <!-- -->June 09, 2026</p>",
         "effective date: june 09, 2026"),
        ("<p>one</p><p>two</p>", "one two"),
        ("<p>API <code>POST</code>, and more</p>", "api post, and more"),
    ]
    for body, want in texts:
        got = page_text(body)
        if got != want:
            bad += 1
            print("FAIL text %r -> %r, wanted %r" % (body, got, want))
    # Wake ab4e216b: single-quoted spans, and the apostrophes that must not
    # be read as quotation marks.
    qs = [
        ("the builder 'generates a full site from a chat' and more", ["generates a full site from a chat"]),
        ("the vendor's page says the agent's tool is the agents' own", []),
        ("'your site must run on Wix\u2019s servers', 'the content belongs to you'",
         ["your site must run on Wix\u2019s servers", "the content belongs to you"]),
        ('"double quoted span here" and \'single quoted span here\'',
         ["double quoted span here", "single quoted span here"]),
        ("a short 'one' and the vendor's 'fourteen char!' no", []),
    ]
    for text, want in qs:
        got = quoted(text)
        if got != want:
            bad += 1
            print("FAIL quoted %r -> %r, wanted %r" % (text, got, want))
    # FLAPPING, the --confirm pass. No network: fetch is stubbed per round,
    # and these drive the REAL confirm() rather than a copy of its loop, for
    # the reason the twin rules in profiles.py spell out -- a harness that
    # re-implements the function under test passes while the function is
    # broken. The rule that matters is the FOUND one: hardening only the
    # absences leaves the verdict that SUPPRESSES a row on one coin flip.
    import contextlib
    import io

    flaps = []

    def run_confirm(verdict, schedule, readable1="all"):
        global fetch
        real, calls = fetch, [1]

        def stub(url):  # noqa: ARG001
            calls[0] += 1
            served = schedule[min(calls[0], max(schedule))]
            # A (code, body) pair lets a rule FAIL a round, not merely serve
            # it differently. A bare string is a 200, which is what every
            # rule written before wake f4f33561 assumes.
            return served if isinstance(served, tuple) else (200, served)

        row = dict(who="v", job="j", quote="the words", verdict=verdict,
                   scope=["https://example.invalid/p"], parts=["the words"], on=None)
        try:
            fetch = stub
            with contextlib.redirect_stdout(io.StringIO()):
                confirm([row], max(schedule),
                        readable1=None if readable1 == "all" else readable1)
        finally:
            fetch = real
        return row

    HAS, HASNT = "x the words x", "x nothing here x"
    DEAD, GONE = (None, "ConnectionResetError: x"), (503, "")
    flap_cases = [
        ("a FOUND row whose page stops carrying it FLAPS",
         run_confirm("FOUND", {1: HAS, 2: HASNT, 3: HASNT})["verdict"], "FLAPPING"),
        ("a NOT FOUND row whose page turns out to carry it flaps",
         run_confirm("NOT FOUND", {1: HASNT, 2: HAS, 3: HASNT})["verdict"], "FLAPPING"),
        ("a row absent every round keeps its verdict",
         run_confirm("NOT FOUND", {1: HASNT, 2: HASNT, 3: HASNT})["verdict"], "NOT FOUND"),
        ("a row present every round stays FOUND",
         run_confirm("FOUND", {1: HAS, 2: HAS, 3: HAS})["verdict"], "FOUND"),
        ("a SHELL row that answers once flaps rather than staying SHELL",
         run_confirm("SHELL", {1: HASNT, 2: HAS, 3: HASNT})["verdict"], "FLAPPING"),
        ("a flapping row carries its round count",
         (lambda r: (r["rounds_found"], r["rounds"]))(
             run_confirm("FOUND", {1: HAS, 2: HASNT, 3: HASNT})), (1, 3)),
        # THE CONFOUND FIXED AT WAKE f4f33561: a round that did not ANSWER is
        # not a round that answered and said no. Every rule here read
        # FLAPPING or kept a verdict it had no evidence for, before the fix.
        ("a FOUND row whose later rounds FAIL is not flapping: the page never said no",
         run_confirm("FOUND", {1: HAS, 2: DEAD, 3: DEAD})["verdict"], "FOUND"),
        ("a NOT FOUND row whose later rounds FAIL keeps its verdict, it does not flap",
         run_confirm("NOT FOUND", {1: HASNT, 2: DEAD, 3: GONE})["verdict"], "NOT FOUND"),
        ("found once, absent once and failed once DOES flap: two decisive rounds disagreed",
         run_confirm("FOUND", {1: HAS, 2: HASNT, 3: DEAD})["verdict"], "FLAPPING"),
        ("a flapping row counts DECISIVE rounds and carries the unreadable one",
         (lambda r: (r["rounds_found"], r["rounds_decisive"], r["rounds_unreadable"]))(
             run_confirm("FOUND", {1: HAS, 2: HASNT, 3: DEAD})), (1, 2, 1)),
        ("a hit is decisive on its own: a 503 cannot unfind a span the page served",
         (lambda r: (r["rounds_found"], r["rounds_decisive"]))(
             run_confirm("FOUND", {1: HAS, 2: HAS, 3: GONE})), (2, 2)),
        ("a row no round could read is UNREADABLE, never NOT FOUND: two failed "
         "reads are not evidence of absence",
         run_confirm("NOT FOUND", {1: DEAD, 2: DEAD, 3: GONE},
                     readable1=set())["verdict"], "UNREADABLE"),
        ("round 1's own failure counts, so an UNREADABLE row names every round",
         (lambda r: (r["rounds_decisive"], r["rounds_unreadable"], r["rounds"]))(
             run_confirm("NOT FOUND", {1: DEAD, 2: DEAD, 3: GONE}, readable1=set())), (0, 3, 3)),
        ("round 1 failing does not sink a row its later rounds could read: confirm() "
         "can now UPGRADE a verdict, which it never could before",
         run_confirm("NOT FOUND", {1: DEAD, 2: HAS, 3: HAS},
                     readable1=set())["verdict"], "FOUND"),
        ("a SHELL row read cleanly in every decisive round becomes FOUND",
         run_confirm("SHELL", {1: HASNT, 2: HAS, 3: HAS}, readable1=set())["verdict"], "FOUND"),
    ]
    for label, got, want in flap_cases:
        flaps.append(label)
        if got != want:
            bad += 1
            print("FAIL flap %s -> %r, wanted %r" % (label, got, want))

    print("selftest: %d rule(s), %d failure(s)"
          % (len(cases) + len(cues) + len(texts) + len(qs) + len(flaps), bad))
    return 1 if bad else 0


def claims(clone):
    out = []
    for path in sorted(glob.glob(os.path.join(clone, "registry/tools/*/tool.json"))
                       + glob.glob(os.path.join(clone, "registry/agents/*/agent.json"))):
        d = json.load(open(path))
        who = d.get("slug") or d.get("handle")
        for j in d.get("jobs") or []:
            if not isinstance(j, dict):
                continue
            # Wake ab4e216b: grafana-cloud-mcp and mintlify-docs-mcp file
            # their claims with `sources` and no `source`, and this skipped
            # them for as long as they have been on main. The first listed
            # page stands in for `source`; the rest stay `extra`.
            if not j.get("source"):
                srcs = [u for u in (j.get("sources") or []) if isinstance(u, str)]
                if not srcs:
                    continue
                j = dict(j, source=srcs[0], sources=srcs[1:])
            quotes = quoted(j.get("summary") or "")
            # #103 (2026-09-18): a claim may name the other pages it quotes in
            # `sources`; a phrase is verbatim if it is on ANY page the claim
            # names, and the sweep says which. Before this the two Intercom
            # price sentences on fin.ai/pricing read as misses against fin.ai/.
            extra = [u for u in (j.get("sources") or []) if isinstance(u, str)]
            # wake 3dbf9edb: and the pages the summary names in prose, which is
            # where almost every claim actually puts them.
            summary = j.get("summary") or ""
            extra += [u for u in summary_urls(summary)
                      if u != j["source"] and u not in extra]
            if quotes:
                out.append((who, j["job"], j["source"], extra, quotes, summary))
    return out


def confirm(results, rounds, readable1=None):
    """Re-ask every row and name the ones whose verdict is a coin flip.

    Written at wake fe48cdd6 (2026-11-03), the same wake the identical
    defect was measured in profiles.py: THREE runs of that sweep over one
    tree inside eleven seconds disagreed, and across the whole registry 131
    of 1,012 quotation verdicts did not survive three rounds -- 12.9%,
    spread over 37 of 48 vendors. claims.py reads the same kind of vendor
    page with the same single fetch, and this sweep's own notes already
    name the fin.ai/pricing pair as "the known hydration flappers": a
    verdict that flaps was being reported as a verdict.

    EVERY row is re-asked, FOUND ones included. Hardening only the
    absences leaves the verdict that SUPPRESSES a row (FOUND) on one coin
    flip; that was the first draft of the twin function in profiles.py and
    on the real corpus it missed seven of the nine rows that mattered.

    THAT CONFOUND IS FIXED AS OF WAKE f4f33561, in the same shape as
    profiles.py and for the same reason: a FAILED fetch left empty text,
    indistinguishable from a client-rendered shell, so a round that never
    answered was scored as a round that answered and said no. A round is
    DECISIVE for a row or it is not scored:

      found                            -> decisive, and found
      not found, every scope page
        ANSWERED                       -> decisive, and not found
      not found, some scope page
        did not answer                 -> UNREADABLE, scored neither way

    FLAPPING is computed over decisive rounds only. A row with no decisive
    round is UNREADABLE, not NOT FOUND. And a row whose INCOMING verdict
    rested on a round that did not answer can now be upgraded, which is
    the half of the fix that is easy to miss: before this, confirm() could
    only ever make a verdict worse, which was safe exactly as long as
    round 1 was assumed to have answered.

    ONE DIVERGENCE FROM profiles.py, deliberate and narrow. This sweep
    scores a page's TEXT whenever the fetch returned any status at all
    (`and code`), where profiles.py requires `code < 400`; so a 4xx body
    carrying the span still reads FOUND here. That is pre-existing
    behaviour and this change does not move it. What the strict test
    governs is only whether a round is DECISIVE, so the two sweeps now
    agree about which rounds count even where they disagree about which
    bodies to read. Noted rather than silently unified: unifying it would
    change verdicts on the real corpus and belongs in its own change.

    AND THE FLAP SET ITSELF FLAPS. Two runs of this sweep minutes apart on
    2026-11-03 printed 2 flapping rows and then 4, all intercom. So a
    flapper count at N rounds is a LOWER BOUND on the flapping set, never
    an enumeration of it: N rounds proves flapping exists and does not list
    who flaps. Do not diff two runs' flap sets and call the difference a
    change in the world.
    """
    if not results:
        return
    urls = sorted({u for r in results for u in r["scope"]})
    print("\nconfirming %d row(s) over %d page(s), %d more round(s): a verdict that does not "
          "survive its own repetition is not a measurement" % (len(results), len(urls), rounds - 1))
    for r in results:
        scope_ok = True if readable1 is None else all(u in readable1 for u in r["scope"])
        found1 = r["verdict"] == "FOUND"
        r["rounds"] = 1
        r["rounds_found"] = 1 if found1 else 0
        r["rounds_decisive"] = 1 if (found1 or scope_ok) else 0
        r["rounds_unreadable"] = 0 if (found1 or scope_ok) else 1
    for _ in range(2, rounds + 1):
        texts, answered = {}, set()
        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
            for url, (code, body) in zip(urls, ex.map(fetch, urls)):
                texts[url] = page_text(body) if isinstance(body, str) and code else ""
                if isinstance(body, str) and code is not None and code < 400:
                    answered.add(url)
        for r in results:
            r["rounds"] += 1
            hit = next((u for u in r["scope"]
                        if u in texts and all(normalise(p) in texts[u] for p in r["parts"])), None)
            if hit:
                r["rounds_decisive"] += 1
                r["rounds_found"] += 1
                r["on"] = r["on"] or hit
            elif all(u in answered for u in r["scope"]):
                r["rounds_decisive"] += 1
            else:
                r["rounds_unreadable"] += 1
    flapped = unreadable = upgraded = 0
    for r in sorted(results, key=lambda r: (r["who"], r["job"], r["quote"])):
        unread = r["rounds_unreadable"]
        if r["rounds_decisive"] == 0:
            unreadable += 1
            r["verdict"] = "UNREADABLE"
            print("UNREADABLE %-14s %-34s %s [no round answered on every page this quotation "
                  "is cited from, %d of %d rounds unreadable]"
                  % (r["who"], r["job"], r["quote"][:60], unread, r["rounds"]))
        elif r["rounds_found"] and r["rounds_found"] == r["rounds_decisive"]:
            if r["verdict"] != "FOUND":
                upgraded += 1
                print("FOUND    %-14s %-34s %s [came in %s on a round that did not answer; "
                      "found in all %d decisive round(s)]"
                      % (r["who"], r["job"], r["quote"][:60], r["verdict"], r["rounds_decisive"]))
            r["verdict"] = "FOUND"
        elif 0 < r["rounds_found"] < r["rounds_decisive"]:
            flapped += 1
            r["verdict"] = "FLAPPING"
            print("FLAPPING %-14s %-34s %s [found in %d of %d DECISIVE rounds%s: a one-shot "
                  "verdict on this page is a coin flip]"
                  % (r["who"], r["job"], r["quote"][:60], r["rounds_found"],
                     r["rounds_decisive"], ", %d unreadable" % unread if unread else ""))
    partial = sum(1 for r in results if r["rounds_unreadable"] and r["rounds_decisive"])
    print("%d of %d row(s) flapped across their decisive rounds; %d could not be read at all; "
          "%d were upgraded off a round that did not answer; the other %d held their verdict "
          "every time" % (flapped, len(results), unreadable, upgraded,
                          len(results) - flapped - unreadable - upgraded))
    print("%d row(s) lost at least one round to a page that did not answer and were still "
          "decided on the rounds that did; before wake f4f33561 every one of those rounds "
          "counted as an absence." % partial)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--clone", default="/tmp/reg")
    ap.add_argument("--json", default=None)
    ap.add_argument("--confirm", type=int, default=1,
                    help="re-ask every row this many times total; disagreement is FLAPPING")
    ap.add_argument("--selftest", action="store_true",
                    help="check the prose-URL and attribution-cue rules against "
                         "fixed cases and exit")
    ap.add_argument("--ref", default="origin/main",

                    help="the tree this sweep claims to measure; the clone must be "

                         "checked out at exactly this (wake fe04f262)")
    args = ap.parse_args()
    if args.selftest:
        raise SystemExit(_selftest())

    corpus.require_clone(args.clone, "the claims sweep")
    # A sweep that says "the repository" must say WHICH tree, and refuse
    # anything else. Wake fe04f262: three instruments read a wrong corpus
    # in one hour, and only one of the three was loud about it.
    head = corpus.require_ref(args.clone, args.ref, "the claims sweep")
    rows = claims(args.clone)
    corpus.require_rows(len(rows), "quoting job claims", args.clone)
    when = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    nq = sum(len(r[4]) for r in rows)
    print("registry %s, %d job claims quote their source verbatim (%d phrases), measured %s"
          % (head, len(rows), nq, when))

    urls = sorted({r[2] for r in rows} | {u for r in rows for u in r[3]})
    pages = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
        for url, res in zip(urls, ex.map(fetch, urls)):
            pages[url] = res

    results = []
    for who, job, src, extra, quotes, summary in rows:
        texts = []
        for u in [src] + extra:
            code, body = pages[u]
            texts.append((u, code, page_text(body) if isinstance(body, str) and code else ""))
        code, text = texts[0][1], texts[0][2]
        # A page serving real prose has thousands of words; a shell has a few.
        shell = all(len(t.split()) < 120 for _, _, t in texts)
        for q in quotes:
            # A quotation may elide with an ellipsis ("WISMO ... queries are
            # deflected"): each side is the vendor's words, the join is not.
            # Check the fragments, or the whole phrase when there is no elision.
            # Found at wake 09fd8d02, when FourKites' Cassie quote read as drift
            # and was nothing but my own literal search meeting a legal elision.
            parts = [p.strip(" .") for p in re.split(r"\s*(?:\.\.\.|…)\s*", q)]
            parts = [p for p in parts if len(p) >= 12] or [q]
            on = next((u for u, _, t in texts if all(normalise(p) in t for p in parts)), None)
            found = on is not None
            verdict = "FOUND" if found else ("SHELL" if shell else "NOT FOUND")
            cue = None if found else attribution(summary, q)
            results.append(dict(who=who, job=job, source=src, sources=extra, on=on, quote=q,
                                code=code, verdict=verdict, words=len(text.split()),
                                attributed=cue, scope=[src] + extra, parts=list(parts)))
            where = "" if on in (None, src) else "  [on %s]" % on
            if cue:
                where += '  [the summary hands this to: "%s" -- read it, it is not a pass]' % cue
            print("%-7s %-14s %-34s %s%s" % (verdict, who, job, q[:70], where))

    if args.confirm > 1:
        confirm(results, args.confirm)

    found = sum(1 for r in results if r["verdict"] == "FOUND")
    shell = sum(1 for r in results if r["verdict"] == "SHELL")
    # Wake c8b1f644: "on the page" over-claimed and the word changed to
    # "in the delivered body" after fin.ai/pricing was found to SEND a
    # paragraph four times with its real prices and then DELETE it at
    # hydration -- 0 occurrences in the DOM's textContent, not merely hidden.
    # A FOUND here has always been a statement about the body a GET returns.
    # It is not a statement about what a reader sees, and now it says so.
    print("\n%d of %d quoted phrases are still in the DELIVERED BODY of the page"
          "\n  they are attributed to (a GET's body, NOT what a reader sees:"
          "\n  fin.ai/pricing sends a paragraph it deletes at hydration)"
          % (found, len(results)))
    if shell:
        print("%d could not be checked: the page served a shell, not prose "
              "(client-rendered; a miss there means nothing)" % shell)
    elsewhere = sum(1 for r in results
                    if r["verdict"] == "FOUND" and r["on"] not in (None, r["source"]))
    if elsewhere:
        print("%d of the found ones are on a page the claim's own summary or "
              "`sources` names, not on its `source`" % elsewhere)
    cued = [r for r in results if r["verdict"] == "NOT FOUND" and r["attributed"]]
    if cued:
        print("%d of the misses carry a prose cue handing the words to another "
              "document; a cue is NEVER a pass, so each is still a miss until "
              "that document is read:" % len(cued))
        for r in cued:
            print('   %-14s %-34s "%s" <- %s' % (r["who"], r["job"], r["quote"][:46],
                                                 r["attributed"]))

    if args.json:
        json.dump({"head": head, "measured": when, "results": results},
                  open(args.json, "w"), indent=1)


if __name__ == "__main__":
    main()