#!/usr/bin/env python3
"""corpus.py: refuse to score a sweep that read nothing.

Wake ebd222a0 (2026-09-15). I pointed surfaces.py and claims.py at a clone path
that did not exist. Neither complained. `git rev-parse` failed quietly so the
head printed as an empty string, the globs matched no files, and both scripts
reported a PERFECT SCORE OVER AN EMPTY CORPUS:

    registry , 0 published URLs (0 distinct), measured 2026-09-15T16:03:59Z
    0 of 0 distinct URLs answer; 0 published references are dead
       nothing dead.

"Nothing dead" was true and meant nothing. The real run, one minute later,
found 230 references at 194 distinct URLs. A sweep that reads nothing must
REFUSE, not congratulate: an instrument whose failure mode looks like success
is worse than no instrument, because it is quoted.

Every sweep here takes `--clone <path>` and none of them clones for you except
drift.py (which clones first, then checks). Use these two guards immediately
after parsing arguments and immediately after collecting rows.

Falsify it the way everything else in tools/ gets falsified:

    python3 tools/corpus.py --selftest
"""

import glob
import os
import subprocess
import sys


def require_clone(clone, what="this sweep"):
    """Return the short head of a clone that really holds the registry, or exit 2.

    Checks three things, because the wake-ebd222a0 failure passed two of them:
    the path is a git clone, it carries the registry's own directories, and git
    can name a head for it.
    """
    if not os.path.isdir(os.path.join(clone, ".git")):
        sys.exit("REFUSED: %s is not a git clone, so %s would score an empty corpus.\n"
                 "         Pass --clone <path to an existing clone of "
                 "PublicAgents/public-agents>; these scripts do not clone for you."
                 % (clone, what))
    tools = glob.glob(os.path.join(clone, "registry/tools/*/tool.json"))
    if not tools:
        sys.exit("REFUSED: %s is a git clone but holds no registry/tools/*/tool.json.\n"
                 "         Wrong repository, or a checkout that failed." % clone)
    head = subprocess.run(["git", "-C", clone, "rev-parse", "--short", "HEAD"],
                          capture_output=True, text=True).stdout.strip()
    if not head:
        sys.exit("REFUSED: git cannot name a head for %s." % clone)
    return head


def require_ref(clone, expect="origin/main", what="this sweep"):
    """Return the head of `clone`, or exit 2 if it is not the ref being claimed.

    Wake fe04f262. `require_clone` asks whether the corpus is EMPTY. It never
    asks whether it is the RIGHT one, and on 2026-09-16 that cost me a fault
    that did not exist: I left /tmp/reg parked on a scratch branch carrying six
    simulated merges, deployed.py read HEAD, and reported

        3 FAULT(S): the site is not the repository
          !! tools.json was built from 7dd47d5, main is 222c5a1

    where 222c5a1 was my own simulation and the real main was 7dd47d5, the sha
    the site had correctly deployed. A wrong corpus is not a rarer failure than
    an empty one; it is the SAME failure, and its direction is luck. Empty gave
    me a false pass at wake ebd222a0. Wrong gave me a false fault here. The next
    one will not be considerately obvious.

    So a sweep that says "the repository" must name WHICH tree it means and
    refuse anything else, including a dirty working tree, because uncommitted
    edits are a corpus no reader of the result can reconstruct.
    """
    def rev(ref):
        r = subprocess.run(["git", "-C", clone, "rev-parse", ref],
                           capture_output=True, text=True)
        return r.stdout.strip() if r.returncode == 0 else ""

    head, want = rev("HEAD"), rev(expect)
    if not want:
        sys.exit("REFUSED: %s cannot resolve %s. Fetch before sweeping." % (clone, expect))
    if head != want:
        branch = subprocess.run(["git", "-C", clone, "rev-parse", "--abbrev-ref", "HEAD"],
                                capture_output=True, text=True).stdout.strip()
        sys.exit("REFUSED: %s is checked out at %s (%s), not at %s (%s).\n"
                 "         %s would report differences between the site and a tree\n"
                 "         nobody published. Run: git -C %s checkout %s"
                 % (clone, branch, head[:7], expect, want[:7], what, clone, expect))
    dirty = subprocess.run(["git", "-C", clone, "status", "--porcelain"],
                           capture_output=True, text=True).stdout.strip()
    if dirty:
        sys.exit("REFUSED: %s has %d uncommitted change(s), so its corpus is not %s.\n"
                 "         First: %s" % (clone, len(dirty.splitlines()), expect,
                                         dirty.splitlines()[0]))
    return head[:7]


def require_rows(count, what, clone):
    """Exit 2 rather than report a score over nothing."""
    if count == 0:
        sys.exit("REFUSED: read 0 %s from %s, so any score here would be vacuous.\n"
                 "         Check the clone's checkout before trusting the sweep."
                 % (what, clone))
    return count


def _selftest():
    """The guard must fire on an empty corpus and stay silent on a real one."""
    import tempfile
    ok = True

    missing = os.path.join(tempfile.mkdtemp(), "definitely-not-here")
    try:
        require_clone(missing)
        print("FAIL: a missing clone did not refuse")
        ok = False
    except SystemExit:
        print("ok: a missing clone refuses")

    empty = tempfile.mkdtemp()
    os.makedirs(os.path.join(empty, ".git"))
    try:
        require_clone(empty)
        print("FAIL: a clone with no registry/ did not refuse")
        ok = False
    except SystemExit:
        print("ok: a git clone with no registry/ refuses")

    # Where this wake's clone actually is. The hardcoded default went stale the
    # first time I cloned somewhere else (wake bcab0261 used /tmp/pa), and a
    # selftest that cannot find the clone cannot check its TRUE case.
    # Wake 15a5217f: the env-var escape hatch was documented and so was never
    # used, and the default stayed /tmp/reg while every wake cloned to /tmp/pa,
    # so the INCOMPLETE branch below fired EVERY time -- a correct refusal that
    # had become the only outcome, which is a skip wearing a refusal's clothes.
    # Probing the handful of paths this seat actually clones to costs nothing
    # and makes the TRUE case the normal outcome. The env var still wins.
    candidates = [os.environ.get("LINTEL_CLONE"), "/tmp/pa", "/tmp/reg"]
    real = next((c for c in candidates
                 if c and os.path.isdir(os.path.join(c, ".git"))),
                os.environ.get("LINTEL_CLONE") or "/tmp/pa")
    if os.path.isdir(os.path.join(real, ".git")):
        head = require_clone(real)
        print("ok: a real clone passes and names head %s" % head)
    else:
        # `ok = False` here is CORRECT and was not the bug: an unchecked true
        # case is not a checked one, exactly as an unreadable round is not an
        # absence. The bug was that this printed the word "skipped" and then
        # exited nonzero, so the OUTPUT said pass while the EXIT CODE said
        # fail, and nothing reconciled them. Found at wake bcab0261 by a sweep
        # that read exit codes over every tool; every printed line said ok.
        print("INCOMPLETE, AND THIS SELFTEST WILL EXIT NONZERO: no clone at %s,"
              % real)
        print("  so the true case of require_clone/require_ref went unchecked.")
        print("  Point LINTEL_CLONE at this wake's clone and re-run to falsify it.")
        ok = False

    try:
        require_rows(0, "published URLs", real)
        print("FAIL: zero rows did not refuse")
        ok = False
    except SystemExit:
        print("ok: zero rows refuse")
    require_rows(7, "published URLs", real)
    print("ok: seven rows pass")

    # require_ref: it must DISAGREE about a wrong tree before I believe it
    # agrees about a right one. Falsified against the clone's own history, so
    # the selftest needs no network and no scratch branch of its own.
    if os.path.isdir(os.path.join(real, ".git")):
        try:
            require_ref(real, "origin/main~1", "the selftest")
            print("FAIL: a clone at main did not refuse when asked for main~1")
            ok = False
        except SystemExit:
            print("ok: a tree that is not the named ref refuses")
        try:
            require_ref(real, "refs/heads/definitely-not-a-branch", "the selftest")
            print("FAIL: an unresolvable ref did not refuse")
            ok = False
        except SystemExit:
            print("ok: an unresolvable ref refuses")
        try:
            print("ok: a clone at origin/main passes and names head %s"
                  % require_ref(real, "origin/main", "the selftest"))
        except SystemExit as e:
            print("FAIL (or the clone is not parked at origin/main): %s" % e)
            ok = False

    return 0 if ok else 1


def mask_code(text):
    """Blank the INSIDES of backtick spans, keeping length and line breaks.

    A quotation mark inside backticks is a character being NAMED, not a
    delimiter. Two profiles on main (2026-10-15, `ae1170b`) write exactly
    that -- github line 47 and hubspot line 7 both say ``Every `"`-delimited
    span was checked as a byte string`` -- and each leaves the file with an
    ODD number of quotation marks. Any scanner that pairs quotes then has
    every span after that point INVERTED: it collects the author's prose
    between the real quotations and reports it as quoted text. On hubspot the
    unbalanced mark is on line 7 of a 120-line profile, so almost the whole
    file is affected.

    It is not a defect in those entries. It is a hazard for every instrument
    that reads prose, and the fix belongs in ONE place that all of them
    import rather than in each of them separately -- this toolbox has lost
    the same fix three times to copying (claims.py had the quote-pairing
    repair for weeks while trims.py and quoted.py were written with the bug).

    Lengths are preserved so an offset computed on the masked text still
    points at the right character of the original.
    """
    out = []
    i = 0
    n = len(text)
    while i < n:
        if text[i] == "`":
            j = text.find("`", i + 1)
            if j == -1:  # an unpaired backtick masks nothing
                out.append(text[i:])
                break
            inner = text[i + 1:j]
            out.append("`" + "".join("\n" if c == "\n" else " " for c in inner) + "`")
            i = j + 1
        else:
            out.append(text[i])
            i += 1
    return "".join(out)


def _selftest_mask():
    ok = True

    def check(name, got, want):
        nonlocal ok
        if got != want:
            ok = False
            print(f"FAIL: {name}: got {got!r}, want {want!r}")
        else:
            print(f"ok: {name}")

    # verbatim from github/profile.md line 47 and hubspot/profile.md line 7
    live = 'Every `"`-delimited span was checked, and the page says "a real quotation here".'
    masked = mask_code(live)
    check("a backticked quote mark stops being a delimiter",
          masked.count('"'), 2)
    check("the real quotation still pairs",
          [s for i, s in enumerate(masked.split('"')) if i % 2 == 1],
          ["a real quotation here"])
    check("masking preserves length", len(masked), len(live))
    check("a backticked identifier keeps its delimiters",
          mask_code("call `get_order` now"), "call `         ` now")
    check("an unpaired backtick masks nothing after it",
          mask_code('a `b " c'), 'a `b " c')
    check("line breaks survive masking",
          mask_code("a `x\ny` b").count("\n"), 1)
    return 0 if ok else 1


def decode_body(headers, raw):
    """Decode a response body by its Content-Encoding (wake 51fa3bb7, 2026-10-25).

    Plumb's reading on #278 and #271: docs.datadoghq.com (nine documents) and
    legal.ironcladapp.com (three) answer `content-encoding: gzip` to a request
    that sends NO Accept-Encoding, and urllib decodes nothing, so every sweep
    here that counted words in those bodies was counting compressed bytes and
    reporting 0. Confirmed from this seat at 22:04Z: credit_card/ 93,016 bytes
    gzip, terms-of-service 253,056 bytes gzip. curl without --compressed sees
    the same. A fetcher that does not read this header is not reading the page.

    Returns (bytes, note): note is '' for identity, the encoding name when it
    was decoded, '<enc>?' when the body claimed an encoding it could not be
    decoded from (left as delivered), and 'br-undecoded' when brotli is
    announced and no decoder is installed. Truncated gzip/deflate input (a
    capped read) decodes to the prefix it covers rather than raising.
    """
    import gzip as _gzip  # noqa: F401  (kept for readers grepping the module)
    import zlib
    enc = ""
    try:
        enc = (headers.get("Content-Encoding") or "").strip().lower()
    except Exception:
        enc = ""
    if enc in ("", "identity"):
        return raw, ""
    if enc in ("gzip", "x-gzip"):
        try:
            return zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(raw), enc
        except zlib.error:
            return raw, enc + "?"
    if enc == "deflate":
        for wbits in (zlib.MAX_WBITS, -zlib.MAX_WBITS):
            try:
                return zlib.decompressobj(wbits).decompress(raw), enc
            except zlib.error:
                continue
        return raw, enc + "?"
    if enc == "br":
        try:
            import brotli  # type: ignore
            return brotli.decompress(raw), enc
        except Exception:
            return raw, "br-undecoded"
    return raw, enc + "?"


def _selftest_decode():
    import gzip, zlib
    ok = True

    def check(name, got, want):
        nonlocal ok
        if got != want:
            ok = False
            print(f"FAIL: {name}: got {got!r}, want {want!r}")

    body = b"<p>Only one payment method is stored on file at a time</p>" * 50
    gz = gzip.compress(body)
    check("gzip decodes to the page", decode_body({"Content-Encoding": "gzip"}, gz), (body, "gzip"))
    check("x-gzip is gzip", decode_body({"Content-Encoding": "x-gzip"}, gz)[0], body)
    check("header case and spaces do not matter", decode_body({"Content-Encoding": " GZIP "}, gz)[0], body)
    check("identity passes bytes through untouched", decode_body({}, body), (body, ""))
    check("explicit identity too", decode_body({"Content-Encoding": "identity"}, body), (body, ""))
    check("zlib deflate decodes", decode_body({"Content-Encoding": "deflate"}, zlib.compress(body))[0], body)
    rawdef = zlib.compressobj(wbits=-zlib.MAX_WBITS); rd = rawdef.compress(body) + rawdef.flush()
    check("raw deflate decodes", decode_body({"Content-Encoding": "deflate"}, rd)[0], body)
    check("a truncated gzip body yields the prefix, not an exception",
          decode_body({"Content-Encoding": "gzip"}, gz[: len(gz) // 2])[0][:3], b"<p>")
    check("a body that lies about gzip is left as delivered and marked",
          decode_body({"Content-Encoding": "gzip"}, body), (body, "gzip?"))
    check("an unknown encoding is left as delivered and marked",
          decode_body({"Content-Encoding": "zstd"}, body), (body, "zstd?"))
    got = decode_body({"Content-Encoding": "br"}, body)
    check("brotli without a decoder says so", got[0] == body and got[1] in ("br-undecoded", "br"), True)
    # the real shape urllib hands over
    import email.message
    m = email.message.Message(); m["Content-Encoding"] = "gzip"
    check("an http.client HTTPMessage works as headers", decode_body(m, gz)[0], body)
    print("selftest(decode): %s" % ("ok" if ok else "FAIL"))
    return 0 if ok else 1


if __name__ == "__main__":
    if "--selftest" in sys.argv:
        # BOTH, always: `a() or b()` short-circuits, and _selftest() returns 1
        # whenever /tmp/reg is absent, so the masking rules silently never ran
        # the first time I wired them up here.
        sys.exit(max(_selftest(), _selftest_mask(), _selftest_decode()))
    sys.exit("corpus.py is a guard imported by the sweeps; run it with --selftest.")