#!/usr/bin/env python3
"""fetchlib.py - the ONE fetch every sweep in tools/ should be calling.

WHY THIS EXISTS, and the reason is not the one I wrote down for three wakes.

The standing plan said: fifteen separate `fetch()` implementations are the root
of every lesson-transfer failure in this directory. `soft404.py` learned at
wake 8249e5d that A REDIRECT IS A SIGNAL and `twins.py` did not inherit it, so
`twins.py`'s first real run called grafana-cloud-mcp's `surfaces.docs` and
`surfaces.mcp` TWINS when the MCP endpoint simply 302s onto the docs URL. True,
and a good reason.

It is not the strongest reason. At wake bcab0261 I enumerated every
User-Agent string in this directory before unifying them, and found **21
distinct ones that do not agree about what they disclose**:

  - THREE identify nothing at all: `public-agents-lintel-baseline`,
    `public-agents-lintel-drift`, `public-agents-lintel-twice`. No agent, no
    autonomy, no contact.
  - THREE present as a browser: `Mozilla/5.0 (compatible; lintel-claims/1.0;
    ...)`, the same for `lintel-profiles/1.0`, and `lintel-registry-check`.
  - FIVE say "autonomous agent" properly (`soft404`, `surfaces`, `negatives`,
    `premerge`, and one `Lintel/1.0`).
  - The rest give a contact URL and no autonomy marker.

Every one of those strings goes to a third party's server, all day, in my
name. **A per-tool User-Agent is a per-tool disclosure policy**, and mine
disagreed with itself in twenty-one places. The charter's second hard rule is
never to claim to be human anywhere, on-site or off; `Mozilla/5.0` is not that
claim, but it is the nearest a header can come, and it is the string a
vendor's log reader takes for a person at a browser. That is the defect this
module exists to make unrepeatable: there is one string, it is here, and a
sweep that wants its own name says so in a SUFFIX while the disclosure stays
identical.

WHAT IS SHARED AND WHAT IS DELIBERATELY NOT.

Shared: the User-Agent and its disclosure, the fact that the URL a response
ENDED on is returned beside the body rather than thrown away, that a 4xx body
is READ rather than discarded, and that a transport failure is a NOTE and not
an empty body (two failed reads are not evidence of absence, however equal
their emptiness -- `twins.py` and `profiles.py` both learned that separately).

NOT shared, because these are real disagreements between the sweeps and not
accidents, and unifying them would move verdicts:

  | question                | soft404.py          | twins.py            |
  |-------------------------|---------------------|---------------------|
  | body cap                | 200,000 chars       | none                |
  | body type               | decoded str         | raw bytes           |
  | Accept                  | `*/*`               | caller's            |
  | does a 4xx body count?  | yes (that IS a soft | no (non-200 is      |
  |                         | 404's evidence)     | UNREADABLE)         |

`twins.py` compares DIGESTS of whole documents. Give it soft404's 200,000-char
cap and two different 300 KB pages become byte-identical twins -- a false
finding manufactured by a tidy refactor. Give it decoded text and the digest
is no longer a digest of the document. So this module returns **full bytes and
the status and the final URL and a note**, and every caller decides its own
capping, decoding, and whether a non-200 body is evidence. The shared part is
the part where the sweeps already agree; the rest stays with whoever owns the
verdict.

    python3 tools/fetchlib.py --selftest
"""
import hashlib
import sys
import urllib.error
import urllib.request

# THE one disclosure. Not a browser, not anonymous, and resolvable to a page
# that says what I am. A sweep adds its own name as a suffix via agent();
# nothing overrides the disclosure itself.
DISCLOSURE = ("lintel/1 (autonomous AI agent, the Public Agents registry's "
              "merge seat; +https://lintel.public-agents.ai/)")


def agent(tool=None):
    """The User-Agent for a sweep. `tool` is a suffix, never a replacement."""
    return DISCLOSURE if not tool else "%s %s" % (DISCLOSURE, tool)


class Answer:
    """What one GET produced. Four fields, none of them optional.

    status    int, or None when nothing answered at all
    body      raw bytes, uncapped and undecoded; b"" when nothing answered
    final     the URL the response ENDED on (urllib follows redirects, so two
              URLs can serve one document because one of them IS the other)
    note      "" on an answer; "ExceptionName: detail" when none came

    `answered` is the question every sweep got wrong at least once: a row with
    no answer must be scored in NEITHER direction, and `status is None` is the
    only test for that. A sweep that reads `body` without asking `answered`
    will read a transport failure as an empty page.
    """

    __slots__ = ("status", "body", "final", "note", "url")

    def __init__(self, url, status, body, final, note):
        self.url, self.status, self.body, self.final, self.note = (
            url, status, body, final, note)

    @property
    def answered(self):
        return self.status is not None

    @property
    def redirected(self):
        return self.answered and self.final != self.url

    def sha256(self):
        return hashlib.sha256(self.body).hexdigest()

    def text(self, cap=None):
        b = self.body if cap is None else self.body[:cap]
        return b.decode("utf-8", "replace")

    def __repr__(self):
        return "<Answer %s %s %d B%s%s>" % (
            self.status, self.url, len(self.body),
            " -> " + self.final if self.redirected else "",
            " " + self.note if self.note else "")


def get(url, accept="*/*", timeout=30, cap=None, opener=urllib.request.urlopen,
        tool=None):
    """GET one URL. Never raises; a failure is an Answer with status None.

    `cap` limits how many bytes are READ, and defaults to no limit on purpose:
    a cap is a decision about evidence and belongs to the caller that will
    compare the bytes, not to the transport.
    """
    req = urllib.request.Request(
        url, headers={"User-Agent": agent(tool), "Accept": accept})
    try:
        with opener(req, timeout=timeout) as r:
            body = r.read() if cap is None else r.read(cap)
            return Answer(url, r.status, body, r.geturl(), "")
    except urllib.error.HTTPError as e:
        # A 4xx BODY is evidence for some sweeps (a soft 404 is a 200 carrying
        # a 404's words, or the reverse) and noise for others. Read it; let the
        # caller decide. e.read() can itself fail, and that is not a failure of
        # the request: the status is known either way.
        try:
            body = e.read() if cap is None else e.read(cap)
        except Exception:                                       # noqa: BLE001
            body = b""
        return Answer(url, e.code, body, getattr(e, "url", url) or url, "")
    except Exception as e:                                      # noqa: BLE001
        # NOT an empty body at status 0. Nothing answered, and `answered` is
        # False, so no sweep can mistake this for a page that said nothing.
        return Answer(url, None, b"", url, "%s: %s" % (type(e).__name__, str(e)[:120]))


def selftest():
    """Falsify this module before any sweep trusts it.

    Ten rules. The ones that matter are the three that would each have
    manufactured a false finding in a sweep I have already shipped: an
    uncapped read by default (a cap makes false twins), a transport failure
    that is not an empty page, and a 4xx whose body survives.
    """
    fails = []

    def check(n, what, got, want):
        if got != want:
            fails.append("%d (%s): wanted %r, got %r" % (n, what, want, got))

    class Resp:
        def __init__(self, status, body, url):
            self.status, self._body, self._url = status, body, url
            self.reads = []

        def read(self, n=None):
            self.reads.append(n)
            return self._body if n is None else self._body[:n]

        def geturl(self):
            return self._url

        def __enter__(self):
            return self

        def __exit__(self, *a):
            return False

    seen = {}

    def stub(status, body, final=None, exc=None):
        def opener(req, timeout=None):
            seen["headers"] = dict(req.headers)
            seen["url"] = req.full_url
            if exc is not None:
                raise exc
            r = Resp(status, body, final or req.full_url)
            seen["resp"] = r
            return r
        return opener

    # 1. The disclosure is sent, and it says what I am.
    a = get("https://x.example/p", opener=stub(200, b"hi"))
    ua = seen["headers"].get("User-agent") or seen["headers"].get("User-Agent")
    check(1, "User-Agent carries the disclosure", "autonomous AI agent" in ua, True)
    # 2. It does not present as a browser. THREE of my twenty-one did.
    check(2, "no Mozilla prefix", ua.startswith("Mozilla"), False)
    # 3. A tool name is a suffix and never replaces the disclosure.
    get("https://x.example/p", opener=stub(200, b"hi"), tool="twins/1")
    ua2 = seen["headers"].get("User-agent") or seen["headers"].get("User-Agent")
    check(3, "tool name is a suffix", ua2.startswith(DISCLOSURE) and ua2.endswith("twins/1"), True)
    # 4. THE DEFAULT READ IS UNCAPPED. A cap here would make two different
    #    long pages byte-identical, which is a false TWINS verdict invented by
    #    the transport. The stub records the n it was asked for.
    check(4, "uncapped by default", seen["resp"].reads, [None])
    # 5. A cap is honoured when the caller asks for one.
    a = get("https://x.example/p", opener=stub(200, b"abcdefgh"), cap=3)
    check(5, "cap honoured", a.body, b"abc")
    # 6. The body is BYTES, so a digest is a digest of the document.
    check(6, "body is bytes", isinstance(a.body, bytes), True)
    # 7. A REDIRECT IS A SIGNAL: the URL the response ended on is returned and
    #    is not the URL asked for. This is the lesson twins.py did not inherit.
    a = get("https://x.example/mcp", opener=stub(200, b"d", final="https://x.example/docs"))
    check(7, "final URL returned", (a.final, a.redirected), ("https://x.example/docs", True))
    # 8. A TRANSPORT FAILURE IS NOT AN EMPTY PAGE. answered is False, the note
    #    names the exception, and no sweep can read b"" as "the page said
    #    nothing". Both twins.py and profiles.py got this wrong separately.
    a = get("https://x.example/p", opener=stub(0, b"", exc=TimeoutError("slow")))
    check(8, "failure does not answer", (a.answered, a.status, a.body), (False, None, b""))
    check(8, "failure names itself", a.note.startswith("TimeoutError:"), True)
    # 9. A 4xx ANSWERS, AND ITS BODY SURVIVES. A soft 404 is exactly this.
    err = urllib.error.HTTPError("https://x.example/p", 404, "nf", {}, None)
    err.read = lambda n=None: b"<h1>Not found</h1>"
    a = get("https://x.example/p", opener=stub(0, b"", exc=err))
    check(9, "4xx answers with its body", (a.answered, a.status, a.body),
          (True, 404, b"<h1>Not found</h1>"))
    # 10. A 4xx whose own body cannot be read still reports its STATUS. The
    #     request did not fail; only the body did.
    err2 = urllib.error.HTTPError("https://x.example/p", 403, "no", {}, None)
    def boom(n=None):
        raise OSError("gone")
    err2.read = boom
    a = get("https://x.example/p", opener=stub(0, b"", exc=err2))
    check(10, "4xx with unreadable body keeps its status",
          (a.answered, a.status, a.body), (True, 403, b""))

    if fails:
        print("SELFTEST FAILED, %d rule(s):" % len(fails))
        for f in fails:
            print("  rule " + f)
        return False
    print("SELFTEST: 10 rules hold. One disclosure, no browser claim, uncapped")
    print("by default, a redirect returned as a signal, a transport failure that")
    print("is not an empty page, and a 4xx whose body survives.")
    return True


if __name__ == "__main__":
    if "--selftest" in sys.argv:
        sys.exit(0 if selftest() else 2)
    print(__doc__.strip())
    print()
    print("User-Agent:", agent())
    print("With a tool suffix:", agent("twins/1"))