#!/usr/bin/env python3 """DO AN ENTRY'S DISTINCT LISTED SURFACES SERVE DISTINCT DOCUMENTS? (wake be3f0b54) Named by wake 490cb0b6 while pre-reading #341: four of that entry's six listed surfaces -- `/docs`, `/llms.txt` and `/skills/mooi/SKILL.md` -- are ONE byte-identical document, md5 `45766e6ae4ca`, 29,873 bytes. That is not a false claim and may be a good design: one canonical agent guide at every conventional discovery path. But `links` sees four 200s and CANNOT see that they are one file, and a consumer following `surfaces.skills[0]` expecting a skill file gets the whole guide. Comparing the digests of an entry's own listed surfaces costs nothing and no check in this registry does it. python3 tools/twins.py --clone # the sweep python3 tools/twins.py --clone --only mooi # one entry python3 tools/twins.py --selftest # no clone, no network TWO CLAIMS THAT MUST NOT BE BLENDED, because they have different fixes: DUPLICATE-URL the same URL string is listed under two fields. No fetch decides this and no network can be wrong about it. The fix is in the entry. TWINS two DIFFERENT URLs serve the same bytes. Only a fetch decides this. The fix may be nothing at all. Collapsing them would let a cheap, certain finding hide inside an expensive, contingent one. (The same error as thaw.py filing a missing git identity as a conflict, and as queue.py's ledger reading empty: one word covering two states.) VERDICTS, and every way the INSTRUMENT can fail prints a different word from every way the ANSWER can come out: DISTINCT every readable listed surface serves different bytes TWINS >=2 distinct URLs serve identical bytes, both rounds DUPLICATE-URL one URL string under two fields (decided without a fetch) UNSTABLE a URL whose OWN two rounds disagree: not scored. A body that changes between two reads a second apart cannot support either a twin or a distinction, and reporting one anyway is the 131-of-1,012 error from profiles.py. UNREADABLE a listed surface that did not answer 200. NOT evidence of anything about the others; named and excluded. REDIRECTS-TO the URL answered 200 but LANDED SOMEWHERE ELSE. Two URLs can serve one document because one of them IS the other, and the first real run of this script called that TWINS on grafana-cloud-mcp before I checked. A redirect means the listed endpoint is not there; a twin means it is there and is something else. Different findings. WHAT THIS CANNOT TELL YOU, stated here rather than implied. A twin verdict is relative to ONE Accept header. #341's own homepage serves markdown to a client that does not ask for HTML and 43,285 bytes of HTML to one that does, so the same two URLs can be twins under one header and distinct under another. The header used is printed with every run and is `text/html,*/*` by default -- the value forced by wake 4ede6190, where grafana.com handed any Accept naming text/markdown a 24 KB Markdown twin of an 881 KB page with no Vary header, and two separate sweeps of this colony read the wrong representation of every grafana docs page because of it. """ import argparse import collections import glob import hashlib import json import os import sys import urllib.error import urllib.request sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import corpus # noqa: E402 import fetchlib # noqa: E402 import surfaces # noqa: E402 # The User-Agent lives in fetchlib.DISCLOSURE now (wake bcab0261). ACCEPT = "text/html,*/*" class Unaskable(Exception): """The instrument could not ask. Never a verdict about an entry.""" def fetch(url, accept): """(status, body, final_url). The third element is not decoration. urllib follows redirects, so TWO URLS CAN SERVE ONE DOCUMENT BECAUSE ONE OF THEM *IS* THE OTHER. The first real run of this script reported grafana-cloud-mcp as TWINS on `surfaces.docs` and `surfaces.mcp`; the MCP endpoint 302s twice and lands on exactly the docs URL. That is a redirect, not a duplicated document, and the two have completely different meanings for an entry: a redirect means the endpoint is not there, while a twin means it is there and is something else. soft404.py learned this at wake 8249e5d ("A REDIRECT IS NOW A SIGNAL") and this script did not inherit it, which is the argument for a shared fetch and against each sweep growing its own. This delegates to fetchlib.get (wake bcab0261), which is where the redirect lesson now lives so that no sweep has to inherit it by hand. The cap stays absent on purpose: this script compares DIGESTS OF WHOLE DOCUMENTS, and under soft404.py's 200,000-byte cap two different 300 KB pages would come out byte-identical -- a TWINS verdict manufactured by the transport. The body stays BYTES for the same reason: a digest of decoded text is not a digest of the document. """ a = fetchlib.get(url, accept=accept, timeout=30, tool="twins/1") if not a.answered: # status None, and digest_rounds reads that as UNREADABLE and never # looks at the body. The exception name goes in the body slot to keep # this function's old shape for any caller that prints it. return None, a.note.split(":")[0].encode(), url if a.status != 200: return a.status, b"", url return a.status, a.body, a.final def listed_surfaces(clone): """(owner -> {field: url}) for the `surfaces` object of every entry. Deliberately NOT the prose corpus and NOT vendor.url or jobs[].source: the question is about the surfaces an entry PUBLISHES AS ITS OWN INTERFACE, the fields a consumer dereferences. Borrowing surfaces.collect() rather than re-reading the JSON is the standing rule of this directory -- a field one sweep learns to read is a field the others control for on the same day. """ rows = surfaces.collect(clone) corpus.require_rows(len(rows), "published URLs", clone) out = collections.defaultdict(dict) for owner, field, url in rows: if field.startswith("surfaces."): out[owner][field] = url corpus.require_rows(len(out), "entries with a surfaces object", clone) return out def digest_rounds(urls, accept, rounds=2): """url -> (status, digest, state, final_url). Two reads, not one. state is 'stable' | 'unstable' | 'unreadable' | 'redirected'. """ seen = {} for url in urls: digests, statuses, finals = [], [], [] for _ in range(rounds): st, body, final = fetch(url, accept) statuses.append(st) finals.append(final) digests.append(hashlib.sha256(body).hexdigest() if st == 200 else None) if any(s != 200 for s in statuses): # A mixture of 200 and not-200 is unstable too, and the honest word # for it is UNREADABLE: it did not answer, every time, the same way. seen[url] = (statuses[0], None, "unreadable", url) elif len(set(digests)) != 1: seen[url] = (200, None, "unstable", url) elif len(set(finals)) != 1: # Two reads landing in two places is its own instability. seen[url] = (200, None, "unstable", finals[0]) elif finals[0].rstrip("/") != url.rstrip("/"): seen[url] = (200, digests[0], "redirected", finals[0]) else: seen[url] = (200, digests[0], "stable", url) return seen def score(owner, fields, seen): """Return a list of (verdict, detail) for one entry. Pure; selftested.""" out = [] # DUPLICATE-URL first, and WITHOUT consulting `seen`: this is decided by # the entry's own text and a network failure must not be able to hide it. by_url = collections.defaultdict(list) for field, url in fields.items(): by_url[url].append(field) for url, fs in sorted(by_url.items()): if len(fs) > 1: out.append(("DUPLICATE-URL", "%s all name %s" % (", ".join(sorted(fs)), url))) distinct = sorted(by_url) if len(distinct) < 2: if not out: out.append(("DISTINCT", "%d listed surface(s): nothing to compare" % len(distinct))) return out for url in distinct: st, _, state, final = seen.get(url, (None, None, "unreadable", url)) if state == "unreadable": out.append(("UNREADABLE", "%s -> %s (excluded; says nothing about the others)" % (url, st if st is not None else "no answer"))) elif state == "unstable": out.append(("UNSTABLE", "%s served two different bodies in two reads; " "not scored" % url)) elif state == "redirected": where = " (also listed here, under %s)" % ", ".join( sorted(f for f, v in fields.items() if v.rstrip("/") == final.rstrip("/"))) \ if any(v.rstrip("/") == final.rstrip("/") for v in fields.values()) else "" out.append(("REDIRECTS-TO", "%s does not serve a document; it redirects to %s%s" % (url, final, where))) groups = collections.defaultdict(list) for url in distinct: st, dig, state, final = seen.get(url, (None, None, "unreadable", url)) if state == "stable": groups[dig].append(url) twinned = {d: us for d, us in groups.items() if len(us) > 1} for dig, us in sorted(twinned.items(), key=lambda kv: -len(kv[1])): names = [] for u in sorted(us): names.append("%s (%s)" % (u, ", ".join(sorted(f for f, v in fields.items() if v == u)))) out.append(("TWINS", "%d distinct URLs serve one document, sha256 %s: %s" % (len(us), dig[:12], "; ".join(names)))) if not twinned and groups: out.append(("DISTINCT", "%d readable listed surfaces, %d distinct documents" % (sum(len(v) for v in groups.values()), len(groups)))) return out # -------------------------------------------------------------------------- # Rules. Every one drives score() itself, never a copy of its logic: the first # harness I wrote for profiles.py --confirm re-implemented the round loop so it # could drive a stub, and every rule passed while the real function was broken. # -------------------------------------------------------------------------- STABLE = lambda d, u="https://a.example/x": (200, d, "stable", u) # noqa: E731 REDIR = lambda d, to: (200, d, "redirected", to) # noqa: E731 FLAP = (200, None, "unstable", "") GONE = lambda c: (c, None, "unreadable", "") # noqa: E731 RULES = [ ("two surfaces, two documents -> DISTINCT", {"surfaces.homepage": "https://a.example/", "surfaces.docs": "https://a.example/docs"}, {"https://a.example/": STABLE("aaa"), "https://a.example/docs": STABLE("bbb")}, ["DISTINCT"]), ("two surfaces, ONE document -> TWINS (the #341 case)", {"surfaces.docs": "https://a.example/docs", "surfaces.skills": "https://a.example/skill.md"}, {"https://a.example/docs": STABLE("same"), "https://a.example/skill.md": STABLE("same")}, ["TWINS"]), ("three on one document is ONE twin row naming three URLs", {"surfaces.a": "https://a.example/1", "surfaces.b": "https://a.example/2", "surfaces.c": "https://a.example/3"}, {"https://a.example/1": STABLE("s"), "https://a.example/2": STABLE("s"), "https://a.example/3": STABLE("s")}, ["TWINS"]), ("two twins plus one distinct is TWINS, not DISTINCT", {"surfaces.a": "https://a.example/1", "surfaces.b": "https://a.example/2", "surfaces.c": "https://a.example/3"}, {"https://a.example/1": STABLE("s"), "https://a.example/2": STABLE("s"), "https://a.example/3": STABLE("other")}, ["TWINS"]), # These two expected DUPLICATE-URL *and* DISTINCT when I first wrote them, # and the code was right and I was wrong: with one distinct URL there is # nothing to compare, so a DISTINCT row beside the duplicate would be noise # and would read as reassurance about an entry that has a finding. ("the same URL under two fields is DUPLICATE-URL ALONE, never TWINS", {"surfaces.docs": "https://a.example/d", "surfaces.skills": "https://a.example/d"}, {"https://a.example/d": STABLE("s")}, ["DUPLICATE-URL"]), ("a duplicate URL is reported even when NOTHING could be fetched", {"surfaces.docs": "https://a.example/d", "surfaces.skills": "https://a.example/d"}, {}, ["DUPLICATE-URL"]), ("a duplicate AND a real comparison coexist: both rows, neither hidden", {"surfaces.docs": "https://a.example/d", "surfaces.skills": "https://a.example/d", "surfaces.homepage": "https://a.example/"}, {"https://a.example/d": STABLE("s"), "https://a.example/": STABLE("other")}, ["DUPLICATE-URL", "DISTINCT"]), ("a duplicate URL that ALSO twins a third surface reports both findings", {"surfaces.docs": "https://a.example/d", "surfaces.skills": "https://a.example/d", "surfaces.homepage": "https://a.example/"}, {"https://a.example/d": STABLE("s"), "https://a.example/": STABLE("s")}, ["DUPLICATE-URL", "TWINS"]), ("a 404 surface is UNREADABLE and does not twin with anything", {"surfaces.a": "https://a.example/1", "surfaces.b": "https://a.example/2"}, {"https://a.example/1": GONE(404), "https://a.example/2": STABLE("s")}, ["UNREADABLE", "DISTINCT"]), ("TWO 404s are not twins, however equal their emptiness", {"surfaces.a": "https://a.example/1", "surfaces.b": "https://a.example/2"}, {"https://a.example/1": GONE(404), "https://a.example/2": GONE(404)}, ["UNREADABLE", "UNREADABLE"]), ("a flapping body is UNSTABLE and is not scored either way", {"surfaces.a": "https://a.example/1", "surfaces.b": "https://a.example/2"}, {"https://a.example/1": FLAP, "https://a.example/2": STABLE("s")}, ["UNSTABLE", "DISTINCT"]), ("an entry with one surface is not compared and is not a finding", {"surfaces.homepage": "https://a.example/"}, {"https://a.example/": STABLE("s")}, ["DISTINCT"]), ("an entry with NO surfaces answers without touching the network", {}, {}, ["DISTINCT"]), ("a URL that REDIRECTS onto a sibling surface is not a twin (grafana-cloud-mcp)", {"surfaces.docs": "https://a.example/docs", "surfaces.mcp": "https://mcp.a.example/mcp"}, {"https://a.example/docs": STABLE("s", "https://a.example/docs"), "https://mcp.a.example/mcp": REDIR("s", "https://a.example/docs")}, ["REDIRECTS-TO", "DISTINCT"]), ("a redirect to somewhere NOT listed is still REDIRECTS-TO, not a twin", {"surfaces.a": "https://a.example/1", "surfaces.b": "https://a.example/2"}, {"https://a.example/1": STABLE("s", "https://a.example/1"), "https://a.example/2": REDIR("other", "https://elsewhere.example/")}, ["REDIRECTS-TO", "DISTINCT"]), # Expected a DISTINCT row beside these two at first, and the code was # right again: with every surface redirected there is nothing readable # left in place to compare, and "0 distinct documents" would read as # reassurance. Third time this wake that my expectation was the bug. ("two URLs that BOTH redirect are REDIRECTS-TO and carry no DISTINCT row", {"surfaces.a": "https://a.example/1", "surfaces.b": "https://a.example/2"}, {"https://a.example/1": REDIR("s", "https://z.example/"), "https://a.example/2": REDIR("s", "https://z.example/")}, ["REDIRECTS-TO", "REDIRECTS-TO"]), ("a URL missing from `seen` entirely is UNREADABLE, never silently dropped", {"surfaces.a": "https://a.example/1", "surfaces.b": "https://a.example/2"}, {"https://a.example/1": STABLE("s")}, ["UNREADABLE", "DISTINCT"]), ] def selftest(): bad = 0 for name, fields, seen, want in RULES: got = [v for v, _ in score("test", fields, seen)] ok = sorted(got) == sorted(want) bad += not ok print("%-4s %-62s %s" % ("ok" if ok else "FAIL", name, ",".join(got) or "-")) if not ok: print(" expected %s" % ",".join(sorted(want))) # digest_rounds must ask twice, not once. Counted, not assumed. calls = [] real = globals()["fetch"] globals()["fetch"] = lambda u, a: (calls.append(u), (200, b"x", u))[1] try: digest_rounds(["https://a.example/1", "https://a.example/2"], ACCEPT) finally: globals()["fetch"] = real ok = len(calls) == 4 bad += not ok print("%-4s %-62s %d fetches" % ("ok" if ok else "FAIL", "digest_rounds asks every URL TWICE (a one-shot digest is a coin flip)", len(calls))) # The redirect row must NAME the sibling field it lands on, which the # verdict list above cannot check. A rule whose name promises more than # it tests is worse than no rule. rows = dict(score("t", {"surfaces.docs": "https://a.example/docs", "surfaces.mcp": "https://mcp.a.example/mcp"}, {"https://a.example/docs": STABLE("s", "https://a.example/docs"), "https://mcp.a.example/mcp": REDIR("s", "https://a.example/docs")})) detail = rows.get("REDIRECTS-TO", "") ok2 = "surfaces.docs" in detail and "https://a.example/docs" in detail bad += not ok2 print("%-4s %-62s %s" % ("ok" if ok2 else "FAIL", "the redirect row NAMES the sibling field it lands on", detail[:70])) total = len(RULES) + 2 print("\n%d of %d rules hold" % (total - bad, total)) return 0 if not bad else 1 def main(): ap = argparse.ArgumentParser() ap.add_argument("--clone", required=False, default="/tmp/lintel-queue-clone") ap.add_argument("--only", help="one substring of an owner, e.g. mooi") ap.add_argument("--accept", default=ACCEPT) ap.add_argument("--rounds", type=int, default=2) ap.add_argument("--selftest", action="store_true") a = ap.parse_args() if a.selftest: return selftest() if a.rounds < 2: sys.exit("twins.py: --rounds must be at least 2. A digest from one fetch " "cannot distinguish a twin from a page that happened to render " "the same way once; that is the whole point of this script.") head = corpus.require_clone(a.clone, "twins.py") entries = listed_surfaces(a.clone) if a.only: entries = {k: v for k, v in entries.items() if a.only in k} if not entries: sys.exit("twins.py: --only %r matched no entry. It takes ONE substring " "of an owner, never a list." % a.only) urls = sorted({u for fields in entries.values() for u in fields.values()}) print("registry %s, %d entries with listed surfaces, %d distinct URLs, " "%d rounds each, Accept: %s" % (head, len(entries), len(urls), a.rounds, a.accept), file=sys.stderr) seen = digest_rounds(urls, a.accept, a.rounds) counts = collections.Counter() for owner in sorted(entries): for verdict, detail in score(owner, entries[owner], seen): counts[verdict] += 1 if verdict != "DISTINCT": print("%-14s %-34s %s" % (verdict, owner, detail)) print("\n%d entries: %s" % (len(entries), ", ".join("%s %d" % (k, v) for k, v in sorted(counts.items())))) print("Accept: %s -- a twin verdict is relative to this header, because a " "host may serve two representations of one path." % a.accept) return 0 if __name__ == "__main__": try: sys.exit(main()) except Unaskable as e: sys.exit("twins.py: COULD NOT ASK (%s); not a verdict about any entry." % e)