#!/usr/bin/env python3 """Render the Accept-Encoding lab tables from results.json (no hand-typed numbers). python3 render_tables.py [results.json] # English matrix.md to stdout python3 render_tables.py results.json --out matrix.md python3 render_tables.py results.json --strings ru.json --out matrix.ru.md python3 render_tables.py results.json --section answer # one section only (for a guide body) python3 render_tables.py --emit-strings > strings.ru.json # English strings: the translation template python3 render_tables.py results.json --parity ru.json # same data in both locales? python3 render_tables.py results.json --self-test # pseudo-locale parity + checks python3 render_tables.py harness/out/results.json --out my-tables.md # tables of your own run Every byte count, header value, version and outcome comes from results.json. The script only chooses which records back each row and how to word the labels. The words live in a strings map (English built in, other locales via --strings) whose {placeholders} and `code spans` must survive translation unchanged; --parity compares the data tokens (code spans and numbers) of two renders line by line. Each answer row also carries checks against the data (for example "curl --compressed decodes" or "curl's default sends no header"). If a rerun changes one of them, the script names the row and exits 1 instead of rendering advice the data no longer supports. It also renders a run's own output (harness/out/results.json from ./run.sh): the parts the published file adds (CONNECT pairs, verify-snippet runs, coverage gaps, reviewed anomalies) are computed from its records or read from harness/out/snippets/, and not-run cells never back a row (a row whose records were not run stops the render with the reason). Sections: header, answer, provenance, detail, bytes, timing, pitfalls, snippets, coverage (or "all", the default). Standard library only; Python 3.9+. """ from __future__ import annotations import argparse import collections import copy import fnmatch import json import os import re import sys # ---------------------------------------------------------------------------- strings (English) # Typography lives in "meta": list_sep joins list items (client names, statuses, the versions inside a row label), # sentence_sep joins sentences in running text ("" for zh, whose full stop carries its own spacing). # rows..label is the row's "Versions run" label; {v}, {g}, {l} are the versions the data supplies. EN: dict = { "meta": {"locale": "en", "group_sep": ",", "decimal_sep": ".", "list_sep": ", ", "times": "×", "sentence_sep": " "}, "doc": { "title": "Accept-Encoding defaults: lab matrix", "synthetic": "**Synthetic loopback results.** A 127.0.0.1 origin and a counting proxy served local files. " "No public site was measured, and no byte count here is a provider bill.", "run_window": "Runs {start} to {end} (UTC), {records} records ({counts}). Harness tree SHA-256 {tree}.", "status": {"run": "run", "source-read": "source-read", "not-run": "not run"}, "fixture_note": "Byte counts use the synthetic fixture, {identity} B of HTML. Its ratios are a fixture ratio, " "not typical.", "wire_note": "Wire body bytes exclude response headers, TLS records, the CONNECT exchange and retries, so " "they are not what a provider bills.", }, "answer": { "heading": "Who asks for compression by default", "group_none": "Sends no Accept-Encoding, or identity", "group_none_intro": "A server that compresses only when asked sends these clients uncompressed text.", "sent_none": "No header: {clients}.", "sent_value": "{value}: {clients}.", "cols_none": ["Client", "Fix"], "group_compresses": "Already asks for compression", "group_compresses_intro": "Nothing to change: these clients ask for compression and decode the reply.", "cols_compresses": ["Client", "Header sent"], "none": "(none)", "qual_sep": "; ", "qual": "{qual}: {value}", "versions": "Versions run: {list}.", "versions_sep": "; ", }, "rows": { "curl": {"name": "curl", "label": "curl {v}", "fix": "Add {flag}", "note": "curl: a hand-set {manual} is sent, but the body stays compressed."}, "wget": {"name": "Wget", "label": "Wget {v}", "fix": "Add {flag}", "note": "Wget: with {flag} it sent {sent} and decoded the reply."}, "pystdlib": {"name": "urllib, http.client", "label": "urllib.request, http.client (Python {v})", "fix": "Use Requests or httpx", "note": "urllib.request and http.client return compressed bodies as they arrived."}, "nodehttp": {"name": "node:http, node:https", "label": "node:http, node:https (Node.js {v})", "fix": "Use {fetch}, got or axios", "note": "node:http and node:https return compressed bodies as they arrived."}, "phpcurl": {"name": "PHP ext-curl", "label": "PHP {v} ext-curl (libcurl {l})", "fix": "Set {opt} to {empty}", "note": "PHP ext-curl: with {opt} set to {empty}, this libcurl sent {sent}. A hand-set header is " "sent but not decoded."}, "guzzle": {"name": "Guzzle, Laravel Http", "label": "Guzzle {g}, Laravel Http {l}", "fix": "Set {opt} to {value}", "note": "Laravel Http: {laravel}."}, "java": {"name": "Java HttpClient", "label": "Java {v} HttpClient", "fix": "Send {hdr}, unzip with {gz}", "note": "Java HttpClient never decodes: wrap the body in {gz} when the reply is gzip."}, "requests": {"name": "Requests", "label": "Requests {v}", "note": "Requests: {br} adds br; on Python 3.13, {zstd} adds zstd."}, "httpx": {"name": "httpx", "label": "httpx {v}", "note": "httpx: {br} adds br and {zstd} adds zstd."}, "aiohttp": {"name": "aiohttp", "label": "aiohttp {v}", "note": "aiohttp: {br} adds br; on Python 3.13, {zstd} adds zstd."}, "scrapy": {"name": "Scrapy", "label": "Scrapy {v}"}, "nodefetch": {"name": "Node.js built-in fetch", "label": "built-in fetch (Node.js {v})", "note_other": "Built-in fetch over {scheme}: {value}."}, "go": {"name": "Go net/http", "label": "Go {v} net/http", "note": "Go: setting the header yourself turns decoding off."}, }, "provenance": { "heading": "Where each row comes from", "cols": ["Row", "Basis", "Records", "Cell definitions"], }, "detail": { "heading": "Per-client detail", "intro": "One row per setup. Paths that behaved the same are merged into one row.", "cols": ["Setup", "Paths", "Sent", "Server sent (fixture)", "Your code got"], "served_identity": "not compressed, {n} B", "served": "{coding}, {n} B", "got_decoded": "decoded, {n} B", "got_raw": "still {coding}, {n} B", "got_plain": "{n} B", "got_first_frame": "first zstd frame only, {n} B", "got_error": "error", "got_none": "no body", }, "paths": {"direct-http": "direct http://", "direct-https": "direct https://", "absolute-form": "proxy http://", "absolute-form-tunnelled": "proxy http:// (sent as CONNECT)", "CONNECT": "proxy CONNECT"}, "families": { "curl": "curl", "wget": "Wget", "pystdlib": "Python standard library", "requests": "Requests", "httpx": "httpx", "aiohttp": "aiohttp", "scrapy": "Scrapy", "nodefetch": "Node.js built-in fetch", "nodehttp": "Node.js node:http and node:https", "nodelibs": "Node.js libraries", "go": "Go net/http", "php": "PHP: ext-curl, Guzzle and Laravel Http", "java": "Java java.net.http.HttpClient", }, "bytes": { "heading": "Bytes with and without compression", "pairs_heading": "Fixture over CONNECT, same client", "pairs_intro": "Fixture ratio, not typical. The proxy leg adds response headers, TLS records and the " "CONNECT reply to the body, so its ratio is lower.", "pairs_cols": ["Client", "Body bytes: without → with", "Proxy → client bytes: without → with", "Ratio: body / proxy leg"], "arrow": "{a} → {b}", "ratio_pair": "{a} / {b}", "corpus_heading": "Corpus: {files} CPython {version} documentation pages", "corpus_intro": "This corpus only, not typical of the web. Each row is one client fetching all {files} pages " "through CONNECT.", "corpus_cols": ["Client", "Coding", "Body bytes: identity → wire", "Ratio"], "coding_identity": "not compressed", }, "timing": { "heading": "What Node.js fetch reports about body bytes", "intro": "After reading the body, the client looked up the {api} entry for its URL. Proxy → client bytes " "are the counting proxy's own count for the same request.", "cols": ["Node.js", "Path", "Server sent", "Wire body bytes", "`encodedBodySize`", "`decodedBodySize`", "`transferSize`", "Proxy → client bytes"], "enc_all": "In all {n} rows, {enc} equals the body bytes the origin wrote and {dec} equals the bytes your " "code received.", "enc_some": "{enc} differs from the origin's body bytes in {bad} of {n} rows.", "transfer_one": "{transfer} was {enc} + {diff} B in every row, whether the request went direct or through " "the proxy: a fixed allowance in Node.js, not a count of header, TLS or CONNECT bytes.", "transfer_many": "{transfer} minus {enc} took the values {diffs} B.", "late": "In every row the entry was not yet visible synchronously right after the body was read; it " "appeared after {turns} event-loop turn(s) ({wait}).", "late_mixed": "The entry was visible synchronously after the body in {n} of {total} rows.", "none": "—", }, "pitfalls": { "heading": "Pitfalls", "unasked_heading": "The server compresses without being asked", "unasked_intro": "The origin sent gzip even though the client asked for none.", "unasked_cols": ["Client", "Sent", "Your code got"], "decoders_heading": "Which codings each client decodes", "decoders_intro": "The origin forced each coding regardless of the request. \"zstd ×4\" is four concatenated " "zstd frames, which RFC 8878 allows.", "decoders_cols": ["Client", "gzip", "br", "zstd", "zstd ×4"], "copied_heading": "A copied browser header", "copied_intro": "The client was given {header} by hand.", "copied_cols": ["Client", "Server chose zstd", "Server forced br"], "manual_heading": "A hand-set header and decoding", "manual_cols": ["Client", "Sent", "Your code got"], "errors_heading": "Exact error text", "errors_intro": "As recorded, except that loopback URLs and ports are shortened to …", "errors_item": "{n} record(s), for example {ids}:", "range_heading": "HEAD and Range requests", "range_cols": ["Client", "Request", "Sent"], "req_head": "HEAD", "req_range": "GET with {range}", "proxy_heading": "What an HTTP proxy can see", "proxy_cols": ["Client", "http:// through the proxy", "https:// through CONNECT"], "proxy_saw": "proxy saw {value}", "proxy_saw_none": "proxy saw no header", "proxy_tunnel": "sent through CONNECT: not visible", "proxy_connect": "not visible", "proxy_quirk": "request failed at the proxy (see below)", "tls_heading": "Certificate checks stayed on", "tls_intro": "The same https requests without the lab CA. Every client refused the certificate. Loopback URLs " "and ports are shortened to …", "tls_cols": ["Client", "Result without the lab CA"], "cell_decoded": "decoded", "cell_raw": "raw bytes", "cell_error": "error", "cell_first_frame": "first frame only", "cell_not_run": "—", }, "snippets": { "heading": "Verify snippets", "intro": "Each published snippet ran unchanged except that {placeholder} became a loopback URL.", "cols": ["Snippet", "Client", "Printed", "Matches the origin log"], "yes": "yes", "no": "no", "missing": "No snippet results were found next to this results.json.", "curl_min": "`%header{}` in curl -w needs curl {version} or later (source-read from the man page shipped with curl {source}; no older curl was run).", }, "coverage": { "heading": "Not run, dropped and reviewed anomalies", "not_run_cols": ["Item", "Status", "Why"], "anomalies_heading": "Anomalies the runner flagged, reviewed", "anomalies_cols": ["Record", "Flag", "Review"], "unreviewed": "not reviewed: this is a runner's own output, not the published results", }, "variants": { "default": "default", "bare": "bare install", "stdlib": "standard library", "default deps": "default dependencies", "+brotli": "+ `brotli`", "+Brotli": "+ `Brotli`", "+zstandard": "+ `zstandard`", "+backports.zstd": "+ `backports.zstd`", "--compressed": "`--compressed`", "--compressed --no-compressed": "`--compressed --no-compressed`", "--compression=auto": "`--compression=auto`", "-H 'Accept-Encoding: gzip'": "`-H 'Accept-Encoding: gzip'`", "CURLOPT_ACCEPT_ENCODING ''": "`CURLOPT_ACCEPT_ENCODING => ''`", "CURLOPT_HTTPHEADER Accept-Encoding: gzip": "`CURLOPT_HTTPHEADER`, `Accept-Encoding: gzip`", "default (CURLOPT_ACCEPT_ENCODING not set)": "default", "default (decode_content true)": "default", "decode_content 'gzip'": "`'decode_content' => 'gzip'`", "decode_content 'gzip, deflate, br, zstd'": "`'decode_content' => 'gzip, deflate, br, zstd'`", "StreamHandler, default": "default", "StreamHandler, decode_content 'gzip'": "`'decode_content' => 'gzip'`", "StreamHandler, default + stream_context http.request_fulluri true": "default + `request_fulluri`", "default (Http::get equivalent)": "default", "withHeaders(['Accept-Encoding' => 'gzip'])": "`withHeaders(['Accept-Encoding' => 'gzip'])`", "withOptions(['decode_content' => 'gzip'])": "`withOptions(['decode_content' => 'gzip'])`", "header Accept-Encoding: gzip + application GZIPInputStream": "`Accept-Encoding: gzip` + `GZIPInputStream`", "default (http.DefaultTransport clone)": "default", "Transport.DisableCompression = true": "`DisableCompression: true`", "req.Header.Set(\"Accept-Encoding\", \"gzip\")": "`req.Header.Set(\"Accept-Encoding\", \"gzip\")`", "transitional.advertiseZstdAcceptEncoding: true": "`advertiseZstdAcceptEncoding: true`", "default header": "default", "default header (http://)": "default (http://)", "default (asks for gzip)": "default", "default (no header)": "default", "default, server compresses unasked": "default", "StreamHandler, server compresses unasked": "default", "DisableCompression = true": "`DisableCompression: true`", "decode_content false + header gzip": "`'decode_content' => false` + `Accept-Encoding: gzip`", "header Accept-Encoding: gzip, no application decode": "`Accept-Encoding: gzip`, no `GZIPInputStream`", "copied browser header": "copied header", "copied browser header 'gzip, deflate, br, zstd'": "copied header", "copied browser header via CURLOPT_HTTPHEADER": "copied header", "copied browser header via 'headers' (decode_content default)": "copied header", "-H 'Accept-Encoding: gzip, deflate, br, zstd' (copied browser header)": "copied header", "StreamHandler + copied browser header": "copied header", "StreamHandler, default, http:// URL through an http proxy (no request_fulluri)": "default, http:// through a proxy, no `request_fulluri`", }, } # ---------------------------------------------------------------------------- spec (structure, not words) C = lambda s: s # marks a literal code token in the spec (rendered as a code span) ANSWER_ROWS = [ # id, category, version lookups for the label (strings rows..label), default-header records, qualifier dims, # tokens, checks; # optional primary=(field, value): the header cell shows only those default records, the rest become a note dict(id="curl", cat="none", ver=[("v", "curl-*-default--*", "client_version", r"(.+)")], default=["curl-system-default--*", "curl-brew-default--*"], dims=[], tokens={"flag": C("--compressed"), "manual": C("-H 'Accept-Encoding: gzip'")}, checks=[("curl-*-compressed--*", "decoded"), ("curl-*-manual-gzip--*", "raw"), ("curl-*-no-compressed", "absent"), ("pit-curl-*-default-unasked", "raw")]), dict(id="wget", cat="none", ver=[("v", "wget-default--*", "client_version", r"(.+)")], default=["wget-default--*"], dims=[], tokens={"flag": C("--compression=auto"), "sent": ("header", "wget-compression-auto--*")}, checks=[("wget-compression-auto--*", "decoded"), ("pit-wget-default-unasked", "raw")]), dict(id="pystdlib", cat="none", ver=[("v", "py31?-stdlib-*--*", "runtime_version", r"(.+)")], default=["py31?-stdlib-urllib--*", "py31?-stdlib-httpclient--*"], dims=[], tokens={}, checks=[("pit-py31?-stdlib-urllib-decoders--*", "raw"), ("pit-py31?-stdlib-urllib-copied-header--*", "raw"), ("py31?-requests-bare--*", "decoded"), ("py31?-httpx-bare--*", "decoded")]), dict(id="nodehttp", cat="none", ver=[("v", "node*-http--*", "runtime_version", r"(.+)")], default=["node*-http--*"], dims=[], tokens={"fetch": C("fetch")}, checks=[("pit-node*-http-unasked--*", "raw"), ("pit-node*-http-copied-header", "raw"), ("node*-fetch--*", "decoded"), ("node*-got--*", "decoded"), ("node*-axios--*", "decoded")]), dict(id="phpcurl", cat="none", ver=[("v", "php-curl-default--*", "runtime_version", r"^(\S+)"), ("l", "php-curl-default--*", "client_version", r"libcurl (\S+?)\)")], default=["php-curl-default--*"], dims=[], tokens={"opt": C("CURLOPT_ACCEPT_ENCODING"), "empty": C("''"), "sent": ("header", "php-curl-accept-encoding-empty--*")}, checks=[("php-curl-accept-encoding-empty--*", "decoded"), ("php-curl-manual-gzip--*", "raw"), ("pit-php-curl-default-unasked", "raw")]), dict(id="guzzle", cat="none", ver=[("g", "guzzle-default--*", "client_version", r"guzzle (\S+)"), ("l", "laravel-http-default--*", "client_version", r"illuminate/http v(\S+)")], default=["guzzle-default--*", "guzzle-stream-default--*", "guzzle-stream-default-fulluri", "laravel-http-default--*"], dims=[], tokens={"opt": C("decode_content"), "value": C("'gzip'"), "laravel": C("withOptions(['decode_content' => 'gzip'])")}, checks=[("guzzle-decode-content-gzip--*", "decoded"), ("guzzle-stream-decode-content-gzip--*", "decoded"), ("laravel-http-decode-content-gzip", "decoded"), ("laravel-http-with-headers-gzip--*", "decoded")]), dict(id="java", cat="none", ver=[("v", "java-httpclient-default--*", "client_version", r"JDK (\S+?)[-+)]")], default=["java-httpclient-default--*"], dims=[], tokens={"hdr": C("Accept-Encoding: gzip"), "gz": C("GZIPInputStream")}, checks=[("java-httpclient-gzip-gunzip--*", "decoded"), ("pit-java-httpclient-manual-gzip", "raw"), ("pit-java-httpclient-default-unasked", "raw")]), dict(id="requests", cat="compresses", ver=[("v", "py31?-requests-bare--*", "client_version", r"(.+)")], default=["py31?-requests-bare--*"], dims=[("runtime_version", "Python {v}")], tokens={"br": C("brotli"), "zstd": C("backports.zstd")}, checks=[("py313-requests-brotli--*", "has:br"), ("py313-requests-zstd--*", "has:zstd"), ("py31?-requests-*--*", "decoded")]), dict(id="httpx", cat="compresses", ver=[("v", "py31?-httpx-bare--*", "client_version", r"(.+)")], default=["py31?-httpx-bare--*"], dims=[("runtime_version", "Python {v}")], tokens={"br": C("brotli"), "zstd": C("zstandard")}, checks=[("py31?-httpx-brotli--*", "has:br"), ("py31?-httpx-zstandard--*", "has:zstd"), ("py31?-httpx-*--*", "decoded")]), dict(id="aiohttp", cat="compresses", ver=[("v", "py31?-aiohttp-bare--*", "client_version", r"(.+)")], default=["py31?-aiohttp-bare--*"], dims=[("runtime_version", "Python {v}")], tokens={"br": C("Brotli"), "zstd": C("backports.zstd")}, checks=[("py31?-aiohttp-brotli--*", "has:br"), ("py313-aiohttp-zstd--*", "has:zstd"), ("py31?-aiohttp-*--*", "decoded")]), dict(id="scrapy", cat="compresses", ver=[("v", "py31?-scrapy--*", "client_version", r"(.+)")], default=["py31?-scrapy--*"], dims=[("runtime_version", "Python {v}")], tokens={}, checks=[("py31?-scrapy--*", "decoded"), ("pit-py31?-scrapy-decoders--*", "decoded")]), dict(id="nodefetch", cat="compresses", ver=[("v", "node*-fetch--*", "runtime_version", r"(.+)")], default=["node*-fetch--*"], dims=[("runtime_version", "Node.js {v}")], tokens={}, # the table cell shows https:// (what a proxied scraper mostly fetches); other schemes go to a note primary=("scheme", "https"), checks=[("node*-fetch--*", "decoded")]), dict(id="go", cat="compresses", ver=[("v", "go-default--*", "runtime_version", r"(.+)")], default=["go-default--*"], dims=[], tokens={}, checks=[("go-default--*", "decoded"), ("go-manual-gzip--*", "raw"), ("go-disable-compression--*", "absent")]), ] FAMILIES = [ ("curl", ["curl-*"]), ("wget", ["wget-*"]), ("pystdlib", ["py31?-stdlib-*"]), ("requests", ["py31?-requests-*"]), ("httpx", ["py31?-httpx-*"]), ("aiohttp", ["py31?-aiohttp-*"]), ("scrapy", ["py31?-scrapy-*"]), ("nodefetch", ["node*-fetch--*"]), ("nodehttp", ["node*-http--*"]), ("nodelibs", ["node*-axios*", "node*-got--*", "node*-node-fetch*"]), ("go", ["go-*"]), ("php", ["php-curl-*", "guzzle-*", "laravel-http-*"]), ("java", ["java-httpclient-*"]), ] UNASKED = ["pit-curl-system-default-unasked", "pit-curl-brew-default-unasked", "pit-wget-default-unasked", "pit-py31?-stdlib-urllib-decoders--forced-gzip", "pit-node*-http-unasked--forced-gzip", "pit-php-curl-default-unasked", "pit-guzzle-default-decoders--forced-gzip", "pit-guzzle-stream-decoders--forced-gzip", "pit-laravel-http-default-unasked", "pit-java-httpclient-default-unasked", "pit-go-disable-compression-unasked"] DECODER_COLS = ["forced-gzip", "forced-br", "forced-zstd", "forced-zstd-multiframe"] MANUAL = ["curl-system-manual-gzip--direct", "curl-brew-manual-gzip--direct", "php-curl-manual-gzip--direct", "go-manual-gzip--direct", "pit-java-httpclient-manual-gzip", "pit-guzzle-no-decode-manual-gzip", "laravel-http-with-headers-gzip--direct"] RANGE = ["pit-go-head", "pit-go-range", "pit-node*-fetch-range"] PROXY_ROWS = ["curl-brew-default", "curl-brew-compressed", "wget-default", "py314-requests-bare", "py314-stdlib-urllib", "node22-fetch", "node24-fetch", "node26-fetch", "node26-http", "node26-axios", "go-default", "php-curl-default", "guzzle-default", "java-httpclient-default"] CORPUS_ORDER = ["curl (Homebrew) | default | CONNECT", "curl (macOS system) | --compressed | CONNECT", "curl (Homebrew) | --compressed | CONNECT"] # ---------------------------------------------------------------------------- helpers class RenderError(Exception): pass def nat_key(s: str): return [int(t) if t.isdigit() else t for t in re.split(r"(\d+)", s)] def code(s: str) -> str: runs = [len(m) for m in re.findall(r"`+", s)] fence = "`" * (max(runs) + 1 if runs else 1) pad = " " if s.startswith("`") or s.endswith("`") else "" return f"{fence}{pad}{s}{pad}{fence}" def mask_loopback(t: str) -> str: """Shorten run-specific loopback URLs and ports to … so a rerun renders the same text.""" t = re.sub(r"url='[^']*'", "url='…'", t) t = re.sub(r"https?://127\.0\.0\.1:\d+/[^\s\"']*", "…", t) t = re.sub(r"\bport=\d+", "port=…", t) return re.sub(r"127\.0\.0\.1:\d+", "127.0.0.1:…", t) def esc(s: str) -> str: """Escape prose outside code spans for GFM (no HTML, no MDX braces).""" out, pos = [], 0 for m in re.finditer(r"(`+)(.+?)\1", s): out.append(re.sub(r"([<>{}\\])", r"\\\1", s[pos:m.start()])) out.append(m.group(0)) pos = m.end() out.append(re.sub(r"([<>{}\\])", r"\\\1", s[pos:])) return "".join(out) def fill(template: str, values: dict) -> str: """Substitute {name} placeholders; values are already Markdown (code spans, numbers).""" def rep(m): k = m.group(1) if k not in values: raise RenderError(f"placeholder {{{k}}} has no value in {template!r}") return values[k] parts, pos = [], 0 for m in re.finditer(r"(`+)(.+?)\1", template): # leave code spans in templates alone parts.append(re.sub(r"\{([a-z_]+)\}", rep, esc(template[pos:m.start()]).replace("\\{", "{").replace("\\}", "}"))) parts.append(m.group(0)) pos = m.end() parts.append(re.sub(r"\{([a-z_]+)\}", rep, esc(template[pos:]).replace("\\{", "{").replace("\\}", "}"))) return "".join(parts) def table(cols: list[str], rows: list[list[str]]) -> list[str]: def cell(s): return s.replace("|", "\\|").replace("\n", " ") lines = ["| " + " | ".join(esc(c) for c in cols) + " |", "|" + "|".join("---" for _ in cols) + "|"] for r in rows: if len(r) != len(cols): raise RenderError(f"table row has {len(r)} cells, header has {len(cols)}") lines.append("| " + " | ".join(cell(c) for c in r) + " |") return lines # ---------------------------------------------------------------------------- data class Data: """The records, with not-run cells kept for the counts but never used to back a row.""" def __init__(self, doc: dict): self.doc = doc self.records = doc["records"] self.usable = [r for r in self.records if r["status"] != "not-run"] self.by_id = {r["id"]: r for r in self.usable} def select(self, *patterns, need=True) -> list[dict]: match = lambda r: any(fnmatch.fnmatchcase(r["id"], p) for p in patterns) out = [r for r in self.usable if match(r)] if need and not out: skipped = [r for r in self.records if match(r)] if skipped: r = skipped[0] why = mask_loopback((r.get("exception_text") or "no reason recorded").strip().splitlines()[0]) raise RenderError(f"{patterns}: {len(skipped)} record(s), none run (for example {r['id']}: {why}); " "this section needs them") raise RenderError(f"no record matches {patterns}; this section needs those cells (a subset run " "cannot fill it: run the whole matrix with ./run.sh, or render one --section it covers)") return out # Default vs fixed configuration of the same client over CONNECT (fixture); the same pairs as the # published summaries.derived.connect_pairs (evidence/lab/finalize_results.py). CONNECT_PAIRS = [ ("curl-system-default--connect", "curl-system-compressed--connect"), ("curl-brew-default--connect", "curl-brew-compressed--connect"), ("wget-default--connect", "wget-compression-auto--connect"), ("php-curl-default--connect", "php-curl-accept-encoding-empty--connect"), ("guzzle-default--connect", "guzzle-decode-content-gzip--connect"), ("laravel-http-default--connect", "laravel-http-with-headers-gzip--connect"), ("java-httpclient-default--connect", "java-httpclient-gzip-gunzip--connect"), ("go-disable-compression--connect", "go-default--connect"), ] def complete_runner_output(doc: dict, path: str) -> str | None: """Let the tables render from a run's own results.json (harness/out/results.json). The published results.json is finalized: it adds the CONNECT pairs, the verify-snippet runs, the reviewed anomalies and the coverage gaps. A runner's output has only the records and the runner's summaries, so compute the pairs from the records, read the snippet results that ./run.sh wrote next to it (snippets/*.json), list its own not-run cells as coverage gaps and mark its anomalies unreviewed. Returns a one-line note, or None for a finalized file.""" if "run" in doc: return None usable = {r["id"]: r for r in doc["records"] if r.get("status") == "run"} summaries = doc.setdefault("summaries", {}) if "derived" not in summaries: pairs = [] for a, b in CONNECT_PAIRS: ra, rb = usable.get(a), usable.get(b) if not ra or not rb or not all(isinstance(r.get(k), int) and r.get(k) > 0 for r in (ra, rb) for k in ("origin_wire_body_bytes", "proxy_bytes_down")): continue pairs.append({"without_compression": a, "with_compression": b, "ratio_wire_body": round(ra["origin_wire_body_bytes"] / rb["origin_wire_body_bytes"], 3), "ratio_proxy_down": round(ra["proxy_bytes_down"] / rb["proxy_bytes_down"], 3)}) summaries["derived"] = {"connect_pairs": pairs} snip_dir = os.path.join(os.path.dirname(os.path.abspath(path)), "snippets") if "snippet_runs" not in doc: runs, meta = [], {} for name in ("snippets-results.json", "snippets-node-go-results.json"): p = os.path.join(snip_dir, name) if os.path.exists(p): with open(p, encoding="utf-8") as fh: d = json.load(fh) runs += d.get("runs", []) if "curl_header_write_out" in d: meta["curl_header_write_out"] = d["curl_header_write_out"] doc["snippet_runs"] = dict(meta, runs=runs) if "not_run" not in doc: doc["not_run"] = [{"item": r["id"], "status": r["status"], "reason": r.get("exception_text") or ""} for r in doc["records"] if r.get("status") == "not-run"] if "reviewed_anomalies" not in doc: doc["reviewed_anomalies"] = [{"id": r["id"], "anomaly": "; ".join(r["anomalies"]), "review": None} for r in usable.values() if r.get("anomalies")] counts = collections.Counter(r.get("status") for r in doc["records"]) return (f"{path} is a run's own output, not the published results: {len(doc['records'])} records " f"({', '.join(f'{v} {k}' for k, v in sorted(counts.items()))}); CONNECT pairs computed from the " f"records, {len(doc['snippet_runs']['runs'])} snippet run(s) read from {snip_dir}, anomalies unreviewed") # ---------------------------------------------------------------------------- renderer class Renderer: def __init__(self, data: Data, strings: dict): self.d = data self.s = strings self.problems: list[str] = [] # numbers ------------------------------------------------------------ def n(self, v: int) -> str: return f"{v:,}".replace(",", self.s["meta"]["group_sep"]) def ratio(self, v: float) -> str: return f"{v:.2f}".replace(".", self.s["meta"]["decimal_sep"]) + self.s["meta"]["times"] # labels ------------------------------------------------------------- def client_label(self, r: dict) -> str: c, cv, rv = r["client"], r["client_version"] or "", r["runtime_version"] or "" if c.startswith("curl"): return f"curl {cv} ({'macOS' if 'system' in c else 'Homebrew'})" if c == "Wget": return f"Wget {cv}" if c in ("urllib.request", "http.client"): return f"{c}, Python {rv}" if c in ("requests", "httpx", "aiohttp", "Scrapy"): return f"{'Requests' if c == 'requests' else c} {cv}, Python {rv}" if r["runtime"] == "Node.js": if cv.startswith("built-in fetch"): name = "fetch" elif cv.startswith("node:"): name = cv.split(" ")[0] else: name = re.sub(r"\s*\(.*\)$", "", cv) return f"{name}, Node.js {rv}" if r["runtime"] == "Go": return f"Go {rv}" if c == "PHP ext-curl": lib = re.search(r"libcurl (\S+?)\)", cv) return f"PHP {rv.split(' ')[0]} ext-curl (libcurl {lib.group(1) if lib else '?'})" if c.startswith("Guzzle"): ver = re.search(r"guzzle (\S+)", cv) return f"Guzzle {ver.group(1) if ver else '?'}" + (" StreamHandler" if "Stream" in c else "") if c.startswith("Laravel"): ver = re.search(r"illuminate/http v(\S+)", cv) return f"Laravel Http {ver.group(1) if ver else '?'}" if c.startswith("Java"): ver = re.search(r"JDK (\S+?)[-+)]", cv) return f"Java {ver.group(1) if ver else '?'} HttpClient" return c def variant(self, r: dict, first_only=True) -> str: v = r["variant"].split(";")[0].strip() if first_only else r["variant"] return self.s["variants"].get(v, v if v.startswith("`") else esc(v)) def setup(self, r: dict) -> str: return esc(self.client_label(r)) + " · " + self.variant(r) def header(self, r: dict) -> str: return code(r["header_received"]) if r["header_present"] else self.s["answer"]["none"] def served(self, r: dict) -> str: t = self.s["detail"] if r["content_encoding_served"] is None: return fill(t["served_identity"], {"n": self.n(r["origin_wire_body_bytes"])}) return fill(t["served"], {"coding": code(r["content_encoding_served"]), "n": self.n(r["origin_wire_body_bytes"])}) def got(self, r: dict) -> str: t = self.s["detail"] dc = r["decode_check"] if dc == "no-output": return t["got_error"] if r["auto_decoded"] is True: return fill(t["got_decoded"], {"n": self.n(r["decoded_bytes"])}) if dc == "wire-bytes-not-decoded": return fill(t["got_raw"], {"coding": code(r["content_encoding_served"]), "n": self.n(r["decoded_bytes"])}) if dc == "first-frame-only": return fill(t["got_first_frame"], {"n": self.n(r["decoded_bytes"])}) if r["decoded_bytes"] is None or (dc == "other" and not r["decoded_bytes"]): return t["got_none"] return fill(t["got_plain"], {"n": self.n(r["decoded_bytes"])}) def outcome_cell(self, r: dict | None) -> str: p = self.s["pitfalls"] if r is None: return p["cell_not_run"] if r["decode_check"] == "no-output": return p["cell_error"] if r["auto_decoded"] is True: return p["cell_decoded"] if r["decode_check"] == "first-frame-only": return p["cell_first_frame"] if r["decode_check"] == "wire-bytes-not-decoded": return p["cell_raw"] return esc(r["decode_check"] or "?") def path_key(self, r: dict) -> str: if r["via"] == "absolute-form" and "CONNECT" in (r["extra"].get("proxy_modes_seen") or []): return "absolute-form-tunnelled" return f"direct-{r['scheme']}" if r["via"] == "direct" else r["via"] # checks ------------------------------------------------------------- def verify_row(self, spec: dict, defaults: list[dict]) -> None: cat = spec["cat"] for r in defaults: h = r["header_received"] if r["header_present"] else None asks = h is not None and any(tok in h for tok in ("gzip", "br", "zstd", "deflate")) if cat == "none" and asks: self.problems.append(f"row {spec['id']}: {r['id']} now sends {h!r}; the row says it asks for no compression") if cat == "compresses" and (not asks or r["auto_decoded"] is not True): self.problems.append(f"row {spec['id']}: {r['id']} sent {h!r}, decoded={r['auto_decoded']}; " "the row says it already compresses") for pattern, kind in spec["checks"]: for r in self.d.select(pattern): ok = {"decoded": r["auto_decoded"] is True, "raw": r["decode_check"] == "wire-bytes-not-decoded", "absent": r["header_present"] is False}.get(kind) if kind.startswith("has:"): ok = bool(r["header_present"]) and kind[4:] in [t.strip() for t in r["header_received"].split(",")] if not ok: self.problems.append(f"row {spec['id']}: check {kind} failed for {r['id']}") # sections ----------------------------------------------------------- def section_header(self) -> list[str]: t, doc = self.s["doc"], self.d.doc counts = collections.Counter(r["status"] for r in self.d.records) cstr = self.s["meta"]["list_sep"].join(f"{self.n(v)} {t['status'][k]}" for k, v in sorted(counts.items())) return [f"# {esc(t['title'])}", "", t["synthetic"], "", fill(t["run_window"], {"start": doc["started_utc"], "end": doc["finished_utc"], "records": self.n(len(self.d.records)), "counts": cstr, "tree": code(doc["harness_source"]["tree_sha256"][:16])}), "", fill(t["fixture_note"], {"identity": self.n(doc["fixture"]["bytes"])}), "", t["wire_note"], ""] def answer_rows(self): if getattr(self, "_answer_rows", None) is not None: return self._answer_rows out = [] for spec in ANSWER_ROWS: vals = {} for key, pattern, field, rx in spec["ver"]: found = sorted({m.group(1) for r in self.d.select(pattern) for m in [re.search(rx, r[field] or "")] if m}, key=nat_key) if not found: raise RenderError(f"row {spec['id']}: no version for {key} in {pattern}") vals[key] = self.s["meta"]["list_sep"].join(found) # plain text, escaped where it is placed (versions line, provenance table), so no fill() here template = self.s["rows"][spec["id"]]["label"] missing = sorted(set(placeholders(template)) - set(vals)) if missing: raise RenderError(f"row {spec['id']}: label {template!r} has no version for {missing}") label = re.sub(r"\{([a-z_]+)\}", lambda m: vals[m.group(1)], template) defaults = self.d.select(*spec["default"]) self.verify_row(spec, defaults) out.append((spec, label, defaults)) self._answer_rows = out return out def default_cell(self, spec: dict, defaults: list[dict]) -> str: a = self.s["answer"] groups = collections.OrderedDict() for r in sorted(defaults, key=lambda r: (r["header_received"] or "")): groups.setdefault(self.header(r), []).append(r) if len(groups) == 1: return next(iter(groups)) dims = spec["dims"] if not dims: self.problems.append(f"row {spec['id']}: defaults differ ({list(groups)}) but the row has no qualifier dims") return a["qual_sep"].join(groups) parts = [] all_vals = {f: sorted({r[f] for r in defaults}, key=nat_key) for f, _ in dims} for value, rs in groups.items(): quals = [] for f, fmt in dims: vals = sorted({r[f] for r in rs}, key=nat_key) if vals != all_vals[f]: quals.append(fmt.format(v="/".join(vals))) parts.append(fill(a["qual"], {"qual": esc(", ".join(quals)), "value": value}) if quals else value) return a["qual_sep"].join(parts) def row_tokens(self, spec: dict) -> dict: toks = {} for k, v in spec["tokens"].items(): if isinstance(v, tuple) and v[0] == "header": hs = {r["header_received"] for r in self.d.select(v[1])} if len(hs) != 1 or None in hs: raise RenderError(f"row {spec['id']}: token {k} needs one header, found {hs}") toks[k] = code(hs.pop()) else: toks[k] = code(v) return toks def section_answer(self, heading_level=2) -> list[str]: """Two narrow tables, so the decision column fits a phone: clients that send no Accept-Encoding (or identity) with the fix, and clients that already ask for compression with the header they sent. Detail that would widen a cell goes into the notes under each table.""" a, words = self.s["answer"], self.s["rows"] sub = "#" * (heading_level + 1) + " " out = ["#" * heading_level + " " + esc(a["heading"]), ""] rows_by_cat = collections.defaultdict(list) for spec, label, defaults in self.answer_rows(): rows_by_cat[spec["cat"]].append((spec, label, defaults)) # clients that ask for no compression: who sends which header, then client | fix by_header = collections.OrderedDict() for spec, label, defaults in rows_by_cat["none"]: by_header.setdefault(self.default_cell(spec, defaults), []).append(esc(words[spec["id"]]["name"])) sent = [] for value, names in sorted(by_header.items(), key=lambda kv: kv[0] != a["none"]): names = self.s["meta"]["list_sep"].join(names) sent.append(fill(a["sent_none"], {"clients": names}) if value == a["none"] else fill(a["sent_value"], {"value": value, "clients": names})) versions = lambda cat: fill(a["versions"], {"list": esc(a["versions_sep"].join(label for _, label, _ in rows_by_cat[cat]))}) rows, notes = [], [] for spec, label, defaults in rows_by_cat["none"]: toks = self.row_tokens(spec) rows.append([esc(words[spec["id"]]["name"]), fill(words[spec["id"]]["fix"], toks)]) if words[spec["id"]].get("note"): notes.append("- " + fill(words[spec["id"]]["note"], toks)) out += [sub + esc(a["group_none"]), "", self.s["meta"]["sentence_sep"].join([a["group_none_intro"]] + sent), ""] out += table(a["cols_none"], rows) + ["", versions("none"), ""] + notes + [""] # clients that already ask for compression: client | header sent rows, notes = [], [] for spec, label, defaults in rows_by_cat["compresses"]: toks = self.row_tokens(spec) shown, others = defaults, [] if spec.get("primary"): field, want = spec["primary"] shown = [r for r in defaults if r[field] == want] others = [r for r in defaults if r[field] != want] if not shown: raise RenderError(f"row {spec['id']}: no default record with {field} = {want}") rows.append([esc(words[spec["id"]]["name"]), self.default_cell(spec, shown)]) if words[spec["id"]].get("note"): notes.append("- " + fill(words[spec["id"]]["note"], toks)) groups = collections.OrderedDict() for r in others: groups.setdefault(r[spec["primary"][0]], []).append(r) for other, rs in groups.items(): notes.append("- " + fill(words[spec["id"]]["note_other"], {"scheme": code(f"{other}://"), "value": self.default_cell(spec, rs)})) out += [sub + esc(a["group_compresses"]), "", a["group_compresses_intro"], ""] out += table(a["cols_compresses"], rows) + ["", versions("compresses"), ""] + notes + [""] return out def section_provenance(self) -> list[str]: p = self.s["provenance"] rows = [] for spec, label, defaults in self.answer_rows(): recs = list(defaults) for pattern, _ in spec["checks"]: recs += self.d.select(pattern) ids = sorted({r["id"] for r in recs}) statuses = collections.Counter(self.d.by_id[i]["status"] for i in ids) basis = self.s["meta"]["list_sep"].join(self.s["doc"]["status"][k] for k in sorted(statuses)) locs = sorted({self.d.by_id[i]["locator"].split("#")[0] for i in ids}) rows.append([esc(label), basis, self.n(len(ids)), self.s["meta"]["list_sep"].join(code(l) for l in locs)]) return ["### " + esc(p["heading"]), ""] + table(p["cols"], rows) + [""] def section_detail(self) -> list[str]: t = self.s["detail"] out = ["## " + esc(t["heading"]), "", t["intro"], ""] for fam, patterns in FAMILIES: recs = [r for r in self.d.select(*patterns) if r["group"] in ("core",) and r["body"] == "fixture"] groups = collections.OrderedDict() for r in recs: key = (self.client_label(r), r["variant"].split(";")[0].strip(), self.header(r), r["content_encoding_served"], r["origin_wire_body_bytes"], self.got(r)) groups.setdefault(key, []).append(r) rows = [] for key, rs in groups.items(): order = {"direct-http": 0, "direct-https": 1, "absolute-form": 2, "absolute-form-tunnelled": 2, "CONNECT": 3} paths = sorted({self.path_key(r) for r in rs}, key=lambda k: order[k]) rows.append([self.setup(rs[0]), self.s["meta"]["list_sep"].join(self.s["paths"][p] for p in paths), self.header(rs[0]), self.served(rs[0]), self.got(rs[0])]) out += ["### " + esc(self.s["families"][fam]), ""] + table(t["cols"], rows) + [""] return out def section_bytes(self) -> list[str]: b = self.s["bytes"] doc = self.d.doc out = ["## " + esc(b["heading"]), "", "### " + esc(b["pairs_heading"]), "", b["pairs_intro"], ""] rows = [] for p in doc["summaries"]["derived"]["connect_pairs"]: ra, rb = self.d.by_id[p["without_compression"]], self.d.by_id[p["with_compression"]] # recompute from the records, never trust a stored ratio wa, wb, da, db = (ra["origin_wire_body_bytes"], rb["origin_wire_body_bytes"], ra["proxy_bytes_down"], rb["proxy_bytes_down"]) if round(wa / wb, 3) != p["ratio_wire_body"] or round(da / db, 3) != p["ratio_proxy_down"]: self.problems.append(f"bytes: stored ratio for {p['with_compression']} disagrees with its records") label = esc(self.client_label(ra)) + ": " + self.variant(ra) + " → " + self.variant(rb) coding = code(rb["content_encoding_served"]) rows.append([label + f" ({coding})", fill(b["arrow"], {"a": self.n(wa), "b": self.n(wb)}), fill(b["arrow"], {"a": self.n(da), "b": self.n(db)}), fill(b["ratio_pair"], {"a": self.ratio(wa / wb), "b": self.ratio(da / db)})]) out += table(b["pairs_cols"], rows) + [""] corpus = doc["summaries"]["corpus"] tot = corpus["encoder_totals"] manifest_ver = "3.14.7" m = re.search(r"cpython-(\d+\.\d+\.\d+)-docs", json.dumps(self.d.select("corpus-*")[0]["body"])) if m: manifest_ver = m.group(1) out += ["### " + fill(b["corpus_heading"], {"files": self.n(tot["files"]), "version": manifest_ver}), "", fill(b["corpus_intro"], {"files": self.n(tot["files"])}), ""] rows = [] groups = corpus["client_totals"] keys = [k for k in CORPUS_ORDER if k in groups] + sorted(k for k in groups if k not in CORPUS_ORDER) for key in keys: g = groups[key] client, variant, via = [x.strip() for x in key.split("|")] recs = [r for r in self.d.select("corpus-*") if r["client"] == client and r["variant"] == variant and r["via"] == via] wire = sum(r["origin_wire_body_bytes"] for r in recs) ident = sum(r["identity_body_bytes"] for r in recs) if wire != g["origin_wire_body_bytes"] or ident != g["identity_bytes"] or len(recs) != g["cells"]: self.problems.append(f"corpus: stored totals for {key} disagree with its records") codings = g["encodings"] ctext = b["coding_identity"] if codings == ["identity"] else ", ".join(code(c) for c in codings) rows.append([esc(self.client_label(recs[0])) + " · " + self.variant(recs[0]), ctext, fill(b["arrow"], {"a": self.n(ident), "b": self.n(wire)}), self.ratio(ident / wire)]) out += table(b["corpus_cols"], rows) + [""] return out def section_timing(self) -> list[str]: t = self.s["timing"] recs = [r for r in self.d.usable if r["group"] == "resource-timing"] if not recs: return [] stored = {x["id"]: x for x in self.d.doc["summaries"].get("resource_timing", {}).get("rows", [])} order = {"direct-http": 0, "direct-https": 1, "absolute-form": 2, "absolute-form-tunnelled": 2, "CONNECT": 3} recs.sort(key=lambda r: (nat_key(r["runtime_version"]), order[self.path_key(r)], r["mode"] != "negotiate")) rows, diffs, bad, sync, turns = [], set(), 0, 0, set() for r in recs: rt = r["extra"].get("resource_timing") or {} enc, dec, tsz = rt.get("encodedBodySize"), rt.get("decodedBodySize"), rt.get("transferSize") num = lambda v: self.n(v) if isinstance(v, int) and not isinstance(v, bool) else (esc(str(v)) if v is not None else t["none"]) if enc != r["origin_wire_body_bytes"] or dec != r["decoded_bytes"]: bad += 1 if isinstance(tsz, int) and isinstance(enc, int): diffs.add(tsz - enc) sync += 1 if rt.get("visible_synchronously_after_body") else 0 turns.add(rt.get("event_loop_turns_waited")) s = stored.get(r["id"]) if s and (s["encodedBodySize"], s["decodedBodySize"], s["transferSize"]) != (enc, dec, tsz): self.problems.append(f"timing: stored summary for {r['id']} disagrees with its record") rows.append([esc(r["runtime_version"]), self.s["paths"][self.path_key(r)], self.served(r), self.n(r["origin_wire_body_bytes"]), num(enc), num(dec), num(tsz), self.n(r["proxy_bytes_down"]) if r["proxy_bytes_down"] is not None else t["none"]]) out = ["## " + esc(t["heading"]), "", fill(t["intro"], {"api": code("performance.getEntriesByType('resource')")}), ""] out += table(t["cols"], rows) + [""] codes = {"enc": code("encodedBodySize"), "dec": code("decodedBodySize"), "transfer": code("transferSize")} if bad == 0: out += [fill(t["enc_all"], dict(codes, n=self.n(len(recs)))), ""] else: out += [fill(t["enc_some"], dict(codes, bad=self.n(bad), n=self.n(len(recs)))), ""] if len(diffs) == 1: out += [fill(t["transfer_one"], dict(codes, diff=self.n(next(iter(diffs))))), ""] elif diffs: out += [fill(t["transfer_many"], dict(codes, diffs=", ".join(self.n(d) for d in sorted(diffs)))), ""] if sync == 0: tv = sorted(x for x in turns if isinstance(x, int)) out += [fill(t["late"], {"turns": "/".join(self.n(x) for x in tv), "wait": code("await new Promise(setImmediate)")}), ""] else: out += [fill(t["late_mixed"], {"n": self.n(sync), "total": self.n(len(recs))}), ""] return out def section_pitfalls(self) -> list[str]: p = self.s["pitfalls"] out = ["## " + esc(p["heading"]), ""] # unasked rows = [[self.setup(r), self.header(r), self.got(r)] for r in self.d.select(*UNASKED)] out += ["### " + esc(p["unasked_heading"]), "", p["unasked_intro"], ""] + table(p["unasked_cols"], rows) + [""] # decoders bases = collections.OrderedDict() for r in self.d.select("pit-*-decoders--*"): bases.setdefault(r["id"].split("--")[0], {})[r["mode"]] = r rows = [] for base, modes in bases.items(): any_r = next(iter(modes.values())) rows.append([self.setup(any_r)] + [self.outcome_cell(modes.get(m)) for m in DECODER_COLS]) out += ["### " + esc(p["decoders_heading"]), "", p["decoders_intro"], ""] + table(p["decoders_cols"], rows) + [""] # copied header copied = collections.OrderedDict() headers = set() for r in self.d.select("pit-*-copied-header*"): copied.setdefault(r["id"].split("--")[0], {})[r["mode"]] = r headers.add(r["header_received"]) if len(headers) != 1: raise RenderError(f"copied-header cells sent different headers: {headers}") rows = [] for base, modes in copied.items(): any_r = next(iter(modes.values())) rows.append([self.setup(any_r), self.outcome_cell(modes.get("negotiate")), self.outcome_cell(modes.get("forced-br"))]) out += ["### " + esc(p["copied_heading"]), "", fill(p["copied_intro"], {"header": code(headers.pop())}), ""] out += table(p["copied_cols"], rows) + [""] # hand-set header rows = [[self.setup(r), self.header(r), self.got(r)] for r in self.d.select(*MANUAL)] out += ["### " + esc(p["manual_heading"]), ""] + table(p["manual_cols"], rows) + [""] # exact errors errs = collections.OrderedDict() for r in self.d.select("pit-*"): if r["exception_text"]: txt = mask_loopback(r["exception_text"].strip()) errs.setdefault(txt, []).append(r["id"]) out += ["### " + esc(p["errors_heading"]), "", esc(p["errors_intro"]), ""] for txt, ids in errs.items(): out += ["- " + fill(p["errors_item"], {"n": self.n(len(ids)), "ids": ", ".join(code(i) for i in ids[:2])}), "", " ```text", " " + txt.replace("\n", "\n "), " ```", ""] # HEAD / Range rows = [] for r in self.d.select(*RANGE): req = p["req_head"] if r["extra"].get("method") == "HEAD" else fill(p["req_range"], {"range": code("Range: bytes=0-")}) rows.append([esc(self.client_label(r)), req, self.header(r)]) out += ["### " + esc(p["range_heading"]), ""] + table(p["range_cols"], rows) + [""] # proxy visibility rows = [] for base in PROXY_ROWS: af, cn = self.d.by_id.get(f"{base}--absolute-form"), self.d.by_id.get(f"{base}--connect") if not af or not cn: raise RenderError(f"proxy row {base}: absolute-form or CONNECT record missing") if "CONNECT" in (af["extra"].get("proxy_modes_seen") or []): left = p["proxy_tunnel"] elif af["proxy_saw_header"] in (None, "(absent)"): left = p["proxy_saw_none"] else: left = fill(p["proxy_saw"], {"value": code(af["proxy_saw_header"])}) if cn["proxy_saw_header"] is not None: raise RenderError(f"{cn['id']}: a header was visible inside CONNECT") rows.append([self.setup(af), left, p["proxy_connect"]]) quirk = self.d.select("pit-guzzle-stream-http-proxy-origin-form")[0] rows.append([self.setup(quirk), p["proxy_quirk"], p["cell_not_run"]]) out += ["### " + esc(p["proxy_heading"]), ""] + table(p["proxy_cols"], rows) + [""] # TLS controls rows = [] for r in self.d.select("tls-*"): first = mask_loopback((r["exception_text"] or "").strip().splitlines()[0]) first = first if len(first) <= 110 else first[:107] + "…" rows.append([esc(self.client_label(r)), code(first)]) out += ["### " + esc(p["tls_heading"]), "", p["tls_intro"], ""] + table(p["tls_cols"], rows) + [""] return out def section_snippets(self) -> list[str]: t = self.s["snippets"] sr = self.d.doc.get("snippet_runs") or {"runs": []} rows = [] for run in sr["runs"]: printed = " / ".join(code(l) for l in (run.get("stdout") or "").strip().splitlines()) or esc( run.get("reason") or run.get("status") or "") cv = run.get("client_version") or "" cv = re.sub(r" \$LAB_[A-Z]+_DIR\S*", "", cv) # drop the interpreter path cv = re.sub(r" (?:~|/)\S*$", "", cv) # (a run's own output: home written as ~, or absolute) cv = re.sub(r"^(curl \S+).*", r"\1", cv) rows.append([code(run["snippet"]), esc(cv), printed, t["yes"] if run.get("consistent_with_origin") else t["no"]]) out = ["## " + esc(t["heading"]), "", fill(t["intro"], {"placeholder": code("https://example.com/")}), ""] if not rows: return out + [t["missing"], ""] out += table(t["cols"], rows) + [""] cm = sr.get("curl_header_write_out") if cm: builds = {r["client_version"]: r for r in self.d.select("curl-*-default--direct")} src = [v for v, r in builds.items() if "Homebrew" in r["client"] and "homebrew" in cm["source"]] if len(src) != 1: raise RenderError(f"cannot tell which curl build shipped {cm['source']}") out += [fill(t["curl_min"], {"version": code(cm["min_version"]), "source": esc(src[0])}), ""] return out def section_coverage(self) -> list[str]: c = self.s["coverage"] doc = self.d.doc rows = [[esc(x["item"]), esc(x["status"]), esc(x["reason"])] for x in doc.get("not_run", [])] out = ["## " + esc(c["heading"]), ""] + table(c["not_run_cols"], rows) + [""] rows = [[code(a["id"]), code(a["anomaly"]), esc(a["review"]) if a.get("review") else c["unreviewed"]] for a in doc.get("reviewed_anomalies", [])] out += ["### " + esc(c["anomalies_heading"]), ""] + table(c["anomalies_cols"], rows) + [""] return out def render(self, section: str = "all") -> str: secs = {"header": self.section_header, "answer": self.section_answer, "provenance": self.section_provenance, "detail": self.section_detail, "bytes": self.section_bytes, "timing": self.section_timing, "pitfalls": self.section_pitfalls, "snippets": self.section_snippets, "coverage": self.section_coverage} order = ["header", "answer", "provenance", "detail", "bytes", "timing", "pitfalls", "snippets", "coverage"] if section != "all" and section not in secs: raise RenderError(f"unknown section {section!r}; choose from {', '.join(order)} or all") lines = [] for name in (order if section == "all" else [section]): lines += secs[name]() text = "\n".join(lines).rstrip() + "\n" if self.problems: probs = list(dict.fromkeys(self.problems)) raise RenderError("the data no longer supports the rendered advice:\n " + "\n ".join(probs)) return text # ---------------------------------------------------------------------------- locales def placeholders(s: str) -> list[str]: no_code = re.sub(r"(`+)(.+?)\1", "", s) return sorted(re.findall(r"\{([a-z_]+)\}", no_code)) def code_spans(s: str) -> list[str]: return sorted(m.group(2) for m in re.finditer(r"(`+)(.+?)\1", s)) def merge_strings(base: dict, over: dict, path="") -> tuple[dict, list[str]]: """Overlay a locale on English; report missing keys and broken placeholders/code spans.""" problems = [] out = copy.deepcopy(base) for k, v in base.items(): where = f"{path}{k}" if k not in over: if path.startswith("variants"): continue # untranslated variant labels fall back to English problems.append(f"missing key {where}") continue o = over[k] if isinstance(v, dict): out[k], p = merge_strings(v, o if isinstance(o, dict) else {}, where + ".") problems += p elif isinstance(v, list): if not isinstance(o, list) or len(o) != len(v): problems.append(f"{where}: needs a list of {len(v)} strings") else: out[k] = o else: if not isinstance(o, str): problems.append(f"{where}: needs a string") continue if placeholders(o) != placeholders(v): problems.append(f"{where}: placeholders {placeholders(o)} != English {placeholders(v)}") if code_spans(o) != code_spans(v): problems.append(f"{where}: code spans {code_spans(o)} != English {code_spans(v)}") out[k] = o for k in over: if k not in base and not path.startswith("variants"): problems.append(f"unknown key {path}{k}") return out, problems def data_signature(md: str, strings: dict) -> list[tuple]: """Per line: the multiset of code spans and numbers. Only what the renderer formats per locale is canonicalised: ratios (1.23×) and grouped integers (100,129). Every other digit run (versions, IPs, dates) is compared literally.""" meta = strings["meta"] g, dsep, times = re.escape(meta["group_sep"]), re.escape(meta["decimal_sep"]), re.escape(meta["times"]) num = re.compile(rf"(?P\d+{dsep}\d+{times})|(?P\d{{1,3}}(?:{g}\d{{3}})+)(?!\d)|(?P\d+(?:\.\d+)*)") def canon(m): if m.group("ratio"): return "R" + m.group("ratio").replace(meta["decimal_sep"], ".").replace(meta["times"], "") if m.group("grouped"): return m.group("grouped").replace(meta["group_sep"], "") return m.group("raw") sig = [] for line in md.splitlines(): spans = [m.group(2) for m in re.finditer(r"(`+)(.+?)\1", line)] rest = re.sub(r"(`+)(.+?)\1", " ", line) nums = [canon(m) for m in num.finditer(rest)] if spans or nums or line.startswith("|"): sig.append((tuple(sorted(spans)), tuple(sorted(nums)), line.count("|"))) return sig def pseudo_locale(en: dict) -> dict: """A test locale: prose wrapped in ⟦ ⟧, other separators; code spans and placeholders untouched.""" def tr(s: str) -> str: parts = re.split(r"((`+).+?\2|\{[a-z_]+\}|\*\*)", s) out = [] for i, part in enumerate(parts): if part is None or re.fullmatch(r"`+", part or "x"): continue if part.startswith("`") or re.fullmatch(r"\{[a-z_]+\}|\*\*", part) or not part.strip(): out.append(part) else: out.append("⟦" + part + "⟧") return "".join(out) def walk(o): if isinstance(o, dict): return {k: walk(v) for k, v in o.items()} if isinstance(o, list): return [walk(x) for x in o] return tr(o) loc = walk(copy.deepcopy(en)) loc["meta"] = {"locale": "xx-pseudo", "group_sep": "\u202f", "decimal_sep": ",", "list_sep": "; ", "times": "×", "sentence_sep": "\u2009"} return loc # ---------------------------------------------------------------------------- main def load_strings(path: str | None) -> tuple[dict, list[str]]: if not path: return EN, [] with open(path, encoding="utf-8") as fh: return merge_strings(EN, json.load(fh)) def main(argv=None) -> int: ap = argparse.ArgumentParser(description="Render the Accept-Encoding lab tables from results.json.") ap.add_argument("results", nargs="?", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "results.json")) ap.add_argument("--strings", help="locale strings JSON (keys as in --emit-strings)") ap.add_argument("--section", default="all") ap.add_argument("--out", help="write here instead of stdout") ap.add_argument("--emit-strings", action="store_true", help="print the English strings map and exit") ap.add_argument("--parity", metavar="STRINGS", help="render English and this locale; compare data tokens") ap.add_argument("--self-test", action="store_true", help="render a pseudo-locale and check parity with English") a = ap.parse_args(argv) if a.emit_strings: print(json.dumps(EN, indent=2, ensure_ascii=False)) return 0 try: with open(a.results, encoding="utf-8") as fh: doc = json.load(fh) note = complete_runner_output(doc, a.results) data = Data(doc) except (OSError, json.JSONDecodeError, KeyError, TypeError, AttributeError) as exc: print(f"render_tables: cannot read {a.results}: {type(exc).__name__}: {exc}", file=sys.stderr) return 2 if note: print(f"render_tables: {note}", file=sys.stderr) def render_with(strings): return Renderer(data, strings).render(a.section) try: if a.parity or a.self_test: if a.self_test: other, problems = merge_strings(EN, pseudo_locale(EN)) label = "pseudo-locale" else: other, problems = load_strings(a.parity) label = a.parity if problems: print("render_tables: locale strings problems:\n " + "\n ".join(problems), file=sys.stderr) return 1 en_md, other_md = render_with(EN), render_with(other) s1, s2 = data_signature(en_md, EN), data_signature(other_md, other) if s1 != s2: for i, (x, y) in enumerate(zip(s1, s2)): if x != y: print(f"render_tables: parity FAIL at data line {i}: {x} != {y}", file=sys.stderr) break else: print(f"render_tables: parity FAIL: {len(s1)} vs {len(s2)} data lines", file=sys.stderr) return 1 print(f"render_tables: parity OK against {label}: {len(s1)} data lines, " f"{sum(len(x[0]) for x in s1)} code spans, {sum(len(x[1]) for x in s1)} numbers identical") if a.out: with open(a.out, "w", encoding="utf-8") as fh: fh.write(other_md) return 0 strings, problems = load_strings(a.strings) if problems: print("render_tables: locale strings problems:\n " + "\n ".join(problems), file=sys.stderr) return 1 md = render_with(strings) except RenderError as exc: print(f"render_tables: {exc}", file=sys.stderr) return 1 except (KeyError, TypeError, ValueError, IndexError, AttributeError, ZeroDivisionError) as exc: # a results.json this renderer does not understand: one line, never a traceback print(f"render_tables: cannot render section {a.section!r} from {a.results}: {type(exc).__name__}: {exc}", file=sys.stderr) return 1 if a.out: with open(a.out, "w", encoding="utf-8") as fh: fh.write(md) print(f"render_tables: wrote {a.out}", file=sys.stderr) else: sys.stdout.write(md) return 0 if __name__ == "__main__": sys.exit(main())