#!/usr/bin/env python3
"""Run every case of the start_requests() lab and write results.json.

    python run_lab.py LABEL=/path/to/venv [LABEL=/path/to/venv ...]

Each LABEL names a virtual environment that has Scrapy and pytest installed, for example
    python run_lab.py 2.19=venv-219 2.15=venv-215 2.12=venv-212

The runner starts `python -m http.server` on 127.0.0.1:8047 and forward_proxy.py on 127.0.0.1:8048,
then runs each spider with `scrapy runspider` in a new interpreter, with a clean environment, an empty
working directory and an empty home directory. Nothing leaves the machine.
"""
import json
import os
import pathlib
import re
import subprocess
import sys
import tempfile
import time
import urllib.request

HERE = pathlib.Path(__file__).resolve().parent
TARGET_PORT, PROXY_PORT = 8047, 8048
TARGET = f"http://127.0.0.1:{TARGET_PORT}/"
PROXY = f"http://user:pass@127.0.0.1:{PROXY_PORT}"

# (case, extra spider arguments)
CASES = [
    ("a_start_requests_only", {}),
    ("a2_start_urls_only", {}),
    ("b_async_start", {}),
    ("b2_start_urls_headers_meta", {}),
    ("b3_proxy_meta", {"proxy": PROXY}),
    ("c_both", {}),
    ("c2_both_delegating", {}),
    ("d1_mw_process_start_requests", {}),
    ("d1b_old_spider_old_mw", {}),
    ("d1c_mw_process_start", {}),
    ("d1d_mw_both", {}),
    ("d2_mw_sync_output", {}),
    ("d2b_mw_async_output", {}),
    ("d2c_mw_async_output_spider_arg", {}),
    ("d2d_mw_universal", {}),
    ("e1_start_not_async", {}),
    ("e2_start_returns_list", {}),
]
STAT_KEYS = [
    "downloader/request_count", "downloader/response_count", "response_received_count",
    "item_scraped_count", "scheduler/enqueued", "finish_reason",
    "log_count/INFO", "log_count/WARNING", "log_count/ERROR",
]
NOTABLE = re.compile(r"WARNING|ERROR|CRITICAL|Traceback|Error\b|Deprecat|parse\(\) got|Spider closed|Spider opened|Crawled |^\{'url'")


def wait_for(url):
    opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
    for _ in range(50):
        try:
            opener.open(url, timeout=1).read()
            return
        except Exception:
            time.sleep(0.1)
    raise SystemExit(f"fixture did not answer: {url}")


def stats_of(log):
    block = log.split("Dumping Scrapy stats:", 1)
    if len(block) < 2:
        return None
    found = {}
    for key in STAT_KEYS:
        match = re.search(r"'%s': ('[^']*'|\d+)" % re.escape(key), block[1])
        if match:
            value = match.group(1)
            found[key] = value.strip("'") if value.startswith("'") else int(value)
    return found


def run(command, venv, label, extra_env=None):
    """Run one command in a clean environment. Absolute lab and venv paths are shortened in what is recorded."""
    def short(text):
        text = text.replace(str(venv), f"venv-{label}").replace(str(HERE), ".")
        text = re.sub(r"(?:\.\./)+" + re.escape(str(HERE).lstrip("/")), ".", text)
        return re.sub(r"/tmp/lab-run-\w+", "/tmp/lab-run", text)

    with tempfile.TemporaryDirectory(prefix="lab-run-") as work:
        home = pathlib.Path(work, "home")
        cwd = pathlib.Path(work, "cwd")
        home.mkdir()
        cwd.mkdir()
        env = {"PATH": "/usr/bin:/bin", "HOME": str(home), "LANG": "C.UTF-8", "PYTHONDONTWRITEBYTECODE": "1"}
        env.update(extra_env or {})
        done = subprocess.run(command, cwd=cwd, env=env, capture_output=True, text=True, timeout=120)
    log = short(done.stdout + done.stderr)
    return {
        "command": short(" ".join(str(part) for part in command)),
        "returncode": done.returncode,
        "stats": stats_of(log),
        "log": log.splitlines(),
    }


def show(label, name, result):
    stats = result["stats"] or {}
    print(f"\n### {label} {name}: exit {result['returncode']}, requests {stats.get('downloader/request_count')}, "
          f"responses {stats.get('response_received_count')}, items {stats.get('item_scraped_count')}, "
          f"finish_reason {stats.get('finish_reason')}, WARNING {stats.get('log_count/WARNING')}, ERROR {stats.get('log_count/ERROR')}")
    for line in result["log"]:
        if NOTABLE.search(line):
            print("   ", line[:420])


def main():
    venvs = dict(argument.split("=", 1) for argument in sys.argv[1:])
    if not venvs:
        raise SystemExit(__doc__)
    proxy_log = pathlib.Path(tempfile.mkdtemp(prefix="lab-proxy-")) / "proxy.jsonl"
    proxy_log.write_text("")
    fixtures = [
        subprocess.Popen([sys.executable, "-m", "http.server", str(TARGET_PORT), "--bind", "127.0.0.1", "--directory", str(HERE / "site")],
                         stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL),
        subprocess.Popen([sys.executable, str(HERE / "forward_proxy.py"), str(PROXY_PORT), str(proxy_log)]),
    ]
    results = {"target": TARGET, "proxy": PROXY, "environments": {}}
    try:
        wait_for(TARGET)
        for label, venv in venvs.items():
            venv = pathlib.Path(venv).absolute()
            python = venv / "bin" / "python"
            versions = json.loads(subprocess.run(
                [python, "-c", "import json, platform, scrapy, twisted; print(json.dumps({'scrapy': scrapy.__version__, 'twisted': twisted.__version__, 'python': platform.python_version()}))"],
                capture_output=True, text=True, check=True).stdout)
            print(f"\n================ {label}: {versions}")
            environment = {"versions": versions, "version_commands": {}, "cases": {}, "ci": {}}
            for name, command in (("scrapy version", [venv / "bin" / "scrapy", "version"]), ("python -m pip show scrapy", [python, "-m", "pip", "show", "scrapy"])):
                result = run(command, venv, label)
                environment["version_commands"][name] = result["log"][:2]
                print(f"    $ {name}: {result['log'][:2]}")
            for name, arguments in CASES:
                command = [python, "-m", "scrapy", "runspider", HERE / "spiders" / f"{name}.py", "-a", f"url={TARGET}"]
                for key, value in arguments.items():
                    command += ["-a", f"{key}={value}"]
                before = len(proxy_log.read_text().splitlines())
                result = run(command, venv, label)
                result["proxy_log"] = [json.loads(line) for line in proxy_log.read_text().splitlines()[before:]]
                environment["cases"][name] = result
                show(label, name, result)
                if result["proxy_log"]:
                    print("    proxy saw:", result["proxy_log"])
            for project in ("old", "new"):
                command = [python, "-m", "pytest", "-q", "-p", "no:cacheprovider", HERE / "ci" / "test_crawl.py"]
                result = run(command, venv, label, {"PYTHONPATH": str(HERE / "ci" / project)})
                environment["ci"][project] = result
                print(f"\n### {label} pytest with ci/{project}: exit {result['returncode']}")
                for line in result["log"]:
                    if re.search(r"^E  |passed|failed|^>", line):
                        print("   ", line[:300])
            results["environments"][label] = environment
    finally:
        for fixture in fixtures:
            fixture.terminate()
    (HERE / "results.json").write_text(json.dumps(results, indent=1) + "\n", encoding="utf-8")
    print("\nwrote results.json")


if __name__ == "__main__":
    main()