"""Cases for the Scrapy proxy lab. Each case is a complete spider file.

A case is (name, environment variables, code). The runner writes the code to <name>.py and runs
`python -m scrapy runspider <name>.py` in a new interpreter with a clean environment, so a case sees
only the variables listed for it. Constants and the LabSpider base come from lab_common.py.
"""

HEADER = "from lab_common import *\n"


def case(requests: str, settings: str = "", attributes: str = "", before: str = "") -> str:
    """A spider whose start() runs the given lines."""
    lines = [HEADER]
    if before:
        lines += [before.strip("\n"), "", ""]
    lines += ["class Case(LabSpider):", '    name = "case"']
    if settings:
        lines.append(f"    custom_settings = {settings}")
    if attributes:
        lines.append(f"    {attributes}")
    lines += ["", "    async def start(self):"]
    lines += ["        " + line for line in requests.strip("\n").splitlines()]
    return "\n".join(lines) + "\n"


def get(url: str, meta: str = "", args: str = "", errback: bool = True, **kwargs: str) -> str:
    """One GET; meta and args are spliced in verbatim."""
    parts = [url]
    if meta:
        parts.append(f"meta={meta}")
    if args:
        parts.append(args)
    if errback:
        parts.append("errback=self.on_error")
    return case(f"yield scrapy.Request({', '.join(parts)})", **kwargs)


ROTATE_WITH_HEADER = '''
from itertools import cycle

PROXIES = cycle([PROXY, PROXY_B])


class RotateWithHeader:
    """Rotation that sets a proxy URL without credentials and a Proxy-Authorization header by hand."""

    def process_request(self, request):
        request.meta["proxy"] = next(PROXIES)
        request.headers["Proxy-Authorization"] = BASIC
'''

# The first proxy rejects the header (its password is different), so RetryMiddleware sends the
# request through the middlewares again and the rotation picks the next proxy.
ROTATE_WITH_HEADER_RETRY = ROTATE_WITH_HEADER.replace("cycle([PROXY, PROXY_B])", 'cycle(["http://" + PROXY_C_HOSTPORT, PROXY])')

ROTATE_WITH_URL = '''
from itertools import cycle

PROXIES = cycle([PROXY_AUTH, PROXY_B_AUTH])


class RotateProxyMiddleware:
    """Rotation that puts the credentials in the proxy URL."""

    def process_request(self, request):
        request.meta["proxy"] = next(PROXIES)
'''

SWITCH_ALWAYS = '''
class SwitchProxy:
    """Moves every request to the second proxy before HttpProxyMiddleware sees it."""

    def process_request(self, request):
        request.meta["proxy"] = PROXY_B
'''

SWITCH_ON_REDIRECT = '''
class SwitchProxyOnRedirect:
    """Moves a request to the second proxy only when it is a redirected request."""

    def process_request(self, request):
        if request.meta.get("redirect_times"):
            request.meta["proxy"] = PROXY_B
'''

EXPLICIT_ONLY = '''
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware


class ExplicitProxyMiddleware(HttpProxyMiddleware):
    """HttpProxyMiddleware without the proxies read from the environment."""

    def __init__(self, auth_encoding="latin-1"):
        super().__init__(auth_encoding)
        self.proxies = {}
'''
EXPLICIT_ONLY_SETTINGS = ('{"DOWNLOADER_MIDDLEWARES": {"scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": None, '
                          "ExplicitProxyMiddleware: 750}}")

FOLLOW_UP = HEADER + '''
class Case(LabSpider):
    name = "case"

    async def start(self):
        yield scrapy.Request(HTTP_URL, meta={"proxy": PROXY_AUTH}, errback=self.on_error)

    def parse(self, response):
        super().parse(response)
        if response.url == HTTP_URL:
            yield response.follow("/dir/", errback=self.on_error)
'''

MANUAL_HEADER = 'headers={"Proxy-Authorization": BASIC}'
HTTPX = '{"DOWNLOAD_HANDLERS": HTTPX_HANDLERS}'
ENV_AUTH = "http://user:pass@127.0.0.1:8888"

START_REQUESTS_ONLY = HEADER + '''
class Case(LabSpider):
    name = "case"

    def start_requests(self):
        print("start_requests() was called", flush=True)
        yield scrapy.Request(HTTP_URL, errback=self.on_error)
'''

START_AND_START_REQUESTS = HEADER + '''
class Case(LabSpider):
    name = "case"

    async def start(self):
        print("start() was called", flush=True)
        yield scrapy.Request(HTTP_URL, errback=self.on_error)

    def start_requests(self):
        print("start_requests() was called", flush=True)
        yield scrapy.Request(HTTP_URL, errback=self.on_error)
'''

START_URLS_ONLY = HEADER + '''
class Case(LabSpider):
    name = "case"
    start_urls = [HTTP_URL]
'''

CASES = [
    # --- credentials in meta["proxy"] ---------------------------------------------------------
    ("https_no_credentials", {}, get("HTTPS_URL", '{"proxy": PROXY}')),
    ("https_wrong_password", {}, get("HTTPS_URL", '{"proxy": "http://user:wrong@127.0.0.1:8888"}')),
    ("https_credentials_in_url", {}, get("HTTPS_URL", '{"proxy": PROXY_AUTH}')),
    ("http_credentials_in_url", {}, get("HTTP_URL", '{"proxy": PROXY_AUTH}')),
    ("http_no_credentials", {}, get("HTTP_URL", '{"proxy": PROXY}')),
    ("http_no_credentials_no_errback", {}, get("HTTP_URL", '{"proxy": PROXY}', errback=False)),
    ("http_wrong_password", {}, get("HTTP_URL", '{"proxy": "http://user:wrong@127.0.0.1:8888"}')),
    ("https_no_credentials_no_errback", {}, get("HTTPS_URL", '{"proxy": PROXY}', errback=False)),
    ("https_no_credentials_dont_retry", {}, get("HTTPS_URL", '{"proxy": PROXY, "dont_retry": True}')),
    ("proxy_url_without_scheme", {}, get("HTTPS_URL", '{"proxy": "user:pass@127.0.0.1:8888"}')),
    ("password_percent_encoded", {}, get("HTTPS_URL", '{"proxy": "http://user:p%40ss%3Aw%2Frd@" + PROXY_C_HOSTPORT}')),
    ("password_not_encoded", {}, get("HTTPS_URL", '{"proxy": "http://user:p@ss:w/rd@" + PROXY_C_HOSTPORT}')),
    ("https_proxy_url_https_target", {}, get("HTTPS_URL", '{"proxy": "https://user:pass@127.0.0.1:8888"}')),
    ("redirect_keeps_proxy", {}, get("REDIRECT_URL", '{"proxy": PROXY_AUTH}')),
    ("follow_up_request_without_meta", {}, FOLLOW_UP),
    # --- handling the 407 response on http:// ---------------------------------------------------
    ("http_407_handle_httpstatus_list_meta", {}, get("HTTP_URL", '{"proxy": PROXY, "handle_httpstatus_list": [407]}')),
    ("http_407_handle_httpstatus_list_attribute", {}, get("HTTP_URL", '{"proxy": PROXY}', attributes="handle_httpstatus_list = [407]")),
    ("http_407_httperror_allowed_codes", {}, get("HTTP_URL", '{"proxy": PROXY}', settings='{"HTTPERROR_ALLOWED_CODES": [407]}')),
    ("https_407_handle_httpstatus_list_meta", {}, get("HTTPS_URL", '{"proxy": PROXY, "handle_httpstatus_list": [407]}')),
    # --- environment variables ------------------------------------------------------------------
    ("env_http_proxy", {"HTTP_PROXY": ENV_AUTH}, get("HTTP_URL")),
    ("env_http_proxy_lowercase", {"http_proxy": ENV_AUTH}, get("HTTP_URL")),
    ("env_https_proxy", {"HTTPS_PROXY": ENV_AUTH}, get("HTTPS_URL")),
    ("env_http_proxy_only_https_target", {"HTTP_PROXY": ENV_AUTH}, get("HTTPS_URL")),
    ("env_all_proxy", {"ALL_PROXY": ENV_AUTH}, get("HTTP_URL")),
    ("env_no_proxy", {"HTTP_PROXY": ENV_AUTH, "NO_PROXY": "127.0.0.1"}, get("HTTP_URL")),
    ("env_no_proxy_with_meta_proxy", {"NO_PROXY": "127.0.0.1"}, get("HTTP_URL", '{"proxy": PROXY_AUTH}')),
    ("env_http_proxy_meta_proxy_wins", {"HTTP_PROXY": ENV_AUTH}, get("HTTP_URL", '{"proxy": PROXY_B_AUTH}')),
    ("env_http_proxy_httpproxy_disabled", {"HTTP_PROXY": ENV_AUTH}, get("HTTP_URL", settings='{"HTTPPROXY_ENABLED": False}')),
    ("env_http_proxy_meta_proxy_none", {"HTTP_PROXY": ENV_AUTH}, get("HTTP_URL", '{"proxy": None}')),
    ("env_http_proxy_socks_url", {"HTTP_PROXY": "socks5://127.0.0.1:1080"}, get("HTTP_URL")),
    ("env_http_proxy_explicit_only_middleware", {"HTTP_PROXY": ENV_AUTH}, get("HTTP_URL", settings=EXPLICIT_ONLY_SETTINGS, before=EXPLICIT_ONLY)),
    ("env_http_proxy_explicit_only_middleware_meta_proxy", {"HTTP_PROXY": ENV_AUTH},
     get("HTTP_URL", '{"proxy": PROXY_B_AUTH}', settings=EXPLICIT_ONLY_SETTINGS, before=EXPLICIT_ONLY)),
    ("httpproxy_disabled_meta_proxy_with_credentials", {}, get("HTTP_URL", '{"proxy": PROXY_AUTH}', settings='{"HTTPPROXY_ENABLED": False}')),
    ("httpproxy_disabled_meta_proxy_https_target", {}, get("HTTPS_URL", '{"proxy": PROXY_AUTH}', settings='{"HTTPPROXY_ENABLED": False}')),
    # --- a hand-set Proxy-Authorization header ----------------------------------------------------
    ("manual_header_https", {}, get("HTTPS_URL", '{"proxy": PROXY}', MANUAL_HEADER)),
    ("manual_header_http", {}, get("HTTP_URL", '{"proxy": PROXY}', MANUAL_HEADER)),
    ("manual_header_same_proxy_redirect", {}, get("REDIRECT_URL", '{"proxy": PROXY}', MANUAL_HEADER)),
    ("manual_header_no_proxy", {}, get("HEADERS_URL", "", MANUAL_HEADER)),
    ("manual_header_proxy_changed_first_pass", {},
     get("HTTP_URL", '{"proxy": PROXY}', MANUAL_HEADER, settings='{"DOWNLOADER_MIDDLEWARES": {SwitchProxy: 740}}', before=SWITCH_ALWAYS)),
    ("manual_header_proxy_changed_on_redirect", {},
     get("REDIRECT_URL", '{"proxy": PROXY}', MANUAL_HEADER, settings='{"DOWNLOADER_MIDDLEWARES": {SwitchProxyOnRedirect: 740}}', before=SWITCH_ON_REDIRECT)),
    ("rotation_header_redirect", {},
     get("REDIRECT_URL", settings='{"DOWNLOADER_MIDDLEWARES": {RotateWithHeader: 740}}', before=ROTATE_WITH_HEADER)),
    ("rotation_header_redirect_407_handled", {},
     get("REDIRECT_URL", '{"handle_httpstatus_list": [407]}', settings='{"DOWNLOADER_MIDDLEWARES": {RotateWithHeader: 740}}', before=ROTATE_WITH_HEADER)),
    ("rotation_header_retry_https", {},
     get("HTTPS_URL", settings='{"DOWNLOADER_MIDDLEWARES": {RotateWithHeader: 740}}', before=ROTATE_WITH_HEADER_RETRY)),
    ("rotation_url_credentials_redirect", {},
     get("REDIRECT_URL", settings='{"DOWNLOADER_MIDDLEWARES": {RotateProxyMiddleware: 740}}', before=ROTATE_WITH_URL)),
    # --- SOCKS ------------------------------------------------------------------------------------
    ("socks5_http_target", {}, get("HTTP_URL", '{"proxy": SOCKS}')),
    ("socks5_https_target", {}, get("HTTPS_URL", '{"proxy": SOCKS}')),
    ("socks5h_http_target", {}, get("HTTP_URL", '{"proxy": "socks5h://127.0.0.1:1080"}')),
    ("socks5_no_errback", {}, get("HTTP_URL", '{"proxy": SOCKS}', errback=False)),
    ("httpx_handler_socks5_http_target", {}, get("HTTP_URL", '{"proxy": SOCKS}', settings=HTTPX)),
    ("httpx_handler_socks5_https_target", {}, get("HTTPS_URL", '{"proxy": SOCKS}', settings=HTTPX)),
    ("httpx_handler_socks5h_http_target", {}, get("HTTP_URL", '{"proxy": "socks5h://127.0.0.1:1080"}', settings=HTTPX)),
    ("httpx_handler_socks5_hostname", {}, get('"http://localhost:8080/"', '{"proxy": SOCKS}', settings=HTTPX)),
    ("httpx_handler_http_proxy_https_target", {}, get("HTTPS_URL", '{"proxy": PROXY_AUTH}', settings=HTTPX)),
    ("httpx_handler_http_proxy_http_target", {}, get("HTTP_URL", '{"proxy": PROXY_AUTH}', settings=HTTPX)),
    ("httpx_handler_https_no_credentials", {}, get("HTTPS_URL", '{"proxy": PROXY}', settings=HTTPX)),
    ("httpx_handler_http_no_credentials", {}, get("HTTP_URL", '{"proxy": PROXY}', settings=HTTPX)),
    # --- start() and start_requests() -----------------------------------------------------------
    ("start_requests_only", {}, START_REQUESTS_ONLY),
    ("start_and_start_requests", {}, START_AND_START_REQUESTS),
    ("start_urls_only", {}, START_URLS_ONLY),
    # --- retries ------------------------------------------------------------------------------------
    ("post_lost_response", {},
     case('yield scrapy.Request(JOBS_URL, method="POST", body="job-retried", meta={"proxy": PROXY_AUTH}, errback=self.on_error)')),
    ("post_lost_response_dont_retry", {},
     case('yield scrapy.Request(JOBS_URL, method="POST", body="job-not-retried", meta={"proxy": PROXY_AUTH, "dont_retry": True}, errback=self.on_error)')),
    ("post_lost_response_no_proxy", {},
     case('yield scrapy.Request(JOBS_URL, method="POST", body="job-direct", errback=self.on_error)')),
]