{ "recorded_at": "2026-10-07T10:36:10Z", "python": "3.14.4", "platform": "Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "versions": { "Scrapy": "2.19.0", "Twisted": "26.4.0", "pyOpenSSL": "26.4.0", "cryptography": "50.0.2", "w3lib": "2.5.0", "httpx2": "2.13.1", "httpcore2": "2.13.1", "socksio": "1.0.0" }, "ports": { "http_target": 8080, "https_target": 8443, "jobs_target": 8081, "proxy": 8888, "proxy_b": 8889, "proxy_c": 8890, "socks": 1080 }, "settings": { "LOG_FORMAT": "%(levelname)s [%(name)s] %(message)s", "TELNETCONSOLE_ENABLED": "False", "REMOTE_CONTROL_ENABLED": "False", "DOWNLOAD_TIMEOUT": "15" }, "lab_common": "\"\"\"Shared pieces of the Scrapy proxy lab cases: the loopback fixture URLs and a spider base that\nprints one line per callback or errback. Every case file starts with `from lab_common import *`.\n\"\"\"\nimport scrapy\n\nHTTP_URL = \"http://127.0.0.1:8080/\"\nHTTPS_URL = \"https://127.0.0.1:8443/\"\nREDIRECT_URL = \"http://127.0.0.1:8080/dir\" # python -m http.server answers 301 with Location: /dir/\nHEADERS_URL = \"http://127.0.0.1:8081/headers\" # logs the names of the request headers it received\nJOBS_URL = \"http://127.0.0.1:8081/jobs\" # loses the response to the first POST of each body\n\nPROXY = \"http://127.0.0.1:8888\" # wants user / pass\nPROXY_AUTH = \"http://user:pass@127.0.0.1:8888\"\nPROXY_B = \"http://127.0.0.1:8889\" # second proxy, same credentials\nPROXY_B_AUTH = \"http://user:pass@127.0.0.1:8889\"\nPROXY_C_HOSTPORT = \"127.0.0.1:8890\" # third proxy, password p@ss:w/rd\nSOCKS = \"socks5://127.0.0.1:1080\"\nBASIC = \"Basic dXNlcjpwYXNz\" # base64 of user:pass\n\nHTTPX_HANDLERS = {\n \"http\": \"scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler\",\n \"https\": \"scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler\",\n}\n\n\nclass LabSpider(scrapy.Spider):\n \"\"\"Base for the cases. It has no name, so `scrapy runspider` only sees the case's own spider.\"\"\"\n\n def parse(self, response):\n challenge = response.headers.get(\"Proxy-Authenticate\")\n print(\n f\"CALLBACK status={response.status} url={response.url} body={response.text.strip()!r} \"\n f\"meta_proxy={response.meta.get('proxy')!r} \"\n f\"proxy_authenticate={challenge.decode() if challenge else None!r}\",\n flush=True,\n )\n\n def on_error(self, failure):\n print(f\"ERRBACK {failure.type.__module__}.{failure.type.__qualname__}: {failure.getErrorMessage()}\", flush=True)\n", "cases": [ { "case": "https_no_credentials", "env": {}, "command": "python -m scrapy runspider https_no_credentials.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.core.downloader.handlers.http11.TunnelError: Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 2 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "ERROR [scrapy.downloadermiddlewares.retry] Gave up retrying (failed 3 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.core.downloader.handlers.http11.TunnelError": "3", "downloader/request_count": "3", "downloader/request_method_count/GET": "3", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/ERROR": "1", "log_count/INFO": "2", "retry/count": "2", "retry/max_reached": "1", "retry/reason_count/scrapy.core.downloader.handlers.http11.TunnelError": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] } ], "socks_log": [], "target_log": [] }, { "case": "https_wrong_password", "env": {}, "command": "python -m scrapy runspider https_wrong_password.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": \"http://user:wrong@127.0.0.1:8888\"}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.core.downloader.handlers.http11.TunnelError: Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 2 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "ERROR [scrapy.downloadermiddlewares.retry] Gave up retrying (failed 3 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.core.downloader.handlers.http11.TunnelError": "3", "downloader/request_count": "3", "downloader/request_method_count/GET": "3", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/ERROR": "1", "log_count/INFO": "2", "retry/count": "2", "retry/max_reached": "1", "retry/reason_count/scrapy.core.downloader.handlers.http11.TunnelError": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "wrong", "header_names": [ "Host", "Proxy-Authorization" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "wrong", "header_names": [ "Host", "Proxy-Authorization" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "wrong", "header_names": [ "Host", "Proxy-Authorization" ] } ], "socks_log": [], "target_log": [] }, { "case": "https_credentials_in_url", "env": {}, "command": "python -m scrapy runspider https_credentials_in_url.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=https://127.0.0.1:8443/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "ok", "header_names": [ "Host", "Proxy-Authorization" ] } ], "socks_log": [], "target_log": [] }, { "case": "http_credentials_in_url", "env": {}, "command": "python -m scrapy runspider http_credentials_in_url.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "http_no_credentials", "env": {}, "command": "python -m scrapy runspider http_no_credentials.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.spidermiddlewares.httperror.HttpError: Ignoring non-200 response" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (407) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "http_no_credentials_no_errback", "env": {}, "command": "python -m scrapy runspider http_no_credentials_no_errback.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY})", "exit_code": 0, "stdout": [], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (407) (referer: None)", "INFO [scrapy.spidermiddlewares.httperror] Ignoring response <407 http://127.0.0.1:8080/>: HTTP status code is not handled or not allowed", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "httperror/response_ignored_count": "1", "httperror/response_ignored_status_count/407": "1", "log_count/DEBUG": "1", "log_count/INFO": "3", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "http_wrong_password", "env": {}, "command": "python -m scrapy runspider http_wrong_password.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": \"http://user:wrong@127.0.0.1:8888\"}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.spidermiddlewares.httperror.HttpError: Ignoring non-200 response" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (407) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "wrong", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "https_no_credentials_no_errback", "env": {}, "command": "python -m scrapy runspider https_no_credentials_no_errback.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY})", "exit_code": 0, "stdout": [], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 2 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "ERROR [scrapy.downloadermiddlewares.retry] Gave up retrying (failed 3 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "ERROR [scrapy.core.scraper] Error downloading ", "Traceback (most recent call last):", " File \"/lib/python3.14/site-packages/twisted/internet/defer.py\", line 1834, in _inlineCallbacks", " result = context.run(", " cast(Failure, result).throwExceptionIntoGenerator, gen", " )", " File \"/lib/python3.14/site-packages/twisted/python/failure.py\", line 467, in throwExceptionIntoGenerator", " return g.throw(self.value.with_traceback(self.tb))", " ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/engine.py\", line 514, in _download", " result = yield self.downloader.fetch(request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/twisted/internet/defer.py\", line 1834, in _inlineCallbacks", " result = context.run(", " cast(Failure, result).throwExceptionIntoGenerator, gen", " )", " File \"/lib/python3.14/site-packages/twisted/python/failure.py\", line 467, in throwExceptionIntoGenerator", " return g.throw(self.value.with_traceback(self.tb))", " ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/__init__.py\", line 226, in fetch", " result: Response | Request = yield (", " ^^^^^^^", " ...<3 lines>...", " )", " ^", " File \"/lib/python3.14/site-packages/twisted/internet/defer.py\", line 1247, in adapt", " extracted: _SelfResultT | Failure = result.result()", " ~~~~~~~~~~~~~^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/middleware.py\", line 87, in download_async", " result = await self._process_exception(ex, request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/middleware.py\", line 156, in _process_exception", " raise exception", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/middleware.py\", line 80, in download_async", " result: Response | Request = await self._process_request(", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " request, download_func", " ^^^^^^^^^^^^^^^^^^^^^^", " )", " ^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/middleware.py\", line 114, in _process_request", " return await download_func(request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/__init__.py\", line 281, in _enqueue_request", " return await maybe_deferred_to_future(d) # fired in _wait_for_download()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/__init__.py\", line 348, in _wait_for_download", " response = await self._download(slot, request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/__init__.py\", line 318, in _download", " response: Response = await self.handlers.download_request_async(request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/handlers/__init__.py\", line 168, in download_request_async", " return await handler.download_request(request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/handlers/http11.py\", line 140, in download_request", " return await maybe_deferred_to_future(agent.download_request(request))", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", "scrapy.core.downloader.handlers.http11.TunnelError: Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.core.downloader.handlers.http11.TunnelError": "3", "downloader/request_count": "3", "downloader/request_method_count/GET": "3", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/ERROR": "2", "log_count/INFO": "2", "retry/count": "2", "retry/max_reached": "1", "retry/reason_count/scrapy.core.downloader.handlers.http11.TunnelError": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] } ], "socks_log": [], "target_log": [] }, { "case": "https_no_credentials_dont_retry", "env": {}, "command": "python -m scrapy runspider https_no_credentials_dont_retry.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY, \"dont_retry\": True}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.core.downloader.handlers.http11.TunnelError: Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.core.downloader.handlers.http11.TunnelError": "1", "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] } ], "socks_log": [], "target_log": [] }, { "case": "proxy_url_without_scheme", "env": {}, "command": "python -m scrapy runspider proxy_url_without_scheme.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": \"user:pass@127.0.0.1:8888\"}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=https://127.0.0.1:8443/ body='lab target ok' meta_proxy='//127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "ok", "header_names": [ "Host", "Proxy-Authorization" ] } ], "socks_log": [], "target_log": [] }, { "case": "password_percent_encoded", "env": {}, "command": "python -m scrapy runspider password_percent_encoded.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": \"http://user:p%40ss%3Aw%2Frd@\" + PROXY_C_HOSTPORT}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=https://127.0.0.1:8443/ body='lab target ok' meta_proxy='http://127.0.0.1:8890' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8890, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "ok", "header_names": [ "Host", "Proxy-Authorization" ] } ], "socks_log": [], "target_log": [] }, { "case": "password_not_encoded", "env": {}, "command": "python -m scrapy runspider password_not_encoded.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": \"http://user:p@ss:w/rd@\" + PROXY_C_HOSTPORT}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK builtins.ValueError: Port could not be cast to integer value as 'w'" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/builtins.ValueError": "1", "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "https_proxy_url_https_target", "env": {}, "command": "python -m scrapy runspider https_proxy_url_https_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": \"https://user:pass@127.0.0.1:8888\"}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK builtins.NotImplementedError: HTTPS proxies for HTTPS destinations are not supported" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/builtins.NotImplementedError": "1", "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "redirect_keeps_proxy", "env": {}, "command": "python -m scrapy runspider redirect_keeps_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(REDIRECT_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/dir/ body='redirected ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.redirect] Redirecting (301) to from ", "DEBUG [scrapy.core.engine] Crawled (200) (referer: http://127.0.0.1:8080/dir)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "2", "downloader/request_method_count/GET": "2", "downloader/response_status_count/200": "1", "downloader/response_status_count/301": "1", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] }, { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "Referer", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "follow_up_request_without_meta", "env": {}, "command": "python -m scrapy runspider follow_up_request_without_meta.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)\n\n def parse(self, response):\n super().parse(response)\n if response.url == HTTP_URL:\n yield response.follow(\"/dir/\", errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None", "CALLBACK status=200 url=http://127.0.0.1:8080/dir/ body='redirected ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "DEBUG [scrapy.core.engine] Crawled (200) (referer: http://127.0.0.1:8080/)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "2", "downloader/request_method_count/GET": "2", "downloader/response_status_count/200": "2", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "http_407_handle_httpstatus_list_meta", "env": {}, "command": "python -m scrapy runspider http_407_handle_httpstatus_list_meta.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY, \"handle_httpstatus_list\": [407]}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=407 url=http://127.0.0.1:8080/ body='' meta_proxy='http://127.0.0.1:8888' proxy_authenticate='Basic realm=\"lab\"'" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (407) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "http_407_handle_httpstatus_list_attribute", "env": {}, "command": "python -m scrapy runspider http_407_handle_httpstatus_list_attribute.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n handle_httpstatus_list = [407]\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=407 url=http://127.0.0.1:8080/ body='' meta_proxy='http://127.0.0.1:8888' proxy_authenticate='Basic realm=\"lab\"'" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (407) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "http_407_httperror_allowed_codes", "env": {}, "command": "python -m scrapy runspider http_407_httperror_allowed_codes.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"HTTPERROR_ALLOWED_CODES\": [407]}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=407 url=http://127.0.0.1:8080/ body='' meta_proxy='http://127.0.0.1:8888' proxy_authenticate='Basic realm=\"lab\"'" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (407) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "https_407_handle_httpstatus_list_meta", "env": {}, "command": "python -m scrapy runspider https_407_handle_httpstatus_list_meta.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY, \"handle_httpstatus_list\": [407]}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.core.downloader.handlers.http11.TunnelError: Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 2 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "ERROR [scrapy.downloadermiddlewares.retry] Gave up retrying (failed 3 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.core.downloader.handlers.http11.TunnelError": "3", "downloader/request_count": "3", "downloader/request_method_count/GET": "3", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/ERROR": "1", "log_count/INFO": "2", "retry/count": "2", "retry/max_reached": "1", "retry/reason_count/scrapy.core.downloader.handlers.http11.TunnelError": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] } ], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy", "env": { "HTTP_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_http_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy_lowercase", "env": { "http_proxy": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_http_proxy_lowercase.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "env_https_proxy", "env": { "HTTPS_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_https_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=https://127.0.0.1:8443/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "ok", "header_names": [ "Host", "Proxy-Authorization" ] } ], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy_only_https_target", "env": { "HTTP_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_http_proxy_only_https_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=https://127.0.0.1:8443/ body='lab target ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "env_all_proxy", "env": { "ALL_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_all_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "env_no_proxy", "env": { "HTTP_PROXY": "http://user:pass@127.0.0.1:8888", "NO_PROXY": "127.0.0.1" }, "command": "python -m scrapy runspider env_no_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "env_no_proxy_with_meta_proxy", "env": { "NO_PROXY": "127.0.0.1" }, "command": "python -m scrapy runspider env_no_proxy_with_meta_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy_meta_proxy_wins", "env": { "HTTP_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_http_proxy_meta_proxy_wins.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY_B_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8889' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8889, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy_httpproxy_disabled", "env": { "HTTP_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_http_proxy_httpproxy_disabled.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"HTTPPROXY_ENABLED\": False}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy_meta_proxy_none", "env": { "HTTP_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_http_proxy_meta_proxy_none.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": None}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy_socks_url", "env": { "HTTP_PROXY": "socks5://127.0.0.1:1080" }, "command": "python -m scrapy runspider env_http_proxy_socks_url.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.exceptions.UnsupportedURLSchemeError: Unsupported scheme: b'socks5'" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.UnsupportedURLSchemeError": "1", "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy_explicit_only_middleware", "env": { "HTTP_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_http_proxy_explicit_only_middleware.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nfrom scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware\n\n\nclass ExplicitProxyMiddleware(HttpProxyMiddleware):\n \"\"\"HttpProxyMiddleware without the proxies read from the environment.\"\"\"\n\n def __init__(self, auth_encoding=\"latin-1\"):\n super().__init__(auth_encoding)\n self.proxies = {}\n\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOADER_MIDDLEWARES\": {\"scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware\": None, ExplicitProxyMiddleware: 750}}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "env_http_proxy_explicit_only_middleware_meta_proxy", "env": { "HTTP_PROXY": "http://user:pass@127.0.0.1:8888" }, "command": "python -m scrapy runspider env_http_proxy_explicit_only_middleware_meta_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nfrom scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware\n\n\nclass ExplicitProxyMiddleware(HttpProxyMiddleware):\n \"\"\"HttpProxyMiddleware without the proxies read from the environment.\"\"\"\n\n def __init__(self, auth_encoding=\"latin-1\"):\n super().__init__(auth_encoding)\n self.proxies = {}\n\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOADER_MIDDLEWARES\": {\"scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware\": None, ExplicitProxyMiddleware: 750}}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY_B_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8889' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8889, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "httpproxy_disabled_meta_proxy_with_credentials", "env": {}, "command": "python -m scrapy runspider httpproxy_disabled_meta_proxy_with_credentials.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"HTTPPROXY_ENABLED\": False}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK builtins.ValueError: invalid hostname: user:pass@127.0.0.1" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/builtins.ValueError": "1", "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "httpproxy_disabled_meta_proxy_https_target", "env": {}, "command": "python -m scrapy runspider httpproxy_disabled_meta_proxy_https_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"HTTPPROXY_ENABLED\": False}\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.core.downloader.handlers.http11.TunnelError: Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 2 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "ERROR [scrapy.downloadermiddlewares.retry] Gave up retrying (failed 3 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.core.downloader.handlers.http11.TunnelError": "3", "downloader/request_count": "3", "downloader/request_method_count/GET": "3", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/ERROR": "1", "log_count/INFO": "2", "retry/count": "2", "retry/max_reached": "1", "retry/reason_count/scrapy.core.downloader.handlers.http11.TunnelError": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] } ], "socks_log": [], "target_log": [] }, { "case": "manual_header_https", "env": {}, "command": "python -m scrapy runspider manual_header_https.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY}, headers={\"Proxy-Authorization\": BASIC}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=https://127.0.0.1:8443/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "ok", "header_names": [ "Host", "Proxy-Authorization" ] } ], "socks_log": [], "target_log": [] }, { "case": "manual_header_http", "env": {}, "command": "python -m scrapy runspider manual_header_http.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY}, headers={\"Proxy-Authorization\": BASIC}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "manual_header_same_proxy_redirect", "env": {}, "command": "python -m scrapy runspider manual_header_same_proxy_redirect.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(REDIRECT_URL, meta={\"proxy\": PROXY}, headers={\"Proxy-Authorization\": BASIC}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/dir/ body='redirected ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.redirect] Redirecting (301) to from ", "DEBUG [scrapy.core.engine] Crawled (200) (referer: http://127.0.0.1:8080/dir)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "2", "downloader/request_method_count/GET": "2", "downloader/response_status_count/200": "1", "downloader/response_status_count/301": "1", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] }, { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "Referer", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "manual_header_no_proxy", "env": {}, "command": "python -m scrapy runspider manual_header_no_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HEADERS_URL, headers={\"Proxy-Authorization\": BASIC}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8081/headers body='headers logged' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [ { "method": "GET", "path": "/headers", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "User-Agent" ] } ] }, { "case": "manual_header_proxy_changed_first_pass", "env": {}, "command": "python -m scrapy runspider manual_header_proxy_changed_first_pass.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass SwitchProxy:\n \"\"\"Moves every request to the second proxy before HttpProxyMiddleware sees it.\"\"\"\n\n def process_request(self, request):\n request.meta[\"proxy\"] = PROXY_B\n\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOADER_MIDDLEWARES\": {SwitchProxy: 740}}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY}, headers={\"Proxy-Authorization\": BASIC}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8889' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8889, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "manual_header_proxy_changed_on_redirect", "env": {}, "command": "python -m scrapy runspider manual_header_proxy_changed_on_redirect.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass SwitchProxyOnRedirect:\n \"\"\"Moves a request to the second proxy only when it is a redirected request.\"\"\"\n\n def process_request(self, request):\n if request.meta.get(\"redirect_times\"):\n request.meta[\"proxy\"] = PROXY_B\n\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOADER_MIDDLEWARES\": {SwitchProxyOnRedirect: 740}}\n\n async def start(self):\n yield scrapy.Request(REDIRECT_URL, meta={\"proxy\": PROXY}, headers={\"Proxy-Authorization\": BASIC}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.spidermiddlewares.httperror.HttpError: Ignoring non-200 response" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.redirect] Redirecting (301) to from ", "DEBUG [scrapy.core.engine] Crawled (407) (referer: http://127.0.0.1:8080/dir)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "2", "downloader/request_method_count/GET": "2", "downloader/response_status_count/301": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] }, { "proxy_port": 8889, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Referer", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "rotation_header_redirect", "env": {}, "command": "python -m scrapy runspider rotation_header_redirect.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nfrom itertools import cycle\n\nPROXIES = cycle([PROXY, PROXY_B])\n\n\nclass RotateWithHeader:\n \"\"\"Rotation that sets a proxy URL without credentials and a Proxy-Authorization header by hand.\"\"\"\n\n def process_request(self, request):\n request.meta[\"proxy\"] = next(PROXIES)\n request.headers[\"Proxy-Authorization\"] = BASIC\n\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOADER_MIDDLEWARES\": {RotateWithHeader: 740}}\n\n async def start(self):\n yield scrapy.Request(REDIRECT_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.spidermiddlewares.httperror.HttpError: Ignoring non-200 response" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.redirect] Redirecting (301) to from ", "DEBUG [scrapy.core.engine] Crawled (407) (referer: http://127.0.0.1:8080/dir)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "2", "downloader/request_method_count/GET": "2", "downloader/response_status_count/301": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] }, { "proxy_port": 8889, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Referer", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "rotation_header_redirect_407_handled", "env": {}, "command": "python -m scrapy runspider rotation_header_redirect_407_handled.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nfrom itertools import cycle\n\nPROXIES = cycle([PROXY, PROXY_B])\n\n\nclass RotateWithHeader:\n \"\"\"Rotation that sets a proxy URL without credentials and a Proxy-Authorization header by hand.\"\"\"\n\n def process_request(self, request):\n request.meta[\"proxy\"] = next(PROXIES)\n request.headers[\"Proxy-Authorization\"] = BASIC\n\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOADER_MIDDLEWARES\": {RotateWithHeader: 740}}\n\n async def start(self):\n yield scrapy.Request(REDIRECT_URL, meta={\"handle_httpstatus_list\": [407]}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=407 url=http://127.0.0.1:8080/dir/ body='' meta_proxy='http://127.0.0.1:8889' proxy_authenticate='Basic realm=\"lab\"'" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.redirect] Redirecting (301) to from ", "DEBUG [scrapy.core.engine] Crawled (407) (referer: http://127.0.0.1:8080/dir)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "2", "downloader/request_method_count/GET": "2", "downloader/response_status_count/301": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] }, { "proxy_port": 8889, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Referer", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "rotation_header_retry_https", "env": {}, "command": "python -m scrapy runspider rotation_header_retry_https.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nfrom itertools import cycle\n\nPROXIES = cycle([\"http://\" + PROXY_C_HOSTPORT, PROXY])\n\n\nclass RotateWithHeader:\n \"\"\"Rotation that sets a proxy URL without credentials and a Proxy-Authorization header by hand.\"\"\"\n\n def process_request(self, request):\n request.meta[\"proxy\"] = next(PROXIES)\n request.headers[\"Proxy-Authorization\"] = BASIC\n\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOADER_MIDDLEWARES\": {RotateWithHeader: 740}}\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.core.downloader.handlers.http11.TunnelError: Could not open CONNECT tunnel with proxy 127.0.0.1:8890 [{'status': 407, 'reason': b'Proxy Authentication Required'}]" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8890 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 2 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8888 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "ERROR [scrapy.downloadermiddlewares.retry] Gave up retrying (failed 3 times): Could not open CONNECT tunnel with proxy 127.0.0.1:8890 [{'status': 407, 'reason': b'Proxy Authentication Required'}]", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.core.downloader.handlers.http11.TunnelError": "3", "downloader/request_count": "3", "downloader/request_method_count/GET": "3", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/ERROR": "1", "log_count/INFO": "2", "retry/count": "2", "retry/max_reached": "1", "retry/reason_count/scrapy.core.downloader.handlers.http11.TunnelError": "2" }, "proxy_log": [ { "proxy_port": 8890, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "wrong", "header_names": [ "Host", "Proxy-Authorization" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Host" ] }, { "proxy_port": 8890, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "wrong", "header_names": [ "Host", "Proxy-Authorization" ] } ], "socks_log": [], "target_log": [] }, { "case": "rotation_url_credentials_redirect", "env": {}, "command": "python -m scrapy runspider rotation_url_credentials_redirect.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nfrom itertools import cycle\n\nPROXIES = cycle([PROXY_AUTH, PROXY_B_AUTH])\n\n\nclass RotateProxyMiddleware:\n \"\"\"Rotation that puts the credentials in the proxy URL.\"\"\"\n\n def process_request(self, request):\n request.meta[\"proxy\"] = next(PROXIES)\n\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOADER_MIDDLEWARES\": {RotateProxyMiddleware: 740}}\n\n async def start(self):\n yield scrapy.Request(REDIRECT_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/dir/ body='redirected ok' meta_proxy='http://127.0.0.1:8889' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.redirect] Redirecting (301) to from ", "DEBUG [scrapy.core.engine] Crawled (200) (referer: http://127.0.0.1:8080/dir)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "2", "downloader/request_method_count/GET": "2", "downloader/response_status_count/200": "1", "downloader/response_status_count/301": "1", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "User-Agent" ] }, { "proxy_port": 8889, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/dir/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Host", "Proxy-Authorization", "Referer", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "socks5_http_target", "env": {}, "command": "python -m scrapy runspider socks5_http_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": SOCKS}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.exceptions.UnsupportedURLSchemeError: Unsupported scheme: b'socks5'" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.UnsupportedURLSchemeError": "1", "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "socks5_https_target", "env": {}, "command": "python -m scrapy runspider socks5_https_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": SOCKS}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.exceptions.DownloadTimeoutError: Getting https://127.0.0.1:8443/ took longer than 15.0 seconds." ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): Getting https://127.0.0.1:8443/ took longer than 15.0 seconds.", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 2 times): Getting https://127.0.0.1:8443/ took longer than 15.0 seconds.", "ERROR [scrapy.downloadermiddlewares.retry] Gave up retrying (failed 3 times): Getting https://127.0.0.1:8443/ took longer than 15.0 seconds.", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.DownloadTimeoutError": "3", "downloader/request_count": "3", "downloader/request_method_count/GET": "3", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/ERROR": "1", "log_count/INFO": "2", "retry/count": "2", "retry/max_reached": "1", "retry/reason_count/scrapy.exceptions.DownloadTimeoutError": "2" }, "proxy_log": [], "socks_log": [ "NOT_SOCKS5 first_bytes=b'CONNECT 127.0.0.1:8443 H'", "NOT_SOCKS5 first_bytes=b'CONNECT 127.0.0.1:8443 H'", "NOT_SOCKS5 first_bytes=b'CONNECT 127.0.0.1:8443 H'" ], "target_log": [] }, { "case": "socks5h_http_target", "env": {}, "command": "python -m scrapy runspider socks5h_http_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": \"socks5h://127.0.0.1:1080\"}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.exceptions.UnsupportedURLSchemeError: Unsupported scheme: b'socks5h'" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.UnsupportedURLSchemeError": "1", "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "socks5_no_errback", "env": {}, "command": "python -m scrapy runspider socks5_no_errback.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": SOCKS})", "exit_code": 0, "stdout": [], "log": [ "INFO [scrapy.core.engine] Spider opened", "ERROR [scrapy.core.scraper] Error downloading ", "Traceback (most recent call last):", " File \"/lib/python3.14/site-packages/scrapy/utils/_download_handlers.py\", line 55, in wrap_twisted_exceptions", " yield", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/handlers/http11.py\", line 140, in download_request", " return await maybe_deferred_to_future(agent.download_request(request))", " ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/handlers/http11.py\", line 512, in download_request", " d: Deferred[IResponse] = agent.request(", " ~~~~~~~~~~~~~^", " method,", " ^^^^^^^", " ...<2 lines>...", " cast(\"IBodyProducer\", bodyproducer),", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " )", " ^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/handlers/http11.py\", line 418, in request", " endpoint=self._getEndpoint(self._proxyURI), # type: ignore[no-untyped-call]", " ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/twisted/web/client.py\", line 1181, in _getEndpoint", " return self._endpointFactory.endpointForURI(uri)", " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^", " File \"/lib/python3.14/site-packages/twisted/web/client.py\", line 1057, in endpointForURI", " raise SchemeNotSupported(f\"Unsupported scheme: {uri.scheme!r}\")", "twisted.web.error.SchemeNotSupported: Unsupported scheme: b'socks5'", "", "The above exception was the direct cause of the following exception:", "", "Traceback (most recent call last):", " File \"/lib/python3.14/site-packages/twisted/internet/defer.py\", line 1834, in _inlineCallbacks", " result = context.run(", " cast(Failure, result).throwExceptionIntoGenerator, gen", " )", " File \"/lib/python3.14/site-packages/twisted/python/failure.py\", line 467, in throwExceptionIntoGenerator", " return g.throw(self.value.with_traceback(self.tb))", " ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/engine.py\", line 514, in _download", " result = yield self.downloader.fetch(request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/twisted/internet/defer.py\", line 1834, in _inlineCallbacks", " result = context.run(", " cast(Failure, result).throwExceptionIntoGenerator, gen", " )", " File \"/lib/python3.14/site-packages/twisted/python/failure.py\", line 467, in throwExceptionIntoGenerator", " return g.throw(self.value.with_traceback(self.tb))", " ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/__init__.py\", line 226, in fetch", " result: Response | Request = yield (", " ^^^^^^^", " ...<3 lines>...", " )", " ^", " File \"/lib/python3.14/site-packages/twisted/internet/defer.py\", line 1247, in adapt", " extracted: _SelfResultT | Failure = result.result()", " ~~~~~~~~~~~~~^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/middleware.py\", line 87, in download_async", " result = await self._process_exception(ex, request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/middleware.py\", line 156, in _process_exception", " raise exception", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/middleware.py\", line 80, in download_async", " result: Response | Request = await self._process_request(", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " request, download_func", " ^^^^^^^^^^^^^^^^^^^^^^", " )", " ^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/middleware.py\", line 114, in _process_request", " return await download_func(request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/__init__.py\", line 281, in _enqueue_request", " return await maybe_deferred_to_future(d) # fired in _wait_for_download()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/__init__.py\", line 348, in _wait_for_download", " response = await self._download(slot, request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/__init__.py\", line 318, in _download", " response: Response = await self.handlers.download_request_async(request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/handlers/__init__.py\", line 168, in download_request_async", " return await handler.download_request(request)", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/core/downloader/handlers/http11.py\", line 139, in download_request", " with wrap_twisted_exceptions():", " ~~~~~~~~~~~~~~~~~~~~~~~^^", " File \"/usr/lib/python3.14/contextlib.py\", line 162, in __exit__", " self.gen.throw(value)", " ~~~~~~~~~~~~~~^^^^^^^", " File \"/lib/python3.14/site-packages/scrapy/utils/_download_handlers.py\", line 57, in wrap_twisted_exceptions", " raise UnsupportedURLSchemeError(str(e)) from e", "scrapy.exceptions.UnsupportedURLSchemeError: Unsupported scheme: b'socks5'", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.UnsupportedURLSchemeError": "1", "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "finish_reason": "'finished'", "log_count/ERROR": "1", "log_count/INFO": "2" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "httpx_handler_socks5_http_target", "env": {}, "command": "python -m scrapy runspider httpx_handler_socks5_http_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOAD_HANDLERS\": HTTPX_HANDLERS}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": SOCKS}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='socks5://127.0.0.1:1080' proxy_authenticate=None" ], "log": [ "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "INFO [scrapy.core.engine] Spider opened", "INFO [httpx2] HTTP Request: GET http://127.0.0.1:8080/ \"HTTP/1.0 200 OK\"", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "15", "log_count/INFO": "3", "response_received_count": "1" }, "proxy_log": [], "socks_log": [ "CONNECT atyp=IPv4 dest=127.0.0.1:8080" ], "target_log": [] }, { "case": "httpx_handler_socks5_https_target", "env": {}, "command": "python -m scrapy runspider httpx_handler_socks5_https_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOAD_HANDLERS\": HTTPX_HANDLERS}\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": SOCKS}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=https://127.0.0.1:8443/ body='lab target ok' meta_proxy='socks5://127.0.0.1:1080' proxy_authenticate=None" ], "log": [ "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "INFO [scrapy.core.engine] Spider opened", "INFO [httpx2] HTTP Request: GET https://127.0.0.1:8443/ \"HTTP/1.0 200 OK\"", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "17", "log_count/INFO": "3", "response_received_count": "1" }, "proxy_log": [], "socks_log": [ "CONNECT atyp=IPv4 dest=127.0.0.1:8443" ], "target_log": [] }, { "case": "httpx_handler_socks5h_http_target", "env": {}, "command": "python -m scrapy runspider httpx_handler_socks5h_http_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOAD_HANDLERS\": HTTPX_HANDLERS}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": \"socks5h://127.0.0.1:1080\"}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='socks5h://127.0.0.1:1080' proxy_authenticate=None" ], "log": [ "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "INFO [scrapy.core.engine] Spider opened", "INFO [httpx2] HTTP Request: GET http://127.0.0.1:8080/ \"HTTP/1.0 200 OK\"", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "15", "log_count/INFO": "3", "response_received_count": "1" }, "proxy_log": [], "socks_log": [ "CONNECT atyp=IPv4 dest=127.0.0.1:8080" ], "target_log": [] }, { "case": "httpx_handler_socks5_hostname", "env": {}, "command": "python -m scrapy runspider httpx_handler_socks5_hostname.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOAD_HANDLERS\": HTTPX_HANDLERS}\n\n async def start(self):\n yield scrapy.Request(\"http://localhost:8080/\", meta={\"proxy\": SOCKS}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://localhost:8080/ body='lab target ok' meta_proxy='socks5://127.0.0.1:1080' proxy_authenticate=None" ], "log": [ "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "INFO [scrapy.core.engine] Spider opened", "INFO [httpx2] HTTP Request: GET http://localhost:8080/ \"HTTP/1.0 200 OK\"", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "15", "log_count/INFO": "3", "response_received_count": "1" }, "proxy_log": [], "socks_log": [ "CONNECT atyp=DOMAIN dest=localhost:8080" ], "target_log": [] }, { "case": "httpx_handler_http_proxy_https_target", "env": {}, "command": "python -m scrapy runspider httpx_handler_http_proxy_https_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOAD_HANDLERS\": HTTPX_HANDLERS}\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=https://127.0.0.1:8443/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "INFO [scrapy.core.engine] Spider opened", "INFO [httpx2] HTTP Request: GET https://127.0.0.1:8443/ \"HTTP/1.0 200 OK\"", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "21", "log_count/INFO": "3", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "ok", "header_names": [ "Accept", "Host", "Proxy-Authorization" ] } ], "socks_log": [], "target_log": [] }, { "case": "httpx_handler_http_proxy_http_target", "env": {}, "command": "python -m scrapy runspider httpx_handler_http_proxy_http_target.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOAD_HANDLERS\": HTTPX_HANDLERS}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "INFO [scrapy.core.engine] Spider opened", "INFO [httpx2] HTTP Request: GET http://127.0.0.1:8080/ \"HTTP/1.0 200 OK\"", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "13", "log_count/INFO": "3", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Connection", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "httpx_handler_https_no_credentials", "env": {}, "command": "python -m scrapy runspider httpx_handler_https_no_credentials.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOAD_HANDLERS\": HTTPX_HANDLERS}\n\n async def start(self):\n yield scrapy.Request(HTTPS_URL, meta={\"proxy\": PROXY}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.exceptions.DownloadConnectionRefusedError: 407 Proxy Authentication Required" ], "log": [ "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): 407 Proxy Authentication Required", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 2 times): 407 Proxy Authentication Required", "ERROR [scrapy.downloadermiddlewares.retry] Gave up retrying (failed 3 times): 407 Proxy Authentication Required", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.DownloadConnectionRefusedError": "3", "downloader/request_count": "3", "downloader/request_method_count/GET": "3", "finish_reason": "'finished'", "log_count/DEBUG": "32", "log_count/ERROR": "1", "log_count/INFO": "2", "retry/count": "2", "retry/max_reached": "1", "retry/reason_count/scrapy.exceptions.DownloadConnectionRefusedError": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Accept", "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Accept", "Host" ] }, { "proxy_port": 8888, "event": "request", "method": "CONNECT", "target": "127.0.0.1:8443", "auth": "missing", "header_names": [ "Accept", "Host" ] } ], "socks_log": [], "target_log": [] }, { "case": "httpx_handler_http_no_credentials", "env": {}, "command": "python -m scrapy runspider httpx_handler_http_no_credentials.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n custom_settings = {\"DOWNLOAD_HANDLERS\": HTTPX_HANDLERS}\n\n async def start(self):\n yield scrapy.Request(HTTP_URL, meta={\"proxy\": PROXY}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.spidermiddlewares.httperror.HttpError: Ignoring non-200 response" ], "log": [ "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "WARNING [scrapy.core.downloader.handlers._base_streaming] HttpxDownloadHandler is experimental and is not recommended for production use.", "INFO [scrapy.core.engine] Spider opened", "INFO [httpx2] HTTP Request: GET http://127.0.0.1:8080/ \"HTTP/1.1 407 Proxy Authentication Required\"", "DEBUG [scrapy.core.engine] Crawled (407) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/407": "1", "finish_reason": "'finished'", "log_count/DEBUG": "13", "log_count/INFO": "3", "response_received_count": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "GET", "target": "http://127.0.0.1:8080/", "auth": "missing", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Connection", "Host", "User-Agent" ] } ], "socks_log": [], "target_log": [] }, { "case": "start_requests_only", "env": {}, "command": "python -m scrapy runspider start_requests_only.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n def start_requests(self):\n print(\"start_requests() was called\", flush=True)\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "start_and_start_requests", "env": {}, "command": "python -m scrapy runspider start_and_start_requests.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n print(\"start() was called\", flush=True)\n yield scrapy.Request(HTTP_URL, errback=self.on_error)\n\n def start_requests(self):\n print(\"start_requests() was called\", flush=True)\n yield scrapy.Request(HTTP_URL, errback=self.on_error)", "exit_code": 0, "stdout": [ "start() was called", "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "start_urls_only", "env": {}, "command": "python -m scrapy runspider start_urls_only.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n start_urls = [HTTP_URL]", "exit_code": 0, "stdout": [ "CALLBACK status=200 url=http://127.0.0.1:8080/ body='lab target ok' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.core.engine] Crawled (200) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/request_count": "1", "downloader/request_method_count/GET": "1", "downloader/response_status_count/200": "1", "finish_reason": "'finished'", "log_count/DEBUG": "1", "log_count/INFO": "2", "response_received_count": "1" }, "proxy_log": [], "socks_log": [], "target_log": [] }, { "case": "post_lost_response", "env": {}, "command": "python -m scrapy runspider post_lost_response.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(JOBS_URL, method=\"POST\", body=\"job-retried\", meta={\"proxy\": PROXY_AUTH}, errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=201 url=http://127.0.0.1:8081/jobs body='created job-retried' meta_proxy='http://127.0.0.1:8888' proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): []", "DEBUG [scrapy.core.engine] Crawled (201) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.DownloadFailedError": "1", "downloader/request_count": "2", "downloader/request_method_count/POST": "2", "downloader/response_status_count/201": "1", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "1", "retry/count": "1", "retry/reason_count/scrapy.exceptions.DownloadFailedError": "1" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "POST", "target": "http://127.0.0.1:8081/jobs", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Content-Length", "Host", "Proxy-Authorization", "User-Agent" ] }, { "proxy_port": 8888, "event": "request", "method": "POST", "target": "http://127.0.0.1:8081/jobs", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Content-Length", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [ { "method": "POST", "path": "/jobs", "body": "job-retried", "answered": false }, { "method": "POST", "path": "/jobs", "body": "job-retried", "answered": true } ] }, { "case": "post_lost_response_dont_retry", "env": {}, "command": "python -m scrapy runspider post_lost_response_dont_retry.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(JOBS_URL, method=\"POST\", body=\"job-not-retried\", meta={\"proxy\": PROXY_AUTH, \"dont_retry\": True}, errback=self.on_error)", "exit_code": 0, "stdout": [ "ERRBACK scrapy.exceptions.DownloadFailedError: []" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.DownloadFailedError": "1", "downloader/request_count": "1", "downloader/request_method_count/POST": "1", "finish_reason": "'finished'", "log_count/INFO": "2" }, "proxy_log": [ { "proxy_port": 8888, "event": "request", "method": "POST", "target": "http://127.0.0.1:8081/jobs", "auth": "ok", "header_names": [ "Accept", "Accept-Encoding", "Accept-Language", "Content-Length", "Host", "Proxy-Authorization", "User-Agent" ] } ], "socks_log": [], "target_log": [ { "method": "POST", "path": "/jobs", "body": "job-not-retried", "answered": false } ] }, { "case": "post_lost_response_no_proxy", "env": {}, "command": "python -m scrapy runspider post_lost_response_no_proxy.py -s LOG_FORMAT=%(levelname)s [%(name)s] %(message)s -s TELNETCONSOLE_ENABLED=False -s REMOTE_CONTROL_ENABLED=False -s DOWNLOAD_TIMEOUT=15", "code": "from lab_common import *\n\nclass Case(LabSpider):\n name = \"case\"\n\n async def start(self):\n yield scrapy.Request(JOBS_URL, method=\"POST\", body=\"job-direct\", errback=self.on_error)", "exit_code": 0, "stdout": [ "CALLBACK status=201 url=http://127.0.0.1:8081/jobs body='created job-direct' meta_proxy=None proxy_authenticate=None" ], "log": [ "INFO [scrapy.core.engine] Spider opened", "DEBUG [scrapy.downloadermiddlewares.retry] Retrying (failed 1 times): []", "DEBUG [scrapy.core.engine] Crawled (201) (referer: None)", "INFO [scrapy.core.engine] Closing spider (finished)", "INFO [scrapy.core.engine] Spider closed (finished)" ], "stats": { "downloader/exception_type_count/scrapy.exceptions.DownloadFailedError": "1", "downloader/request_count": "2", "downloader/request_method_count/POST": "2", "downloader/response_status_count/201": "1", "finish_reason": "'finished'", "log_count/DEBUG": "2", "log_count/INFO": "2", "response_received_count": "1", "retry/count": "1", "retry/reason_count/scrapy.exceptions.DownloadFailedError": "1" }, "proxy_log": [], "socks_log": [], "target_log": [ { "method": "POST", "path": "/jobs", "body": "job-direct", "answered": false }, { "method": "POST", "path": "/jobs", "body": "job-direct", "answered": true } ] } ] }