{ "target": "http://127.0.0.1:8047/", "proxy": "http://user:pass@127.0.0.1:8048", "environments": { "2.19": { "versions": { "scrapy": "2.19.0", "twisted": "26.4.0", "python": "3.14.4" }, "version_commands": { "scrapy version": [ "Scrapy 2.19.0" ], "python -m pip show scrapy": [ "Name: Scrapy", "Version: 2.19.0" ] }, "cases": { "a_start_requests_only": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/a_start_requests_only.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:33 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:33 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:33 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:33 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:33 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:33 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:33 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:33 [scrapy.extensions.telnet] INFO: Telnet Password: c84b3d0f8a640e9f", "2026-10-09 12:33:33 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:33 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:33 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:33 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:33 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:33 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:33 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:33 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:33 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 43793 (job 499122-ee1d82dfbedc4560b320ab9f05bf065f)", "2026-10-09 12:33:33 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:33 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.002526537049561739,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 33, 861621, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/INFO': 4,", " 'memusage/max': 90951680,", " 'memusage/startup': 90951680,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 33, 859092, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:33 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "a2_start_urls_only": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/a2_start_urls_only.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:34 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:34 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:34 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:34 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:34 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:34 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:34 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:34 [scrapy.extensions.telnet] INFO: Telnet Password: b07e6dd412f1ff0f", "2026-10-09 12:33:34 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:34 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:34 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:34 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:34 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:34 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:34 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:34 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:34 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 39925 (job 499124-25ea781d02224b33bfce5899a87348da)", "2026-10-09 12:33:34 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:34 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:34 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:34 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:34 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.007947327103465796,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 34, 536922, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91049984,", " 'memusage/startup': 91049984,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 34, 528972, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:34 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b_async_start": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/b_async_start.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:34 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:34 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:34 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:34 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:34 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:34 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:34 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:34 [scrapy.extensions.telnet] INFO: Telnet Password: 5c9c83d279e7a4ad", "2026-10-09 12:33:34 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:34 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:35 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:35 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:35 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:35 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:35 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:35 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:35 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 42131 (job 499133-8f67ed6f7baa44b4aaa6918e58e6e185)", "2026-10-09 12:33:35 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:35 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:35 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:35 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.007903959602117538,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 35, 48923, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91095040,", " 'memusage/startup': 91095040,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 35, 41017, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:35 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b2_start_urls_headers_meta": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/b2_start_urls_headers_meta.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:35 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:35 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:35 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:35 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:35 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:35 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:35 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:35 [scrapy.extensions.telnet] INFO: Telnet Password: d9d39ec31a904515", "2026-10-09 12:33:35 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:35 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:35 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:35 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:35 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:35 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:35 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:35 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:35 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 45309 (job 499137-af7e6d2747cd4aeeb4bbd0b205687db6)", "2026-10-09 12:33:35 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:35 [lab] INFO: parse() got 200 Accept-Language=en-AU page=1", "2026-10-09 12:33:35 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:35 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:35 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 223,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.008365944027900696,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 35, 633947, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91238400,", " 'memusage/startup': 91238400,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 35, 625579, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:35 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b3_proxy_meta": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/b3_proxy_meta.py -a url=http://127.0.0.1:8047/ -a proxy=http://user:pass@127.0.0.1:8048", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:36 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:36 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:36 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:36 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:36 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:36 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:36 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:36 [scrapy.extensions.telnet] INFO: Telnet Password: 8a4decae4298501e", "2026-10-09 12:33:36 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:36 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:36 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:36 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:36 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:36 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:36 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:36 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:36 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 41303 (job 499141-590f6585dd73402d94c9e1a77c02acc3)", "2026-10-09 12:33:36 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:36 [lab] INFO: parse() got 200 via http://127.0.0.1:8048", "2026-10-09 12:33:36 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:36 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:36 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 261,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 337,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.009326072875410318,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 36, 209155, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91193344,", " 'memusage/startup': 91193344,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 36, 199828, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:36 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [ { "request_line": "GET http://127.0.0.1:8047/ HTTP/1.1", "proxy_authorization": "valid" } ] }, "c_both": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/c_both.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:36 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:36 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:36 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:36 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:36 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:36 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:36 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:36 [scrapy.extensions.telnet] INFO: Telnet Password: 61f98a09a456b613", "2026-10-09 12:33:36 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:36 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:36 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:36 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:36 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:36 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:36 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:36 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:36 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 45803 (job 499146-2c14024aa34e498f928d805656bd7748)", "2026-10-09 12:33:36 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:36 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:36 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:36 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:36 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.008564492221921682,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 36, 772151, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91025408,", " 'memusage/startup': 91025408,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 36, 763586, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:36 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "c2_both_delegating": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/c2_both_delegating.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:37 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:37 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:37 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:37 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:37 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:37 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:37 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:37 [scrapy.extensions.telnet] INFO: Telnet Password: a750803d0dd1f862", "2026-10-09 12:33:37 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:37 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:37 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:37 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:37 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:37 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:37 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:37 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:37 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 46529 (job 499163-1d12cfb079e4462cb9d5933f56def48f)", "2026-10-09 12:33:37 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:37 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:37 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.008060286287218332,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 37, 339778, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91131904,", " 'memusage/startup': 91131904,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 37, 331715, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:37 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1_mw_process_start_requests": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/d1_mw_process_start_requests.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:37 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:37 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:37 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:37 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:37 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:37 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:37 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:37 [scrapy.extensions.telnet] INFO: Telnet Password: c68a6b91eaa8ef0d", "2026-10-09 12:33:37 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:37 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:37 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:37 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:37 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:37 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:37 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:37 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:37 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 45643 (job 499192-b8a5ba6d51824ebfa85d41fdcd4a7a20)", "2026-10-09 12:33:37 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:37 [lab] INFO: parse() got 200 tagged=None", "2026-10-09 12:33:37 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:37 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.008221286814659834,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 37, 852980, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91103232,", " 'memusage/startup': 91103232,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 37, 844758, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:37 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1b_old_spider_old_mw": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/d1b_old_spider_old_mw.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:38 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:38 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:38 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:38 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:38 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:38 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:38 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:38 [scrapy.extensions.telnet] INFO: Telnet Password: 04b5db7c8135ca60", "2026-10-09 12:33:38 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:38 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:38 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:38 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:38 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:38 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:38 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:38 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:38 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 45153 (job 499196-e010fae8e8494cd8b2c903a9e53741da)", "2026-10-09 12:33:38 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:38 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.0021102940663695335,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 38, 375928, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/INFO': 4,", " 'memusage/max': 91049984,", " 'memusage/startup': 91049984,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 38, 373816, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:38 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1c_mw_process_start": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/d1c_mw_process_start.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:38 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:38 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:38 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:38 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:38 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:38 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:38 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:38 [scrapy.extensions.telnet] INFO: Telnet Password: 89e60cdc15118316", "2026-10-09 12:33:38 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:38 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:38 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:38 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:38 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:38 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:38 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:38 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:38 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 43023 (job 499198-df4ac7c803244ad6a241f8e5137543a2)", "2026-10-09 12:33:38 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:38 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:33:38 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:38 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:38 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.00947251496836543,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 38, 900517, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91222016,", " 'memusage/startup': 91222016,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 38, 891043, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:38 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1d_mw_both": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/d1d_mw_both.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:39 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:39 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:39 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:39 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:39 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:39 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:39 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:39 [scrapy.extensions.telnet] INFO: Telnet Password: 5e03168f63a38881", "2026-10-09 12:33:39 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:39 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:39 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:39 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:39 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:39 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:39 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:39 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:39 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 45823 (job 499202-9847da9c839e4fbb8babb0a9af189cfd)", "2026-10-09 12:33:39 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:39 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:33:39 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:39 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:39 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.008409939240664244,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 39, 430034, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91172864,", " 'memusage/startup': 91172864,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 39, 421622, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:39 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2_mw_sync_output": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/d2_mw_sync_output.py -a url=http://127.0.0.1:8047/", "returncode": 1, "stats": null, "log": [ "2026-10-09 12:33:39 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:39 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:39 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:39 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:39 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:39 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:39 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:39 [scrapy.extensions.telnet] INFO: Telnet Password: 6499480bca453f14", "2026-10-09 12:33:39 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:39 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:39 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:39 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:39 [asyncio] ERROR: Task exception was never retrieved", "future: exception=TypeError(\"Middleware d2_mw_sync_output.StampMiddleware doesn't support asynchronous spider output. Its process_spider_output() method should be an async generator function or it should additionally define a process_spider_output_async() method.\")>", "Traceback (most recent call last):", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/crawler.py\", line 675, in _crawl_and_track", " await crawler.crawl_async(*args, **kwargs)", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/crawler.py\", line 313, in crawl_async", " self.engine = self._create_engine()", " ~~~~~~~~~~~~~~~~~~~^^", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/crawler.py\", line 330, in _create_engine", " return ExecutionEngine(self, lambda _: self.stop_async())", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/engine.py\", line 150, in __init__", " self.scraper: Scraper = Scraper(crawler)", " ~~~~~~~^^^^^^^^^", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/scraper.py\", line 109, in __init__", " self.spidermw: SpiderMiddlewareManager = build_from_crawler(", " ~~~~~~~~~~~~~~~~~~^", " SpiderMiddlewareManager, crawler", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " )", " ^", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/utils/misc.py\", line 190, in build_from_crawler", " instance = objcls.from_crawler(crawler, *args, **kwargs)", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/middleware.py\", line 114, in from_crawler", " return cls(*middlewares, crawler=crawler)", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/middleware.py\", line 56, in __init__", " self._add_middleware(mw)", " ~~~~~~~~~~~~~~~~~~~~^^^^", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 66, in _add_middleware", " process_spider_output = self._get_process_spider_output(mw)", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 280, in _get_process_spider_output", " raise TypeError(", " ...<4 lines>...", " )", "TypeError: Middleware d2_mw_sync_output.StampMiddleware doesn't support asynchronous spider output. Its process_spider_output() method should be an async generator function or it should additionally define a process_spider_output_async() method." ], "proxy_log": [] }, "d2b_mw_async_output": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/d2b_mw_async_output.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:40 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:40 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:40 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:40 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:40 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:40 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:40 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:40 [scrapy.extensions.telnet] INFO: Telnet Password: e9cbd628ecfae4a1", "2026-10-09 12:33:40 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:40 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:40 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:40 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:40 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:40 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:40 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:40 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:40 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 39897 (job 499216-a45ec83b864740c5a55c16d3eb9d2e14)", "2026-10-09 12:33:40 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:40 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:40 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': True}", "2026-10-09 12:33:40 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:40 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.008729408960789442,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 40, 507689, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91271168,", " 'memusage/startup': 91271168,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 40, 498958, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:40 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2c_mw_async_output_spider_arg": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/d2c_mw_async_output_spider_arg.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:40 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:40 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:40 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:40 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:40 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:40 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:40 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:40 [scrapy.extensions.telnet] INFO: Telnet Password: 3ed617f2f4bd5e6c", "2026-10-09 12:33:40 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:40 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:41 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:41 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:41 [py.warnings] WARNING: venv-2.19/lib/python3.14/site-packages/scrapy/core/spidermw.py:69: ScrapyDeprecationWarning: StampMiddleware.process_spider_output() requires a spider argument, this is deprecated and the argument will not be passed in future Scrapy versions. If you need to access the spider instance you can save the crawler instance passed to from_crawler() and use its spider attribute.", " self._check_mw_method_spider_arg(process_spider_output)", "", "2026-10-09 12:33:41 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:41 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:41 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:41 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:41 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 34877 (job 499220-71692e2188c14ffaad0bf4594f9dbe4d)", "2026-10-09 12:33:41 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:41 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:41 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': True}", "2026-10-09 12:33:41 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:41 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.00775348674505949,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 41, 87060, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91181056,", " 'memusage/startup': 91181056,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 41, 79305, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:41 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2d_mw_universal": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/d2d_mw_universal.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 5 }, "log": [ "2026-10-09 12:33:41 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:41 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:41 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:41 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:41 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:41 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:41 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:41 [scrapy.extensions.telnet] INFO: Telnet Password: f0e096085fd53a4a", "2026-10-09 12:33:41 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:41 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:41 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:41 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:41 [py.warnings] WARNING: venv-2.19/lib/python3.14/site-packages/scrapy/core/spidermw.py:69: ScrapyDeprecationWarning: StampMiddleware.process_spider_output_async() requires a spider argument, this is deprecated and the argument will not be passed in future Scrapy versions. If you need to access the spider instance you can save the crawler instance passed to from_crawler() and use its spider attribute.", " self._check_mw_method_spider_arg(process_spider_output)", "", "2026-10-09 12:33:41 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:41 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:41 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:41 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:41 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 39161 (job 499224-2173622444ad4999b66dda838844af02)", "2026-10-09 12:33:41 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:41 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:41 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': 'async'}", "2026-10-09 12:33:41 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:41 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 220,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.008428136818110943,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 41, 645105, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 5,", " 'memusage/max': 91381760,", " 'memusage/startup': 91250688,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 41, 636676, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:41 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "e1_start_not_async": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/e1_start_not_async.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "start_error", "log_count/INFO": 4, "log_count/ERROR": 1 }, "log": [ "2026-10-09 12:33:42 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:42 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:42 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:42 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:42 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:42 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:42 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:42 [scrapy.extensions.telnet] INFO: Telnet Password: 1067d73c7730b529", "2026-10-09 12:33:42 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:42 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:42 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:42 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:42 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:42 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:42 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:42 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:42 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 33895 (job 499230-c23e3d43fa0941c1b5f018ab98dfafac)", "2026-10-09 12:33:42 [scrapy.core.engine] ERROR: Error while reading start items and requests: e1_start_not_async.LabSpider.start() must be an asynchronous generator, i.e. an async def method with yield statements, got .", "Traceback (most recent call last):", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/engine.py\", line 284, in _process_start_next", " item_or_request = await anext(self._start)", " ^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 2 more times]", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 262, in _reject_start", " raise TypeError(", " ...<3 lines>...", " )", "TypeError: e1_start_not_async.LabSpider.start() must be an asynchronous generator, i.e. an async def method with yield statements, got ", "Traceback (most recent call last):", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/engine.py\", line 284, in _process_start_next", " item_or_request = await anext(self._start)", " ^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 2 more times]", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 262, in _reject_start", " raise TypeError(", " ...<3 lines>...", " )", "TypeError: e1_start_not_async.LabSpider.start() must be an asynchronous generator, i.e. an async def method with yield statements, got ", "2026-10-09 12:33:42 [scrapy.core.engine] INFO: Closing spider (start_error)", "2026-10-09 12:33:42 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.003534529358148575,", " 'finish_reason': 'start_error',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 42, 227916, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/ERROR': 1,", " 'log_count/INFO': 4,", " 'memusage/max': 90992640,", " 'memusage/startup': 90992640,", " 'responses_per_minute': None,", " 'spider_exceptions/TypeError': 1,", " 'spider_exceptions/count': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 42, 224379, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:42 [scrapy.core.engine] INFO: Spider closed (start_error)" ], "proxy_log": [] }, "e2_start_returns_list": { "command": "venv-2.19/bin/python -m scrapy runspider ./spiders/e2_start_returns_list.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "start_error", "log_count/INFO": 4, "log_count/ERROR": 1 }, "log": [ "2026-10-09 12:33:42 [scrapy.utils.log] INFO: Scrapy 2.19.0 started (bot: scrapybot)", "2026-10-09 12:33:42 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:42 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:42 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:42 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:42 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:42 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:42 [scrapy.extensions.telnet] INFO: Telnet Password: 8d02e06b9c2c6df1", "2026-10-09 12:33:42 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "2026-10-09 12:33:42 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:42 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:42 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "2026-10-09 12:33:42 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:42 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:42 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:42 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:42 [scrapy.extensions.remote_control] INFO: Remote control HTTP server listening on port 45599 (job 499240-3e3dfe7e0fc04c0e8e6f2503423699a9)", "2026-10-09 12:33:42 [scrapy.core.engine] ERROR: Error while reading start items and requests: e2_start_returns_list.LabSpider.start() must be an asynchronous generator, i.e. an async def method with yield statements, got .", "Traceback (most recent call last):", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/engine.py\", line 284, in _process_start_next", " item_or_request = await anext(self._start)", " ^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 2 more times]", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 262, in _reject_start", " raise TypeError(", " ...<3 lines>...", " )", "TypeError: e2_start_returns_list.LabSpider.start() must be an asynchronous generator, i.e. an async def method with yield statements, got ", "Traceback (most recent call last):", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/engine.py\", line 284, in _process_start_next", " item_or_request = await anext(self._start)", " ^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 2 more times]", " File \"venv-2.19/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 262, in _reject_start", " raise TypeError(", " ...<3 lines>...", " )", "TypeError: e2_start_returns_list.LabSpider.start() must be an asynchronous generator, i.e. an async def method with yield statements, got ", "2026-10-09 12:33:42 [scrapy.core.engine] INFO: Closing spider (start_error)", "2026-10-09 12:33:42 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.003307959996163845,", " 'finish_reason': 'start_error',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 42, 755763, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/ERROR': 1,", " 'log_count/INFO': 4,", " 'memusage/max': 91217920,", " 'memusage/startup': 91217920,", " 'responses_per_minute': None,", " 'spider_exceptions/TypeError': 1,", " 'spider_exceptions/count': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 42, 752453, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:42 [scrapy.core.engine] INFO: Spider closed (start_error)" ], "proxy_log": [] } }, "ci": { "old": { "command": "venv-2.19/bin/python -m pytest -q -p no:cacheprovider ./ci/test_crawl.py", "returncode": 1, "stats": { "finish_reason": "finished" }, "log": [ "F [100%]", "=================================== FAILURES ===================================", "__________________________ test_spider_sends_requests __________________________", "", " def test_spider_sends_requests():", " process = CrawlerProcess(settings={\"LOG_LEVEL\": \"WARNING\"})", " crawler = process.create_crawler(QuotesSpider)", " process.crawl(crawler)", " process.start()", "> assert crawler.stats.get_value(\"downloader/request_count\", 0) > 0", "E AssertionError: assert 0 > 0", "E + where 0 = ('downloader/request_count', 0)", "E + where = .get_value", "E + where = .stats", "", "./ci/test_crawl.py:11: AssertionError", "------------------------------ Captured log call -------------------------------", "INFO scrapy.utils.log:log.py:222 Scrapy 2.19.0 started (bot: scrapybot)", "INFO scrapy.utils.log:log.py:230 Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "DEBUG scrapy.crawler:crawler.py:858 Using CrawlerProcess", "INFO scrapy.addons:addons.py:49 Enabled addons:", "[]", "DEBUG asyncio:selector_events.py:64 Using selector: EpollSelector", "DEBUG scrapy.utils.log:log.py:236 Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "DEBUG scrapy.utils.log:log.py:238 Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "INFO scrapy.extensions.telnet:telnet.py:64 Telnet Password: 7cb4de834e13fb3d", "INFO scrapy.middleware:middleware.py:106 Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats',", " 'scrapy.extensions.remote_control.RemoteControl']", "INFO scrapy.crawler:crawler.py:214 Overridden settings:", "{'LOG_LEVEL': 'WARNING'}", "INFO scrapy.middleware:middleware.py:106 Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "INFO scrapy.middleware:middleware.py:106 Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware',", " 'scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware']", "INFO scrapy.middleware:middleware.py:106 Enabled item pipelines:", "[]", "INFO scrapy.core.engine:engine.py:553 Spider opened", "INFO scrapy.extensions.logstats:logstats.py:65 Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "INFO scrapy.extensions.telnet:telnet.py:76 Telnet console listening on 127.0.0.1:6023", "INFO scrapy.extensions.remote_control:remote_control.py:178 Remote control HTTP server listening on port 44463 (job 499242-c1b4a70380754a3ebaacf28b4ac7ef1e)", "INFO scrapy.core.engine:engine.py:645 Closing spider (finished)", "INFO scrapy.statscollectors:statscollectors.py:107 Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001959038432687521,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 43, 452962, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'memusage/max': 100294656,", " 'memusage/startup': 100294656,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 43, 451000, tzinfo=datetime.timezone.utc)}", "INFO scrapy.core.engine:engine.py:705 Spider closed (finished)", "=========================== short test summary info ============================", "FAILED ../../.../ci/test_crawl.py::test_spider_sends_requests", "1 failed in 0.42s" ] }, "new": { "command": "venv-2.19/bin/python -m pytest -q -p no:cacheprovider ./ci/test_crawl.py", "returncode": 0, "stats": null, "log": [ ". [100%]", "1 passed in 0.36s" ] } } }, "2.16": { "versions": { "scrapy": "2.16.0", "twisted": "26.4.0", "python": "3.14.4" }, "version_commands": { "scrapy version": [ "Scrapy 2.16.0" ], "python -m pip show scrapy": [ "Name: Scrapy", "Version: 2.16.0" ] }, "cases": { "a_start_requests_only": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/a_start_requests_only.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 3 }, "log": [ "2026-10-09 12:33:45 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:45 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:45 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:45 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:45 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:45 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:45 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:45 [scrapy.extensions.telnet] INFO: Telnet Password: 57664678a899ec0a", "2026-10-09 12:33:45 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:45 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:45 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:45 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:45 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:45 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:45 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:45 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:45 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:45 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.0013318280689418316,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 45, 590171, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/INFO': 3,", " 'memusage/max': 80187392,", " 'memusage/startup': 80187392,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 45, 588835, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:45 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "a2_start_urls_only": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/a2_start_urls_only.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:45 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:45 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:45 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:45 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:45 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:45 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:45 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:45 [scrapy.extensions.telnet] INFO: Telnet Password: 567c34c7e365ffb9", "2026-10-09 12:33:45 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:45 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:46 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:46 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:46 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:46 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:46 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:46 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:46 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:46 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:46 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:46 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:46 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10847482923418283,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 46, 137080, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80158720,", " 'memusage/startup': 80158720,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 46, 28602, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:46 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b_async_start": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/b_async_start.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:46 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:46 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:46 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:46 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:46 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:46 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:46 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:46 [scrapy.extensions.telnet] INFO: Telnet Password: 8e6ab2bfb1af3177", "2026-10-09 12:33:46 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:46 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:46 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:46 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:46 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:46 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:46 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:46 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:46 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:46 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:46 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:46 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:46 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.11125621944665909,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 46, 675154, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80076800,", " 'memusage/startup': 80076800,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 46, 563895, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:46 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b2_start_urls_headers_meta": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/b2_start_urls_headers_meta.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:47 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:47 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:47 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:47 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:47 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:47 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:47 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:47 [scrapy.extensions.telnet] INFO: Telnet Password: 95d9f7e34d483591", "2026-10-09 12:33:47 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:47 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:47 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:47 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:47 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:47 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:47 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:47 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:47 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:47 [lab] INFO: parse() got 200 Accept-Language=en-AU page=1", "2026-10-09 12:33:47 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:47 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:47 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 213,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.11196443997323513,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 47, 224422, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80203776,", " 'memusage/startup': 80203776,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 47, 112454, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:47 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b3_proxy_meta": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/b3_proxy_meta.py -a url=http://127.0.0.1:8047/ -a proxy=http://user:pass@127.0.0.1:8048", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:47 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:47 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:47 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:47 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:47 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:47 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:47 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:47 [scrapy.extensions.telnet] INFO: Telnet Password: 1d4d482a6e9cb26e", "2026-10-09 12:33:47 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:47 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:47 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:47 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:47 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:47 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:47 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:47 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:47 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:47 [lab] INFO: parse() got 200 via http://127.0.0.1:8048", "2026-10-09 12:33:47 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:47 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:47 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 251,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 337,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10948632471263409,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 47, 824586, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80289792,", " 'memusage/startup': 80289792,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 47, 715088, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:47 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [ { "request_line": "GET http://127.0.0.1:8047/ HTTP/1.1", "proxy_authorization": "valid" } ] }, "c_both": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/c_both.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:48 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:48 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:48 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:48 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:48 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:48 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:48 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:48 [scrapy.extensions.telnet] INFO: Telnet Password: 7f7029a9657d0bc9", "2026-10-09 12:33:48 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:48 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:48 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:48 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:48 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:48 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:48 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:48 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:48 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:48 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:48 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:48 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:48 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10823075193911791,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 48, 401407, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80097280,", " 'memusage/startup': 80097280,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 48, 293173, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:48 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "c2_both_delegating": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/c2_both_delegating.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:48 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:48 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:48 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:48 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:48 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:48 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:48 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:48 [scrapy.extensions.telnet] INFO: Telnet Password: ce52fd1b3047b05a", "2026-10-09 12:33:48 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:48 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:48 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:48 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:48 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:48 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:48 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:48 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:48 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:48 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:48 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:48 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:48 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10899003222584724,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 48, 962945, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80154624,", " 'memusage/startup': 80154624,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 48, 853950, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:48 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1_mw_process_start_requests": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/d1_mw_process_start_requests.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:49 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:49 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:49 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:49 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:49 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:49 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:49 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:49 [scrapy.extensions.telnet] INFO: Telnet Password: 815585b8644e2458", "2026-10-09 12:33:49 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:49 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:49 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:49 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:49 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:49 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:49 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:49 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:49 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:49 [lab] INFO: parse() got 200 tagged=None", "2026-10-09 12:33:49 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:49 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:49 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10847331676632166,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 49, 521295, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80203776,", " 'memusage/startup': 80203776,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 49, 412817, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:49 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1b_old_spider_old_mw": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/d1b_old_spider_old_mw.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 3 }, "log": [ "2026-10-09 12:33:49 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:49 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:49 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:49 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:49 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:49 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:49 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:49 [scrapy.extensions.telnet] INFO: Telnet Password: a738e2808490b404", "2026-10-09 12:33:49 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:49 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:50 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:50 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:50 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:50 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:50 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:50 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:50 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:50 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.0021208790130913258,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 50, 14662, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/INFO': 3,", " 'memusage/max': 80031744,", " 'memusage/startup': 80031744,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 50, 12536, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:50 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1c_mw_process_start": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/d1c_mw_process_start.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:50 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:50 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:50 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:50 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:50 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:50 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:50 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:50 [scrapy.extensions.telnet] INFO: Telnet Password: 0a1d38f84012b388", "2026-10-09 12:33:50 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:50 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:50 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:50 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:50 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:50 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:50 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:50 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:50 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:50 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:33:50 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:50 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:50 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10965546825900674,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 50, 584359, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80162816,", " 'memusage/startup': 80162816,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 50, 474700, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:50 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1d_mw_both": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/d1d_mw_both.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:50 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:50 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:50 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:50 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:50 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:50 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:50 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:50 [scrapy.extensions.telnet] INFO: Telnet Password: 5d1849fc5ba58bec", "2026-10-09 12:33:50 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:50 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:51 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:51 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:51 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:51 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:51 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:51 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:51 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:51 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:33:51 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:51 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:51 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10880565410479903,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 51, 172040, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80216064,", " 'memusage/startup': 80216064,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 51, 63231, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:51 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2_mw_sync_output": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/d2_mw_sync_output.py -a url=http://127.0.0.1:8047/", "returncode": 1, "stats": null, "log": [ "2026-10-09 12:33:51 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:51 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:51 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:51 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:51 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:51 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:51 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:51 [scrapy.extensions.telnet] INFO: Telnet Password: cc4dfd604f86f867", "2026-10-09 12:33:51 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:51 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:51 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:51 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:51 [asyncio] ERROR: Task exception was never retrieved", "future: exception=TypeError(\"Middleware d2_mw_sync_output.StampMiddleware doesn't support asynchronous spider output. Its process_spider_output() method should be an async generator function or it should additionally define a process_spider_output_async() method.\")>", "Traceback (most recent call last):", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/crawler.py\", line 576, in _crawl_and_track", " await crawler.crawl_async(*args, **kwargs)", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/crawler.py\", line 221, in crawl_async", " self.engine = self._create_engine()", " ~~~~~~~~~~~~~~~~~~~^^", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/crawler.py\", line 234, in _create_engine", " return ExecutionEngine(self, lambda _: self.stop_async())", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/core/engine.py\", line 149, in __init__", " self.scraper: Scraper = Scraper(crawler)", " ~~~~~~~^^^^^^^^^", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/core/scraper.py\", line 105, in __init__", " self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler(", " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^", " crawler", " ^^^^^^^", " )", " ^", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/middleware.py\", line 114, in from_crawler", " return cls(*middlewares, crawler=crawler)", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/middleware.py\", line 56, in __init__", " self._add_middleware(mw)", " ~~~~~~~~~~~~~~~~~~~~^^^^", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 65, in _add_middleware", " process_spider_output = self._get_process_spider_output(mw)", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 260, in _get_process_spider_output", " raise TypeError(", " ...<4 lines>...", " )", "TypeError: Middleware d2_mw_sync_output.StampMiddleware doesn't support asynchronous spider output. Its process_spider_output() method should be an async generator function or it should additionally define a process_spider_output_async() method." ], "proxy_log": [] }, "d2b_mw_async_output": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/d2b_mw_async_output.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:51 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:51 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:51 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:51 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:51 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:51 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:51 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:51 [scrapy.extensions.telnet] INFO: Telnet Password: a1a0391f4c313278", "2026-10-09 12:33:52 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:52 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:52 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:52 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:52 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:52 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:52 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:52 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:52 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:52 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:52 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': True}", "2026-10-09 12:33:52 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:52 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.11035463726148009,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 52, 216481, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80171008,", " 'memusage/startup': 80171008,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 52, 106122, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:52 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2c_mw_async_output_spider_arg": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/d2c_mw_async_output_spider_arg.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:52 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:52 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:52 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:52 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:52 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:52 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:52 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:52 [scrapy.extensions.telnet] INFO: Telnet Password: ebba83855334d773", "2026-10-09 12:33:52 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:52 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:52 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:52 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:52 [py.warnings] WARNING: venv-2.16/lib/python3.14/site-packages/scrapy/core/spidermw.py:68: ScrapyDeprecationWarning: StampMiddleware.process_spider_output() requires a spider argument, this is deprecated and the argument will not be passed in future Scrapy versions. If you need to access the spider instance you can save the crawler instance passed to from_crawler() and use its spider attribute.", " self._check_mw_method_spider_arg(process_spider_output)", "", "2026-10-09 12:33:52 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:52 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:52 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:52 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:52 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:52 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:52 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': True}", "2026-10-09 12:33:52 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:52 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.11017768271267414,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 52, 803991, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80175104,", " 'memusage/startup': 80175104,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 52, 693807, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:52 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2d_mw_universal": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/d2d_mw_universal.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:53 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:53 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:53 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:53 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:53 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:53 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:53 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:53 [scrapy.extensions.telnet] INFO: Telnet Password: 036956b1b50ccd12", "2026-10-09 12:33:53 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:53 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:53 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:53 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:53 [py.warnings] WARNING: venv-2.16/lib/python3.14/site-packages/scrapy/core/spidermw.py:68: ScrapyDeprecationWarning: StampMiddleware.process_spider_output_async() requires a spider argument, this is deprecated and the argument will not be passed in future Scrapy versions. If you need to access the spider instance you can save the crawler instance passed to from_crawler() and use its spider attribute.", " self._check_mw_method_spider_arg(process_spider_output)", "", "2026-10-09 12:33:53 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:53 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:53 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:53 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:53 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:53 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:53 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': 'async'}", "2026-10-09 12:33:53 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:53 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.11151410592719913,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 53, 398860, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80248832,", " 'memusage/startup': 80248832,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 53, 287342, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:53 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "e1_start_not_async": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/e1_start_not_async.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 3, "log_count/ERROR": 1 }, "log": [ "2026-10-09 12:33:53 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:53 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:53 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:53 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:53 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:53 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:53 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:53 [scrapy.extensions.telnet] INFO: Telnet Password: 45c8aa5dfc29d127", "2026-10-09 12:33:53 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:53 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:53 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:53 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:53 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:53 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:53 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:53 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:53 [scrapy.core.engine] ERROR: Error while reading start items and requests: 'async for' requires an object with __aiter__ method, got generator.", "Traceback (most recent call last):", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/core/engine.py\", line 277, in _process_start_next", " item_or_request = await anext(self._start)", " ^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 1 more time]", "TypeError: 'async for' requires an object with __aiter__ method, got generator", "Traceback (most recent call last):", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/core/engine.py\", line 277, in _process_start_next", " item_or_request = await anext(self._start)", " ^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 1 more time]", "TypeError: 'async for' requires an object with __aiter__ method, got generator", "2026-10-09 12:33:53 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:53 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.0024783462285995483,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 53, 903955, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/ERROR': 1,", " 'log_count/INFO': 3,", " 'memusage/max': 80334848,", " 'memusage/startup': 80334848,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 53, 901470, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:53 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "e2_start_returns_list": { "command": "venv-2.16/bin/python -m scrapy runspider ./spiders/e2_start_returns_list.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 3, "log_count/WARNING": 1, "log_count/ERROR": 1 }, "log": [ "2026-10-09 12:33:54 [scrapy.utils.log] INFO: Scrapy 2.16.0 started (bot: scrapybot)", "2026-10-09 12:33:54 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:54 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:54 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:54 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:54 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:54 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:54 [scrapy.extensions.telnet] INFO: Telnet Password: 59aedb628af43740", "2026-10-09 12:33:54 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:54 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:54 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:54 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:54 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:54 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:54 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:54 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:54 [scrapy.core.engine] ERROR: Error while reading start items and requests: 'async for' requires an object with __aiter__ method, got coroutine.", "Traceback (most recent call last):", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/core/engine.py\", line 277, in _process_start_next", " item_or_request = await anext(self._start)", " ^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 1 more time]", "TypeError: 'async for' requires an object with __aiter__ method, got coroutine", "Traceback (most recent call last):", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/core/engine.py\", line 277, in _process_start_next", " item_or_request = await anext(self._start)", " ^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.16/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 45, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 1 more time]", "TypeError: 'async for' requires an object with __aiter__ method, got coroutine", "2026-10-09 12:33:54 [py.warnings] WARNING: venv-2.16/lib/python3.14/site-packages/scrapy/core/engine.py:283: RuntimeWarning: coroutine 'LabSpider.start' was never awaited", " logger.error(", "", "2026-10-09 12:33:54 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:54 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.002491594757884741,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 54, 363849, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/ERROR': 1,", " 'log_count/INFO': 3,", " 'log_count/WARNING': 1,", " 'memusage/max': 80281600,", " 'memusage/startup': 80281600,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 54, 361353, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:54 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] } }, "ci": { "old": { "command": "venv-2.16/bin/python -m pytest -q -p no:cacheprovider ./ci/test_crawl.py", "returncode": 1, "stats": { "finish_reason": "finished" }, "log": [ "F [100%]", "=================================== FAILURES ===================================", "__________________________ test_spider_sends_requests __________________________", "", " def test_spider_sends_requests():", " process = CrawlerProcess(settings={\"LOG_LEVEL\": \"WARNING\"})", " crawler = process.create_crawler(QuotesSpider)", " process.crawl(crawler)", " process.start()", "> assert crawler.stats.get_value(\"downloader/request_count\", 0) > 0", "E AssertionError: assert 0 > 0", "E + where 0 = ('downloader/request_count', 0)", "E + where = .get_value", "E + where = .stats", "", "./ci/test_crawl.py:11: AssertionError", "------------------------------ Captured log call -------------------------------", "INFO scrapy.utils.log:log.py:188 Scrapy 2.16.0 started (bot: scrapybot)", "INFO scrapy.utils.log:log.py:196 Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '26.4.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "DEBUG scrapy.crawler:crawler.py:754 Using CrawlerProcess", "INFO scrapy.addons:addons.py:49 Enabled addons:", "[]", "DEBUG asyncio:selector_events.py:64 Using selector: EpollSelector", "DEBUG scrapy.utils.log:log.py:202 Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "DEBUG scrapy.utils.log:log.py:204 Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "INFO scrapy.extensions.telnet:telnet.py:64 Telnet Password: fbdab70682fd92a5", "INFO scrapy.middleware:middleware.py:106 Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "INFO scrapy.crawler:crawler.py:150 Overridden settings:", "{'LOG_LEVEL': 'WARNING'}", "INFO scrapy.middleware:middleware.py:106 Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "INFO scrapy.middleware:middleware.py:106 Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "INFO scrapy.middleware:middleware.py:106 Enabled item pipelines:", "[]", "INFO scrapy.core.engine:engine.py:536 Spider opened", "INFO scrapy.extensions.logstats:logstats.py:66 Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "INFO scrapy.extensions.telnet:telnet.py:76 Telnet console listening on 127.0.0.1:6023", "INFO scrapy.core.engine:engine.py:612 Closing spider (finished)", "INFO scrapy.statscollectors:statscollectors.py:93 Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.0013095950707793236,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 54, 890389, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'memusage/max': 89526272,", " 'memusage/startup': 89526272,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 54, 889074, tzinfo=datetime.timezone.utc)}", "INFO scrapy.core.engine:engine.py:674 Spider closed (finished)", "=========================== short test summary info ============================", "FAILED ../../.../ci/test_crawl.py::test_spider_sends_requests", "1 failed in 0.31s" ] }, "new": { "command": "venv-2.16/bin/python -m pytest -q -p no:cacheprovider ./ci/test_crawl.py", "returncode": 0, "stats": null, "log": [ ". [100%]", "1 passed in 0.41s" ] } } }, "2.15": { "versions": { "scrapy": "2.15.2", "twisted": "25.5.0", "python": "3.14.4" }, "version_commands": { "scrapy version": [ "Scrapy 2.15.2" ], "python -m pip show scrapy": [ "Name: Scrapy", "Version: 2.15.2" ] }, "cases": { "a_start_requests_only": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/a_start_requests_only.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:56 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:33:57 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:57 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:57 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:57 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:57 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:57 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:57 [scrapy.extensions.telnet] INFO: Telnet Password: 912470b7af9abb57", "2026-10-09 12:33:57 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:57 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:57 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:57 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:57 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:57 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:57 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py:454: ScrapyDeprecationWarning: a_start_requests_only.LabSpider defines the deprecated start_requests() method. start_requests() has been deprecated in favor of a new method, start(), to support asynchronous code execution. start_requests() will stop being called in a future version of Scrapy. If you use Scrapy 2.13 or higher only, replace start_requests() with start(); note that start() is a coroutine (async def). If you need to maintain compatibility with lower Scrapy versions, when overriding start_requests() in a spider class, override start() as well; you can use super() to reuse the inherited start() implementation without copy-pasting. See the release notes of Scrapy 2.13 for details: https://docs.scrapy.org/en/2.13/news.html", " self._check_deprecated_start_requests_use()", "", "2026-10-09 12:33:57 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:57 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:57 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:57 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:57 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:57 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:57 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.110036,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 57, 209459, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80138240,", " 'memusage/startup': 80138240,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 57, 99423, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:57 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "a2_start_urls_only": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/a2_start_urls_only.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:57 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:33:57 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:57 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:57 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:57 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:57 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:57 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:57 [scrapy.extensions.telnet] INFO: Telnet Password: b2297eadc84f9fd9", "2026-10-09 12:33:57 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:57 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:57 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:57 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:57 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:57 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:57 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:57 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:57 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:57 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:57 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:57 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:57 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.109886,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 57, 825264, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80052224,", " 'memusage/startup': 80052224,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 57, 715378, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:57 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b_async_start": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/b_async_start.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:58 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:33:58 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:58 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:58 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:58 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:58 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:58 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:58 [scrapy.extensions.telnet] INFO: Telnet Password: 3029bdd872d3694d", "2026-10-09 12:33:58 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:58 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:58 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:58 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:58 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:58 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:58 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:58 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:58 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:58 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:33:58 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:58 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:58 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.109044,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 58, 474390, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 79929344,", " 'memusage/startup': 79929344,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 58, 365346, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:58 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b2_start_urls_headers_meta": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/b2_start_urls_headers_meta.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:58 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:33:58 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:58 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:58 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:58 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:58 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:58 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:58 [scrapy.extensions.telnet] INFO: Telnet Password: f97eccf197e26b28", "2026-10-09 12:33:58 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:58 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:58 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:58 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:58 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:58 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:58 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:58 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:58 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:59 [lab] INFO: parse() got 200 Accept-Language=en-AU page=1", "2026-10-09 12:33:59 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:59 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:59 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 213,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.107985,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 59, 66623, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80093184,", " 'memusage/startup': 80093184,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 58, 958638, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:59 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b3_proxy_meta": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/b3_proxy_meta.py -a url=http://127.0.0.1:8047/ -a proxy=http://user:pass@127.0.0.1:8048", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:33:59 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:33:59 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:33:59 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:33:59 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:33:59 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:33:59 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:33:59 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:33:59 [scrapy.extensions.telnet] INFO: Telnet Password: 0f1ef3e3c010cb66", "2026-10-09 12:33:59 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:33:59 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:33:59 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:33:59 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:33:59 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:33:59 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:33:59 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:33:59 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:33:59 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:33:59 [lab] INFO: parse() got 200 via http://127.0.0.1:8048", "2026-10-09 12:33:59 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:33:59 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:33:59 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 251,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 337,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.108973,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 33, 59, 706289, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80171008,", " 'memusage/startup': 80171008,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 33, 59, 597316, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:33:59 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [ { "request_line": "GET http://127.0.0.1:8047/ HTTP/1.1", "proxy_authorization": "valid" } ] }, "c_both": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/c_both.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:00 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:00 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:00 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:00 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:00 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:00 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:00 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:00 [scrapy.extensions.telnet] INFO: Telnet Password: a5b26e4c3825a275", "2026-10-09 12:34:00 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:00 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:00 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:00 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:00 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:00 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:00 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:00 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:00 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:00 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:00 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:00 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.11498,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 0, 429331, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80105472,", " 'memusage/startup': 80105472,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 0, 314351, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:00 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "c2_both_delegating": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/c2_both_delegating.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:00 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:00 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:00 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:00 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:00 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:00 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:00 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:00 [scrapy.extensions.telnet] INFO: Telnet Password: 2d171f1cbe00c388", "2026-10-09 12:34:00 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:00 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:00 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:00 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:00 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:00 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:01 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:01 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:01 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:01 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:01 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:01 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:01 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.112214,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 1, 113152, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 79798272,", " 'memusage/startup': 79798272,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 1, 938, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:01 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1_mw_process_start_requests": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/d1_mw_process_start_requests.py -a url=http://127.0.0.1:8047/", "returncode": 1, "stats": {}, "log": [ "2026-10-09 12:34:01 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:01 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:01 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:01 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:01 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:01 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:01 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:01 [scrapy.extensions.telnet] INFO: Telnet Password: 0b79aa4fd89ca258", "2026-10-09 12:34:01 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:01 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:01 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:01 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:01 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py:64: ScrapyDeprecationWarning: The following enabled spider middlewares, directly or through their parent classes, define the deprecated process_start_requests() method: d1_mw_process_start_requests.TagStartMiddleware. process_start_requests() has been deprecated in favor of a new method, process_start(), to support asynchronous code execution. process_start_requests() will stop being called in a future version of Scrapy. If you use Scrapy 2.13 or higher only, replace process_start_requests() with process_start(); note that process_start() is a coroutine (async def). If you need to maintain compatibility with lower Scrapy versions, when defining process_start_requests() in a spider middleware class, define process_start() as well. See the release notes of Scrapy 2.13 for details: https://docs.scrapy.org/en/2.13/news.html", " self._check_deprecated_process_start_requests_use(middlewares)", "", "2026-10-09 12:34:01 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:01 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:01 [scrapy.core.engine] INFO: Closing spider (shutdown)", "2026-10-09 12:34:01 [scrapy.core.engine] ERROR: Scraper close failure", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/crawler.py\", line 221, in crawl_async", " await self.engine.open_spider_async()", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 540, in open_spider_async", " self._start = await self.scraper.spidermw.process_start()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 454, in process_start", " self._check_deprecated_start_requests_use()", " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 520, in _check_deprecated_start_requests_use", " raise ValueError(", " ...<14 lines>...", " )", "ValueError: d1_mw_process_start_requests.LabSpider does not define the deprecated start_requests() method. However, one or more of your enabled spider middlewares (reported in an earlier deprecation warning) define the process_start_requests() method, and not the process_start() method, making them only compatible with (deprecated) spiders that define the start_requests() method. To solve this issue, disable the offending spider middlewares, upgrade them as described in that earlier deprecation warning, or make your spider compatible with deprecated spider middlewares (and earlier Scrapy versions) by defining a sync start_requests() method that works similarly to its existing start() method. See the release notes of Scrapy 2.13 for details: https://docs.scrapy.org/en/2.13/news.html", "", "During handling of the above exception, another exception occurred:", "", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 629, in close_spider_async", " await self.scraper.close_spider_async()", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/scraper.py\", line 197, in close_spider_async", " raise RuntimeError(\"Scraper slot not assigned\")", "RuntimeError: Scraper slot not assigned", "2026-10-09 12:34:01 [scrapy.core.engine] ERROR: Scheduler close failure", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/crawler.py\", line 221, in crawl_async", " await self.engine.open_spider_async()", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 540, in open_spider_async", " self._start = await self.scraper.spidermw.process_start()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 454, in process_start", " self._check_deprecated_start_requests_use()", " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 520, in _check_deprecated_start_requests_use", " raise ValueError(", " ...<14 lines>...", " )", "ValueError: d1_mw_process_start_requests.LabSpider does not define the deprecated start_requests() method. However, one or more of your enabled spider middlewares (reported in an earlier deprecation warning) define the process_start_requests() method, and not the process_start() method, making them only compatible with (deprecated) spiders that define the start_requests() method. To solve this issue, disable the offending spider middlewares, upgrade them as described in that earlier deprecation warning, or make your spider compatible with deprecated spider middlewares (and earlier Scrapy versions) by defining a sync start_requests() method that works similarly to its existing start() method. See the release notes of Scrapy 2.13 for details: https://docs.scrapy.org/en/2.13/news.html", "", "During handling of the above exception, another exception occurred:", "", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 635, in close_spider_async", " if (d := self._slot.scheduler.close(reason)) is not None:", " ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/scheduler.py\", line 361, in close", " if self.dqs is not None:", " ^^^^^^^^", "AttributeError: 'Scheduler' object has no attribute 'dqs'", "2026-10-09 12:34:01 [scrapy.utils.signal] ERROR: Error caught on signal handler: >", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/utils/signal.py\", line 192, in handler", " robustApply(", " ~~~~~~~~~~~^", " receiver, *arguments, signal=signal, sender=sender, **named", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " ),", " ^", " File \"venv-2.15/lib/python3.14/site-packages/pydispatch/robustapply.py\", line 55, in robustApply", " return receiver(*arguments, **named)", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/extensions/corestats.py\", line 41, in spider_closed", " assert self.start_time is not None", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^", "AssertionError", "2026-10-09 12:34:01 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'items_per_minute': None, 'responses_per_minute': None}", "2026-10-09 12:34:01 [scrapy.core.engine] INFO: Spider closed (shutdown)", "2026-10-09 12:34:01 [asyncio] ERROR: Task exception was never retrieved", "future: ._crawl_and_track() done, defined at venv-2.15/lib/python3.14/site-packages/scrapy/crawler.py:577> exception=ValueError('d1_mw_process_start_requests.LabSpider does not define the deprecated start_requests() method. However, one or more of your enabled spider middlewares (reported in an earlier deprecation warning) define the process_start_requests() method, and not the process_start() method, making them only compatible with (deprecated) spiders that define the start_requests() method. To solve this issue, disable the offending spider middlewares, upgrade them as described in that earlier deprecation warning, or make your spider compatible with deprecated spider middlewares (and earlier Scrapy versions) by defining a sync start_requests() method that works similarly to its existing start() method. See the release notes of Scrapy 2.13 for details: https://docs.scrapy.org/en/2.13/news.html')>", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/crawler.py\", line 579, in _crawl_and_track", " await crawler.crawl_async(*args, **kwargs)", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/crawler.py\", line 221, in crawl_async", " await self.engine.open_spider_async()", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 540, in open_spider_async", " self._start = await self.scraper.spidermw.process_start()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 454, in process_start", " self._check_deprecated_start_requests_use()", " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 520, in _check_deprecated_start_requests_use", " raise ValueError(", " ...<14 lines>...", " )", "ValueError: d1_mw_process_start_requests.LabSpider does not define the deprecated start_requests() method. However, one or more of your enabled spider middlewares (reported in an earlier deprecation warning) define the process_start_requests() method, and not the process_start() method, making them only compatible with (deprecated) spiders that define the start_requests() method. To solve this issue, disable the offending spider middlewares, upgrade them as described in that earlier deprecation warning, or make your spider compatible with deprecated spider middlewares (and earlier Scrapy versions) by defining a sync start_requests() method that works similarly to its existing start() method. See the release notes of Scrapy 2.13 for details: https://docs.scrapy.org/en/2.13/news.html" ], "proxy_log": [] }, "d1b_old_spider_old_mw": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/d1b_old_spider_old_mw.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:02 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:02 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:02 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:02 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:02 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:02 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:02 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:02 [scrapy.extensions.telnet] INFO: Telnet Password: aa86a5e39c5b8487", "2026-10-09 12:34:02 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:02 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:02 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:02 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:02 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py:64: ScrapyDeprecationWarning: The following enabled spider middlewares, directly or through their parent classes, define the deprecated process_start_requests() method: d1b_old_spider_old_mw.TagStartMiddleware. process_start_requests() has been deprecated in favor of a new method, process_start(), to support asynchronous code execution. process_start_requests() will stop being called in a future version of Scrapy. If you use Scrapy 2.13 or higher only, replace process_start_requests() with process_start(); note that process_start() is a coroutine (async def). If you need to maintain compatibility with lower Scrapy versions, when defining process_start_requests() in a spider middleware class, define process_start() as well. See the release notes of Scrapy 2.13 for details: https://docs.scrapy.org/en/2.13/news.html", " self._check_deprecated_process_start_requests_use(middlewares)", "", "2026-10-09 12:34:02 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:02 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:02 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py:454: ScrapyDeprecationWarning: d1b_old_spider_old_mw.LabSpider defines the deprecated start_requests() method. start_requests() has been deprecated in favor of a new method, start(), to support asynchronous code execution. start_requests() will stop being called in a future version of Scrapy. If you use Scrapy 2.13 or higher only, replace start_requests() with start(); note that start() is a coroutine (async def). If you need to maintain compatibility with lower Scrapy versions, when overriding start_requests() in a spider class, override start() as well; you can use super() to reuse the inherited start() implementation without copy-pasting. See the release notes of Scrapy 2.13 for details: https://docs.scrapy.org/en/2.13/news.html", " self._check_deprecated_start_requests_use()", "", "2026-10-09 12:34:02 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:02 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:02 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:02 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:34:02 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:02 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:02 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.111149,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 2, 295139, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80191488,", " 'memusage/startup': 80191488,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 2, 183990, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:02 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1c_mw_process_start": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/d1c_mw_process_start.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:02 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:02 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:02 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:02 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:02 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:02 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:02 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:02 [scrapy.extensions.telnet] INFO: Telnet Password: c283164951f45c77", "2026-10-09 12:34:02 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:02 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:02 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:02 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:02 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:02 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:02 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:02 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:02 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:02 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:34:02 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:02 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:02 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.111236,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 2, 941933, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80228352,", " 'memusage/startup': 80228352,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 2, 830697, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:02 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1d_mw_both": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/d1d_mw_both.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:03 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:03 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:03 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:03 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:03 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:03 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:03 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:03 [scrapy.extensions.telnet] INFO: Telnet Password: 8091688df4ccce9e", "2026-10-09 12:34:03 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:03 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:03 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:03 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:03 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:03 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:03 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:03 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:03 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:03 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:34:03 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:03 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:03 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.113242,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 3, 610107, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80236544,", " 'memusage/startup': 80105472,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 3, 496865, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:03 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2_mw_sync_output": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/d2_mw_sync_output.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:04 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:04 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:04 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:04 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:04 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:04 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:04 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:04 [scrapy.extensions.telnet] INFO: Telnet Password: dc51bc3fdd2c5f42", "2026-10-09 12:34:04 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:04 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:04 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:04 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:04 [scrapy.core.spidermw] WARNING: Middleware d2_mw_sync_output.StampMiddleware doesn't support asynchronous spider output, this is deprecated and will stop working in a future version of Scrapy. The middleware should be updated to support it. Please see https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users for more information.", "2026-10-09 12:34:04 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py:138: ScrapyDeprecationWarning: StampMiddleware.process_spider_output() requires a spider argument, this is deprecated and the argument will not be passed in future Scrapy versions. If you need to access the spider instance you can save the crawler instance passed to from_crawler() and use its spider attribute.", " self._check_mw_method_spider_arg(process_spider_output)", "", "2026-10-09 12:34:04 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:04 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:04 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:04 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:04 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:04 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:04 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': True}", "2026-10-09 12:34:04 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:04 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.111872,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 4, 273929, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80203776,", " 'memusage/startup': 80203776,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 4, 162057, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:04 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2b_mw_async_output": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/d2b_mw_async_output.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:04 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:04 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:04 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:04 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:04 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:04 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:04 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:04 [scrapy.extensions.telnet] INFO: Telnet Password: 419020a4ae5a02a5", "2026-10-09 12:34:04 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:04 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:04 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:04 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:04 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:04 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:04 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:04 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:04 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:04 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:04 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': True}", "2026-10-09 12:34:04 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:04 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.1116,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 4, 964814, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80097280,", " 'memusage/startup': 80097280,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 4, 853214, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:04 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2c_mw_async_output_spider_arg": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/d2c_mw_async_output_spider_arg.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:05 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:05 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:05 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:05 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:05 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:05 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:05 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:05 [scrapy.extensions.telnet] INFO: Telnet Password: 97e890ad08632d64", "2026-10-09 12:34:05 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:05 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:05 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:05 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:05 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py:138: ScrapyDeprecationWarning: StampMiddleware.process_spider_output() requires a spider argument, this is deprecated and the argument will not be passed in future Scrapy versions. If you need to access the spider instance you can save the crawler instance passed to from_crawler() and use its spider attribute.", " self._check_mw_method_spider_arg(process_spider_output)", "", "2026-10-09 12:34:05 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:05 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:05 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:05 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:05 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': True}", "2026-10-09 12:34:05 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:05 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10992,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 5, 648460, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80179200,", " 'memusage/startup': 80179200,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 5, 538540, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:05 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2d_mw_universal": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/d2d_mw_universal.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 4 }, "log": [ "2026-10-09 12:34:06 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:06 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:06 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:06 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:06 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:06 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:06 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:06 [scrapy.extensions.telnet] INFO: Telnet Password: 7324485331ef1d53", "2026-10-09 12:34:06 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:06 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:06 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:06 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:06 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py:141: ScrapyDeprecationWarning: StampMiddleware.process_spider_output() requires a spider argument, this is deprecated and the argument will not be passed in future Scrapy versions. If you need to access the spider instance you can save the crawler instance passed to from_crawler() and use its spider attribute.", " self._check_mw_method_spider_arg(m)", "", "2026-10-09 12:34:06 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/spidermw.py:141: ScrapyDeprecationWarning: StampMiddleware.process_spider_output_async() requires a spider argument, this is deprecated and the argument will not be passed in future Scrapy versions. If you need to access the spider instance you can save the crawler instance passed to from_crawler() and use its spider attribute.", " self._check_mw_method_spider_arg(m)", "", "2026-10-09 12:34:06 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:06 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:06 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:06 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:06 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:06 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:06 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/', 'stamped': 'sync'}", "2026-10-09 12:34:06 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:06 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.113394,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 6, 278106, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/INFO': 4,", " 'memusage/max': 80334848,", " 'memusage/startup': 80334848,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 6, 164712, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:06 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "e1_start_not_async": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/e1_start_not_async.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 3, "log_count/ERROR": 1 }, "log": [ "2026-10-09 12:34:06 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:06 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:06 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:06 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:06 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:06 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:06 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:06 [scrapy.extensions.telnet] INFO: Telnet Password: d1194c3a2b8fd956", "2026-10-09 12:34:06 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:06 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:06 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:06 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:06 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:06 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:06 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:06 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:06 [scrapy.core.engine] ERROR: Error while reading start items and requests: 'async for' requires an object with __aiter__ method, got generator.", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 276, in _process_start_next", " item_or_request = await self._start.__anext__()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 1 more time]", "TypeError: 'async for' requires an object with __aiter__ method, got generator", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 276, in _process_start_next", " item_or_request = await self._start.__anext__()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 1 more time]", "TypeError: 'async for' requires an object with __aiter__ method, got generator", "2026-10-09 12:34:06 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:06 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.002451,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 6, 796692, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/ERROR': 1,", " 'log_count/INFO': 3,", " 'memusage/max': 80023552,", " 'memusage/startup': 80023552,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 6, 794241, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:06 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "e2_start_returns_list": { "command": "venv-2.15/bin/python -m scrapy runspider ./spiders/e2_start_returns_list.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 3, "log_count/WARNING": 1, "log_count/ERROR": 1 }, "log": [ "2026-10-09 12:34:07 [scrapy.utils.log] INFO: Scrapy 2.15.2 started (bot: scrapybot)", "2026-10-09 12:34:07 [scrapy.utils.log] INFO: Versions:", "{'lxml': '6.1.3',", " 'libxml2': '2.14.6',", " 'cssselect': '1.5.0',", " 'parsel': '1.12.1',", " 'w3lib': '2.5.0',", " 'Twisted': '25.5.0',", " 'Python': '3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0]',", " 'pyOpenSSL': '26.4.0 (OpenSSL 4.0.3 29 Sep 2026)',", " 'cryptography': '50.0.2',", " 'Platform': 'Linux-7.0.0-30-generic-x86_64-with-glibc2.43'}", "2026-10-09 12:34:07 [scrapy.crawler] DEBUG: Using AsyncCrawlerProcess", "2026-10-09 12:34:07 [asyncio] DEBUG: Using selector: EpollSelector", "2026-10-09 12:34:07 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:07 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", "2026-10-09 12:34:07 [scrapy.utils.log] DEBUG: Using asyncio event loop: asyncio.unix_events._UnixSelectorEventLoop", "2026-10-09 12:34:07 [scrapy.extensions.telnet] INFO: Telnet Password: 8745c76683b8bbf2", "2026-10-09 12:34:07 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.logcount.LogCount',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:07 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_CLASS': }", "2026-10-09 12:34:07 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:07 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.start.StartSpiderMiddleware',", " 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:07 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:07 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:07 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:07 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:07 [scrapy.core.engine] ERROR: Error while reading start items and requests: 'async for' requires an object with __aiter__ method, got coroutine.", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 276, in _process_start_next", " item_or_request = await self._start.__anext__()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 1 more time]", "TypeError: 'async for' requires an object with __aiter__ method, got coroutine", "Traceback (most recent call last):", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py\", line 276, in _process_start_next", " item_or_request = await self._start.__anext__()", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " File \"venv-2.15/lib/python3.14/site-packages/scrapy/spidermiddlewares/base.py\", line 52, in process_start", " async for o in start:", " if (o := self._get_processed(o, None)) is not None:", " yield o", " [Previous line repeated 1 more time]", "TypeError: 'async for' requires an object with __aiter__ method, got coroutine", "2026-10-09 12:34:07 [py.warnings] WARNING: venv-2.15/lib/python3.14/site-packages/scrapy/core/engine.py:282: RuntimeWarning: coroutine 'LabSpider.start' was never awaited", " logger.error(", "", "2026-10-09 12:34:07 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:07 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.002509,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 7, 501776, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/ERROR': 1,", " 'log_count/INFO': 3,", " 'log_count/WARNING': 1,", " 'memusage/max': 80031744,", " 'memusage/startup': 80031744,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 7, 499267, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:07 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] } }, "ci": { "old": { "command": "venv-2.15/bin/python -m pytest -q -p no:cacheprovider ./ci/test_crawl.py", "returncode": 0, "stats": null, "log": [ ". [100%]", "1 passed in 0.40s" ] }, "new": { "command": "venv-2.15/bin/python -m pytest -q -p no:cacheprovider ./ci/test_crawl.py", "returncode": 0, "stats": null, "log": [ ". [100%]", "1 passed in 0.40s" ] } } }, "2.12": { "versions": { "scrapy": "2.12.0", "twisted": "24.11.0", "python": "3.14.4" }, "version_commands": { "scrapy version": [ "Scrapy 2.12.0" ], "python -m pip show scrapy": [ "Name: Scrapy", "Version: 2.12.0" ] }, "cases": { "a_start_requests_only": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/a_start_requests_only.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 11 }, "log": [ "2026-10-09 12:34:10 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:10 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:10 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:10 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:10 [scrapy.extensions.telnet] INFO: Telnet Password: ec9686f1caa6f48d", "2026-10-09 12:34:10 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:10 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:10 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:10 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:10 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:10 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:10 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:10 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:10 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:10 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:10 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:10 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:10 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.111211,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 10, 584182, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 3,", " 'log_count/INFO': 11,", " 'memusage/max': 76288000,", " 'memusage/startup': 76288000,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 10, 472971, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:10 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "a2_start_urls_only": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/a2_start_urls_only.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 11 }, "log": [ "2026-10-09 12:34:10 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:10 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:10 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:10 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:10 [scrapy.extensions.telnet] INFO: Telnet Password: 8d1a1e5f387c28bf", "2026-10-09 12:34:10 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:10 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:10 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:10 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:10 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:10 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:10 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:10 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:10 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:11 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:11 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:11 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:11 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.10712,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 11, 91113, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 3,", " 'log_count/INFO': 11,", " 'memusage/max': 76226560,", " 'memusage/startup': 76226560,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 10, 983993, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:11 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b_async_start": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/b_async_start.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:11 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:11 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:11 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:11 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:11 [scrapy.extensions.telnet] INFO: Telnet Password: 2aeeae3dfde7645a", "2026-10-09 12:34:11 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:11 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:11 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:11 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:11 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:11 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:11 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:11 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:11 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:11 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001296,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 11, 525060, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76312576,", " 'memusage/startup': 76312576,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 11, 523764, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:11 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b2_start_urls_headers_meta": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/b2_start_urls_headers_meta.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 10, "log_count/ERROR": 1 }, "log": [ "2026-10-09 12:34:11 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:11 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:11 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:11 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:11 [scrapy.extensions.telnet] INFO: Telnet Password: aa71eb91f7b53ee4", "2026-10-09 12:34:11 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:11 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:11 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:11 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:11 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:11 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:11 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:11 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:11 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:12 [scrapy.core.scraper] ERROR: Spider error processing (referer: None)", "Traceback (most recent call last):", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/utils/defer.py\", line 327, in iter_errback", " yield next(it)", " ~~~~^^^^", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/utils/python.py\", line 368, in __next__", " return next(self.data)", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/utils/python.py\", line 368, in __next__", " return next(self.data)", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 106, in process_sync", " yield from iterable", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/spidermiddlewares/referer.py\", line 379, in ", " return (self._set_referer(r, response) for r in result)", " ^^^^^^", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 106, in process_sync", " yield from iterable", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/spidermiddlewares/urllength.py\", line 57, in ", " return (r for r in result if self._filter(r, spider))", " ^^^^^^", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 106, in process_sync", " yield from iterable", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/spidermiddlewares/depth.py\", line 54, in ", " return (r for r in result if self._filter(r, response, spider))", " ^^^^^^", " File \"venv-2.12/lib/python3.14/site-packages/scrapy/core/spidermw.py\", line 106, in process_sync", " yield from iterable", " File \"./spiders/b2_start_urls_headers_meta.py\", line 26, in parse", " response.meta[\"page\"],", " ~~~~~~~~~~~~~^^^^^^^^", "KeyError: 'page'", "2026-10-09 12:34:12 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:12 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.110616,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 12, 52004, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 2,", " 'log_count/ERROR': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76410880,", " 'memusage/startup': 76410880,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'spider_exceptions/KeyError': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 11, 941388, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:12 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "b3_proxy_meta": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/b3_proxy_meta.py -a url=http://127.0.0.1:8047/ -a proxy=http://user:pass@127.0.0.1:8048", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:12 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:12 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:12 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:12 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:12 [scrapy.extensions.telnet] INFO: Telnet Password: d8555af6aece64b7", "2026-10-09 12:34:12 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:12 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:12 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:12 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:12 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:12 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:12 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:12 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:12 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001216,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 12, 505222, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76414976,", " 'memusage/startup': 76414976,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 12, 504006, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:12 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "c_both": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/c_both.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 11 }, "log": [ "2026-10-09 12:34:12 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:12 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:12 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:12 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:12 [scrapy.extensions.telnet] INFO: Telnet Password: 464626dfbcb76184", "2026-10-09 12:34:12 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:12 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:12 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:12 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:12 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:12 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:12 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:12 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:13 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:13 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:13 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:13 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.107718,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 13, 63364, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 3,", " 'log_count/INFO': 11,", " 'memusage/max': 76341248,", " 'memusage/startup': 76341248,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 12, 955646, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:13 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "c2_both_delegating": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/c2_both_delegating.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 11 }, "log": [ "2026-10-09 12:34:13 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:13 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:13 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:13 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:13 [scrapy.extensions.telnet] INFO: Telnet Password: f1fb093fda2295b5", "2026-10-09 12:34:13 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:13 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:13 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:13 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:13 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:13 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:13 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:13 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:13 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:13 [lab] INFO: parse() got 200 http://127.0.0.1:8047/", "2026-10-09 12:34:13 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:13 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:13 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.109701,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 13, 604954, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 3,", " 'log_count/INFO': 11,", " 'memusage/max': 76435456,", " 'memusage/startup': 76435456,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 13, 495253, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:13 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1_mw_process_start_requests": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/d1_mw_process_start_requests.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:13 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:13 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:13 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:13 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:13 [scrapy.extensions.telnet] INFO: Telnet Password: 12e64edbcb580857", "2026-10-09 12:34:13 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:13 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:14 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:14 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:14 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:14 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:14 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:14 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:14 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:14 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001267,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 14, 21052, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76492800,", " 'memusage/startup': 76492800,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 14, 19785, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:14 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1b_old_spider_old_mw": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/d1b_old_spider_old_mw.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 11 }, "log": [ "2026-10-09 12:34:14 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:14 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:14 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:14 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:14 [scrapy.extensions.telnet] INFO: Telnet Password: 7605843ea3f215b5", "2026-10-09 12:34:14 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:14 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:14 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:14 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:14 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:14 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:14 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:14 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:14 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:14 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:34:14 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:14 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:14 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.108176,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 14, 540851, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 3,", " 'log_count/INFO': 11,", " 'memusage/max': 76414976,", " 'memusage/startup': 76414976,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 14, 432675, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:14 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1c_mw_process_start": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/d1c_mw_process_start.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:14 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:14 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:14 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:14 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:14 [scrapy.extensions.telnet] INFO: Telnet Password: b2e1499388d8c231", "2026-10-09 12:34:14 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:14 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:15 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:15 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:15 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:15 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:15 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:15 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:15 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:15 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001298,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 15, 55102, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76292096,", " 'memusage/startup': 76292096,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 15, 53804, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:15 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d1d_mw_both": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/d1d_mw_both.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "downloader/request_count": 1, "downloader/response_count": 1, "response_received_count": 1, "item_scraped_count": 1, "scheduler/enqueued": 1, "finish_reason": "finished", "log_count/INFO": 11 }, "log": [ "2026-10-09 12:34:15 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:15 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:15 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:15 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:15 [scrapy.extensions.telnet] INFO: Telnet Password: 6b2a07a7254da045", "2026-10-09 12:34:15 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:15 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:15 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:15 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:15 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:15 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:15 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:15 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:15 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None)", "2026-10-09 12:34:15 [lab] INFO: parse() got 200 tagged=True", "2026-10-09 12:34:15 [scrapy.core.scraper] DEBUG: Scraped from <200 http://127.0.0.1:8047/>", "{'url': 'http://127.0.0.1:8047/'}", "2026-10-09 12:34:15 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:15 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'downloader/request_bytes': 210,", " 'downloader/request_count': 1,", " 'downloader/request_method_count/GET': 1,", " 'downloader/response_bytes': 385,", " 'downloader/response_count': 1,", " 'downloader/response_status_count/200': 1,", " 'elapsed_time_seconds': 0.109368,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 15, 681182, tzinfo=datetime.timezone.utc),", " 'item_scraped_count': 1,", " 'items_per_minute': None,", " 'log_count/DEBUG': 3,", " 'log_count/INFO': 11,", " 'memusage/max': 76451840,", " 'memusage/startup': 76451840,", " 'response_received_count': 1,", " 'responses_per_minute': None,", " 'scheduler/dequeued': 1,", " 'scheduler/dequeued/memory': 1,", " 'scheduler/enqueued': 1,", " 'scheduler/enqueued/memory': 1,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 15, 571814, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:15 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2_mw_sync_output": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/d2_mw_sync_output.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:15 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:16 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:16 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:16 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:16 [scrapy.extensions.telnet] INFO: Telnet Password: e4575d5c3871e677", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:16 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:16 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:16 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:16 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001202,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 16, 93684, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76443648,", " 'memusage/startup': 76443648,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 16, 92482, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2b_mw_async_output": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/d2b_mw_async_output.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:16 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:16 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:16 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:16 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:16 [scrapy.extensions.telnet] INFO: Telnet Password: b0e8cd88bb4e7610", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:16 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:16 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:16 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:16 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001275,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 16, 546150, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76533760,", " 'memusage/startup': 76533760,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 16, 544875, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2c_mw_async_output_spider_arg": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/d2c_mw_async_output_spider_arg.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:16 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:16 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:16 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:16 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:16 [scrapy.extensions.telnet] INFO: Telnet Password: 2f41a8fb39a22971", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:16 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:16 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:16 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:16 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:16 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001189,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 16, 988790, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76386304,", " 'memusage/startup': 76386304,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 16, 987601, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:16 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "d2d_mw_universal": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/d2d_mw_universal.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:17 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:17 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:17 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:17 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:17 [scrapy.extensions.telnet] INFO: Telnet Password: 5daa964087a209de", "2026-10-09 12:34:17 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:17 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:17 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:17 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " ,", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:17 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:17 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:17 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:17 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:17 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:17 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001183,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 17, 460739, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76492800,", " 'memusage/startup': 76492800,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 17, 459556, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:17 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "e1_start_not_async": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/e1_start_not_async.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:17 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:17 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:17 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:17 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:17 [scrapy.extensions.telnet] INFO: Telnet Password: fcfe1dc9e0392349", "2026-10-09 12:34:17 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:17 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:17 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:17 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:17 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:17 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:17 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:17 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:17 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:17 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001215,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 17, 865483, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76349440,", " 'memusage/startup': 76349440,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 17, 864268, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:17 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] }, "e2_start_returns_list": { "command": "venv-2.12/bin/python -m scrapy runspider ./spiders/e2_start_returns_list.py -a url=http://127.0.0.1:8047/", "returncode": 0, "stats": { "finish_reason": "finished", "log_count/INFO": 10 }, "log": [ "2026-10-09 12:34:18 [scrapy.utils.log] INFO: Scrapy 2.12.0 started (bot: scrapybot)", "2026-10-09 12:34:18 [scrapy.utils.log] INFO: Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "2026-10-09 12:34:18 [scrapy.addons] INFO: Enabled addons:", "[]", "2026-10-09 12:34:18 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.epollreactor.EPollReactor", "2026-10-09 12:34:18 [scrapy.extensions.telnet] INFO: Telnet Password: adaf6845822bc8d0", "2026-10-09 12:34:18 [scrapy.middleware] INFO: Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "2026-10-09 12:34:18 [scrapy.crawler] INFO: Overridden settings:", "{'SPIDER_LOADER_WARN_ONLY': True}", "2026-10-09 12:34:18 [scrapy.middleware] INFO: Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "2026-10-09 12:34:18 [scrapy.middleware] INFO: Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "2026-10-09 12:34:18 [scrapy.middleware] INFO: Enabled item pipelines:", "[]", "2026-10-09 12:34:18 [scrapy.core.engine] INFO: Spider opened", "2026-10-09 12:34:18 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "2026-10-09 12:34:18 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023", "2026-10-09 12:34:18 [scrapy.core.engine] INFO: Closing spider (finished)", "2026-10-09 12:34:18 [scrapy.statscollectors] INFO: Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.001293,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 18, 396923, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/DEBUG': 1,", " 'log_count/INFO': 10,", " 'memusage/max': 76206080,", " 'memusage/startup': 76206080,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 18, 395630, tzinfo=datetime.timezone.utc)}", "2026-10-09 12:34:18 [scrapy.core.engine] INFO: Spider closed (finished)" ], "proxy_log": [] } }, "ci": { "old": { "command": "venv-2.12/bin/python -m pytest -q -p no:cacheprovider ./ci/test_crawl.py", "returncode": 0, "stats": null, "log": [ ". [100%]", "1 passed in 0.39s" ] }, "new": { "command": "venv-2.12/bin/python -m pytest -q -p no:cacheprovider ./ci/test_crawl.py", "returncode": 1, "stats": { "finish_reason": "finished", "log_count/WARNING": 1 }, "log": [ "F [100%]", "=================================== FAILURES ===================================", "__________________________ test_spider_sends_requests __________________________", "", " def test_spider_sends_requests():", " process = CrawlerProcess(settings={\"LOG_LEVEL\": \"WARNING\"})", " crawler = process.create_crawler(QuotesSpider)", " process.crawl(crawler)", " process.start()", "> assert crawler.stats.get_value(\"downloader/request_count\", 0) > 0", "E AssertionError: assert 0 > 0", "E + where 0 = get_value('downloader/request_count', 0)", "E + where get_value = .get_value", "E + where = .stats", "", "./ci/test_crawl.py:11: AssertionError", "----------------------------- Captured stderr call -----------------------------", "2026-10-09 12:34:19 [py.warnings] WARNING: venv-2.12/lib/python3.14/site-packages/scrapy/core/downloader/webclient.py:10: DeprecationWarning: twisted.web.http.HTTPClient was deprecated in Twisted 24.7.0: Use twisted.web.client.Agent instead.", " from twisted.web.http import HTTPClient", "", "------------------------------ Captured log call -------------------------------", "INFO scrapy.utils.log:log.py:173 Scrapy 2.12.0 started (bot: scrapybot)", "INFO scrapy.utils.log:log.py:182 Versions: lxml 6.1.3.0, libxml2 2.14.6, cssselect 1.5.0, parsel 1.12.1, w3lib 2.2.1, Twisted 24.11.0, Python 3.14.4 (main, Aug 20 2026, 10:41:58) [GCC 15.2.0], pyOpenSSL 26.4.0 (OpenSSL 4.0.3 29 Sep 2026), cryptography 50.0.2, Platform Linux-7.0.0-30-generic-x86_64-with-glibc2.43", "INFO scrapy.addons:addons.py:48 Enabled addons:", "[]", "DEBUG scrapy.utils.log:log.py:188 Using reactor: twisted.internet.epollreactor.EPollReactor", "INFO scrapy.extensions.telnet:telnet.py:57 Telnet Password: ba6519ce7abe0042", "INFO scrapy.middleware:middleware.py:101 Enabled extensions:", "['scrapy.extensions.corestats.CoreStats',", " 'scrapy.extensions.telnet.TelnetConsole',", " 'scrapy.extensions.memusage.MemoryUsage',", " 'scrapy.extensions.logstats.LogStats']", "INFO scrapy.crawler:crawler.py:134 Overridden settings:", "{'LOG_LEVEL': 'WARNING'}", "WARNING py.warnings:_py_warnings.py:230 venv-2.12/lib/python3.14/site-packages/scrapy/core/downloader/webclient.py:10: DeprecationWarning: twisted.web.http.HTTPClient was deprecated in Twisted 24.7.0: Use twisted.web.client.Agent instead.", " from twisted.web.http import HTTPClient", "", "INFO scrapy.middleware:middleware.py:101 Enabled downloader middlewares:", "['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware',", " 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',", " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',", " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',", " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',", " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',", " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',", " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',", " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',", " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',", " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',", " 'scrapy.downloadermiddlewares.stats.DownloaderStats']", "INFO scrapy.middleware:middleware.py:101 Enabled spider middlewares:", "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',", " 'scrapy.spidermiddlewares.referer.RefererMiddleware',", " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',", " 'scrapy.spidermiddlewares.depth.DepthMiddleware']", "INFO scrapy.middleware:middleware.py:101 Enabled item pipelines:", "[]", "INFO scrapy.core.engine:engine.py:384 Spider opened", "INFO scrapy.extensions.logstats:logstats.py:65 Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)", "INFO scrapy.extensions.telnet:telnet.py:69 Telnet console listening on 127.0.0.1:6023", "INFO scrapy.core.engine:engine.py:435 Closing spider (finished)", "INFO scrapy.statscollectors:statscollectors.py:61 Dumping Scrapy stats:", "{'elapsed_time_seconds': 0.002551,", " 'finish_reason': 'finished',", " 'finish_time': datetime.datetime(2026, 10, 9, 12, 34, 19, 893277, tzinfo=datetime.timezone.utc),", " 'items_per_minute': None,", " 'log_count/WARNING': 1,", " 'memusage/max': 85315584,", " 'memusage/startup': 85315584,", " 'responses_per_minute': None,", " 'start_time': datetime.datetime(2026, 10, 9, 12, 34, 19, 890726, tzinfo=datetime.timezone.utc)}", "INFO scrapy.core.engine:engine.py:476 Spider closed (finished)", "=========================== short test summary info ============================", "FAILED ../../.../ci/test_crawl.py::test_spider_sends_requests", "1 failed in 0.37s" ] } } } } }