From 21447d25f3d374d01cd3c207f82436543906fbab Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 26 Oct 2026 08:00:05 +0100 Subject: [PATCH 1/2] test: deflake test_stall_waits_for_in_flight_work --- .../crawlers/_basic/test_basic_crawler.py | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index c9e191e935..3b9ad4a762 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -37,7 +37,12 @@ ) from crawlee.events import Event, EventCrawlerStatusData, LocalEventManager from crawlee.http_clients import HttpClient -from crawlee.request_loaders import RequestList, RequestManagerTandem, ThrottlingRequestManager +from crawlee.request_loaders import ( + RequestList, + RequestManagerTandem, + ThrottlingRequestManager, + _throttling_request_manager, +) from crawlee.sessions import Session, SessionPool from crawlee.statistics import FinalStatistics, StatisticsState from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient @@ -51,6 +56,7 @@ from yarl import URL from crawlee._types import JsonSerializable + from crawlee.storage_clients.models import ProcessedRequest async def test_processes_requests_from_explicit_queue() -> None: @@ -2834,11 +2840,35 @@ async def stop_after_stall() -> None: assert stall_reasons[0] is not None -async def test_stall_waits_for_in_flight_work() -> None: +async def test_stall_waits_for_in_flight_work(monkeypatch: pytest.MonkeyPatch) -> None: """A stall isn't raised while a request is in flight, so the work it enqueues is still crawled.""" - throttler = await open_throttler() - # An overloaded system still runs `min_concurrency` tasks, so the throttled request keeps being dispatched while the - # slow one is in flight. + # The throttler runs on a stopped clock, so a backoff can't run out before the crawler checks whether the domain is + # held back. Each deferral moves the clock one backoff forward until the domain has rate-limited for longer than the + # stall window. Then the clock stops, and the throttled request sits out its cooldown in a stalled domain. + backoff = timedelta(minutes=1) + max_domain_stall = backoff * 1.5 + clock = Mock() + clock.now.return_value = datetime.now(timezone.utc) + monkeypatch.setattr(_throttling_request_manager, 'datetime', clock) + throttler = ThrottlingRequestManager( + await RequestQueue.open(), + domains=['throttled.placeholder.com'], + request_manager_opener=RequestQueue.open, + base_delay=backoff, + max_delay=backoff, + max_domain_stall=max_domain_stall, + ) + started_at = clock.now.return_value + reclaim_request = throttler.reclaim_request + + async def reclaim_after_backoff(request: Request, *, forefront: bool = False) -> ProcessedRequest | None: + if clock.now.return_value - started_at <= max_domain_stall: + clock.now.return_value += backoff + return await reclaim_request(request, forefront=forefront) + + monkeypatch.setattr(throttler, 'reclaim_request', reclaim_after_backoff) + # An overloaded system still runs `min_concurrency` tasks, so the throttled request is dispatched while the slow one + # is in flight. crawler = BasicCrawler( request_manager=throttler, concurrency_settings=ConcurrencySettings(min_concurrency=2, desired_concurrency=2), From ab1e102a7c1888065abdbb4ea2aaab5ad4faf6d3 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 26 Oct 2026 08:32:44 +0100 Subject: [PATCH 2/2] test: reuse open_throttler_held_until_reclaim in the in-flight stall test --- .../crawlers/_basic/test_basic_crawler.py | 35 ++----------------- tests/unit/utils.py | 9 ++++- 2 files changed, 11 insertions(+), 33 deletions(-) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 3b9ad4a762..b47666434a 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -37,12 +37,7 @@ ) from crawlee.events import Event, EventCrawlerStatusData, LocalEventManager from crawlee.http_clients import HttpClient -from crawlee.request_loaders import ( - RequestList, - RequestManagerTandem, - ThrottlingRequestManager, - _throttling_request_manager, -) +from crawlee.request_loaders import RequestList, RequestManagerTandem, ThrottlingRequestManager from crawlee.sessions import Session, SessionPool from crawlee.statistics import FinalStatistics, StatisticsState from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient @@ -56,7 +51,6 @@ from yarl import URL from crawlee._types import JsonSerializable - from crawlee.storage_clients.models import ProcessedRequest async def test_processes_requests_from_explicit_queue() -> None: @@ -2842,31 +2836,8 @@ async def stop_after_stall() -> None: async def test_stall_waits_for_in_flight_work(monkeypatch: pytest.MonkeyPatch) -> None: """A stall isn't raised while a request is in flight, so the work it enqueues is still crawled.""" - # The throttler runs on a stopped clock, so a backoff can't run out before the crawler checks whether the domain is - # held back. Each deferral moves the clock one backoff forward until the domain has rate-limited for longer than the - # stall window. Then the clock stops, and the throttled request sits out its cooldown in a stalled domain. - backoff = timedelta(minutes=1) - max_domain_stall = backoff * 1.5 - clock = Mock() - clock.now.return_value = datetime.now(timezone.utc) - monkeypatch.setattr(_throttling_request_manager, 'datetime', clock) - throttler = ThrottlingRequestManager( - await RequestQueue.open(), - domains=['throttled.placeholder.com'], - request_manager_opener=RequestQueue.open, - base_delay=backoff, - max_delay=backoff, - max_domain_stall=max_domain_stall, - ) - started_at = clock.now.return_value - reclaim_request = throttler.reclaim_request - - async def reclaim_after_backoff(request: Request, *, forefront: bool = False) -> ProcessedRequest | None: - if clock.now.return_value - started_at <= max_domain_stall: - clock.now.return_value += backoff - return await reclaim_request(request, forefront=forefront) - - monkeypatch.setattr(throttler, 'reclaim_request', reclaim_after_backoff) + # By its third dispatch, the throttled request's domain has rate-limited for longer than the stall window. + throttler = await open_throttler_held_until_reclaim(monkeypatch, max_domain_stall=timedelta(minutes=1.5)) # An overloaded system still runs `min_concurrency` tasks, so the throttled request is dispatched while the slow one # is in flight. crawler = BasicCrawler( diff --git a/tests/unit/utils.py b/tests/unit/utils.py index aaa15dba87..5709b4e82c 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -68,11 +68,15 @@ async def read_stream() -> AsyncIterator[bytes]: async def open_throttler_held_until_reclaim( monkeypatch: pytest.MonkeyPatch, + *, + max_domain_stall: timedelta = timedelta(minutes=15), ) -> ThrottlingRequestManager[RequestQueue]: """Open a throttler for `throttled.placeholder.com` whose backoff lasts until the crawler gives a request back. The throttler runs on a stopped clock, so a backoff can't run out before the crawler checks whether the domain is held back. Each reclaimed request moves the clock past the backoff, so a deferred request is dispatched right away. + Once the domain has rate-limited for longer than `max_domain_stall`, reclaims stop moving the clock, so a deferred + request sits out its cooldown in a stalled domain. """ backoff = timedelta(minutes=1) clock = Mock() @@ -84,11 +88,14 @@ async def open_throttler_held_until_reclaim( request_manager_opener=RequestQueue.open, base_delay=backoff, max_delay=backoff, + max_domain_stall=max_domain_stall, ) + started_at = clock.now.return_value reclaim_request = throttler.reclaim_request async def reclaim_after_backoff(request: Request, *, forefront: bool = False) -> ProcessedRequest | None: - clock.now.return_value += backoff + if clock.now.return_value - started_at <= max_domain_stall: + clock.now.return_value += backoff return await reclaim_request(request, forefront=forefront) monkeypatch.setattr(throttler, 'reclaim_request', reclaim_after_backoff)