test: deflake test_stall_waits_for_in_flight_work

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -2834,11 +2834,12 @@ async def stop_after_stall() -> None:
     assert stall_reasons[0] is not None
 
 
-async def test_stall_waits_for_in_flight_work() -> None:
+async def test_stall_waits_for_in_flight_work(monkeypatch: pytest.MonkeyPatch) -> None:
     """A stall isn't raised while a request is in flight, so the work it enqueues is still crawled."""
-    throttler = await open_throttler()
-    # An overloaded system still runs `min_concurrency` tasks, so the throttled request keeps being dispatched while the
-    # slow one is in flight.
+    # By its third dispatch, the throttled request's domain has rate-limited for longer than the stall window.
+    throttler = await open_throttler_held_until_reclaim(monkeypatch, max_domain_stall=timedelta(minutes=1.5))
+    # An overloaded system still runs `min_concurrency` tasks, so the throttled request is dispatched while the slow one
+    # is in flight.
     crawler = BasicCrawler(
         request_manager=throttler,
         concurrency_settings=ConcurrencySettings(min_concurrency=2, desired_concurrency=2),

tests/unit/utils.py

@@ -68,11 +68,15 @@ async def read_stream() -> AsyncIterator[bytes]:
 
 async def open_throttler_held_until_reclaim(
     monkeypatch: pytest.MonkeyPatch,
+    *,
+    max_domain_stall: timedelta = timedelta(minutes=15),
 ) -> ThrottlingRequestManager[RequestQueue]:
     """Open a throttler for `throttled.placeholder.com` whose backoff lasts until the crawler gives a request back.
 
     The throttler runs on a stopped clock, so a backoff can't run out before the crawler checks whether the domain is
     held back. Each reclaimed request moves the clock past the backoff, so a deferred request is dispatched right away.
+    Once the domain has rate-limited for longer than `max_domain_stall`, reclaims stop moving the clock, so a deferred
+    request sits out its cooldown in a stalled domain.
     """
     backoff = timedelta(minutes=1)
     clock = Mock()
@@ -84,11 +88,14 @@ async def open_throttler_held_until_reclaim(
         request_manager_opener=RequestQueue.open,
         base_delay=backoff,
         max_delay=backoff,
+        max_domain_stall=max_domain_stall,
     )
+    started_at = clock.now.return_value
     reclaim_request = throttler.reclaim_request
 
     async def reclaim_after_backoff(request: Request, *, forefront: bool = False) -> ProcessedRequest | None:
-        clock.now.return_value += backoff
+        if clock.now.return_value - started_at <= max_domain_stall:
+            clock.now.return_value += backoff
         return await reclaim_request(request, forefront=forefront)
 
     monkeypatch.setattr(throttler, 'reclaim_request', reclaim_after_backoff)