random sample wayback links
+import random
+
from city_scrapers_core.items import Meeting
from scrapy_wayback_middleware import WaybackMiddleware
@@ -10,9 +12,13 @@ class CityScrapersWaybackMiddleware(WaybackMiddleware):
if "legistar" in item["source"] and "Calendar.aspx" not in item["source"]:
links = [item["source"]]
links.extend(
- [link.get("href") for link in item.get("links", [])][:MAX_LINKS]
+ random.sample(
+ [link.get("href") for link in item.get("links", [])], MAX_LINKS
+ )
)
return links
if isinstance(item, dict):
- return [doc.get("url") for doc in item.get("documents", [])][:MAX_LINKS]
+ return random.sample(
+ [doc.get("url") for doc in item.get("documents", [])], MAX_LINKS
+ )
return []
