make update_server_list scrape 1st 25 pages of mc-lists.org for servers
@@ -25,10 +25,13 @@ class Minecraft(commands.Cog):
@tasks.loop(minutes=10)
async def update_server_list(self):
- async with self.ses.get('https://mc-lists.org') as res:
- soup = bs(await res.text(), 'html.parser')
+ self.bot.logger.INFO('Scraping mc-lists.org...')
servers_nice = []
+
+ for i in range(1, 26):
+ async with self.ses.get(f'https://mc-lists.org/pg.{i}') as res:
+ soup = bs(await res.text(), 'html.parser')
elems = soup.find(class_='ui striped table servers serversa').find_all('tr')
for elem in elems:
@@ -39,6 +42,8 @@ class Minecraft(commands.Cog):
self.server_list = list(set(servers_nice))
+ self.bot.logger.INFO('Finished scraping mc-lists.org')
+
@update_server_list.before_loop
async def before_update_server_list(self):
await self.bot.wait_until_ready()
