Python
비동기로 웹크롤링
앗사비
2023. 4. 1. 22:38
728x90
from bs4 import BeautifulSoup as bs
import aiohttp
import asyncio
async def fetch(session, url, i):
async with session.get(url) as res:
html = await res.text()
soup = bs(html, "html.parser")
tag = soup.find("span", class_="title") # 첫번째 글제목만
print(f"{i+1} : {tag.text}")
async def main():
BASE_URL = "https://myinbox.tistory.com/"
urls = [f"{BASE_URL}?page={i}" for i in range(1, 11)]
async with aiohttp.ClientSession() as session:
await asyncio.gather(*[fetch(session, url, i) for i, url in enumerate(urls)])
if __name__ == "__main__":
asyncio.run(main())728x90