WEB scraping with Python (for personal notes)

pip install requests

pip install beautifulsoup4

import requests
from bs4 import BeautifulSoup

#URL you want to get
url = "*********"

#Get HTML by sending HTTP request with url as argument
response = requests.get(url)

#Character code is automatically encoded
response.encoding = response.apparent_encoding

#HTML parsing
bs = BeautifulSoup(response.text, 'html.parser')
#Extract title
title_tag = bs.find('title')
print(title_tag.text)
#Extract h2 element
h2_tags = bs.select('h2')
for h2_tag in h2_tags:
    print(h2_tag.text)

Recommended Posts