[PYTHON] Scraping the Excel file of the list of stores handling regional coupons

Scraping the Excel file of List of stores handling common coupons

Scraping

import datetime
import re
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup

url = "https://biz.goto.jata-net.or.jp/couponlist.html"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
}

r = requests.get(url, headers=headers)
r.raise_for_status()

soup = BeautifulSoup(r.content, "html.parser")

area = {}

for a in soup.select("section.download_couponlist div.button_area > a"):

    s = a.get("aria-label")
    m = re.search("List of stores handling regional coupons_(.+)（(\d{4})Year(\d{1,2})Month(\d{1,2})As of the date)", s)

    year, month, day = map(int, m.group(2, 3, 4))

    area[m.group(1)] = {
        "link": urljoin(url, a.get("href")),
        "date": datetime.date(year, month, day),
    }

area

Data wrangling

import pandas as pd

# area.keys()
# dict_keys(['Hokkaido', 'Tohoku region', 'Kanto region', 'Chubu region', 'Kinki', 'Chugoku region', 'Shikoku region', 'Kyushu / Okinawa region', 'Nationwide'])

df0 = pd.read_excel(area["Nationwide"]["link"])

df0["Paper coupon"] = df0["Coupon type"].str.contains("paper", na=False).astype(int)
df0["Electronic coupon"] = df0["Coupon type"].str.contains("Electronic", na=False).astype(int)

industries = df0["Industry"].str.split(".", expand=True).rename(columns={0: "Industryコード", 1:"Industry名"})

industries["Industry code"] = industries["Industry code"].astype(int)

df1 = pd.concat([df0, industries], axis=1)

pd.crosstab(df1["Industry name"], df1["Prefectures"])