[PYTHON] Scraping PDF of the status of test positives in each prefecture of the Ministry of Health, Labor and Welfare

apt install python3-tk ghostscript
pip install camelot-py[cv]
pip install requests
pip install beautifulsoup4

import re
from urllib.parse import urljoin

import camelot
import pandas as pd

import requests
from bs4 import BeautifulSoup

def get_link(url, text):

    r = requests.get(url)
    r.raise_for_status()

    soup = BeautifulSoup(r.content, "html.parser")

    tag = soup.find("a", text=re.compile(text))

    link = urljoin(url, tag.get("href"))

    return link

def set_col(df, n = 1):

    if n > 1:
        columns = ["".join(i) for i in zip(*(df.head(n).values))]

    else:
        columns = df.iloc[0]

    return df.iloc[n:].set_axis(columns, axis=1).reset_index(drop=True)

url = get_link(
    "https://www.mhlw.go.jp/stf/seisakunitsuite/bunya/0000121431_00086.html",
    "^Current status of new coronavirus infection and response by the Ministry of Health, Labor and Welfare",
)

link = get_link(url, "Attachment 1")

tables = camelot.read_pdf(link, pages="all", split_text=True, strip_text="\n", )

df1 = set_col(tables[0].df, 2)
df2 = set_col(tables[1].df)

df = pd.concat([df1, df2], axis=1)

df.columns = df.columns.str.replace("\s", "").str.replace("※\d", "")

df["Name of prefectures"] = df["Name of prefectures"].str.replace("\s", "").str.replace("※\d", "")

df = df.apply(lambda x: x.str.replace(",", ""))

df.mask(df == "-", inplace=True)

df.to_csv("corona.csv", encoding="utf_8_sig")