[PYTHON] Visualize details of life insurance company complaints

Introduction

Visualization of the number of complaints from life insurance companies continued

import pathlib
from urllib.parse import urljoin

import pandas as pd
import pdfplumber
import requests
from bs4 import BeautifulSoup


def fetch_file(url, dir="."):

    r = requests.get(url)
    r.raise_for_status()

    p = pathlib.Path(dir, pathlib.PurePath(url).name)
    p.parent.mkdir(parents=True, exist_ok=True)

    with p.open(mode="wb") as fw:
        fw.write(r.content)
    return p


url = "https://www.seiho.or.jp/member/complaint/"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
}

r = requests.get(url, headers=headers)
r.raise_for_status()

soup = BeautifulSoup(r.content, "html.parser")

dfs = []

for tag in soup.select("div.headMod04.mt30"):

    name = tag.h3.get_text(strip=True).replace("Co., Ltd.", "")

    href = (
        tag.find_next_sibling("table", class_="mt10")
        .find("a", text="Details of the breakdown of complaints")
        .get("href")
    )

    link = urljoin(url, href)

    p = fetch_file(link)

    with pdfplumber.open(p) as pdf:

        p0 = pdf.pages[0]

        table = p0.extract_table()

    df0 = pd.DataFrame(table)

    #Remove line breaks, whitespace and commas
    df0 = df0.applymap(lambda s: "".join(s.split()).replace(",", "") if s else None)
    df0 = df0.mask(df0.isna())

    df0.iloc[0] = df0.iloc[0].fillna(method="ffill")
    df0.iloc[:, 0] = df0.iloc[:, 0].fillna(method="ffill")

    #DataFrame cannot be converted well, so convert it to CSV once

    p_csv = p.with_suffix(".csv")
    df0.to_csv(p_csv, index=False, header=False)

    df1 = (
        pd.read_csv(p_csv, index_col=[0, 1], header=[0, 1])
        .loc[:, pd.IndexSlice[:, "Total"]]
        .droplevel(level=1, axis=1)
    )

    df2 = df1.reindex(df1.index.drop("Total", level=1, errors="raise"))

    df2[name] = df2.sum(axis=1)

    dfs.append(df2[name])

df = pd.concat(dfs, axis=1)

df

import japanize_matplotlib
import matplotlib as mpl
import matplotlib.pyplot as plt

mpl.rcParams["figure.dpi"] = 200

df.T.plot.barh(stacked=True, figsize=(15, 15))