[PYTHON] Visualisez les détails des plaintes des compagnies d'assurance-vie

introduction

Visualisation du nombre de plaintes des compagnies d'assurance-vie suite

import pathlib
from urllib.parse import urljoin

import pandas as pd
import pdfplumber
import requests
from bs4 import BeautifulSoup


def fetch_file(url, dir="."):

    r = requests.get(url)
    r.raise_for_status()

    p = pathlib.Path(dir, pathlib.PurePath(url).name)
    p.parent.mkdir(parents=True, exist_ok=True)

    with p.open(mode="wb") as fw:
        fw.write(r.content)
    return p


url = "https://www.seiho.or.jp/member/complaint/"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
}

r = requests.get(url, headers=headers)
r.raise_for_status()

soup = BeautifulSoup(r.content, "html.parser")

dfs = []

for tag in soup.select("div.headMod04.mt30"):

    name = tag.h3.get_text(strip=True).replace("Co., Ltd.", "")

    href = (
        tag.find_next_sibling("table", class_="mt10")
        .find("a", text="Détails de la répartition des réclamations")
        .get("href")
    )

    link = urljoin(url, href)

    p = fetch_file(link)

    with pdfplumber.open(p) as pdf:

        p0 = pdf.pages[0]

        table = p0.extract_table()

    df0 = pd.DataFrame(table)

    #Supprimer les sauts de ligne, les espaces et les virgules
    df0 = df0.applymap(lambda s: "".join(s.split()).replace(",", "") if s else None)
    df0 = df0.mask(df0.isna())

    df0.iloc[0] = df0.iloc[0].fillna(method="ffill")
    df0.iloc[:, 0] = df0.iloc[:, 0].fillna(method="ffill")

    #DataFrame ne peut pas être converti correctement, alors convertissez-le une fois en CSV

    p_csv = p.with_suffix(".csv")
    df0.to_csv(p_csv, index=False, header=False)

    df1 = (
        pd.read_csv(p_csv, index_col=[0, 1], header=[0, 1])
        .loc[:, pd.IndexSlice[:, "Total"]]
        .droplevel(level=1, axis=1)
    )

    df2 = df1.reindex(df1.index.drop("Total", level=1, errors="raise"))

    df2[name] = df2.sum(axis=1)

    dfs.append(df2[name])

df = pd.concat(dfs, axis=1)

df

import japanize_matplotlib
import matplotlib as mpl
import matplotlib.pyplot as plt

mpl.rcParams["figure.dpi"] = 200

df.T.plot.barh(stacked=True, figsize=(15, 15))