[PYTHON] Visualize the number of complaints from life insurance companies

Introduction

[Python] Representing the number of complaints from life insurance companies in a bar graph

Created with reference to

Continued

Convert PDF of details of complaint breakdown to csv with pdfplumber and visualize Visualization of details of life insurance company complaints

Scraping

import requests
from bs4 import BeautifulSoup

url = "https://www.seiho.or.jp/member/complaint/"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
}

r = requests.get(url, headers=headers)
r.raise_for_status()

soup = BeautifulSoup(r.content, "html.parser")

result = []

for tag in soup.select("div.headMod04.mt30"):

    data = []

    data.append(tag.h3.get_text(strip=True).replace("Co., Ltd.", ""))

    for table in tag.find_next_siblings("table", class_=["mt10", "mt15"], limit=2):

        for td in table.select("tr > td.taR"):

            f = float(td.get_text(strip=True).rstrip("subject%").replace(",", ""))

            data.append(int(f) if f.is_integer() else f)

    result.append(data)

import pandas as pd

df = pd.DataFrame(
    result,
    columns=[
        "Insurance company name",
        "Number of complaints",
        "New contract relationship",
        "Storage related",
        "Conservation relations",
        "Insurance money",
        "Other",
        "Number of in-force contracts",
        "Number of customers",
    ],
).set_index("Insurance company name")

df

Visualization

import japanize_matplotlib
import matplotlib as mpl
import matplotlib.pyplot as plt

mpl.rcParams["figure.dpi"] = 200

Number of complaints

df["Number of complaints"].plot.barh(figsize=(15, 15))

Complaint ratio

df["Complaint ratio"] = df["Number of complaints"] / df["Number of customers"]

df["Complaint ratio"].plot.barh(figsize=(15, 15))

Breakdown of complaints

df1 = df.reindex(columns = ["New contract relationship", "Storage related", "Conservation relations", "Insurance money", "Other"])

df1.plot.barh(stacked=True, figsize=(15, 15))

Number of in-force contracts

Number of customers

Average number of contracts per person