[PYTHON] Measure and compare temperature with Raspberry Pi and automatically generate graph

import RPi.GPIO as GPIO
import dht11
import time
import datetime
import os
import numpy as np
import pandas as pd

def add_data(filepath):
    #Check if the csv file exists in the specified path
    if os.path.isfile(filepath):
        #Read if it exists
        df = pd.read_csv(filepath, index_col=0)
    else:
        #Create new if it does not exist
        df = pd.DataFrame([], columns=["year","month","day","hour","sensor_temprature", \
                                       "scraping_temprature","read_try"])
    #Create a variable to count the number of attempts
    count=0
    # is_valid()Read temperature data up to 1029 times until the function is True
    while count<=1029:
        result = instance.read()
        count+=1
        if result.is_valid():
           #Store year, month and day in DataFrame for web scraping
            year = datetime.datetime.now().year
            month = datetime.datetime.now().month
            day = datetime.datetime.now().day
            hour = datetime.datetime.now().hour
            temp = result.temperature
            hum = result.humidity
            data=[year,month,day,hour,temp,hum,"",count]
            s = pd.Series(data, index=df.columns)
            df = df.append(s, ignore_index=True)
            break
    return df

# initialize GPIO
GPIO.setwarnings(True)
GPIO.setmode(GPIO.BCM)

instance = dht11.DHT11(pin=14)

filepath = '/home/pi/Desktop/DHT11_Python/data.csv'
df=add_data(filepath)
df.to_csv(filepath)
print(df)

from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
import requests
import datetime
import os
import sys

#Changing the prec and block codes will change the region.
# m_prec=44 represents Tokyo, m_block=1133 represents Fuchu.
def scraping_weather(m_year,m_month,m_day,m_prec=44,m_block=1133):
    url ="https://www.data.jma.go.jp/obd/stats/etrn/view/hourly_a1.php?prec_no={prec}&block_no={block}&year={year}&month={month}&day={day}&view="
    url = url.format(prec=m_prec,block=m_block,year=m_year,month=m_month,day=m_day)
    html=requests.get(url)
    soup = BeautifulSoup(html.content, "html.parser")

    # id='tablefix2'of<table>Extract
    table = soup.find('table', id='tablefix1')  

    #Extract all th in table2
    th_all = table.find_all('th')  
    #Manually store column titles
    table_column = ["Time", "Precipitation","temperature", "Wind speed / direction(m/s)","日照Time間","snow(cm)","wind speed","Wind direction","降snow","積snow"]

    # <table>Extract all trs in.
    tr_all = table.find_all('tr')  

    #The first tr is already extracted, so skip it
    tr_all = tr_all[1:]  

    #Calculate the number of rows and columns and create an ndarray
    number_of_cols = len(table_column)  
    number_of_rows = len(tr_all)  
    table_data = np.zeros((number_of_rows, number_of_cols), dtype=np.float32)  

    #Store the data of each row in ndarray
    for r, tr in enumerate(tr_all):  
        td_all = tr.find_all('td')  
        for c, td in enumerate(td_all):  
            try:  
                table_data[r,c] = td.string  
            except ValueError:  
                table_data[r,c] = np.nan  

    #Generate a DataFrame of the extracted data
    df = pd.DataFrame(data=table_data, columns=table_column)  
    return df

def combine_scraping_data(df):
    date_before=str(0)
    #Search all data
    for i in range(len(df)):
        #Check if web scraping has been done in the past
        if np.isnan(df.loc[i,"scraping_temprature"]):
            year = df.loc[i,"year"]
            month = df.loc[i,"month"]
            day = df.loc[i,"day"]
            
            #There is no scraping data for today, so skip if it is the same as today
            if ((day == datetime.date.today().day)&(month == datetime.date.today().month)):
                continue
            
            #Substitute date as str type
            date_now = str(year)+str(month)+str(day)
            
            #Check if the previous line and date have changed
            if date_now != date_before:
                #If it has changed, scrape it and store it in DataFrame.
                t_df =  scraping_weather(year,month,day)
                # date_Update date of before variable
                date_before=date_now
            for j in range(len(t_df)):
                #Match the scraped data with the sensor data and substitute the scraped temperature
                if df.loc[i,"hour"] == t_df.loc[j,"Time"]:
                    df.loc[i,"scraping_temprature"] = t_df.loc[j,"temperature"]
    return df


filepath = '/home/pi/Desktop/DHT11_Python/data.csv'

if os.path.isfile(filepath):
    df = pd.read_csv(filepath, index_col=0)
else:
    print("No data. This python wll be stopped.")
    sys.exit()


for i in range(len(df)):
    if df.loc[i,'hour']==0:
        df.loc[i,'hour'] =24
        df.loc[i,'day'] -=1

df = combine_scraping_data(df)

df.to_csv(filepath)
print(df['scraping_temprature'])

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os


filepath = '/home/pi/Desktop/DHT11_Python/data.csv'

#Read csv file with DataFrame
if os.path.isfile(filepath):
    df = pd.read_csv(filepath, index_col=0)
else:
    print("No data. This python wll be stopped.")
    sys.exit()

#Since a daily graph is created, list the month / day data and run a loop.
for g_month in df["month"].unique():
    for g_day in df["day"].unique():
        #Target data t_Load into df
        t_df = df[(df['day']==g_day)&(df['month']==g_month)]
        #Skip if no scraping data exists
        if t_df["scraping_temprature"].isnull().any():
            continue
        
        #Create a new folder
        result_filepath = '/home/pi/Desktop/DHT11_Python/dht11_creating_graph/result/{month}_{day}'
        result_filepath = result_filepath.format(month=g_month, day=g_day)
        os.makedirs(result_filepath, exist_ok=True)
        
        #Create and save the path to save the csv file
        result_filename = '/{month}_{day}.csv'
        result_filename = result_filename.format(month=g_month, day=g_day)
        t_df.to_csv(result_filepath+result_filename)
        
        #Create a path to store the graph
        result_graphname = '/{month}_{day}.png'
        result_graphname = result_graphname.format(month=g_month, day=g_day)
        
        #Create a graph
        x=t_df['hour']
        y1=t_df['sensor_temprature']
        y2=t_df['scraping_temprature']
        
        fig = plt.figure()
        p1 = plt.plot(x, y1, linewidth=2)
        p2 = plt.plot(x, y2, linewidth=2, linestyle="dashed")
        plt.legend((p1[0], p2[0]), ('sensor_temprature', 'scraping_temprature'), loc=2)
        plt.xlabel("hour")
        plt.ylabel("temprature")
        
        #Save the graph
        fig.savefig(result_filepath+result_graphname)
        
        #Delete the graphed data from the original data
        df = df[(df['day']!=g_day)|(df['month']!=g_month)]
        #Reindex the original data
        df.reset_index(inplace=True, drop=True)

#Save the original data
df.to_csv(filepath)