[PYTHON] I tried to predict the change in snowfall for 2 years by machine learning

In this case, the following data was used as peripheral data.

• Temperature
• Wind speed
• Yesterday's snowfall
• 1 day ago temperature, 2 days ago temperature, 3 days ago temperature
• Wind speed 1 day ago, wind speed 2 days ago, wind speed 3 days ago

Expressed as an image, it looks like this.

[temperature,wind speed,Yesterday's snowfall,1日前のtemperature,2日前のtemperature,3日前のtemperature, 1日前のwind speed, 2日前のwind speed, 3日前のwind speed]→ Snowfall on the day

[temperature,wind speed,Yesterday's snowfall,1日前のtemperature,2日前のtemperature,3日前のtemperature, 1日前のwind speed, 2日前のwind speed, 3日前のwind speed]→ Snowfall on the day

[temperature,wind speed,Yesterday's snowfall,1日前のtemperature,2日前のtemperature,3日前のtemperature, 1日前のwind speed, 2日前のwind speed, 3日前のwind speed]→ Snowfall on the day

....


[temperature,wind speed,Yesterday's snowfall,1日前のtemperature,2日前のtemperature,3日前のtemperature, 1日前のwind speed, 2日前のwind speed, 3日前のwind speed]→ Snowfall on the day

So, based on this, give only the peripheral data and get the predicted value

[temperature,wind speed,Yesterday's snowfall,1日前のtemperature,2日前のtemperature,3日前のtemperature, 1日前のwind speed, 2日前のwind speed, 3日前のwind speed]→ (Predicted amount of snow on the day)

I did it like this. Basically, the data of the forecast target date is given, but only one yesterday's snowfall amount is the data one day before the forecast target date. And it seemed to have the most impact on the data it gave. Well, when you think about it, it's natural.

As I wrote at the beginning, I will use the data for about 7500 days from the data obtained from the Japan Meteorological Agency for learning, predict the change in snow cover for the remaining 2 years, and compare it with the actual change in snow cover.

Try to predict

The actual code looks like this:

snow_forecaster.py

import csv
import numpy as np
from matplotlib import pyplot
from sklearn import linear_model
from sklearn import cross_validation


class SnowForecast:

    def __init__(self):
        u"""Initialize each instance variable"""
        self.model = None    #Generated learning model
        self.data = []       #Array of training data
        self.target = []     #Array of actual snow cover
        self.predicts = []   #Array of predicted values of snowfall
        self.reals = []      #Array of actual snow cover
        self.day_counts = [] #Array of elapsed dates from the start date
        self.date_list = []
        self.record_count = 0

    def load_csv(self):
        u"""Read a CSV file for learning"""
        with open("sample_data/data.csv", "r") as f:
            reader = csv.reader(f)
            accumulation_yesterday0 = 0
            date_yesterday = ""            
            temp_3days = []
            wind_speed_3days = []

            for row in reader:
                if row[4] == "":
                    continue

                daytime = row[0]               # "yyyy/mmdd HH:MM:SS"
                date = daytime.split(" ")[0]   # "yyyy/mm/dd"
                temp = int(float(row[1]))      #temperature. There is a subtle effect
                wind_speed = float(row[7])     #wind speed. There is a subtle effect
                precipitation = float(row[12]) #Precipitation. no effect
                accumulation = int(row[4])     #The amount of snow. The amount of snowfall yesterday has a big impact

                if len(wind_speed_3days) == 3:
                    #Training data
                    # [temperature,wind speed,Yesterday's snowfall,1日前のtemperature,2日前のtemperature,3日前のtemperature, 1日前のwind speed, 2日前のwind speed, 3日前のwind speed]
                    sample = [temp, wind_speed, accumulation_yesterday0]
                    sample.extend(temp_3days)
                    sample.extend(wind_speed_3days)
                    self.data.append(sample)
                    self.target.append(accumulation)

                if date_yesterday != date:
                    accumulation_yesterday0 = accumulation
                    self.date_list.append(date)

                    wind_speed_3days.insert(0, wind_speed)
                    if len(wind_speed_3days) > 3:
                        wind_speed_3days.pop()

                    temp_3days.insert(0, temp)
                    if len(temp_3days) > 3:
                        temp_3days.pop()

                    date_yesterday = date

        self.record_count = len(self.data)
        return self.data

    def train(self):
        u"""Generate a learning model. Use the training data up to about 7500 days of the original data"""
        x = self.data
        y = self.target
        print(len(x))
        # ElasticNetCV,LassoCV,Select Elastic NetCV with the smallest error from RidgeCV
        model = linear_model.ElasticNetCV(fit_intercept=True)
        model.fit(x[0:self.training_data_count()], y[0:self.training_data_count()])
        self.model = model

    def predict(self):
        u"""Predict using a learning model. Forecast for the last two years"""
        x = self.data
        y = self.target
        model = self.model

        for i, xi in enumerate(x):
            real_val = y[i]

            if i < self.training_data_count() + 1:
                self.predicts.append(0)
                self.reals.append(real_val)
                self.day_counts.append(i)
                continue

            predict_val = int(model.predict([xi])[0])

            #If the snowfall forecast is 0 or less, it is set to 0.
            if predict_val < 0:
                predict_val = 0

            self.predicts.append(predict_val)
            self.reals.append(real_val)
            self.day_counts.append(i)

    def show_graph(self):
        u"""Compare predicted and measured values with a graph"""
        pyplot.plot(self.day_counts[self.predict_start_num():], self.reals[self.predict_start_num():], "b")
        pyplot.plot(self.day_counts[self.predict_start_num():], self.predicts[self.predict_start_num():], "r")
        pyplot.show()

    def check(self):
        u"""Measure the error between training data and forecast data"""
        x = np.array(self.data[self.predict_start_num():])
        y = np.array(self.target[self.predict_start_num():])
        model = self.model
        p = np.array(self.predicts[self.predict_start_num():])
        e = p - np.array(self.reals[self.predict_start_num():])
        error = np.sum(e * e)
        rmse_10cv = np.sqrt(error / len(self.data[self.predict_start_num():]))
        print("RMSE(10-fold CV: {})".format(rmse_10cv))

    def training_data_count(self):
        u"""Leave the last two years and use the data before that as training data. Returns the number of training data"""
        return self.record_count - 365 * 2

    def predict_start_num(self):
        u"""The last two years are predicted and used to measure the error from the measured value. Returns the predicted start position"""
        return self.training_data_count() + 1

if __name__ == "__main__":
    forecaster = SnowForecast()
    forecaster.load_csv()
    forecaster.train()
    forecaster.predict()
    forecaster.check()
    forecaster.show_graph()

The most annoying part was creating training data from raw data as in the previous chapter. Still, it's easy because it's python.

So, the execution result is as follows (blue is the actual amount of snow, red line is the predicted amount of snow). This is the first "result 1" shown.

I'm predicting something like that.

At this point, I suddenly wondered how to do this. "But I'm predicting by giving the amount of snow one day ago, so when I actually try to use it for future prediction, I can only predict the amount of snow tomorrow ...?" b>

import csv
import numpy as np
from matplotlib import pyplot
from sklearn import linear_model
from sklearn import cross_validation


class SnowForecast:

    def __init__(self):
        u"""Initialize each instance variable"""
        self.model = None    #Generated learning model
        self.data = []       #Array of training data
        self.target = []     #Array of actual snow cover
        self.predicts = []   #Array of predicted values of snowfall
        self.reals = []      #Array of actual snow cover
        self.day_counts = [] #Array of elapsed dates from the start date
        self.date_list = []
        self.record_count = 0

    def load_csv(self):
        u"""Read a CSV file for learning"""
        with open("sample_data/data.csv", "r") as f:
            reader = csv.reader(f)
            accumulation_yesterday0 = 0
            date_yesterday = ""            
            temp_3days = []
            wind_speed_3days = []

            for row in reader:
                if row[4] == "":
                    continue

                daytime = row[0]               # "yyyy/mmdd HH:MM:SS"
                date = daytime.split(" ")[0]   # "yyyy/mm/dd"
                temp = int(float(row[1]))      #temperature. There is a subtle effect
                wind_speed = float(row[7])     #wind speed. There is a subtle effect
                precipitation = float(row[12]) #Precipitation. no effect
                accumulation = int(row[4])     #The amount of snow. The amount of snowfall yesterday has a big impact

                if len(wind_speed_3days) == 3:
                    #Training data
                    # [temperature,wind speed,Yesterday's snowfall,1日前のtemperature,2日前のtemperature,3日前のtemperature, 1日前のwind speed, 2日前のwind speed, 3日前のwind speed]
                    sample = [temp, wind_speed, accumulation_yesterday0]
                    sample.extend(temp_3days)
                    sample.extend(wind_speed_3days)
                    self.data.append(sample)
                    self.target.append(accumulation)

                if date_yesterday != date:
                    accumulation_yesterday0 = accumulation
                    self.date_list.append(date)

                    wind_speed_3days.insert(0, wind_speed)
                    if len(wind_speed_3days) > 3:
                        wind_speed_3days.pop()

                    temp_3days.insert(0, temp)
                    if len(temp_3days) > 3:
                        temp_3days.pop()

                    date_yesterday = date

        self.record_count = len(self.data)
        return self.data

    def train(self):
        u"""Generate a learning model. Use the training data up to about 7500 days of the original data"""
        x = self.data
        y = self.target
        print(len(x))
        # ElasticNetCV,LassoCV,Select Elastic NetCV with the smallest error from RidgeCV
        model = linear_model.ElasticNetCV(fit_intercept=True)
        model.fit(x[0:self.training_data_count()], y[0:self.training_data_count()])
        self.model = model

    def predict(self):
        u"""Predict the amount of snowfall using a learning model. Forecast for the last two years"""
        x = self.data
        y = self.target
        model = self.model
        yesterday_predict_val = None #Variable to store yesterday's forecast value

        for i, xi in enumerate(x):
            real_val = y[i]

            if i < self.training_data_count() + 1:
                self.predicts.append(0)
                self.reals.append(real_val)
                self.day_counts.append(i)
                continue

            #Replace yesterday's snowfall with yesterday's forecast
            if yesterday_predict_val != None:
                xi[2] = yesterday_predict_val

            predict_val = int(model.predict([xi])[0])

            #If the snowfall forecast is 0 or less, it is set to 0.
            if predict_val < 0:
                predict_val = 0

            self.predicts.append(predict_val)
            self.reals.append(real_val)
            self.day_counts.append(i)
            yesterday_predict_val = predict_val

    def show_graph(self):
        u"""Compare predicted and measured values with a graph"""
        pyplot.plot(self.day_counts[self.predict_start_num():], self.reals[self.predict_start_num():], "b")
        pyplot.plot(self.day_counts[self.predict_start_num():], self.predicts[self.predict_start_num():], "r")
        pyplot.show()

    def check(self):
        u"""Measure the error between training data and forecast data"""
        x = np.array(self.data[self.predict_start_num():])
        y = np.array(self.target[self.predict_start_num():])
        model = self.model
        p = np.array(self.predicts[self.predict_start_num():])
        e = p - np.array(self.reals[self.predict_start_num():])
        error = np.sum(e * e)
        rmse_10cv = np.sqrt(error / len(self.data[self.predict_start_num():]))
        print("RMSE(10-fold CV: {})".format(rmse_10cv))

    def training_data_count(self):
        u"""Leave the last two years and use the data before that as training data. Returns the number of training data"""
        return self.record_count - 365 * 2

    def predict_start_num(self):
        u"""The last two years are predicted and used to measure the error from the measured value. Returns the predicted start position"""
        return self.training_data_count() + 1

if __name__ == "__main__":
    forecaster = SnowForecast()
    forecaster.load_csv()
    forecaster.train()
    forecaster.predict()
    forecaster.check()
    forecaster.show_graph()