Supervised learning of mnist in the fully connected layer, clustering and evaluating the final stage

#Import required libraries
import keras
from keras.datasets import mnist

import numpy as np
import pandas as pd
import sklearn
#Display plot results in notebook when using Jupyter notebook
import matplotlib.pyplot as plt
%matplotlib inline

Using TensorFlow backend.

feature_dims = range(2, 12)

#Read data with Keras function. Shuffle the data and split it into learning and training data
(x_train, y_train), (x_test, y_test) = mnist.load_data()

#Convert 2D data to numbers
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
#Type conversion
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
#Divide by 255 as a new variable
x_train /= 255
x_test /= 255

# one-Method for hot encoding
from keras.utils.np_utils import to_categorical
#10 classes
num_classes = 10
y_train = y_train.astype('int32')
y_test = y_test.astype('int32')
labels = y_test
# one-hot encoding
y_train = to_categorical(y_train, num_classes)
y_test =  to_categorical(y_test, num_classes)

def fitting(feature_dim, x_train, y_train, x_test, y_test):
    #Use Adam for importing and optimizing required libraries
    from keras.models import Sequential
    from keras.layers import Dense, Dropout
    from keras.optimizers import Adam
    import gc

    #Modeling
    model = Sequential()
    model.add(Dense(512, activation='relu', input_shape=(784,)))
    model.add(Dropout(0.2))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Dense(feature_dim, activation='relu'))  #Added a layer to take out as a feature
    model.add(Dense(10, activation='softmax'))

    model.summary()

    #Batch size, number of epochs
    batch_size = 128
    epochs = 20

    model.compile(loss='categorical_crossentropy',
                optimizer=Adam(),
                metrics=['accuracy'])
    history = model.fit(x_train, y_train,
                        batch_size=batch_size,
                        epochs=epochs,
                        verbose=1,
                        validation_data=(x_test, y_test))
    score = model.evaluate(x_test, y_test, verbose=0)
    print('Test loss:', score[0])
    print('Test accuracy:', score[1])

#    #Accuracy
#    print(history.history.keys())
#    plt.plot(history.history['accuracy'])
#    plt.plot(history.history['val_accuracy'])
#    plt.title('model accuracy')
#    plt.ylabel('accuracy')
#    plt.xlabel('epoch')
#    plt.legend(['train', 'test'], loc='upper left')
#    plt.show()
#    #loss
#    plt.plot(history.history['loss'])
#    plt.plot(history.history['val_loss'])
#    plt.title('model loss')
#    plt.ylabel('loss')
#    plt.xlabel('epoch')
#    plt.legend(['train', 'test'], loc='upper left')
#    plt.show()
    model.pop() #Remove the softmax layer in the final stage and use the feature layer as the final stage.
    model.summary()
    result = model.predict(x_test)
    keras.backend.clear_session() #← This is
    gc.collect()
    from IPython.display import clear_output
    clear_output()
    return (history, model, result)

#model = fitting(10, x_train, y_train, x_test, y_test)
models = [None] * len(feature_dims)
histories = [None] * len(feature_dims)
results = [None] * len(feature_dims)
for i in range(len(feature_dims)):
    (histories[i], models[i], results[i]) = fitting(feature_dims[i], x_train, y_train, x_test, y_test)

#model.save('model/mnist-10')
#model = keras.models.load_model('model/mnist-10')
#for i in range(len(feature_dims)):
#    models[i].pop() #Remove the softmax layer in the final stage and use the feature layer as the final stage.
#    models[i].summary()

#result = model.predict(x_test)
#results = [None] * len(feature_dims)
#for i in range(len(feature_dims)):
#    keras.backend.clear_session()
#    results[i] = models[i].predict(x_test)

def tsne(result):
    #t-Dimensionality reduction with SNE
    from sklearn.manifold import TSNE
    tsne = TSNE(n_components=2, random_state = 0, perplexity = 30, n_iter = 1000)
    return tsne.fit_transform(result)

#tsne = tsne(result)
tsnes = [None] * len(feature_dims)
for i in range(len(feature_dims)):
    tsnes[i] = tsne(results[i])

#df = pd.DataFrame(tsne, columns = ['x', 'y'])
#df['label'] = labels

def km(n_clusters, result):
    # k-Cluster by means
    from sklearn.cluster import KMeans
    return KMeans(n_clusters).fit_predict(result)

#km = km(10, result)
#df['km'] = km
kms = [None] * len(feature_dims)
for i in range(len(feature_dims)):
    kms[i] = km(10, results[i])

def DBSCAN(n_clusters, result):
    from sklearn.cluster import DBSCAN
    db = DBSCAN(eps=0.2, min_samples=n_clusters).fit(result)
    return db.labels_

#dbscan = DBSCAN(20, result)
#df['DBSCAN'] = dbscan

def hierarchy(result):
    from scipy.cluster.hierarchy import linkage, dendrogram
    result1 = linkage(result, 
                  metric = 'braycurtis', 
                  #metric = 'canberra', 
                  #metric = 'chebyshev', 
                  #metric = 'cityblock', 
                  #metric = 'correlation', 
                  #metric = 'cosine', 
                  #metric = 'euclidean', 
                  #metric = 'hamming', 
                  #metric = 'jaccard', 
                  #method= 'single')
                  method = 'average')
                  #method= 'complete')
                  #method='weighted')
    return result1

#hierarchy = hierarchy(result)
#display(hierarchy)

#def cluster_visualization(x, y, label, cluster, method, n_clusters):
def cluster_visualization(x, y, label, cluster):
    plt.figure(figsize = (30, 15))
    plt.subplot(1,2,1)
    plt.scatter(x, y, c=label)
#    for i in range(10):
#        tmp_df = df[df['label'] == i]
#        plt.scatter(tmp_df['x'], tmp_df['y'], label=i)
#    plt.legend(loc='upper left', bbox_to_anchor=(1,1))
    plt.subplot(1,2,2)
    plt.scatter(x, y, c=cluster)
#    for i in range(n_clusters):
#        tmp_df = df[df[method] == i]
#        plt.scatter(tmp_df['x'], tmp_df['y'], label=i)
#    plt.legend(loc='upper left', bbox_to_anchor=(1,1))

for i in range(len(feature_dims)):
    cluster_visualization(tsnes[i][:,0], tsnes[i][:,1], labels, kms[i])

# https://qiita.com/mamika311/items/75c24f6892f85593f7e7
from sklearn.metrics.cluster import adjusted_rand_score
for i in range(len(feature_dims)):
    print("dim:" + str(feature_dims[i]) + " RMI: " + str(adjusted_rand_score(labels, kms[i])))

dim:2 RMI: 0.36573507862590254
dim:3 RMI: 0.49974179932107105
dim:4 RMI: 0.6248257814760337
dim:5 RMI: 0.8225287029746797
dim:6 RMI: 0.8495039832620757
dim:7 RMI: 0.8417680081349097
dim:8 RMI: 0.8423268187793562
dim:9 RMI: 0.8450473012143238
dim:10 RMI: 0.836035505993697
dim:11 RMI: 0.8815919206871302

# https://scikit-learn.org/stable/modules/generated/sklearn.metrics.normalized_mutual_info_score.html
# https://qiita.com/kotap15/items/38289edfe822005e1e44
from sklearn.metrics import normalized_mutual_info_score
#display(normalized_mutual_info_score(labels, df['km']))
for i in range(len(feature_dims)):
    print("dim:" + str(feature_dims[i]) + " NMI: " + str(normalized_mutual_info_score(labels, kms[i])))

dim:2 NMI: 0.5759443563915843
dim:3 NMI: 0.6735454178249051
dim:4 NMI: 0.7745736983918213
dim:5 NMI: 0.8626814016489588
dim:6 NMI: 0.8759626968874756
dim:7 NMI: 0.8766399602087444
dim:8 NMI: 0.8830520742914061
dim:9 NMI: 0.8706715369843739
dim:10 NMI: 0.8721342625213994
dim:11 NMI: 0.8992713472017846

def shilhouette(clusters, x_test):
    from sklearn.metrics import silhouette_samples
    from matplotlib import cm
    plt.figure(figsize = (10, 10))
    cluster_labels=np.unique(clusters)
    n_clusters=cluster_labels.shape[0]
    silhouette_vals=silhouette_samples(x_test,clusters,metric='euclidean')
    y_ax_lower,y_ax_upper=0,0
    yticks=[]
    for i,c in enumerate(cluster_labels):
        c_silhouette_vals=silhouette_vals[clusters==c]
        print(len(c_silhouette_vals))
        c_silhouette_vals.sort()
        y_ax_upper +=len(c_silhouette_vals)
        color=cm.jet(float(i)/n_clusters)
        plt.barh(range(y_ax_lower,y_ax_upper),
                c_silhouette_vals,
                height=1.0,
                edgecolor='none',
                color=color
                )
        yticks.append((y_ax_lower+y_ax_upper)/2.)
        y_ax_lower += len(c_silhouette_vals)

    #If the silhouette coefficient is 1, clustering is good.
    #Also, when the width of the silhouette is equal on average in terms of the number of clusters, it indicates that the entire data can be divided equally.
    #This division width=A possible setting method is to optimize k so that the widths of the silhouette bars are equal and the silhouette coefficient approaches 1..

    #Draw a line at the average position
    silhouette_avg=np.mean(silhouette_vals)
    plt.axvline(silhouette_avg,color="red",linestyle="--")
    plt.ylabel("Cluster")
    plt.xlabel("Silhouette coefficient")

for i in range(len(feature_dims)):
    shilhouette(kms[i], x_test)

[PYTHON] Supervised learning of mnist in the fully connected layer, clustering and evaluating the final stage

Supervised learning of mnist in the fully connected layer, clustering and evaluating the final stage