#!/usr/bin/env python # coding: utf-8 # # TensorFlow__MLP_MNIST-labeled100__pollenjp # # - 論文 # - [Semi-Supervised Learning with Ladder Networks - Antti Rasmus, Harri Valpola, Mikko Honkala, Mathias Berglund, Tapani Raiko](https://arxiv.org/abs/1507.02672) # # - code # - [rinuboney/ladder](https://github.com/rinuboney/ladder) # - [tarvaina/tensorflow-ladder](https://github.com/tarvaina/tensorflow-ladder) # #

Table of Contents

#
# ## Config # ### Import # In[1]: import os,sys print(sys.version) import re from pathlib import Path import math # In[2]: # OPTIONAL: Load the "autoreload" extension so that code can change get_ipython().run_line_magic('load_ext', 'autoreload') # OPTIONAL: always reload modules so that as you change code in src, it gets loaded get_ipython().run_line_magic('autoreload', '2') # If you want to reload manually, add a below line head. get_ipython().run_line_magic('aimport', '') # ref: https://ipython.org/ipython-doc/3/config/extensions/autoreload.html import matplotlib.pyplot as plt get_ipython().run_line_magic('matplotlib', 'inline') import numpy as np import sklearn from sklearn import datasets import tqdm import csv import pandas as pd seed = None np.random.seed(seed=seed) print("numpy ver: {}".format(np.__version__)) print("scikit-learn ver: {}".format(sklearn.__version__)) print("pandas ver: {}".format(pd.__version__)) # ### TensorFlow, Keras # In[3]: #____________________________________________________________________________________________________ # TensorFlow and Keras GPU configures ##________________________________________________________________________________ ## OPTIONAL : set a GPU viewed by TensorFlow ###____________________________________________________________ ### - https://stackoverflow.com/questions/37893755/tensorflow-set-cuda-visible-devices-within-jupyter import os os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" # see issue #152 os.environ["CUDA_VISIBLE_DEVICES"]="1" ##________________________________________________________________________________ ##________________________________________________________________________________ ## TensorFlow ###____________________________________________________________ import tensorflow as tf print("tensorflow ver: {}".format(tf.__version__)) ### eager mode #tf.enable_eager_execution() print("tf.executing_eagerly(): {}".format(tf.executing_eagerly())) # You can double check that you have the correct devices visible to TF # - https://stackoverflow.com/questions/37893755/tensorflow-set-cuda-visible-devices-within-jupyter from tensorflow.python.client import device_lib print(""" ________________________________________ Visible GPUs from TensorFlow ________________________________________""") for _device in device_lib.list_local_devices(): match = re.search(pattern=r'name: "/device:(?P[A-Z]{3}):(?P\d{1})*', string=str(_device)) if match is None: print("Not Match") continue if match.group("name") == "CPU": name, device_num = match.group("name", "device_num") print() print("({}:{})".format(name, device_num)) continue name, device_num = match.group("name", "device_num") match = re.search(pattern=r'.*pci bus id: (?P\d{4}:\d{2}:\d{2}.\d{1}).*', string=str(_device)) if match is None: print("No GPUs") continue print("({}:{}: pci_bus_id: {})".format(name, device_num, match.group("pci_bus_id"))) print("________________________________________") ###____________________________________________________________ ### sessioin global _SESSION config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True) config.gpu_options.allow_growth = True _SESSION = tf.Session(config=config) ###____________________________________________________________ ##________________________________________________________________________________ #____________________________________________________________________________________________________ import keras import keras.backend as K config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True) config.gpu_options.allow_growth = True _SESSION = tf.Session(config=config) K.set_session(_SESSION) # ### Path # In[4]: HOME = Path(os.getcwd()).parent print(HOME) # In[5]: path_list = [] data_Path = HOME / "data" path_list.append(data_Path) raw_Path = data_Path / "raw" path_list.append(raw_Path) plot_images_Path = data_Path / "plot_images" path_list.append(plot_images_Path) src_Path = HOME / "src" path_list.append(src_Path) for _Path in path_list: _path = str(_Path) if not os.path.exists(_path): os.makedirs(name=_path) print("make a directory: \n\t", _path) else: print(os.path.exists(_path), ": ", _path) # ### Import from Local # In[6]: sys.path.append(str(src_Path)) # In[7]: from data import mnist_input_data from utils_tensorflow.tensorflow_graph_in_jupyer import show_computational_graph # ## Load MNIST # In[8]: num_labeled = 100 print( "=== Loading Data ===" ) mnist = mnist_input_data.read_data_sets(train_dir=str(raw_Path / "MNIST_data"), n_labeled=num_labeled, fake_data=False, one_hot=True) # In[9]: X_train = mnist.semi_train.labeled_ds.images y_train = mnist.semi_train.labeled_ds.labels X_test = mnist.semi_test.images y_test = mnist.semi_test.labels print(X_train.shape) print(y_train.shape) print(X_test.shape) print(y_test.shape) # In[10]: y_train[:10] y_train.sum(axis=0) # ## Simple MLP # In[11]: layer_sizes = [784, 1000, 500, 250, 250, 250, 10] L = len(layer_sizes) - 1 # number of layers num_examples = 60000 num_epochs = 300 starter_learning_rate = 0.02 decay_after = 15 # epoch after which to begin learning rate decay batch_size = 100 num_iter = (num_examples//batch_size) * num_epochs # number of loop iterations # In[12]: from keras.models import Sequential from keras import layers from keras import activations # In[13]: K.clear_session() layer_sizes = [784, 1000, 500, 250, 250, 250, 10] model = Sequential() model.add(layer=layers.Dense(units=layer_sizes[1], activation="relu", input_shape=(layer_sizes[0], ))) model.add(layer=layers.BatchNormalization()) model.add(layer=layers.Dense(units=layer_sizes[2], activation="relu")) model.add(layer=layers.BatchNormalization()) model.add(layer=layers.Dense(units=layer_sizes[3], activation="relu")) model.add(layer=layers.BatchNormalization()) model.add(layer=layers.Dense(units=layer_sizes[4], activation="relu")) model.add(layer=layers.BatchNormalization()) model.add(layer=layers.Dense(units=layer_sizes[5], activation="relu")) model.add(layer=layers.BatchNormalization()) model.add(layer=layers.Dense(units=layer_sizes[6], activation="softmax")) # In[14]: model.summary() # In[15]: from keras import optimizers # In[16]: learning_rate = 0.02 metrics = ["accuracy"] opt = optimizers.Adam(lr=learning_rate) model.compile(optimizer=opt, loss="categorical_crossentropy", metrics=metrics) # ### fit # In[17]: model.fit(x=X_train, y=y_train, batch_size=batch_size, epochs=num_epochs, validation_data=(X_test[:], y_test[:]) ) # ### Show Computational Graph # In[18]: show_computational_graph(graph_def=tf.get_default_graph()) # In[ ]: # ## MLP (Batch Normalization) # In[27]: K.clear_session() # In[28]: layer_sizes = [784, 1000, 500, 250, 250, 250, 10] L = len(layer_sizes) - 1 # number of layers num_examples = 60000 num_epochs = 150 starter_learning_rate = 0.02 decay_after = 15 # epoch after which to begin learning rate decay batch_size = 100 num_iter = (num_examples//batch_size) * num_epochs # number of loop iterations # In[ ]: # In[ ]: # In[ ]: # In[ ]: # ### Session # In[31]: print( "=== Starting Session ===" ) sess = tf.Session() # In[41]: i_iter = 0 ckpt = tf.train.get_checkpoint_state('checkpoints/') # get latest checkpoint (if any) if ckpt and ckpt.model_checkpoint_path: # if checkpoint exists, restore the parameters and set epoch_n and i_iter saver.restore(sess, ckpt.model_checkpoint_path) epoch_n = int(ckpt.model_checkpoint_path.split('-')[1]) # EDITED # https://github.com/rinuboney/ladder/issues/12#issuecomment-332497271 #i_iter = (epoch_n+1) * (num_examples/batch_size) i_iter = (epoch_n+1) * (num_examples//batch_size) print( "Restored Epoch ", epoch_n ) else: # no checkpoint exists. create checkpoints directory if it does not exist. if not os.path.exists('checkpoints'): os.makedirs('checkpoints') init = tf.global_variables_initializer() sess.run(init) print( "=== Training ===" ) print( "Initial Accuracy: {}%", sess.run(accuracy, feed_dict={ inputs: mnist.semi_test.images, outputs: mnist.semi_test.labels, training: False}) ) # In[ ]: for i in tqdm.tqdm(range(i_iter, num_iter)): images, labels = mnist.semi_train.next_batch(batch_size) sess.run(train_step, feed_dict={inputs: images, outputs: labels, training: True}) if (i > 1) and ((i+1) % (num_iter//num_epochs) == 0): epoch_n = i//(num_examples//batch_size) if (epoch_n+1) >= decay_after: # decay learning rate # learning_rate = starter_learning_rate * ((num_epochs - epoch_n) / (num_epochs - decay_after)) ratio = 1.0 * (num_epochs - (epoch_n+1)) # epoch_n + 1 because learning rate is set for next epoch ratio = max(0, ratio / (num_epochs - decay_after)) sess.run(learning_rate.assign(starter_learning_rate * ratio)) #saver.save(sess, 'checkpoints/model.ckpt', epoch_n) # print( "Epoch ", epoch_n, ", Accuracy: ", sess.run(accuracy, feed_dict={inputs: mnist.test.images, outputs:mnist.test.labels, training: False}), "%" ) #with open('train_log', 'a') as train_log: # # write test accuracy to file "train_log" # train_log_w = csv.writer(train_log) # log_i = [epoch_n] + sess.run([accuracy], feed_dict={inputs: mnist.test.images, outputs: mnist.test.labels, training: False}) # train_log_w.writerow(log_i) print( "Final Accuracy: ", sess.run(accuracy, feed_dict={ inputs: mnist.semi_test.images, outputs: mnist.semi_test.labels, training: False}), "%" ) sess.close() # ## show Result # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # ## End