#!/usr/bin/env python
# coding: utf-8
# # TensorFlow__MLP_MNIST-labeled100__pollenjp
#
# - 論文
# - [Semi-Supervised Learning with Ladder Networks - Antti Rasmus, Harri Valpola, Mikko Honkala, Mathias Berglund, Tapani Raiko](https://arxiv.org/abs/1507.02672)
#
# - code
# - [rinuboney/ladder](https://github.com/rinuboney/ladder)
# - [tarvaina/tensorflow-ladder](https://github.com/tarvaina/tensorflow-ladder)
#
#
Table of Contents
#
# ## Config
# ### Import
# In[1]:
import os,sys
print(sys.version)
import re
from pathlib import Path
import math
# In[2]:
# OPTIONAL: Load the "autoreload" extension so that code can change
get_ipython().run_line_magic('load_ext', 'autoreload')
# OPTIONAL: always reload modules so that as you change code in src, it gets loaded
get_ipython().run_line_magic('autoreload', '2')
# If you want to reload manually, add a below line head.
get_ipython().run_line_magic('aimport', '')
# ref: https://ipython.org/ipython-doc/3/config/extensions/autoreload.html
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import numpy as np
import sklearn
from sklearn import datasets
import tqdm
import csv
import pandas as pd
seed = None
np.random.seed(seed=seed)
print("numpy ver: {}".format(np.__version__))
print("scikit-learn ver: {}".format(sklearn.__version__))
print("pandas ver: {}".format(pd.__version__))
# ### TensorFlow, Keras
# In[3]:
#____________________________________________________________________________________________________
# TensorFlow and Keras GPU configures
##________________________________________________________________________________
## OPTIONAL : set a GPU viewed by TensorFlow
###____________________________________________________________
### - https://stackoverflow.com/questions/37893755/tensorflow-set-cuda-visible-devices-within-jupyter
import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"]="1"
##________________________________________________________________________________
##________________________________________________________________________________
## TensorFlow
###____________________________________________________________
import tensorflow as tf
print("tensorflow ver: {}".format(tf.__version__))
### eager mode
#tf.enable_eager_execution()
print("tf.executing_eagerly(): {}".format(tf.executing_eagerly()))
# You can double check that you have the correct devices visible to TF
# - https://stackoverflow.com/questions/37893755/tensorflow-set-cuda-visible-devices-within-jupyter
from tensorflow.python.client import device_lib
print("""
________________________________________
Visible GPUs from TensorFlow
________________________________________""")
for _device in device_lib.list_local_devices():
match = re.search(pattern=r'name: "/device:(?P[A-Z]{3}):(?P\d{1})*',
string=str(_device))
if match is None:
print("Not Match")
continue
if match.group("name") == "CPU":
name, device_num = match.group("name", "device_num")
print()
print("({}:{})".format(name, device_num))
continue
name, device_num = match.group("name", "device_num")
match = re.search(pattern=r'.*pci bus id: (?P\d{4}:\d{2}:\d{2}.\d{1}).*',
string=str(_device))
if match is None:
print("No GPUs")
continue
print("({}:{}: pci_bus_id: {})".format(name, device_num, match.group("pci_bus_id")))
print("________________________________________")
###____________________________________________________________
### sessioin
global _SESSION
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)
config.gpu_options.allow_growth = True
_SESSION = tf.Session(config=config)
###____________________________________________________________
##________________________________________________________________________________
#____________________________________________________________________________________________________
import keras
import keras.backend as K
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)
config.gpu_options.allow_growth = True
_SESSION = tf.Session(config=config)
K.set_session(_SESSION)
# ### Path
# In[4]:
HOME = Path(os.getcwd()).parent
print(HOME)
# In[5]:
path_list = []
data_Path = HOME / "data"
path_list.append(data_Path)
raw_Path = data_Path / "raw"
path_list.append(raw_Path)
plot_images_Path = data_Path / "plot_images"
path_list.append(plot_images_Path)
src_Path = HOME / "src"
path_list.append(src_Path)
for _Path in path_list:
_path = str(_Path)
if not os.path.exists(_path):
os.makedirs(name=_path)
print("make a directory: \n\t", _path)
else:
print(os.path.exists(_path), ": ", _path)
# ### Import from Local
# In[6]:
sys.path.append(str(src_Path))
# In[7]:
from data import mnist_input_data
from utils_tensorflow.tensorflow_graph_in_jupyer import show_computational_graph
# ## Load MNIST
# In[8]:
num_labeled = 100
print( "=== Loading Data ===" )
mnist = mnist_input_data.read_data_sets(train_dir=str(raw_Path / "MNIST_data"), n_labeled=num_labeled,
fake_data=False, one_hot=True)
# In[9]:
X_train = mnist.semi_train.labeled_ds.images
y_train = mnist.semi_train.labeled_ds.labels
X_test = mnist.semi_test.images
y_test = mnist.semi_test.labels
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)
# In[10]:
y_train[:10]
y_train.sum(axis=0)
# ## Simple MLP
# In[11]:
layer_sizes = [784, 1000, 500, 250, 250, 250, 10]
L = len(layer_sizes) - 1 # number of layers
num_examples = 60000
num_epochs = 300
starter_learning_rate = 0.02
decay_after = 15 # epoch after which to begin learning rate decay
batch_size = 100
num_iter = (num_examples//batch_size) * num_epochs # number of loop iterations
# In[12]:
from keras.models import Sequential
from keras import layers
from keras import activations
# In[13]:
K.clear_session()
layer_sizes = [784, 1000, 500, 250, 250, 250, 10]
model = Sequential()
model.add(layer=layers.Dense(units=layer_sizes[1], activation="relu", input_shape=(layer_sizes[0], )))
model.add(layer=layers.BatchNormalization())
model.add(layer=layers.Dense(units=layer_sizes[2], activation="relu"))
model.add(layer=layers.BatchNormalization())
model.add(layer=layers.Dense(units=layer_sizes[3], activation="relu"))
model.add(layer=layers.BatchNormalization())
model.add(layer=layers.Dense(units=layer_sizes[4], activation="relu"))
model.add(layer=layers.BatchNormalization())
model.add(layer=layers.Dense(units=layer_sizes[5], activation="relu"))
model.add(layer=layers.BatchNormalization())
model.add(layer=layers.Dense(units=layer_sizes[6], activation="softmax"))
# In[14]:
model.summary()
# In[15]:
from keras import optimizers
# In[16]:
learning_rate = 0.02
metrics = ["accuracy"]
opt = optimizers.Adam(lr=learning_rate)
model.compile(optimizer=opt, loss="categorical_crossentropy", metrics=metrics)
# ### fit
# In[17]:
model.fit(x=X_train, y=y_train, batch_size=batch_size, epochs=num_epochs,
validation_data=(X_test[:], y_test[:]) )
# ### Show Computational Graph
# In[18]:
show_computational_graph(graph_def=tf.get_default_graph())
# In[ ]:
# ## MLP (Batch Normalization)
# In[27]:
K.clear_session()
# In[28]:
layer_sizes = [784, 1000, 500, 250, 250, 250, 10]
L = len(layer_sizes) - 1 # number of layers
num_examples = 60000
num_epochs = 150
starter_learning_rate = 0.02
decay_after = 15 # epoch after which to begin learning rate decay
batch_size = 100
num_iter = (num_examples//batch_size) * num_epochs # number of loop iterations
# In[ ]:
# In[ ]:
# In[ ]:
# In[ ]:
# ### Session
# In[31]:
print( "=== Starting Session ===" )
sess = tf.Session()
# In[41]:
i_iter = 0
ckpt = tf.train.get_checkpoint_state('checkpoints/') # get latest checkpoint (if any)
if ckpt and ckpt.model_checkpoint_path:
# if checkpoint exists, restore the parameters and set epoch_n and i_iter
saver.restore(sess, ckpt.model_checkpoint_path)
epoch_n = int(ckpt.model_checkpoint_path.split('-')[1])
# EDITED
# https://github.com/rinuboney/ladder/issues/12#issuecomment-332497271
#i_iter = (epoch_n+1) * (num_examples/batch_size)
i_iter = (epoch_n+1) * (num_examples//batch_size)
print( "Restored Epoch ", epoch_n )
else:
# no checkpoint exists. create checkpoints directory if it does not exist.
if not os.path.exists('checkpoints'):
os.makedirs('checkpoints')
init = tf.global_variables_initializer()
sess.run(init)
print( "=== Training ===" )
print( "Initial Accuracy: {}%", sess.run(accuracy,
feed_dict={
inputs: mnist.semi_test.images,
outputs: mnist.semi_test.labels,
training: False}) )
# In[ ]:
for i in tqdm.tqdm(range(i_iter, num_iter)):
images, labels = mnist.semi_train.next_batch(batch_size)
sess.run(train_step, feed_dict={inputs: images, outputs: labels, training: True})
if (i > 1) and ((i+1) % (num_iter//num_epochs) == 0):
epoch_n = i//(num_examples//batch_size)
if (epoch_n+1) >= decay_after:
# decay learning rate
# learning_rate = starter_learning_rate * ((num_epochs - epoch_n) / (num_epochs - decay_after))
ratio = 1.0 * (num_epochs - (epoch_n+1)) # epoch_n + 1 because learning rate is set for next epoch
ratio = max(0, ratio / (num_epochs - decay_after))
sess.run(learning_rate.assign(starter_learning_rate * ratio))
#saver.save(sess, 'checkpoints/model.ckpt', epoch_n)
# print( "Epoch ", epoch_n, ", Accuracy: ", sess.run(accuracy, feed_dict={inputs: mnist.test.images, outputs:mnist.test.labels, training: False}), "%" )
#with open('train_log', 'a') as train_log:
# # write test accuracy to file "train_log"
# train_log_w = csv.writer(train_log)
# log_i = [epoch_n] + sess.run([accuracy], feed_dict={inputs: mnist.test.images, outputs: mnist.test.labels, training: False})
# train_log_w.writerow(log_i)
print( "Final Accuracy: ", sess.run(accuracy,
feed_dict={
inputs: mnist.semi_test.images,
outputs: mnist.semi_test.labels,
training: False}), "%" )
sess.close()
# ## show Result
# In[ ]:
# In[ ]:
# In[ ]:
# In[ ]:
# In[ ]:
# ## End