import os
from nlp4kor.config import MNIST_DATA_DIR, MNIST_CNN_MODEL_DIR

mnist_data = MNIST_DATA_DIR # input
mnist_model = MNIST_CNN_MODEL_DIR # output
if not os.path.exists(mnist_model):
    os.makedirs(mnist_model)
    print('%s created' % mnist_model)
print('mnist_data:', mnist_data) # input
print('mnist_model:', mnist_model) # output

!ls -lh ~/workspace/nlp4kor-mnist/MNIST_data

!echo $HOME

# Ubuntu16.04 + Anaconda3(4.3) + Python3.5 + Tensorflow1.1 + Jupyter Notebook
!lsb_release -a | grep --color=never "Description"
!conda info | grep --color=never "environment\|conda version\|python"
!pip freeze | grep --color=never "tensorflow\|jupyterthemes"

# Jupyter 기본 환경
!grep "^[^#;]" /home/bage/.jupyter/jupyter_notebook_config.py # Ubuntu
#!grep "^[^#;]" /Users/bage/.jupyter/jupyter_notebook_config.py # OSX

# Jupyter - 자주 쓰는 명령은 자동실행되도록 설정해 두었습니다.
!grep "^[^#;]" /home/bage/.ipython/profile_default/ipython_config.py 

# 자주 쓰는 명령은 자동실행
!cat /home/bage/.ipython/bage_notebook_config.py 

!grep "^[^ #;]" /home/bage/.config/matplotlib/matplotlibrc # Ubuntu
# !grep "^[^ #;]" /Users/bage/.matplotlib/matplotlibrc # OSX

# from matplotlib import gridspec
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets(mnist_data, one_hot=True)

# mnist gz 파일 읽기 (binary 라서 안 보임.)
!zcat MNIST_data/train-images-idx3-ubyte.gz | head

# mnist Datasets 객체의 타입 확인
print(type(mnist))

# mnist Datasets(mnist) 객체의 메소드 훑어 보기
[('mnist.%s' % k, type(eval('mnist.%s' % k))) for k in dir(mnist) if not k.startswith('_')]

# mnist Dataset(mnist.test) 객체의 메소드 훑어 보기
[('mnist.test.%s' % k, type(eval('mnist.test.%s' % k))) for k in dir(mnist.test) if not k.startswith('_')]

type(mnist.test.images), type(mnist.test.labels)

pixels = np.array([[0.5+ i*0.001]*11 for i in range(0, 11)])
label = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0] # one hot vector
print(label, '->', np.argmax(label))
print(pixels.min(), '~', pixels.max())
pprint.pprint(pixels)

show_image(pixels, np.argmax(label), smoothing=True, relative_color=True) # default imshow()

show_image(pixels, np.argmax(label), smoothing=False, relative_color=False) # for image

show_image(pixels, np.argmax(label), smoothing=False, relative_color=True) # for kernel

import matplotlib.image as mpimg
nlp4kor_logo = mpimg.imread('https://github.com/bage79/nlp4kor/blob/master/ipynb/img/nlp4kor.png?raw=trueg')
print(nlp4kor_logo.min(), '~', nlp4kor_logo.max())

show_image(nlp4kor_logo, title='nlp4kor', smoothing=False)
show_image(nlp4kor_logo, title='nlp4kor', smoothing=True, relative_color=True)
show_image(nlp4kor_logo, title='nlp4kor', smoothing=True, relative_color=False)

# check input data
n_sample, height, width, channels = 2, 28, 28, 1 # NHWC
sample_images = mnist.test.images[:n_sample]
sample_labels = mnist.test.labels[:n_sample]
print(sample_images.min(), '~', sample_images.max())
print(sample_images.shape, 'mnist.test.images', sample_images[0][350:400])
print(sample_labels.shape, 'mnist.test.labels', sample_labels[0])

print(sample_images.shape, 'mnist.test.images')
sample_images = sample_images.reshape((n_sample, height, width, channels)) # NHWC
print(sample_images.shape, 'NHWC')

for i, image in enumerate(sample_images):
    image = image.reshape((height, width))
    show_image(image, np.argmax(sample_labels[i]), cmap='Greys') # default 1=black
    show_image(image, np.argmax(sample_labels[i]), cmap='gray') # 1=white

def virtual_kernels(height, width, input_channels, output_channels):
    """
    가상의 커널(feature map)을 생성 (HWNC)
    """
    colors = np.linspace(0, 1, output_channels)
    print('colors:', len(colors), colors)
    
    images = np.ndarray((output_channels, height, width, input_channels)) # CHWN
    for nth, image in enumerate(images):
        image.fill(colors[nth]) # same color in same channel.
    return np.moveaxis(images, 0, 3) # HWC

kernels = virtual_kernels(5, 5, 1, 32)
print(kernels.shape)

show_images(kernels, title='test kernels', is_kernel=True, subtitles=[], relative_color=True) 
# default view on your PC

show_images(kernels, title='test kernels', is_kernel=True, subtitles=[], relative_color=False) 

_w = weight_variable([5, 5, 1, 32]) # HWNC
print(_w)

# tf.session
_w = weight_variable([5, 5, 1, 32]) # HWNC

with tf.Session() as sess: # 세션 초기화, 종료 자동화 (with = __enter__, __exit__)
    sess.run(tf.global_variables_initializer())
    w = sess.run(_w) # ndarray
    
#     w = _w.eval() # InteractiveSession() 에서만 되는거 아닌가?
w = w.flatten() # for plotting
plt.plot(w, 'bo') 

# tf.interative session
sess = tf.InteractiveSession() 
_w = weight_variable([5, 5, 1, 32])
_w.initializer.run()
w = _w.eval() # ndarray
sess.close() # 사용후, close()를 반드시 호출해 줄 것.

w = w.flatten()
plt.plot(w, 'bo')

w = run_node(weight_variable([5, 5, 1, 32])).flatten()
plt.plot(w, 'bo')

def weight_variable2(shape): # random_normal
    initial = tf.random_normal(shape=shape, mean=0.0, stddev=0.1) 
    return tf.Variable(initial)
w2 = run_node(weight_variable2([5, 5, 1, 32])).flatten()
plt.plot(w2, 'ro')

w = run_node(weight_variable([5, 5, 1, 32])).flatten()
# _ = plt.hist(w, bins=100, color='blue') # histogram
df = pd.DataFrame(w)
df.plot(kind='density', color='blue') # density plot
print(w.min(), '~', w.max())

w2 = run_node(weight_variable2([5, 5, 1, 32])).flatten()
# _ = plt.hist(w2, bins=100, color='red') # histogram
df2 = pd.DataFrame(w2)
df2.plot(kind='density', color='red') # density plot
print(w2.min(), '~', w2.max())

_, height, width, n_channel0 = x_image_shape = [-1, 28, 28, 1] # gray scale
# _, height, width, n_channel0 = x_image_shape = [-1, 28, 28, 3] # RGB color
# N = mnist.test.labels.shape[0]
n_classes = mnist.test.labels.shape[1] # 0 ~ 9

# input data (X)
x = tf.placeholder(tf.float32, [None, height * width]) # batch, height*width*channel
x_image = tf.reshape(x, x_image_shape) # batch, height, width, channel (NHWC)
print('mnist.test.images:', mnist.test.images.shape)
print('x:', x)
print('x_image:', x_image)
# print('?(N):', N)
# print('mnist.test.images.image: %s, %s * %s, %s (N, H*W, C)' % (N, width, height, channels))

# input data (Y)
print('n_classes:', n_classes)
y_ = tf.placeholder(tf.float32, [None, n_classes]) # labels, from data
print('y_:', y_)

# conv layer 1
n_channel1 = 32
W_conv1 = weight_variable([5, 5, n_channel0, n_channel1]) # height, width, input channels, output channels
b_conv1 = bias_variable([n_channel1])

h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) # relu ( conv(x, W) + b )
h_pool1 = max_pool_2x2(h_conv1) # 28 * 28 -> 14 * 14
print('W_conv1:', W_conv1)
print('b_conv1:', b_conv1)
print('h_conv1:', h_conv1)
print('h_pool1:', h_pool1)

# conv layer 2 
n_channel2 = 64
W_conv2 = weight_variable([5, 5, n_channel1, n_channel2])
b_conv2 = bias_variable([n_channel2])

h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) # relu ( conv(x, W) + b )
h_pool2 = max_pool_2x2(h_conv2) # 14 * 14 -> 7 * 7
print('W_conv2:', W_conv2)
print('b_conv2:', b_conv2)
print('h_conv2:', h_conv2)
print('h_pool2:', h_pool2)

# fully connected layer 1 = 3136 -> 1024
height_pool2, width_pool2, n_channel2 = int(h_pool2.shape[1]), int(h_pool2.shape[2]), int(h_pool2.shape[3])
print(h_pool2.shape)
print('height_pool2, width_pool2, n_channel2:', height_pool2, width_pool2, n_channel2)

n_features = int(height_pool2) * int(width_pool2) * n_channel2 # input of FFNN
n_hidden = 1024 # neuron of FFNN
h_pool2_flat = tf.reshape(h_pool2, [-1, n_features]) # input of FFNN
W_fc1 = weight_variable([n_features, n_hidden]) # Weight of FFNN
b_fc1 = bias_variable([n_hidden]) # bias of FFNN
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) # output of FFNN

# drop out
keep_prob = tf.placeholder(tf.float32) # keep rate for dropout
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # dropouted output of FFNN

print('h_pool2_flat:', h_pool2_flat)
print('W_fc1:', W_fc1)
print('b_fc1:', b_fc1)
print('h_fc1:', h_fc1)
print('h_fc1_drop:', h_fc1_drop)

len(mnist.train.images), len(mnist.test.images)

cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=1)) # loss (cost)
# cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_))

train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # training

correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) # evaluation
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # evaluation

iteration = 10000 # max iteration
check_accuracy_interval = min(1000, (iteration / 10)) # less than 1000
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())

    print('training...')
    for i in range(1, iteration+1): # 1 ~ 10000
        batch = mnist.train.next_batch(50) # mini batch (size=50)
        batch_x, batch_y = batch[0], batch[1] # images, labels
        train_step.run(feed_dict={x: batch[0], y_: batch_y, keep_prob: 0.5}) # with dropout
        if i % check_accuracy_interval == 0:
            train_accuracy = accuracy.eval(feed_dict={x: batch_x, y_: batch[1], keep_prob: 1.0})
            print("[step %d] training accuracy %.8f" % (i, train_accuracy))
    print('training OK.')

    print('evaluate...')
    _accuracy, _W_conv1, _W_conv2, _h_pool1, _h_pool2 = sess.run(
        [accuracy, W_conv1, W_conv2, h_pool1, h_pool2], 
        feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}) # without dropout
    print("evaluate OK. accuracy %.8f" % _accuracy)

    print('save models to numpy array...')
    np.save(os.path.join(mnist_model, 'W_conv1'), _W_conv1)
    np.save(os.path.join(mnist_model, 'W_conv2'), _W_conv2)
    np.save(os.path.join(mnist_model, 'h_pool1'), _h_pool1)
    np.save(os.path.join(mnist_model, 'h_pool2'), _h_pool2)
    np.save(os.path.join(mnist_model, 'labels'), mnist.test.labels)
    print('save models to numpy array OK.')

!ls -lh MNIST_model/

print('load models...')
labels = np.load(os.path.join(mnist_model, 'labels.npy')) # labels
print('load models OK.')
print('labels', labels.min(), '~', labels.max())
pprint.pprint(labels[:3])

print('load models...')
_W_conv1 = np.load(os.path.join(mnist_model, 'W_conv1.npy')) # kernels of convolution layer #1 (5x5)
print('load models OK.')

print('_W_conv1:', _W_conv1.shape, _W_conv1.min(), '~', _W_conv1.max())
show_images(_W_conv1, is_kernel=True, title='W_conv1 %s' % str(_W_conv1.shape), max_samples=32)

print('load models...')
_h_pool1 = np.load(os.path.join(mnist_model, 'h_pool1.npy')) # kernels of pooling layer #1 (5x5)
print('load models OK.')

print('_h_pool1:', _h_pool1.shape, _h_pool1.min(), '~', _h_pool1.max())
for i, images in enumerate(_h_pool1[:3]):
    show_images(images, is_pooled_image=True, 
                title='h_pool1 label: {label}'.format(label=np.argmax(labels[i])))

print('load models...')
_W_conv2 = np.load(os.path.join(mnist_model, 'W_conv2.npy')) # kernels of convolution layer #2 (5x5)
print('load models OK.')

print('_W_conv2:', _W_conv2.shape, _W_conv2.min(), '~', _W_conv2.max())
show_images(_W_conv2, is_kernel=True, title='W_conv2 %s' % str(_W_conv2.shape))

print('load models...')
_h_pool2 = np.load(os.path.join(mnist_model, 'h_pool2.npy'))
print('load models OK.')

print('_h_pool2(after relu):', _h_pool2.shape, _h_pool2.min(), '~', _h_pool2.max())
for i, images in enumerate(_h_pool2[:3]):
    show_images(images, is_pooled_image=True, 
                title='h_pool2 label: {label}'.format(label=np.argmax(labels[i])))