이 노트북에서는 가중치 초기화가 층의 출력에 어떤 영향을 미치는지 알아 봅니다.
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.initializers import Zeros, RandomNormal, glorot_normal, glorot_uniform
n_input = 784
n_dense = 256
b_init = Zeros()
w_init = RandomNormal(stddev=1.0)
# w_init = glorot_normal()
# w_init = glorot_uniform()
model = Sequential()
model.add(Dense(n_dense,
input_dim=n_input,
kernel_initializer=w_init,
bias_initializer=b_init))
model.add(Activation('sigmoid'))
# model.add(Activation('tanh'))
# model.add(Activation('relu'))
x = np.random.random((1,n_input))
a = model.predict(x)
1/1 [==============================] - 4s 4s/step
a
array([[9.9999976e-01, 2.0004527e-06, 1.0000000e+00, 7.6007054e-06, 7.2447336e-01, 1.0000000e+00, 1.0000000e+00, 7.2155090e-06, 9.9873585e-01, 1.0000000e+00, 9.9276406e-01, 1.7446166e-07, 9.9991155e-01, 1.7818085e-08, 6.0669146e-03, 9.9694365e-01, 5.5367585e-02, 8.4336215e-01, 1.0000000e+00, 9.1620052e-01, 3.7742063e-02, 1.1626942e-01, 6.0207140e-02, 9.9999952e-01, 9.9999988e-01, 1.1345611e-18, 8.5575145e-04, 4.0285154e-12, 2.7216640e-07, 1.2736185e-01, 9.9425274e-01, 9.8856676e-01, 1.0000000e+00, 1.0000000e+00, 8.7992710e-01, 7.5850660e-01, 8.3993769e-01, 1.5167581e-02, 9.9999368e-01, 2.5858393e-08, 3.2360949e-05, 2.6045323e-04, 9.9999988e-01, 6.9003995e-08, 4.6693298e-01, 1.7045186e-05, 1.0000000e+00, 4.6300636e-05, 2.7897401e-02, 1.0000000e+00, 1.0000000e+00, 4.3196255e-01, 9.9982542e-01, 1.7630829e-03, 1.2610410e-05, 9.9998820e-01, 9.9999988e-01, 1.0000000e+00, 1.9858588e-09, 9.9865794e-01, 1.0000000e+00, 9.9999988e-01, 9.9999356e-01, 5.2389276e-01, 1.0000000e+00, 2.2728916e-03, 9.9999940e-01, 1.0000000e+00, 9.6458012e-01, 1.0000000e+00, 9.9812490e-01, 9.9949884e-01, 4.7519954e-04, 3.3310425e-01, 9.0552413e-01, 8.5556632e-01, 9.9999988e-01, 7.2879493e-02, 4.2009432e-10, 1.3394852e-03, 3.3087630e-09, 9.9996769e-01, 5.1238286e-01, 3.7398124e-06, 9.3211509e-02, 5.1276106e-04, 4.3272531e-01, 1.7245174e-03, 4.9460980e-01, 9.0364385e-01, 1.0000000e+00, 4.0471383e-07, 3.6911390e-04, 1.0000000e+00, 2.4616565e-03, 7.4751504e-02, 9.9999809e-01, 9.9999869e-01, 6.0375658e-07, 1.5204600e-17, 1.0145973e-08, 3.2308731e-15, 2.2927362e-23, 1.0000000e+00, 9.9996197e-01, 9.4917184e-01, 2.4504557e-02, 9.9999917e-01, 5.0322893e-08, 4.0138799e-09, 1.0000000e+00, 5.7756886e-02, 9.9624120e-10, 9.6115343e-08, 6.7261944e-04, 9.9793267e-01, 4.0458761e-10, 9.9998069e-01, 1.6944791e-09, 5.7222009e-09, 2.9329160e-06, 9.9243402e-01, 1.0670438e-07, 2.3417340e-10, 9.9979085e-01, 9.9999475e-01, 2.7916862e-11, 9.9081635e-01, 9.9996519e-01, 1.0000000e+00, 3.3258403e-09, 9.9999547e-01, 6.4479635e-04, 1.3639749e-20, 2.5365388e-02, 4.9270267e-12, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 8.1613696e-01, 9.9900800e-01, 3.6039779e-05, 1.0000000e+00, 9.9927586e-01, 5.0434846e-02, 1.0000000e+00, 1.0000000e+00, 3.9598272e-06, 9.8010617e-01, 9.8909539e-01, 7.0482641e-02, 9.8838860e-01, 9.9990189e-01, 6.2087643e-01, 1.1945281e-04, 4.1250487e-07, 1.0000000e+00, 8.5153884e-10, 1.1645834e-01, 1.0000000e+00, 4.0245816e-01, 1.0000000e+00, 9.7169703e-01, 9.9998891e-01, 1.0000000e+00, 1.0000000e+00, 9.9997938e-01, 1.8650668e-11, 1.5423197e-01, 1.0000000e+00, 9.7142178e-01, 9.9932158e-01, 1.0000000e+00, 2.2391982e-02, 9.9999058e-01, 1.4179017e-08, 9.6793604e-09, 9.2963719e-01, 2.3809712e-14, 9.8940842e-11, 1.0000000e+00, 8.0581201e-05, 9.9527836e-01, 9.9998581e-01, 1.1921300e-02, 8.3797222e-01, 7.2229195e-01, 2.0421967e-02, 2.5230616e-05, 2.8043921e-04, 9.1558671e-01, 1.0000000e+00, 9.9992621e-01, 9.9997973e-01, 9.9999952e-01, 1.0000000e+00, 1.0000000e+00, 1.8243941e-02, 6.1026850e-07, 6.0636532e-01, 2.2425574e-04, 1.5451181e-04, 5.9624430e-02, 9.9996448e-01, 9.9997723e-01, 9.9996424e-01, 1.2161328e-07, 2.6257087e-08, 1.3579285e-10, 2.6606099e-08, 8.7479502e-01, 5.0915283e-04, 5.9039354e-07, 5.8859558e-04, 3.1329486e-09, 2.4465851e-03, 9.9977654e-01, 9.9999428e-01, 9.9907744e-01, 9.9978012e-01, 1.0000000e+00, 8.5014262e-06, 9.9947661e-01, 1.3509205e-09, 7.6480833e-10, 4.2969562e-04, 9.1507763e-01, 1.5044299e-09, 8.7820133e-04, 9.8001146e-01, 4.0643033e-01, 4.7102249e-03, 9.9734432e-01, 1.1301318e-03, 6.6847545e-01, 9.9999225e-01, 3.2009734e-07, 1.0000000e+00, 9.9856037e-01, 2.0048836e-01, 2.8661656e-01, 1.5250902e-06, 1.5025468e-01, 1.0000000e+00, 1.1106618e-04, 9.7996815e-07, 5.6953407e-08, 7.7879751e-01, 9.9998009e-01, 3.1049974e-04, 9.9977845e-01, 3.6121635e-03, 3.9790309e-04, 9.9987197e-01, 9.9999952e-01, 4.0278728e-03]], dtype=float32)
_ = plt.hist(np.transpose(a))