LSTM과 GRU 셀

LSTM 신경망 훈련하기

In [1]:
from tensorflow.keras.datasets import imdb
from sklearn.model_selection import train_test_split

(train_input, train_target), (test_input, test_target) = imdb.load_data(
    num_words=500)

train_input, val_input, train_target, val_target = train_test_split(
    train_input, train_target, test_size=0.2, random_state=42)
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz
17465344/17464789 [==============================] - 0s 0us/step
<string>:6: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/datasets/imdb.py:155: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
  x_train, y_train = np.array(xs[:idx]), np.array(labels[:idx])
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/datasets/imdb.py:156: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
  x_test, y_test = np.array(xs[idx:]), np.array(labels[idx:])
In [2]:
from tensorflow.keras.preprocessing.sequence import pad_sequences

train_seq = pad_sequences(train_input, maxlen=100)
val_seq = pad_sequences(val_input, maxlen=100)
In [3]:
from tensorflow import keras

model = keras.Sequential()

model.add(keras.layers.Embedding(500, 16, input_length=100))
model.add(keras.layers.LSTM(8))
model.add(keras.layers.Dense(1, activation='sigmoid'))

model.summary()
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding (Embedding)        (None, 100, 16)           8000      
_________________________________________________________________
lstm (LSTM)                  (None, 8)                 800       
_________________________________________________________________
dense (Dense)                (None, 1)                 9         
=================================================================
Total params: 8,809
Trainable params: 8,809
Non-trainable params: 0
_________________________________________________________________
In [4]:
rmsprop = keras.optimizers.RMSprop(learning_rate=1e-4)
model.compile(optimizer=rmsprop, loss='binary_crossentropy', 
              metrics=['accuracy'])

checkpoint_cb = keras.callbacks.ModelCheckpoint('best-lstm-model.h5', 
                                                save_best_only=True)
early_stopping_cb = keras.callbacks.EarlyStopping(patience=3,
                                                  restore_best_weights=True)

history = model.fit(train_seq, train_target, epochs=100, batch_size=64,
                    validation_data=(val_seq, val_target),
                    callbacks=[checkpoint_cb, early_stopping_cb])
Epoch 1/100
313/313 [==============================] - 11s 9ms/step - loss: 0.6923 - accuracy: 0.5354 - val_loss: 0.6917 - val_accuracy: 0.5468
Epoch 2/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6893 - accuracy: 0.6054 - val_loss: 0.6874 - val_accuracy: 0.6250
Epoch 3/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6806 - accuracy: 0.6608 - val_loss: 0.6725 - val_accuracy: 0.6814
Epoch 4/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6354 - accuracy: 0.7053 - val_loss: 0.5830 - val_accuracy: 0.7200
Epoch 5/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5546 - accuracy: 0.7398 - val_loss: 0.5398 - val_accuracy: 0.7456
Epoch 6/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5201 - accuracy: 0.7614 - val_loss: 0.5112 - val_accuracy: 0.7666
Epoch 7/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4943 - accuracy: 0.7747 - val_loss: 0.4908 - val_accuracy: 0.7744
Epoch 8/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4758 - accuracy: 0.7857 - val_loss: 0.4765 - val_accuracy: 0.7824
Epoch 9/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4632 - accuracy: 0.7929 - val_loss: 0.4689 - val_accuracy: 0.7868
Epoch 10/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4546 - accuracy: 0.7964 - val_loss: 0.4624 - val_accuracy: 0.7908
Epoch 11/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4476 - accuracy: 0.7980 - val_loss: 0.4567 - val_accuracy: 0.7900
Epoch 12/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4416 - accuracy: 0.8024 - val_loss: 0.4535 - val_accuracy: 0.7920
Epoch 13/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4374 - accuracy: 0.8044 - val_loss: 0.4508 - val_accuracy: 0.7940
Epoch 14/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4341 - accuracy: 0.8071 - val_loss: 0.4512 - val_accuracy: 0.7884
Epoch 15/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4305 - accuracy: 0.8069 - val_loss: 0.4507 - val_accuracy: 0.7882
Epoch 16/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4281 - accuracy: 0.8073 - val_loss: 0.4439 - val_accuracy: 0.7960
Epoch 17/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4259 - accuracy: 0.8087 - val_loss: 0.4426 - val_accuracy: 0.7956
Epoch 18/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4233 - accuracy: 0.8105 - val_loss: 0.4441 - val_accuracy: 0.7908
Epoch 19/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4218 - accuracy: 0.8109 - val_loss: 0.4411 - val_accuracy: 0.7932
Epoch 20/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4196 - accuracy: 0.8113 - val_loss: 0.4382 - val_accuracy: 0.7994
Epoch 21/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4185 - accuracy: 0.8130 - val_loss: 0.4404 - val_accuracy: 0.7902
Epoch 22/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4171 - accuracy: 0.8129 - val_loss: 0.4356 - val_accuracy: 0.7998
Epoch 23/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4156 - accuracy: 0.8130 - val_loss: 0.4352 - val_accuracy: 0.8008
Epoch 24/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4144 - accuracy: 0.8141 - val_loss: 0.4347 - val_accuracy: 0.8026
Epoch 25/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4135 - accuracy: 0.8137 - val_loss: 0.4344 - val_accuracy: 0.7966
Epoch 26/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4122 - accuracy: 0.8149 - val_loss: 0.4350 - val_accuracy: 0.7972
Epoch 27/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4111 - accuracy: 0.8149 - val_loss: 0.4416 - val_accuracy: 0.7896
Epoch 28/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4101 - accuracy: 0.8158 - val_loss: 0.4321 - val_accuracy: 0.8048
Epoch 29/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4091 - accuracy: 0.8162 - val_loss: 0.4359 - val_accuracy: 0.7946
Epoch 30/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4079 - accuracy: 0.8177 - val_loss: 0.4319 - val_accuracy: 0.7994
Epoch 31/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4072 - accuracy: 0.8169 - val_loss: 0.4298 - val_accuracy: 0.8024
Epoch 32/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4059 - accuracy: 0.8168 - val_loss: 0.4291 - val_accuracy: 0.8046
Epoch 33/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4051 - accuracy: 0.8181 - val_loss: 0.4318 - val_accuracy: 0.7992
Epoch 34/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4042 - accuracy: 0.8176 - val_loss: 0.4354 - val_accuracy: 0.7942
Epoch 35/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4037 - accuracy: 0.8173 - val_loss: 0.4283 - val_accuracy: 0.8012
Epoch 36/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4027 - accuracy: 0.8202 - val_loss: 0.4319 - val_accuracy: 0.7990
Epoch 37/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4021 - accuracy: 0.8187 - val_loss: 0.4282 - val_accuracy: 0.8068
Epoch 38/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4013 - accuracy: 0.8206 - val_loss: 0.4270 - val_accuracy: 0.8062
Epoch 39/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4001 - accuracy: 0.8203 - val_loss: 0.4273 - val_accuracy: 0.8014
Epoch 40/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3997 - accuracy: 0.8220 - val_loss: 0.4276 - val_accuracy: 0.8056
Epoch 41/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3985 - accuracy: 0.8223 - val_loss: 0.4250 - val_accuracy: 0.8074
Epoch 42/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3978 - accuracy: 0.8228 - val_loss: 0.4257 - val_accuracy: 0.8054
Epoch 43/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3968 - accuracy: 0.8214 - val_loss: 0.4282 - val_accuracy: 0.8102
Epoch 44/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3963 - accuracy: 0.8213 - val_loss: 0.4243 - val_accuracy: 0.8056
Epoch 45/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3954 - accuracy: 0.8236 - val_loss: 0.4291 - val_accuracy: 0.8012
Epoch 46/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3943 - accuracy: 0.8231 - val_loss: 0.4243 - val_accuracy: 0.8078
Epoch 47/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3937 - accuracy: 0.8224 - val_loss: 0.4232 - val_accuracy: 0.8060
Epoch 48/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3929 - accuracy: 0.8241 - val_loss: 0.4293 - val_accuracy: 0.8044
Epoch 49/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3918 - accuracy: 0.8242 - val_loss: 0.4308 - val_accuracy: 0.8022
Epoch 50/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3915 - accuracy: 0.8242 - val_loss: 0.4226 - val_accuracy: 0.8062
Epoch 51/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3907 - accuracy: 0.8261 - val_loss: 0.4229 - val_accuracy: 0.8056
Epoch 52/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3899 - accuracy: 0.8245 - val_loss: 0.4244 - val_accuracy: 0.8030
Epoch 53/100
313/313 [==============================] - 2s 8ms/step - loss: 0.3893 - accuracy: 0.8256 - val_loss: 0.4236 - val_accuracy: 0.8052
In [5]:
import matplotlib.pyplot as plt

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.xlabel('epoch')
plt.ylabel('loss')
plt.legend(['train', 'val'])
plt.show()

순환 층에 드롭아웃 적용하기

In [6]:
model2 = keras.Sequential()

model2.add(keras.layers.Embedding(500, 16, input_length=100))
model2.add(keras.layers.LSTM(8, dropout=0.3))
model2.add(keras.layers.Dense(1, activation='sigmoid'))
In [7]:
rmsprop = keras.optimizers.RMSprop(learning_rate=1e-4)
model2.compile(optimizer=rmsprop, loss='binary_crossentropy', 
               metrics=['accuracy'])

checkpoint_cb = keras.callbacks.ModelCheckpoint('best-dropout-model.h5', 
                                                save_best_only=True)
early_stopping_cb = keras.callbacks.EarlyStopping(patience=3,
                                                  restore_best_weights=True)

history = model2.fit(train_seq, train_target, epochs=100, batch_size=64,
                     validation_data=(val_seq, val_target),
                     callbacks=[checkpoint_cb, early_stopping_cb])
Epoch 1/100
313/313 [==============================] - 5s 9ms/step - loss: 0.6923 - accuracy: 0.5348 - val_loss: 0.6911 - val_accuracy: 0.5792
Epoch 2/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6897 - accuracy: 0.5994 - val_loss: 0.6872 - val_accuracy: 0.6398
Epoch 3/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6813 - accuracy: 0.6522 - val_loss: 0.6703 - val_accuracy: 0.6906
Epoch 4/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6297 - accuracy: 0.6992 - val_loss: 0.5941 - val_accuracy: 0.7140
Epoch 5/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5815 - accuracy: 0.7207 - val_loss: 0.5675 - val_accuracy: 0.7374
Epoch 6/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5612 - accuracy: 0.7374 - val_loss: 0.5486 - val_accuracy: 0.7550
Epoch 7/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5452 - accuracy: 0.7498 - val_loss: 0.5345 - val_accuracy: 0.7590
Epoch 8/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5294 - accuracy: 0.7577 - val_loss: 0.5212 - val_accuracy: 0.7664
Epoch 9/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5155 - accuracy: 0.7670 - val_loss: 0.5094 - val_accuracy: 0.7706
Epoch 10/100
313/313 [==============================] - 3s 8ms/step - loss: 0.5031 - accuracy: 0.7750 - val_loss: 0.5010 - val_accuracy: 0.7712
Epoch 11/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4917 - accuracy: 0.7796 - val_loss: 0.4892 - val_accuracy: 0.7782
Epoch 12/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4846 - accuracy: 0.7831 - val_loss: 0.4822 - val_accuracy: 0.7846
Epoch 13/100
313/313 [==============================] - 3s 8ms/step - loss: 0.4758 - accuracy: 0.7865 - val_loss: 0.4733 - val_accuracy: 0.7884
Epoch 14/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4665 - accuracy: 0.7910 - val_loss: 0.4671 - val_accuracy: 0.7842
Epoch 15/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4603 - accuracy: 0.7924 - val_loss: 0.4619 - val_accuracy: 0.7894
Epoch 16/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4533 - accuracy: 0.7971 - val_loss: 0.4560 - val_accuracy: 0.7904
Epoch 17/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4493 - accuracy: 0.7946 - val_loss: 0.4532 - val_accuracy: 0.7928
Epoch 18/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4433 - accuracy: 0.8002 - val_loss: 0.4501 - val_accuracy: 0.7928
Epoch 19/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4413 - accuracy: 0.8016 - val_loss: 0.4453 - val_accuracy: 0.7936
Epoch 20/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4362 - accuracy: 0.8029 - val_loss: 0.4438 - val_accuracy: 0.7950
Epoch 21/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4333 - accuracy: 0.8063 - val_loss: 0.4409 - val_accuracy: 0.7958
Epoch 22/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4320 - accuracy: 0.8058 - val_loss: 0.4412 - val_accuracy: 0.7948
Epoch 23/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4298 - accuracy: 0.8041 - val_loss: 0.4402 - val_accuracy: 0.7924
Epoch 24/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4288 - accuracy: 0.8035 - val_loss: 0.4375 - val_accuracy: 0.7976
Epoch 25/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4274 - accuracy: 0.8041 - val_loss: 0.4358 - val_accuracy: 0.7992
Epoch 26/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4246 - accuracy: 0.8059 - val_loss: 0.4359 - val_accuracy: 0.7998
Epoch 27/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4235 - accuracy: 0.8077 - val_loss: 0.4386 - val_accuracy: 0.8008
Epoch 28/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4227 - accuracy: 0.8077 - val_loss: 0.4353 - val_accuracy: 0.8018
Epoch 29/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4211 - accuracy: 0.8074 - val_loss: 0.4347 - val_accuracy: 0.7984
Epoch 30/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4222 - accuracy: 0.8067 - val_loss: 0.4338 - val_accuracy: 0.7986
Epoch 31/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4215 - accuracy: 0.8065 - val_loss: 0.4336 - val_accuracy: 0.8008
Epoch 32/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4200 - accuracy: 0.8080 - val_loss: 0.4326 - val_accuracy: 0.7998
Epoch 33/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4195 - accuracy: 0.8084 - val_loss: 0.4324 - val_accuracy: 0.7966
Epoch 34/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4192 - accuracy: 0.8101 - val_loss: 0.4321 - val_accuracy: 0.7970
Epoch 35/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4176 - accuracy: 0.8097 - val_loss: 0.4336 - val_accuracy: 0.8036
Epoch 36/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4185 - accuracy: 0.8096 - val_loss: 0.4330 - val_accuracy: 0.8018
Epoch 37/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4163 - accuracy: 0.8087 - val_loss: 0.4335 - val_accuracy: 0.8030
In [8]:
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.xlabel('epoch')
plt.ylabel('loss')
plt.legend(['train', 'val'])
plt.show()

2개의 층을 연결하기

In [9]:
model3 = keras.Sequential()

model3.add(keras.layers.Embedding(500, 16, input_length=100))
model3.add(keras.layers.LSTM(8, dropout=0.3, return_sequences=True))
model3.add(keras.layers.LSTM(8, dropout=0.3))
model3.add(keras.layers.Dense(1, activation='sigmoid'))

model3.summary()
Model: "sequential_2"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_2 (Embedding)      (None, 100, 16)           8000      
_________________________________________________________________
lstm_2 (LSTM)                (None, 100, 8)            800       
_________________________________________________________________
lstm_3 (LSTM)                (None, 8)                 544       
_________________________________________________________________
dense_2 (Dense)              (None, 1)                 9         
=================================================================
Total params: 9,353
Trainable params: 9,353
Non-trainable params: 0
_________________________________________________________________
In [10]:
rmsprop = keras.optimizers.RMSprop(learning_rate=1e-4)
model3.compile(optimizer=rmsprop, loss='binary_crossentropy', 
               metrics=['accuracy'])

checkpoint_cb = keras.callbacks.ModelCheckpoint('best-2rnn-model.h5', 
                                                save_best_only=True)
early_stopping_cb = keras.callbacks.EarlyStopping(patience=3,
                                                  restore_best_weights=True)

history = model3.fit(train_seq, train_target, epochs=100, batch_size=64,
                     validation_data=(val_seq, val_target),
                     callbacks=[checkpoint_cb, early_stopping_cb])
Epoch 1/100
313/313 [==============================] - 7s 14ms/step - loss: 0.6918 - accuracy: 0.5480 - val_loss: 0.6904 - val_accuracy: 0.5902
Epoch 2/100
313/313 [==============================] - 4s 12ms/step - loss: 0.6833 - accuracy: 0.6177 - val_loss: 0.6735 - val_accuracy: 0.5924
Epoch 3/100
313/313 [==============================] - 4s 12ms/step - loss: 0.6318 - accuracy: 0.6682 - val_loss: 0.5963 - val_accuracy: 0.7110
Epoch 4/100
313/313 [==============================] - 4s 12ms/step - loss: 0.5710 - accuracy: 0.7281 - val_loss: 0.5554 - val_accuracy: 0.7390
Epoch 5/100
313/313 [==============================] - 4s 12ms/step - loss: 0.5418 - accuracy: 0.7456 - val_loss: 0.5355 - val_accuracy: 0.7420
Epoch 6/100
313/313 [==============================] - 4s 12ms/step - loss: 0.5222 - accuracy: 0.7577 - val_loss: 0.5129 - val_accuracy: 0.7610
Epoch 7/100
313/313 [==============================] - 4s 12ms/step - loss: 0.5055 - accuracy: 0.7648 - val_loss: 0.4997 - val_accuracy: 0.7646
Epoch 8/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4947 - accuracy: 0.7703 - val_loss: 0.4932 - val_accuracy: 0.7748
Epoch 9/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4865 - accuracy: 0.7775 - val_loss: 0.4798 - val_accuracy: 0.7774
Epoch 10/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4761 - accuracy: 0.7818 - val_loss: 0.4741 - val_accuracy: 0.7800
Epoch 11/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4707 - accuracy: 0.7847 - val_loss: 0.4716 - val_accuracy: 0.7808
Epoch 12/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4653 - accuracy: 0.7885 - val_loss: 0.4646 - val_accuracy: 0.7884
Epoch 13/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4594 - accuracy: 0.7917 - val_loss: 0.4596 - val_accuracy: 0.7854
Epoch 14/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4570 - accuracy: 0.7908 - val_loss: 0.4574 - val_accuracy: 0.7896
Epoch 15/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4534 - accuracy: 0.7919 - val_loss: 0.4541 - val_accuracy: 0.7894
Epoch 16/100
313/313 [==============================] - 4s 13ms/step - loss: 0.4496 - accuracy: 0.7942 - val_loss: 0.4514 - val_accuracy: 0.7898
Epoch 17/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4463 - accuracy: 0.7945 - val_loss: 0.4488 - val_accuracy: 0.7908
Epoch 18/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4437 - accuracy: 0.7980 - val_loss: 0.4470 - val_accuracy: 0.7926
Epoch 19/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4404 - accuracy: 0.7998 - val_loss: 0.4461 - val_accuracy: 0.7906
Epoch 20/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4379 - accuracy: 0.8023 - val_loss: 0.4440 - val_accuracy: 0.7958
Epoch 21/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4364 - accuracy: 0.7994 - val_loss: 0.4442 - val_accuracy: 0.7886
Epoch 22/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4356 - accuracy: 0.8007 - val_loss: 0.4431 - val_accuracy: 0.7966
Epoch 23/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4344 - accuracy: 0.8008 - val_loss: 0.4411 - val_accuracy: 0.7946
Epoch 24/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4327 - accuracy: 0.8042 - val_loss: 0.4408 - val_accuracy: 0.7912
Epoch 25/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4317 - accuracy: 0.8013 - val_loss: 0.4396 - val_accuracy: 0.7916
Epoch 26/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4290 - accuracy: 0.8016 - val_loss: 0.4389 - val_accuracy: 0.7928
Epoch 27/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4293 - accuracy: 0.8034 - val_loss: 0.4390 - val_accuracy: 0.7916
Epoch 28/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4298 - accuracy: 0.8039 - val_loss: 0.4397 - val_accuracy: 0.8010
Epoch 29/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4265 - accuracy: 0.8054 - val_loss: 0.4387 - val_accuracy: 0.7998
Epoch 30/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4276 - accuracy: 0.8056 - val_loss: 0.4374 - val_accuracy: 0.7946
Epoch 31/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4262 - accuracy: 0.8047 - val_loss: 0.4369 - val_accuracy: 0.7918
Epoch 32/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4230 - accuracy: 0.8054 - val_loss: 0.4361 - val_accuracy: 0.7970
Epoch 33/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4239 - accuracy: 0.8064 - val_loss: 0.4355 - val_accuracy: 0.7970
Epoch 34/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4237 - accuracy: 0.8069 - val_loss: 0.4359 - val_accuracy: 0.8000
Epoch 35/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4207 - accuracy: 0.8084 - val_loss: 0.4357 - val_accuracy: 0.7990
Epoch 36/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4219 - accuracy: 0.8079 - val_loss: 0.4348 - val_accuracy: 0.7994
Epoch 37/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4203 - accuracy: 0.8108 - val_loss: 0.4338 - val_accuracy: 0.7960
Epoch 38/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4196 - accuracy: 0.8087 - val_loss: 0.4333 - val_accuracy: 0.7994
Epoch 39/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4194 - accuracy: 0.8099 - val_loss: 0.4340 - val_accuracy: 0.8006
Epoch 40/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4203 - accuracy: 0.8076 - val_loss: 0.4325 - val_accuracy: 0.8002
Epoch 41/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4179 - accuracy: 0.8105 - val_loss: 0.4326 - val_accuracy: 0.8000
Epoch 42/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4173 - accuracy: 0.8106 - val_loss: 0.4333 - val_accuracy: 0.8014
Epoch 43/100
313/313 [==============================] - 4s 12ms/step - loss: 0.4169 - accuracy: 0.8105 - val_loss: 0.4326 - val_accuracy: 0.8026
In [11]:
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.xlabel('epoch')
plt.ylabel('loss')
plt.legend(['train', 'val'])
plt.show()

GRU 신경망 훈련하기

In [12]:
model4 = keras.Sequential()

model4.add(keras.layers.Embedding(500, 16, input_length=100))
model4.add(keras.layers.GRU(8))
model4.add(keras.layers.Dense(1, activation='sigmoid'))

model4.summary()
Model: "sequential_3"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_3 (Embedding)      (None, 100, 16)           8000      
_________________________________________________________________
gru (GRU)                    (None, 8)                 624       
_________________________________________________________________
dense_3 (Dense)              (None, 1)                 9         
=================================================================
Total params: 8,633
Trainable params: 8,633
Non-trainable params: 0
_________________________________________________________________
In [13]:
rmsprop = keras.optimizers.RMSprop(learning_rate=1e-4)
model4.compile(optimizer=rmsprop, loss='binary_crossentropy', 
               metrics=['accuracy'])

checkpoint_cb = keras.callbacks.ModelCheckpoint('best-gru-model.h5', 
                                                save_best_only=True)
early_stopping_cb = keras.callbacks.EarlyStopping(patience=3,
                                                  restore_best_weights=True)

history = model4.fit(train_seq, train_target, epochs=100, batch_size=64,
                     validation_data=(val_seq, val_target),
                     callbacks=[checkpoint_cb, early_stopping_cb])
Epoch 1/100
313/313 [==============================] - 4s 9ms/step - loss: 0.6922 - accuracy: 0.5236 - val_loss: 0.6912 - val_accuracy: 0.5532
Epoch 2/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6902 - accuracy: 0.5547 - val_loss: 0.6891 - val_accuracy: 0.5754
Epoch 3/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6871 - accuracy: 0.5788 - val_loss: 0.6855 - val_accuracy: 0.5992
Epoch 4/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6820 - accuracy: 0.6043 - val_loss: 0.6796 - val_accuracy: 0.6104
Epoch 5/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6740 - accuracy: 0.6217 - val_loss: 0.6708 - val_accuracy: 0.6196
Epoch 6/100
313/313 [==============================] - 2s 7ms/step - loss: 0.6617 - accuracy: 0.6380 - val_loss: 0.6570 - val_accuracy: 0.6344
Epoch 7/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6423 - accuracy: 0.6572 - val_loss: 0.6344 - val_accuracy: 0.6552
Epoch 8/100
313/313 [==============================] - 2s 8ms/step - loss: 0.6090 - accuracy: 0.6819 - val_loss: 0.5934 - val_accuracy: 0.6918
Epoch 9/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5553 - accuracy: 0.7205 - val_loss: 0.5455 - val_accuracy: 0.7290
Epoch 10/100
313/313 [==============================] - 2s 7ms/step - loss: 0.5232 - accuracy: 0.7462 - val_loss: 0.5282 - val_accuracy: 0.7426
Epoch 11/100
313/313 [==============================] - 2s 8ms/step - loss: 0.5080 - accuracy: 0.7571 - val_loss: 0.5203 - val_accuracy: 0.7430
Epoch 12/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4968 - accuracy: 0.7640 - val_loss: 0.5078 - val_accuracy: 0.7602
Epoch 13/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4870 - accuracy: 0.7696 - val_loss: 0.5005 - val_accuracy: 0.7590
Epoch 14/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4787 - accuracy: 0.7752 - val_loss: 0.4932 - val_accuracy: 0.7664
Epoch 15/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4716 - accuracy: 0.7801 - val_loss: 0.4870 - val_accuracy: 0.7714
Epoch 16/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4652 - accuracy: 0.7837 - val_loss: 0.4836 - val_accuracy: 0.7680
Epoch 17/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4595 - accuracy: 0.7893 - val_loss: 0.4762 - val_accuracy: 0.7760
Epoch 18/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4543 - accuracy: 0.7927 - val_loss: 0.4733 - val_accuracy: 0.7726
Epoch 19/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4488 - accuracy: 0.7954 - val_loss: 0.4670 - val_accuracy: 0.7776
Epoch 20/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4449 - accuracy: 0.7964 - val_loss: 0.4619 - val_accuracy: 0.7828
Epoch 21/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4411 - accuracy: 0.7980 - val_loss: 0.4589 - val_accuracy: 0.7826
Epoch 22/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4379 - accuracy: 0.8001 - val_loss: 0.4558 - val_accuracy: 0.7880
Epoch 23/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4352 - accuracy: 0.8021 - val_loss: 0.4538 - val_accuracy: 0.7858
Epoch 24/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4320 - accuracy: 0.8047 - val_loss: 0.4523 - val_accuracy: 0.7894
Epoch 25/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4299 - accuracy: 0.8074 - val_loss: 0.4489 - val_accuracy: 0.7912
Epoch 26/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4274 - accuracy: 0.8066 - val_loss: 0.4499 - val_accuracy: 0.7898
Epoch 27/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4254 - accuracy: 0.8087 - val_loss: 0.4465 - val_accuracy: 0.7930
Epoch 28/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4233 - accuracy: 0.8095 - val_loss: 0.4456 - val_accuracy: 0.7950
Epoch 29/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4220 - accuracy: 0.8107 - val_loss: 0.4483 - val_accuracy: 0.7958
Epoch 30/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4204 - accuracy: 0.8120 - val_loss: 0.4436 - val_accuracy: 0.7928
Epoch 31/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4196 - accuracy: 0.8109 - val_loss: 0.4432 - val_accuracy: 0.7918
Epoch 32/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4182 - accuracy: 0.8120 - val_loss: 0.4422 - val_accuracy: 0.7930
Epoch 33/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4174 - accuracy: 0.8123 - val_loss: 0.4416 - val_accuracy: 0.7938
Epoch 34/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4166 - accuracy: 0.8146 - val_loss: 0.4422 - val_accuracy: 0.7984
Epoch 35/100
313/313 [==============================] - 2s 7ms/step - loss: 0.4160 - accuracy: 0.8145 - val_loss: 0.4440 - val_accuracy: 0.7974
Epoch 36/100
313/313 [==============================] - 2s 8ms/step - loss: 0.4153 - accuracy: 0.8156 - val_loss: 0.4418 - val_accuracy: 0.7982
In [14]:
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.xlabel('epoch')
plt.ylabel('loss')
plt.legend(['train', 'val'])
plt.show()

마무리

In [15]:
test_seq = pad_sequences(test_input, maxlen=100)

rnn_model = keras.models.load_model('best-2rnn-model.h5')

rnn_model.evaluate(test_seq, test_target)
782/782 [==============================] - 5s 5ms/step - loss: 0.4261 - accuracy: 0.8018
Out[15]:
[0.42607444524765015, 0.801800012588501]