import pickle as pkl
import time
import os
import numpy as np
import sys
from music21 import instrument, note, stream, chord, duration
from models.RNNAttention import create_network, sample_with_temp
import matplotlib.pyplot as plt
Using TensorFlow backend. /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_qint8 = np.dtype([("qint8", np.int8, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_quint8 = np.dtype([("quint8", np.uint8, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_qint16 = np.dtype([("qint16", np.int16, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_quint16 = np.dtype([("quint16", np.uint16, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_qint32 = np.dtype([("qint32", np.int32, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. np_resource = np.dtype([("resource", np.ubyte, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_qint8 = np.dtype([("qint8", np.int8, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_quint8 = np.dtype([("quint8", np.uint8, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_qint16 = np.dtype([("qint16", np.int16, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_quint16 = np.dtype([("quint16", np.uint16, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_qint32 = np.dtype([("qint32", np.int32, 1)]) /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. np_resource = np.dtype([("resource", np.ubyte, 1)])
# 실행 파라미터
section = 'compose'
run_id = '0006'
music_name = 'cello'
run_folder = 'run/{}/'.format(section)
run_folder += '_'.join([run_id, music_name])
# 하이퍼파라미터
embed_size = 100
rnn_units = 256
use_attention = True
store_folder = os.path.join(run_folder, 'store')
with open(os.path.join(store_folder, 'distincts'), 'rb') as filepath:
distincts = pkl.load(filepath)
note_names, n_notes, duration_names, n_durations = distincts
with open(os.path.join(store_folder, 'lookups'), 'rb') as filepath:
lookups = pkl.load(filepath)
note_to_int, int_to_note, duration_to_int, int_to_duration = lookups
weights_folder = os.path.join(run_folder, 'weights')
weights_file = 'weights.h5'
model, att_model = create_network(n_notes, n_durations, embed_size, rnn_units, use_attention)
# 각 노드에 가중치 적재하기
weight_source = os.path.join(weights_folder,weights_file)
model.load_weights(weight_source)
model.summary()
WARNING: Logging before flag parsing goes to stderr. W0923 21:55:11.495095 139658663503680 deprecation_wrapper.py:119] From /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead. W0923 21:55:11.504162 139658663503680 deprecation_wrapper.py:119] From /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead. W0923 21:55:11.506637 139658663503680 deprecation_wrapper.py:119] From /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead. W0923 21:55:11.947371 139658663503680 deprecation_wrapper.py:119] From /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead. W0923 21:55:11.953456 139658663503680 deprecation_wrapper.py:119] From /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead. W0923 21:55:11.994368 139658663503680 deprecation_wrapper.py:119] From /home/haesun/github/GDL_code/env/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.
__________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ================================================================================================== input_1 (InputLayer) (None, None) 0 __________________________________________________________________________________________________ input_2 (InputLayer) (None, None) 0 __________________________________________________________________________________________________ embedding_1 (Embedding) (None, None, 100) 3300 input_1[0][0] __________________________________________________________________________________________________ embedding_2 (Embedding) (None, None, 100) 700 input_2[0][0] __________________________________________________________________________________________________ concatenate_1 (Concatenate) (None, None, 200) 0 embedding_1[0][0] embedding_2[0][0] __________________________________________________________________________________________________ lstm_1 (LSTM) (None, None, 256) 467968 concatenate_1[0][0] __________________________________________________________________________________________________ lstm_2 (LSTM) (None, None, 256) 525312 lstm_1[0][0] __________________________________________________________________________________________________ dense_1 (Dense) (None, None, 1) 257 lstm_2[0][0] __________________________________________________________________________________________________ reshape_1 (Reshape) (None, None) 0 dense_1[0][0] __________________________________________________________________________________________________ activation_1 (Activation) (None, None) 0 reshape_1[0][0] __________________________________________________________________________________________________ repeat_vector_1 (RepeatVector) (None, 256, None) 0 activation_1[0][0] __________________________________________________________________________________________________ permute_1 (Permute) (None, None, 256) 0 repeat_vector_1[0][0] __________________________________________________________________________________________________ multiply_1 (Multiply) (None, None, 256) 0 lstm_2[0][0] permute_1[0][0] __________________________________________________________________________________________________ lambda_1 (Lambda) (None, 256) 0 multiply_1[0][0] __________________________________________________________________________________________________ pitch (Dense) (None, 33) 8481 lambda_1[0][0] __________________________________________________________________________________________________ duration (Dense) (None, 7) 1799 lambda_1[0][0] ================================================================================================== Total params: 1,007,817 Trainable params: 1,007,817 Non-trainable params: 0 __________________________________________________________________________________________________
# 예측용 파라미터
notes_temp=0.5
duration_temp = 0.5
max_extra_notes = 50
max_seq_len = 32
seq_len = 32
# notes = ['START', 'D3', 'D3', 'E3', 'D3', 'G3', 'F#3','D3', 'D3', 'E3', 'D3', 'G3', 'F#3','D3', 'D3', 'E3', 'D3', 'G3', 'F#3','D3', 'D3', 'E3', 'D3', 'G3', 'F#3']
# durations = [0, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2]
# notes = ['START', 'F#3', 'G#3', 'F#3', 'E3', 'F#3', 'G#3', 'F#3', 'E3', 'F#3', 'G#3', 'F#3', 'E3','F#3', 'G#3', 'F#3', 'E3', 'F#3', 'G#3', 'F#3', 'E3', 'F#3', 'G#3', 'F#3', 'E3']
# durations = [0, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2]
notes = ['START']
durations = [0]
if seq_len is not None:
notes = ['START'] * (seq_len - len(notes)) + notes
durations = [0] * (seq_len - len(durations)) + durations
sequence_length = len(notes)
prediction_output = []
notes_input_sequence = []
durations_input_sequence = []
overall_preds = []
for n, d in zip(notes,durations):
note_int = note_to_int[n]
duration_int = duration_to_int[d]
notes_input_sequence.append(note_int)
durations_input_sequence.append(duration_int)
prediction_output.append([n, d])
if n != 'START':
midi_note = note.Note(n)
new_note = np.zeros(128)
new_note[midi_note.pitch.midi] = 1
overall_preds.append(new_note)
att_matrix = np.zeros(shape = (max_extra_notes+sequence_length, max_extra_notes))
for note_index in range(max_extra_notes):
prediction_input = [
np.array([notes_input_sequence])
, np.array([durations_input_sequence])
]
notes_prediction, durations_prediction = model.predict(prediction_input, verbose=0)
if use_attention:
att_prediction = att_model.predict(prediction_input, verbose=0)[0]
att_matrix[(note_index-len(att_prediction)+sequence_length):(note_index+sequence_length), note_index] = att_prediction
new_note = np.zeros(128)
for idx, n_i in enumerate(notes_prediction[0]):
try:
note_name = int_to_note[idx]
midi_note = note.Note(note_name)
new_note[midi_note.pitch.midi] = n_i
except:
pass
overall_preds.append(new_note)
i1 = sample_with_temp(notes_prediction[0], notes_temp)
i2 = sample_with_temp(durations_prediction[0], duration_temp)
note_result = int_to_note[i1]
duration_result = int_to_duration[i2]
prediction_output.append([note_result, duration_result])
notes_input_sequence.append(i1)
durations_input_sequence.append(i2)
if len(notes_input_sequence) > max_seq_len:
notes_input_sequence = notes_input_sequence[1:]
durations_input_sequence = durations_input_sequence[1:]
# print(note_result)
# print(duration_result)
if note_result == 'START':
break
overall_preds = np.transpose(np.array(overall_preds))
print('Generated sequence of {} notes'.format(len(prediction_output)))
Generated sequence of 82 notes
fig, ax = plt.subplots(figsize=(15,15))
ax.set_yticks([int(j) for j in range(35,70)])
plt.imshow(overall_preds[35:70,:], origin="lower", cmap='coolwarm', vmin = -0.5, vmax = 0.5, extent=[0, max_extra_notes, 35,70]
)
<matplotlib.image.AxesImage at 0x7f044011d940>
output_folder = os.path.join(run_folder, 'output')
midi_stream = stream.Stream()
# 모델이 생성한 값을 기반으로 악보와 화음 객체 만들기
for pattern in prediction_output:
note_pattern, duration_pattern = pattern
# 패턴이 화음일 경우
if ('.' in note_pattern):
notes_in_chord = note_pattern.split('.')
chord_notes = []
for current_note in notes_in_chord:
new_note = note.Note(current_note)
new_note.duration = duration.Duration(duration_pattern)
new_note.storedInstrument = instrument.Violoncello()
chord_notes.append(new_note)
new_chord = chord.Chord(chord_notes)
midi_stream.append(new_chord)
elif note_pattern == 'rest':
# 패턴이 쉼표일 경우
new_note = note.Rest()
new_note.duration = duration.Duration(duration_pattern)
new_note.storedInstrument = instrument.Violoncello()
midi_stream.append(new_note)
elif note_pattern != 'START':
# 패턴이 하나의 음표일 경우
new_note = note.Note(note_pattern)
new_note.duration = duration.Duration(duration_pattern)
new_note.storedInstrument = instrument.Violoncello()
midi_stream.append(new_note)
midi_stream = midi_stream.chordify()
timestr = time.strftime("%Y%m%d-%H%M%S")
midi_stream.write('midi', fp=os.path.join(output_folder, 'output-' + timestr + '.mid'))
'run/compose/0006_cello/output/output-20190923-215515.mid'
## 어텐션 그래프
if use_attention:
fig, ax = plt.subplots(figsize=(20,20))
im = ax.imshow(att_matrix[(seq_len-2):,], cmap='coolwarm', interpolation='nearest')
# Minor ticks
ax.set_xticks(np.arange(-.5, len(prediction_output)- seq_len, 1), minor=True);
ax.set_yticks(np.arange(-.5, len(prediction_output)- seq_len, 1), minor=True);
# Gridlines based on minor ticks
ax.grid(which='minor', color='black', linestyle='-', linewidth=1)
# We want to show all ticks...
ax.set_xticks(np.arange(len(prediction_output) - seq_len))
ax.set_yticks(np.arange(len(prediction_output)- seq_len+2))
# ... and label them with the respective list entries
ax.set_xticklabels([n[0] for n in prediction_output[(seq_len):]])
ax.set_yticklabels([n[0] for n in prediction_output[(seq_len - 2):]])
# ax.grid(color='black', linestyle='-', linewidth=1)
ax.xaxis.tick_top()
plt.setp(ax.get_xticklabels(), rotation=90, ha="left", va = "center",
rotation_mode="anchor")
plt.show()