#@title PENGGUNAAN RUNTIME (GOOGLE COLAB/LOKAL) { display-mode: "form" }
#@markdown Centang jika menggunakan sistem lokal.

_IS_LOCALE = False #@param {type:"boolean"}
_server = ['Google Colab', 'LOKAL']
print(':: INFORMASI RUNTIME')
print(':: Buku ini menggunakan RUNTIME: {}'.format(_server[_IS_LOCALE]).upper())

#@title DESKRIPSI PROYEK { display-mode: "form" }
from datetime import datetime, timezone, timedelta

NOTEBOOK_NAME = 'taruma_demo_lstm_rr' #@param {type:"string"}
NOTEBOOK_VERSION = '1.0.0' #@param {type:"string"}
_current_time_jakarta = datetime.now(timezone(timedelta(hours=7)))
_prefix = _current_time_jakarta.strftime('%Y%m%d_%H%M')
project_title = f'{_prefix}_{NOTEBOOK_NAME}_{NOTEBOOK_VERSION.replace(".", "_")}'

print(':: INFORMASI PROYEK/BUKU')
print(f':: [project_title]: {project_title}')

#@title MENGGUNAKAN TENSORFLOW 2.x (Google Colab)

from __future__ import absolute_import, division, print_function, unicode_literals

try:
  # %tensorflow_version only exists in Colab.
  print(':: MENGGUNAKAN TENSORFLOW 2.x (GOOGLE COLAB)')
  %tensorflow_version 2.x
except Exception:
  pass

#@title PERIKSA PAKET HIDROKIT { display-mode: "form" }
#@markdown Memeriksa paket hidrokit dan memasangnya jika diperlukan.

print(':: memeriksa paket hidrokit'.upper())
if _IS_LOCALE:
    try:
        import hidrokit
    except ModuleNotFoundError:
        print('Hidrokit tidak ditemukan')
else:
    try:
        import hidrokit
    except ModuleNotFoundError:
        print(':: instalasi paket hidrokit'.upper())
        !pip install git+https://github.com/taruma/hidrokit.git@latest -q

#@title MENAMPILKAN VERSI PADA SISTEM
def _check_system():
    from numpy import __version__ as numpy_ver
    from pandas import __version__ as pandas_ver
    from matplotlib import __version__ as matplotlib_ver
    from tensorflow import __version__ as tensorflow_ver
    from tensorflow.keras import __version__ as keras_ver
    from hidrokit.__version__ import __version__ as hidrokit_ver
    from sys import version_info as python_info

    ALL_SYSTEM = 'python numpy pandas matplotlib tensorflow keras hidrokit'
    SYSTEM_INFO = {}
    
    print(':: INFORMASI VERSI SISTEM')
    for library in ALL_SYSTEM.split():
        if library.lower() == 'python':
            python_ver = '{}.{}.{}'.format(*python_info[:3])
        print(':: {:>12s} version: {:<10s}'.format(library, eval(library + '_ver')))
        SYSTEM_INFO[library] = eval(library + '_ver')
    
    return SYSTEM_INFO
_SYSTEM_INFO = _check_system()

print(':: LOKASI PENYIMPANAN DATASET')
print(f'DATASET_PATH = {DATASET_PATH}')
print(f'DROP_PATH = {DROP_PATH}')

# IMPORT LIBRARIES
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

print(':: IMPORT LIBRARY (NUMPY, PANDAS, MATPLOTLIB)')

from hidrokit.contrib.taruma import hk43

print(':: MEMBACA DATA HUJAN DARI [DATASET_PATH]')
hujan_raw, hujan_invalid = hk43.read_folder(
    DATASET_PATH, pattern='hujan_*', fmt='uma.hujan', prefix='hujan_',
    invalid=True
)

print(f':: tipe [hujan_raw] = {type(hujan_raw)}')
print(f':: tipe [hujan_invalid] = {type(hujan_invalid)}')

print(':: MEMBACA DATA DEBIT DARI [DATASET_PATH]')
debit_raw, debit_invalid = hk43.read_folder(
    DATASET_PATH, pattern='debit_*', fmt='uma.debit', prefix='debit_',
    invalid=True
)

print(f':: tipe [debit_raw] = {type(debit_raw)}')
print(f':: tipe [debit_invalid] = {type(debit_invalid)}')

from hidrokit.contrib.taruma import hk73

KLIMATOLOGI_PATH = DATASET_PATH / 'klimatologi_serang_1998_2008.xlsx'
print(f':: [KLIMATOLOGI_PATH] = {KLIMATOLOGI_PATH}')

print(':: MEMBACA DATA KLIMATOLOGI DARI [KLIMATOLOGI_PATH]')
df_klimatologi = hk73._read_bmkg(KLIMATOLOGI_PATH)
print(f':: tipe [df_klimatologi] = {type(df_klimatologi)}')

print(':: MENGUBAH [hujan_raw] MENJADI [df_hujan] SEBAGAI DATAFRAME')
df_hujan = pd.DataFrame(hujan_raw, index=pd.date_range('19980101', '20081231'))
print(':: MENAMPILKAN [df_hujan]')
df_hujan.head()

print(':: MEMERIKSA NILAI INVALID PADA [df_hujan]')

print(':: MENAMPILKAN NILAI INVALID')
_hujan_invalid = list(hujan_invalid[list(hujan_invalid.keys())[0]].keys())
print(f':: NILAI INVALID BERUPA = {_hujan_invalid}')

print(':: MENGOREKSI NILAI INVALID PADA [df_hujan]')
print(':: MENGOREKSI NILAI "-" MENJADI 0.0')
df_hujan[df_hujan == '-'] = 0.

print(":: MENGUBAH TIPE DATA PADA DATAFRAME [df_hujan]")
df_hujan = df_hujan.infer_objects()
print(":: MENAMPILKAN INFORMASI DATAFRAME [df_hujan]:")
df_hujan.info()

print(':: MENGISI NILAI HILANG MENGGUNAKAN METODE INTERPOLASI LINEAR')
df_hujan.interpolate(method='linear', inplace=True)
df_hujan.bfill(inplace=True) # Mengatasi nilai hilang pada index [0, 1]

print(':: MEMERIKSA JIKA [df_hujan] MASIH MEMILIKI NILAI YANG HILANG:', end=' ')
print(f'{df_hujan.isnull().values.any()}')

print(':: MENGUBAH [debit_raw] MENJADI [df_debit] SEBAGAI DATAFRAME')
df_debit = pd.DataFrame(debit_raw, index=pd.date_range('19980101', '20081231'))
print(':: MENAMPILKAN [df_debit]')
df_debit.head()

print(':: MEMERIKSA NILAI INVALID PADA [df_debit]')

print(':: MENAMPILKAN NILAI INVALID')
_debit_invalid = list(debit_invalid[list(debit_invalid.keys())[0]].keys())
print(f':: NILAI INVALID BERUPA = {_debit_invalid}')

print(':: MENGOREKSI NILAI 20.9.46 MENJADI 209.46')
df_debit[df_debit == '20.9.46'] = 209.46
print(':: MENGOREKSI NILAI tad MENJADI NaN')
df_debit[df_debit == 'tad'] = np.nan

print(":: MENGUBAH TIPE DATA PADA DATAFRAME [df_debit]")
df_debit = df_debit.infer_objects()
print(":: MENAMPILKAN INFORMASI DATAFRAME [df_debit]:")
df_debit.info()

print(':: MENGISI NILAI HILANG MENGGUNAKAN METODE INTERPOLASI LINEAR')
df_debit.interpolate(method='linear', inplace=True)

print(':: MEMERIKSA JIKA [df_debit] MASIH MEMILIKI NILAI YANG HILANG:', end=' ')
print(f'{df_debit.isnull().values.any()}')

print(':: MEMERIKSA KEKERINGAN PADA DATA DEBIT')

# Menggunakan modul hk73 (lihat pada bagian bmkg untuk lebih jelasnya)
from hidrokit.contrib.taruma import hk73

_debit_zero = hk73._get_index1D((df_debit == 0).values[:, 0])
_debit_zero_date = hk73._group_as_index(
    hk73._group_as_list(_debit_zero), index=df_debit.index,
    date_format='%d %b %Y'
)

print(f':: Kekeringan terjadi pada tanggal: {_debit_zero_date}')

print(':: MENAMPILKAN DATAFRAME [df_klimatologi]:')
df_klimatologi.head()

print(':: MEMBERSIHKAN [df_klimatologi] KE DATAFRAME [df_klimatologi_clean]')

df_klimatologi_clean = df_klimatologi.drop(['ddd_car', 'ff_x', 'ddd_x', 'ff_avg', 'RR'], axis=1)
df_klimatologi_clean.head()

print(':: MEMERIKSA DATA YANG HILANG PADA [df_klimatologi_clean]')

print(f':: [df_klimatologi_clean] memiliki kehilangan data: {hk73._have_nan(df_klimatologi_clean)}')
print(f':: Kolom yang memiliki data hilang: {hk73._get_nan_columns(df_klimatologi_clean)}')

print(':: MEMERIKSA KONDISI DATA HILANG PADA [df_klimatologi_clean].ss')

_klimatologi_nan = hk73._get_nan(df_klimatologi_clean)
_missing_ss = _klimatologi_nan['ss']
_missing_ss_as_list = hk73._group_as_list(_missing_ss)
_missing_ss_as_index = hk73._group_as_index(
    _missing_ss_as_list, index=df_klimatologi_clean.index, 
    date_format='%d %b %Y'
)

print(f':: Tanggal terjadinya kehilangan data: {_missing_ss_as_index}')

print(':: MEMERIKSA DATA YANG TIDAK ADA/TEREKAM PADA [df_klimatologi_clean]')
_klimatologi_ungauged = hk73._get_missing(df_klimatologi_clean)

print(_klimatologi_ungauged)

print(':: MENGISI NILAI YANG HILANG MENGGUNAKAN METODE INTERPOLASI LINEAR')

df_klimatologi_clean.interpolate(method='linear', inplace=True)

print(':: MEMERIKSA JIKA [df_klimatologi_clean] MASIH MEMILIKI NILAI YANG HILANG:', end=' ')
print(f'{df_klimatologi_clean.isnull().values.any()}')

print(':: MENENTUKAN PERIODE DATASET')

_DATASET_FROM = '19980301'
_DATASET_TO = '20081231'
_slicedate_dataset = slice(_DATASET_FROM, _DATASET_TO)
print(f':: PERIODE DATASET DARI {_DATASET_FROM} hingga {_DATASET_TO}')

print(':: DataFrame [data_hujan], [data_debit], [data_klimatologi]')
data_hujan = df_hujan[_slicedate_dataset]
data_debit = df_debit[_slicedate_dataset]
data_klimatologi = df_klimatologi_clean[_slicedate_dataset]

print(':: MENGGABUNGKAN SELURUH DATA DALAM SATU DATAFRAME [dataset]')
dataset = pd.concat([data_hujan, data_klimatologi, data_debit], axis=1)

#@title Menamai ulang kolom

print(':: MENAMAI ULANG NAMA KOLOM')
COLUMNS_NAME = 'ch_A ch_B ch_C ch_D ch_E ch_F ch_G ch_H suhu_min suhu_max suhu_rerata lembab_rerata lama_penyinaran debit' #@param {type:"string"}
dataset.columns = COLUMNS_NAME.split()
print(':: INFO [dataset]:')
dataset.info()

print(':: STATISTIK DESKRIPTIF [dataset]')
dataset.describe().T.drop(['count', '25%', '75%'], axis=1)

import seaborn as sns
g = sns.pairplot(dataset)
g.fig.set_size_inches(15,15)

from hidrokit.viz import graph
graph.subplots(dataset, ncols=1, nrows=14, figsize=(15, 15));

#@title Menentukan `train_set` dan `test_set`

def _TO_NONE(text):
    if text.lower() == 'none':
        return None
    else:
        return text

#@markdown ### `train_set`

_TRAIN_FROM = "None" #@param {type: "string"}
_TRAIN_TO = "20061231" #@param {type: "string"}

#@markdown ### `test_set`
_TEST_FROM = "20070101" #@param {type: "string"}
_TEST_TO = "None" #@param {type: "string"}

_TRAIN_FROM, _TRAIN_TO = map(_TO_NONE, [_TRAIN_FROM, _TRAIN_TO])
_TEST_FROM, _TEST_TO = map(_TO_NONE, [_TEST_FROM, _TEST_TO])

_SLICE_TRAIN = slice(_TRAIN_FROM, _TRAIN_TO)
_SLICE_TEST = slice(_TEST_FROM, _TEST_TO)

print(f':: Pemotongan Train set dari\t: {str(_TRAIN_FROM):8s} sampai {_TRAIN_TO:8s}')
print(f':: Pemotongan Test set dari\t: {_TEST_FROM:8s} sampai {str(_TEST_TO):8s}')

print(':: DIMENSI TRAIN SET DAN TEST SET')
train_set = dataset.loc[_SLICE_TRAIN].copy()
test_set = dataset.loc[_SLICE_TEST].copy()
print(f':: DIMENSI [train_set]: {train_set.shape}')
print(f':: DIMENSI [test_set]: {test_set.shape}')

from sklearn.preprocessing import StandardScaler
sc = StandardScaler()

print(':: NORMALISASI DATAFRAME [train_set] MENJADI [train_set_scale]')
train_set_scale = train_set.copy()
train_set_scale[:] = sc.fit_transform(train_set[:])

print(':: MENAMPILKAN SAMPLE DATASET [train_set_scale]')
train_set_scale.sample(5, axis=0).sample(5, axis=1)

print(':: NORMALISASI DATAFRAME [test_set] MENJADI [test_set_scale]')
test_set_scale = test_set.copy()
test_set_scale[:] = sc.transform(test_set[:])

print(':: MENAMPILKAN SAMPLE DATASET [test_set_scale]')
test_set_scale.sample(5, axis=0).sample(5, axis=1)

#@title Menentukan `TIMESTEPS`

#@markdown Satuan `TIMESTEPS` adalah hari.

TIMESTEPS = 365 #@param {type:"integer"}

print(':: MENENTUKAN [TIMESTEPS]')
print(f':: [TIMESTEPS] = {TIMESTEPS} hari')

print(':: MENENTUKAN INPUT COLUMNS DAN OUTPUT COLUMNS')
feature_columns = dataset.columns.to_list()[:-1]
output_columns = ['debit']
print(f':: INPUT COLUMNS = {feature_columns}')
print(f':: OUTPUT COLUMNS = {output_columns}')

from hidrokit.contrib.taruma import hk53

print(':: TRANSFORMASI TRAIN SET')
print(f':: DIMENSI [train_set_scale] = {train_set_scale.shape}')
print(':: TRANSFORMASI [train_set_scale] MENJADI [X_train] DAN [y_train]')
X_train, y_train = hk53.tensor_array(
    train_set_scale, X_columns=feature_columns, 
    timesteps=TIMESTEPS, y_out=True, y_columns=output_columns
)
print(f':: DIMENSI [X_train] = {X_train.shape}')
print(f':: DIMENSI [y_train] = {y_train.shape}')

print(':: TRANSFORMASI TEST SET')
print(f':: DIMENSI [test_set_scale] = {test_set_scale.shape}')
print(':: TRANSFORMASI [test_set_scale] MENJADI [X_test] DAN [y_test]')
X_test, y_test = hk53.tensor_array(
    test_set_scale, X_columns=feature_columns,
    timesteps=TIMESTEPS, y_out=True, y_columns=output_columns
)
print(f':: DIMENSI [X_test] = {X_test.shape}')
print(f':: DIMENSI [y_test] = {y_test.shape}')

print(':: IMPORT TENSORFLOW.KERAS LIBRARY')
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, Dropout

import tensorflow.keras.backend as K

print(':: MEMBUAT FUNGSI KHUSUS METRIK (NSE, NSE_MOD, R_SQUARED)')

def nse(y_true, y_pred):
    # ref: https://hydroerr.readthedocs.io/en/stable/api/HydroErr.HydroErr.nse.html
    S, O = y_pred, y_true
    O_mean = K.mean(O)
    top = K.sum(K.square(S-O))
    bot = K.sum(K.square(O-O_mean))
    NSE = 1 - (top / bot)
    return NSE

def nse_mod(y_true, y_pred, j=1):
    # ref: https://hydroerr.readthedocs.io/en/stable/api/HydroErr.HydroErr.nse_mod.html
    S, O = y_pred, y_true
    O_mean = K.mean(O)
    abs_top = K.abs(S-O)
    abs_bot = K.abs(O-O_mean)
    top = K.sum(K.pow(abs_top, j))
    bot = K.sum(K.pow(abs_bot, j))
    NSE_mod = 1 - (top / bot)
    return NSE_mod

def r_squared(y_true, y_pred):
    # ref: https://hydroerr.readthedocs.io/en/stable/api/HydroErr.HydroErr.r_squared.html
    S, O = y_pred, y_true
    O_mean = K.mean(O)
    S_mean = K.mean(S)

    _top = (O-O_mean) * (S-S_mean)
    top = K.square(K.sum(_top))

    bot = K.sum(K.square(O-O_mean)) * K.sum(K.square(S-S_mean))

    R_squared = top / bot
    return R_squared

def build_model(
    input_shape, optimizer='adam', activation='sigmoid', p=0.1,
    units=20, loss='mse'
):
    model = Sequential()

    # FIRST LAYER
    model.add(LSTM(units=units, return_sequences=True, input_shape=input_shape))
    model.add(Dropout(rate=p))

    # SECOND LAYER
    model.add(LSTM(units=units, return_sequences=False))
    model.add(Dropout(rate=p))

    # OUTPUT LAYER
    model.add(Dense(1))

    # COMPILE
    model.compile(loss=loss, optimizer=optimizer, metrics=['mae', nse, nse_mod, r_squared])

    return model

print(':: PEMODELAN RNN')
model = build_model(input_shape=(TIMESTEPS, X_train.shape[2]))
print(':: SUMMARY [model]:')
model.summary()

_before_train_time = datetime.now(timezone(timedelta(hours=7)))
print(f':: TRAINING START: {_before_train_time.strftime("%Y%m%d %H:%M")}')

_after_train_time = datetime.now(timezone(timedelta(hours=7)))
print(f':: TRAINING FINISH: {_after_train_time.strftime("%Y%m%d %H:%M")}')

print(':: MODEL SELESAI DILATIH')
_duration_train_time = _after_train_time - _before_train_time
print(f':: DURASI: {_duration_train_time}')

print(':: MENYIMPAN HISTORY METRIK DALAM BENTUK DATAFRAME')
df_metric = pd.DataFrame(history.history)
print(':: METRIK DISIMPAN DI [df_metric]')
df_metric.rename(columns={'loss':'mse'}, inplace=True)
print(':: MENAMPILKAN [df_metric]')
df_metric.head()

print(':: GRAFIK METRIK')
df_metric.plot(figsize=(10,5))
plt.title('METRIK PELATIHAN MODEL')
plt.xlabel('Epoch')
plt.ylabel('Nilai');

print(':: OBJEK StandardScaler UNTUK OUTPUT [y_train] DAN [y_test]')
sc_y = StandardScaler()
sc_y.scale_, sc_y.mean_, sc_y.var_, sc_y.n_sample_seen_ = sc.scale_[-1], sc.mean_[-1], sc.var_[-1], sc.n_samples_seen_

print(':: MENYIMPAN EVALUASI TRAIN SET [df_eval_train]')
df_eval_train = pd.DataFrame(
    data=dict(
        true=sc_y.inverse_transform(y_train),
        pred=sc_y.inverse_transform(model.predict(X_train).flatten())
    ),
    index=train_set.index[TIMESTEPS:]
)
print(':: MENAMPILKAN [df_eval_train]')
df_eval_train.head()

print(':: GRAFIK DEBIT ALIRAN MENGGUNAKAN TRAIN SET')
df_eval_train.plot(figsize=(15,5))
plt.title('PREDIKSI DEBIT ALIRAN MENGGUNAKAN DATA TRAIN SET')
plt.ylabel('Debit $m^3/s$')
plt.xlabel('Waktu');

print(':: MENYIMPAN EVALUASI TEST SET [df_eval_test]')
df_eval_test = pd.DataFrame(
    data=dict(
        true=sc_y.inverse_transform(y_test),
        pred=sc_y.inverse_transform(model.predict(X_test).flatten())
    ),
    index=test_set.index[TIMESTEPS:]
)
print(':: MENAMPILKAN [df_eval_test]')
df_eval_test.head()

print(':: GRAFIK DEBIT ALIRAN MENGGUNAKAN TEST SET')
df_eval_test.plot(figsize=(15,5))
plt.title('PREDIKSI DEBIT ALIRAN MENGGUNAKAN TEST SET')
plt.ylabel('Debit $m^3/s$')
plt.xlabel('Waktu');

#@title Periksa Paket HydroErr { display-mode: "form"}
#@markdown Memeriksa paket hidrokit dan memasangnya jika diperlukan

print(':: MEMERIKSA PAKET HYDROERR')
if _IS_LOCALE:
    try:
        import HydroErr
    except ModuleNotFoundError:
        print('HydroErr tidak ditemukan')
else:
    try:
        import HydroErr
        print(':: PAKET HYDROERR SUDAH TERPASANG')
    except ModuleNotFoundError:
        print(':: INSTALASI PAKET HYDROERR')
        !pip install HydroErr -q

import HydroErr as he

print(':: MENGHITUNG METRIK DARI TEST SET')
# Menggunakan nilai yang telah dinormalisasikan
_simulated = model.predict(X_test).flatten()
_observed = y_test

test_metric = {
    'mse':he.mse(_simulated, _observed),
    'mae':he.mae(_simulated, _observed),
    'nse':he.nse(_simulated, _observed),
    'nse_mod':he.nse_mod(_simulated, _observed),
    'r_squared':he.r_squared(_simulated, _observed)
}

print(':: MENAMPILKAN METRIK TEST SET [test_metric]')
test_metric = pd.Series(test_metric)
test_metric

print(':: MEMBANDINGKAN METRIK TRAIN SET DAN TEST SET')
df_eval_metric = pd.DataFrame(
    data=[df_metric.iloc[-1], test_metric], index=['train', 'test']
)
df_eval_metric.T