import sys
from torch import optim, Tensor
import pandas as pd
import seaborn as sns
sys.path.append('../')
from modules import schedulers
def gen_schedule(num_its:int, scheduler:optim.lr_scheduler._LRScheduler):
lrs = []
its = range(num_its)
for i in its:
lrs.append(scheduler.get_lr())
scheduler.step()
lr_df = pd.DataFrame(lrs, columns=[f'pg_{i}' for i in range(1, len(opt.param_groups) + 1)])
lr_df['iteration'] = its
return lr_df
# triangular: 1 param group
opt = optim.SGD([Tensor()], lr=1)
t_scheduler = schedulers.TriangularScheduler(100, 0.1, 1, opt)
t_df = gen_schedule(1000, t_scheduler)
sns.lineplot(x='iteration', y='pg_1', data=t_df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa169cd44a8>
# triangular: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
{'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.TriangularScheduler(100, 0.1, 1, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa1699759e8>
# triangular: learning rate range test - 1 param group
opt = optim.SGD([Tensor()], lr=1)
scheduler = schedulers.TriangularScheduler(1000, 0.1, 1, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa1699564e0>
# triangular: learning rate range test - 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
{'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.TriangularScheduler(1000, 0.1, 1, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa1698cf550>
# triangular2: 1 param group
opt = optim.SGD([Tensor()], lr=1)
t2_scheduler = schedulers.Triangular2Scheduler(100, 0.1, 1, opt)
t_df = gen_schedule(1000, t2_scheduler)
sns.lineplot(x='iteration', y='pg_1', data=t_df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa16989ad30>
# triangular2: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
{'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.Triangular2Scheduler(100, 0.1, 1, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa169801978>
# exp_range: 1 param group
opt = optim.SGD([Tensor()], lr=1)
scheduler = schedulers.ExpRangeScheduler(0.99, 100, 1, 10, opt)
er_df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=er_df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa1697f3cc0>
# exp_range: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
{'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.ExpRangeScheduler(0.99, 100, 1, 10, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa169745668>
# decay: 1 param group
opt = optim.SGD([Tensor()], lr=1)
scheduler = schedulers.DecayScheduler(100, 1, 10, opt)
d_df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=d_df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa1696f8358>
# decay: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
{'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.DecayScheduler(100, 1, 10, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa1696826a0>
# exp: 1 param group
opt = optim.SGD([Tensor()], lr=1)
scheduler = schedulers.ExpScheduler(0.995, 10, opt)
e_df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=e_df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa16964eba8>
# exp: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
{'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.ExpScheduler(0.995, 10, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)
<matplotlib.axes._subplots.AxesSubplot at 0x7fa169616f60>