In [2]:

import sys
from torch import optim, Tensor
import pandas as pd
import seaborn as sns
sys.path.append('../')
from modules import schedulers

In [3]:

def gen_schedule(num_its:int, scheduler:optim.lr_scheduler._LRScheduler):
    lrs = []
    its = range(num_its)
    for i in its:
        lrs.append(scheduler.get_lr())
        scheduler.step()
    lr_df = pd.DataFrame(lrs, columns=[f'pg_{i}' for i in range(1, len(opt.param_groups) + 1)])
    lr_df['iteration'] = its
    return lr_df

In [4]:

# triangular: 1 param group
opt = optim.SGD([Tensor()], lr=1)
t_scheduler = schedulers.TriangularScheduler(100, 0.1, 1, opt)
t_df = gen_schedule(1000, t_scheduler)
sns.lineplot(x='iteration', y='pg_1', data=t_df)

Out[4]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa169cd44a8>

In [5]:

# triangular: 2 param groups

opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
                  {'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.TriangularScheduler(100, 0.1, 1, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)

Out[5]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa1699759e8>

In [6]:

# triangular: learning rate range test - 1 param group
opt = optim.SGD([Tensor()], lr=1)
scheduler = schedulers.TriangularScheduler(1000, 0.1, 1, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)

Out[6]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa1699564e0>

In [7]:

# triangular: learning rate range test - 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
                  {'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.TriangularScheduler(1000, 0.1, 1, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)

Out[7]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa1698cf550>

In [8]:

# triangular2: 1 param group
opt = optim.SGD([Tensor()], lr=1)
t2_scheduler = schedulers.Triangular2Scheduler(100, 0.1, 1, opt)
t_df = gen_schedule(1000, t2_scheduler)
sns.lineplot(x='iteration', y='pg_1', data=t_df)

Out[8]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa16989ad30>

In [9]:

# triangular2: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
                  {'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.Triangular2Scheduler(100, 0.1, 1, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)

Out[9]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa169801978>

In [10]:

# exp_range: 1 param group
opt = optim.SGD([Tensor()], lr=1)
scheduler = schedulers.ExpRangeScheduler(0.99, 100, 1, 10, opt)
er_df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=er_df)

Out[10]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa1697f3cc0>

In [11]:

# exp_range: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
                  {'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.ExpRangeScheduler(0.99, 100, 1, 10, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)

Out[11]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa169745668>

In [12]:

# decay: 1 param group
opt = optim.SGD([Tensor()], lr=1)
scheduler = schedulers.DecayScheduler(100, 1, 10, opt)
d_df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=d_df)

Out[12]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa1696f8358>

In [13]:

# decay: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
                  {'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.DecayScheduler(100, 1, 10, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)

Out[13]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa1696826a0>

In [14]:

# exp: 1 param group
opt = optim.SGD([Tensor()], lr=1)
scheduler = schedulers.ExpScheduler(0.995, 10, opt)
e_df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=e_df)

Out[14]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa16964eba8>

In [15]:

# exp: 2 param groups
opt = optim.SGD([{'params' : Tensor(), 'lr_mult' : 1.0},
                  {'params' : Tensor(), 'lr_mult' : 2.0}], lr=1)
scheduler = schedulers.ExpScheduler(0.995, 10, opt)
df = gen_schedule(1000, scheduler)
sns.lineplot(x='iteration', y='pg_1', data=df)
sns.lineplot(x='iteration', y='pg_2', data=df)

Out[15]:

<matplotlib.axes._subplots.AxesSubplot at 0x7fa169616f60>