%reload_ext autoreload
%autoreload 2
from nb_004a import *
from ipywidgets import IntProgress, HBox, HTML, VBox
from time import time
def format_time(t):
t = int(t)
h,m,s = t//3600, (t//60)%60, t%60
if h!= 0: return f'{h}:{m:02d}:{s:02d}'
else: return f'{m:02d}:{s:02d}'
We will attempt to build a progress bar that fits our stylistic and informational needs. To build a progress bar in Jupyter Notebook we are going to leverage the ipywidgets package that contains widgets designed to either build interactive features or better display information. In our case the objective will be the latter. For more information on the specific functions we will use, you can find them here: IntProgress, HBox, HTML, VBox.
We will use a HBox (horizontal box) where we will include the IntProgress widget (to display progress in a progress bar) and HTML widget (to write the time and batches information beside the progress bar). We will finally use VBox to fit each HBox, one below the other in a vertical box.
class ProgressBar():
update_every = 0.2
def __init__(self, gen, display=True, leave=True, parent=None):
self._gen,self.total = gen,len(gen)
if parent is None: self.leave,self.display = leave,display
else:
self.leave,self.display=False,False
parent.add_child(self)
self.comment = ''
def on_iter_begin(self): pass
def on_interrupt(self): pass
def on_iter_end(self): pass
def on_update(self, val, text): pass
def __iter__(self):
self.on_iter_begin()
self.update(0)
try:
for i,o in enumerate(self._gen):
yield o
self.update(i+1)
except: self.on_interrupt()
self.on_iter_end()
def update(self, val):
if val == 0:
self.start_t = self.last_t = time()
self.pred_t = 0
self.last_v,self.wait_for = 0,1
self.update_bar(0)
elif val >= self.last_v + self.wait_for:
cur_t = time()
avg_t = (cur_t - self.start_t) / val
self.wait_for = max(int(self.update_every / avg_t),1)
self.pred_t = avg_t * self.total
self.last_v,self.last_t = val,cur_t
self.update_bar(val)
def update_bar(self, val):
elapsed_t = self.last_t - self.start_t
remaining_t = format_time(self.pred_t - elapsed_t)
elapsed_t = format_time(elapsed_t)
end = '' if len(self.comment) == 0 else f' {self.comment}'
self.on_update(val, f'{100 * val/self.total:.2f}% [{val}/{self.total} {elapsed_t}<{remaining_t}{end}]')
class NBProgressBar(ProgressBar):
def __init__(self,gen, display=True, leave=True, parent=None):
self.progress,self.text = IntProgress(min=0, max=len(gen)), HTML()
self.box = HBox([self.progress, self.text])
super().__init__(gen, display, leave, parent)
def on_iter_begin(self):
if self.display: display(self.box)
def on_interrupt(self): self.progress.bar_style = 'danger'
def on_iter_end(self):
if not self.leave: self.box.close()
def on_update(self, val, text):
self.text.value = text
self.progress.value = val
class ConsoleProgressBar(ProgressBar):
length:int=50
fill:str='█'
def __init__(self,gen, display=True, leave=True, parent=None):
self.max_len,self.prefix = 0,''
super().__init__(gen, display, leave, parent)
def on_iter_end(self):
if not self.leave:
print(f'\r{self.prefix}' + ' ' * (self.max_len - len(f'\r{self.prefix}')), end = '\r')
def on_update(self, val, text):
if self.display:
filled_len = int(self.length * val // self.total)
bar = self.fill * filled_len + '-' * (self.length - filled_len)
to_write = f'\r{self.prefix} |{bar}| {text}'
if len(to_write) > self.max_len: self.max_len=len(to_write)
print(to_write, end = '\r')
class MasterBar():
def __init__(self, gen, cls):
self.first_bar = cls(gen, display=False)
def on_iter_begin(self): pass
def __iter__(self):
self.on_iter_begin()
for o in self.first_bar:
yield o
def add_child(self, child): pass
def write(self, line): pass
class NBMasterBar(MasterBar):
def __init__(self, gen):
super().__init__(gen, NBProgressBar)
self.text = HTML()
self.vbox = VBox([self.first_bar.box, self.text])
def on_iter_begin(self): display(self.vbox)
def add_child(self, child):
self.child = child
self.vbox.children = [self.first_bar.box, self.text, child.box]
def write(self, line):
self.text.value += line + '<p>'
class ConsoleMasterBar(MasterBar):
def __init__(self, gen):
super().__init__(gen, ConsoleProgressBar)
def add_child(self, child):
self.child = child
self.child.prefix = f'Epoch {self.first_bar.last_v+1}/{self.first_bar.total} :'
self.child.display = True
def write(self, line):
print(line)
We are defining two types of bar classes: notebook bar and console bar. This is because, if we do not use Jupyter Notebook and instead we use a console bar, the widgets we integrated into the code will not work. Thus, you will see that when building the Console bar, we need to find alternatives to the notebook widgets. For example cannot use IntProgress to build our progress bar and instead we need to literally define the character that will build our bar: '█'.
Note also that when training we will need a master bar that records our progress during all the training phase and we will also need intermediate progress bars that show the progress with each epoch.
For each of these two situations we will use classes as building blocks, leveraging the pythonic concept of inheritance. Notice for example, that both the console and notebook bars will need a progress bar class (i.e. we can use it as a base class for both). Also, the master bar is a specific case of a progress bar (i.e. we can use the NBProgressBar and ConsoleProgressBar as a base class for the corresponding derived master bar class and also use these classes alone when we need to show the progress of each epoch).
The examples below effectively illustrate both concepts.
It is important to point out that the way the classes are built allows for the contents of the generator function that is passed as input to be used in the same loop. In this example this concept is represented by the use of 'j' and 'i' in the loop but during training the contents of our generator will be our training batches' data. This functionality is implemented by leveraging the 'yield' keyword in Python.
Finally, notice that in ProgressBar we are a establishing a minimum time elapsed before we update any bar. This is important since we are interested in seeing progress frequently but not more frequently that we can observe, since printing out slows down the training. Our choice of update_every is in this case 0.2 (seconds).
from time import sleep
mb = NBMasterBar(range(5))
for j in mb:
for i in NBProgressBar(range(0, 50), parent=mb):
sleep(0.01)
mb.child.comment = str(i)
mb.write(f'Epoch {j+1}: accuracy: {7.5*j+5}%')
from time import sleep
mb = ConsoleMasterBar(range(5))
for j in mb:
for i in ConsoleProgressBar(range(0, 50), parent=mb):
sleep(0.01)
mb.child.comment = str(i)
mb.write(f'Epoch {j+1}: accuracy: {7.5*j+5}%')
Inspiration
def print_progress(iteration:int, total:int, prefix:str='', suffix:str='', decimals:int=1, length:int=50, fill:str='█'):
"Call in a loop to create terminal progress bar"
iteration += 1
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
filled_len = int(length * iteration // total)
bar = fill * filled_len + '-' * (length - filled_len)
print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end = '\r')
if iteration == total: print()
for i in range(0, 50):
sleep(0.01)
print_progress(i, 50, suffix=f'Epoch {j+1}: accuracy: {7.5*j+5}%')
Now we are going to use the functions we built to actually train a model and see them in action. For this we have to edit several functions we defined earlier.
In particular, we need to include the progress bar in the training loop and add some printing (loss before each backward pass and epoch, loss and metrics after each epoch is finished).
#export
@dataclass
class DeviceDataLoader():
dl: DataLoader
device: torch.device
half: bool = False
def __len__(self): return len(self.dl)
def __iter__(self):
self.gen = (to_device(self.device,o) for o in self.dl)
if self.half: self.gen = (to_half(o) for o in self.gen)
return iter(self.gen)
@classmethod
def create(cls, *args, device=default_device, **kwargs):
return cls(DataLoader(*args, **kwargs), device=device, half=False)
nb_002b.DeviceDataLoader = DeviceDataLoader
def fit(epochs, model, loss_fn, opt, data, callbacks=None, metrics=None, pbar=None):
cb_handler = CallbackHandler(callbacks)
cb_handler.on_train_begin()
if pbar is None: pbar = NBMasterBar(range(epochs))
for epoch in pbar:
model.train()
cb_handler.on_epoch_begin()
for xb,yb in NBProgressBar(data.train_dl, parent=pbar):
xb, yb = cb_handler.on_batch_begin(xb, yb)
loss,_ = loss_batch(model, xb, yb, loss_fn, opt, cb_handler)
if cb_handler.on_batch_end(loss): break
if hasattr(data,'valid_dl') and data.valid_dl is not None:
model.eval()
with torch.no_grad():
*val_metrics,nums = zip(*[loss_batch(model, xb, yb, loss_fn, cb_handler=cb_handler, metrics=metrics)
for xb,yb in NBProgressBar(data.valid_dl, parent=pbar)])
val_metrics = [np.sum(np.multiply(val,nums)) / np.sum(nums) for val in val_metrics]
else: val_metrics=None
if cb_handler.on_epoch_end(val_metrics): break
cb_handler.on_train_end()
#export
@dataclass
class Learner():
data: DataBunch
model: nn.Module
opt_fn: Callable = optim.SGD
loss_fn: Callable = F.cross_entropy
metrics: Collection[Callable] = None
true_wd: bool = False
def __post_init__(self): self.model = self.model.to(self.data.device)
def fit(self, epochs, lr, wd=0., callbacks=None):
self.opt = OptimWrapper(self.opt_fn(self.model.parameters(), lr), wd=wd, true_wd=self.true_wd)
pbar = NBMasterBar(range(epochs))
self.recorder = Recorder(self.opt, self.data.train_dl, pbar)
if callbacks is None: callbacks = []
callbacks = [self.recorder]+callbacks
fit(epochs, self.model, self.loss_fn, self.opt, self.data, callbacks=callbacks, metrics=self.metrics, pbar=pbar)
def lr_find(self, start_lr=1e-5, end_lr=10, num_it=100):
cb = LRFinder(self, start_lr, end_lr, num_it)
a = int(np.ceil(num_it/len(self.data.train_dl)))
self.fit(a, start_lr, callbacks=[cb])
#export
@dataclass
class Recorder(Callback):
opt: torch.optim
train_dl: DeviceDataLoader = None
pbar: MasterBar = None
def on_train_begin(self, **kwargs):
self.losses,self.val_losses,self.lrs,self.moms,self.metrics,self.nb_batches = [],[],[],[],[],[]
def on_batch_begin(self, **kwargs):
self.lrs.append(self.opt.lr)
self.moms.append(self.opt.mom)
def on_backward_begin(self, smooth_loss, **kwargs):
#We record the loss here before any other callback has a chance to modify it.
self.losses.append(smooth_loss)
if self.pbar is not None and hasattr(self.pbar,'child'):
self.pbar.child.comment = f'{smooth_loss:.4f}'
def on_epoch_end(self, epoch, num_batch, smooth_loss, last_metrics, **kwargs):
self.nb_batches.append(num_batch)
if last_metrics is not None:
self.val_losses.append(last_metrics[0])
if len(last_metrics) > 1: self.metrics.append(last_metrics[1:])
self.pbar.write(f'{epoch}, {smooth_loss}, {last_metrics}')
else: self.pbar.write(f'{epoch}, {smooth_loss}')
def plot_lr(self, show_moms=False):
iterations = list(range(len(learn.recorder.lrs)))
if show_moms:
_, axs = plt.subplots(1,2, figsize=(12,4))
axs[0].plot(iterations, self.lrs)
axs[1].plot(iterations, self.moms)
else: plt.plot(iterations, self.lrs)
def plot(self, skip_start=10, skip_end=5):
lrs = self.lrs[skip_start:-skip_end] if skip_end > 0 else self.lrs[skip_start:]
losses = self.losses[skip_start:-skip_end] if skip_end > 0 else self.losses[skip_start:]
_, ax = plt.subplots(1,1)
ax.plot(lrs, losses)
ax.set_xscale('log')
def plot_losses(self):
_, ax = plt.subplots(1,1)
iterations = list(range(len(self.losses)))
ax.plot(iterations, self.losses)
val_iter = self.nb_batches
val_iter = np.array(val_iter).cumsum()
ax.plot(val_iter, self.val_losses)
def plot_metrics(self):
assert len(self.metrics) != 0, "There is no metrics to plot."
_, axes = plt.subplots(len(self.metrics[0]),1,figsize=(6, 4*len(self.metrics[0])))
val_iter = self.nb_batches
val_iter = np.array(val_iter).cumsum()
axes = axes.flatten() if len(self.metrics[0]) != 1 else [axes]
for i, ax in enumerate(axes):
values = [met[i] for met in self.metrics]
ax.plot(val_iter, values)
DATA_PATH = Path('data')
PATH = DATA_PATH/'cifar10'
data_mean,data_std = map(tensor, ([0.491, 0.482, 0.447], [0.247, 0.243, 0.261]))
cifar_norm = normalize_tfm(mean=data_mean,std=data_std)
tfms = [flip_lr_tfm(p=0.5),
pad_tfm(padding=4),
crop_tfm(size=32, row_pct=(0,1.), col_pct=(0,1.))]
bs = 64
train_ds = ImageDataset.from_folder(PATH/'train', classes=['airplane','dog'])
valid_ds = ImageDataset.from_folder(PATH/'test', classes=['airplane','dog'])
data = DataBunch.create(train_ds, valid_ds, bs=bs, train_tfm=tfms, valid_tfm=[], num_workers=4)
len(data.train_dl), len(data.valid_dl)
model = Darknet([1, 2, 2, 2, 2], num_classes=2, nf=16)
learn = Learner(data, model)
learn.fit(5,0.01)
Now we are going to integrate useful graphs to the progress bar so that, when we finish training, we automatically show how our loss evolved during training. The graph will also be saved in the vertical box of our progress bar object for future reference.
To implement this we need to add a function within the MasterBar base and derived classes that progressively updates the graph and add a Callback that progressively sends this function the necessary information.
class MasterBar():
def __init__(self, gen, cls):
self.first_bar = cls(gen, display=False)
def on_iter_begin(self): pass
def on_iter_end(self): pass
def __iter__(self):
self.on_iter_begin()
for o in self.first_bar:
yield o
self.on_iter_end()
def add_child(self, child): pass
def write(self, line): pass
def update_graph(self, graphs, x_bounds, y_bounds): pass
from ipywidgets import widgets
from IPython.display import clear_output
from ipywidgets.widgets.interaction import show_inline_matplotlib_plots
class NBMasterBar(MasterBar):
names = ['train', 'valid']
def __init__(self, gen, show_graph=True):
super().__init__(gen, NBProgressBar)
self.text = HTML()
if show_graph:
self.out = widgets.Output()
self.vbox = VBox([self.first_bar.box, self.text, self.out])
self.fig, self.ax = plt.subplots(1, figsize=(6,4))
else: self.vbox = VBox([self.first_bar.box, self.text])
self.show_graph = show_graph
def on_iter_begin(self): display(self.vbox)
def on_iter_end(self):
if self.show_graph: self.fig.clear()
def add_child(self, child):
self.child = child
self.vbox.children = [self.first_bar.box, self.text, child.box, self.out]
def write(self, line):
self.text.value += line + '<p>'
def update_graph(self, graphs, x_bounds, y_bounds):
if not self.show_graph: return
self.out = widgets.Output()
self.ax.clear()
for g,n in zip(graphs,self.names): self.ax.plot(*g, label=n)
self.ax.legend(loc='upper right')
self.ax.set_xlim(*x_bounds)
self.ax.set_ylim(*y_bounds)
with self.out:
clear_output(wait=True)
display(self.ax.figure)
self.vbox.children = [self.first_bar.box, self.text, self.out]
#export
@dataclass
class Recorder(Callback):
opt: torch.optim
nb_epoch:int
train_dl: DeviceDataLoader = None
pbar: MasterBar = None
def on_train_begin(self, **kwargs):
self.losses,self.val_losses,self.lrs,self.moms,self.metrics,self.nb_batches = [],[],[],[],[],[]
def on_batch_begin(self, **kwargs):
self.lrs.append(self.opt.lr)
self.moms.append(self.opt.mom)
def on_backward_begin(self, smooth_loss, **kwargs):
#We record the loss here before any other callback has a chance to modify it.
self.losses.append(smooth_loss)
if self.pbar is not None and hasattr(self.pbar,'child'):
self.pbar.child.comment = f'{smooth_loss:.4f}'
def on_epoch_end(self, epoch, num_batch, smooth_loss, last_metrics, **kwargs):
self.nb_batches.append(num_batch)
if last_metrics is not None:
self.val_losses.append(last_metrics[0])
if len(last_metrics) > 1: self.metrics.append(last_metrics[1:])
self.pbar.write(f'{epoch}, {smooth_loss}, {last_metrics}')
self.pbar.update_graph(*self.send_graphs())
else: self.pbar.write(f'{epoch}, {smooth_loss}')
def plot_lr(self, show_moms=False):
iterations = list(range(len(learn.recorder.lrs)))
if show_moms:
_, axs = plt.subplots(1,2, figsize=(12,4))
axs[0].plot(iterations, self.lrs)
axs[1].plot(iterations, self.moms)
else: plt.plot(iterations, self.lrs)
def plot(self, skip_start=10, skip_end=5):
lrs = self.lrs[skip_start:-skip_end] if skip_end > 0 else self.lrs[skip_start:]
losses = self.losses[skip_start:-skip_end] if skip_end > 0 else self.losses[skip_start:]
_, ax = plt.subplots(1,1)
ax.plot(lrs, losses)
ax.set_xscale('log')
def plot_losses(self):
_, ax = plt.subplots(1,1)
iterations = list(range(len(self.losses)))
ax.plot(iterations, self.losses)
val_iter = self.nb_batches
val_iter = np.array(val_iter).cumsum()
ax.plot(val_iter, self.val_losses)
def plot_metrics(self):
assert len(self.metrics) != 0, "There is no metrics to plot."
_, axes = plt.subplots(len(self.metrics[0]),1,figsize=(6, 4*len(self.metrics[0])))
val_iter = np.array(self.nb_batches).cumsum()
axes = axes.flatten() if len(self.metrics[0]) != 1 else [axes]
for i, ax in enumerate(axes):
values = [met[i] for met in self.metrics]
ax.plot(val_iter, values)
def send_graphs(self):
iters = list(range(len(self.losses))) + [None] * (self.nb_epoch * self.nb_batches[-1] - len(self.losses))
losses = self.losses + [None] * (self.nb_epoch * self.nb_batches[-1] - len(self.losses))
val_iter = np.array(self.nb_batches).cumsum()
val_losses = self.val_losses + [None] * (self.nb_epoch - len(val_iter))
val_iter = list(val_iter) + [None] * (self.nb_epoch - len(val_iter))
x_bounds = (0, (self.nb_epoch - len(self.nb_batches)) * self.nb_batches[-1] + len(self.losses))
y_bounds = (0, max((max(self.losses), max(self.val_losses))))
return [(iters, losses), (val_iter, val_losses)], x_bounds, y_bounds
#export
@dataclass
class Learner():
data: DataBunch
model: nn.Module
opt_fn: Callable = optim.SGD
loss_fn: Callable = F.cross_entropy
metrics: Collection[Callable] = None
true_wd: bool = False
def __post_init__(self): self.model = self.model.to(self.data.device)
def fit(self, epochs, lr, wd=0., callbacks=None):
self.opt = OptimWrapper(self.opt_fn(self.model.parameters(), lr), wd=wd, true_wd=self.true_wd)
pbar = NBMasterBar(range(epochs))
self.recorder = Recorder(self.opt, epochs, self.data.train_dl, pbar)
if callbacks is None: callbacks = []
callbacks = [self.recorder]+callbacks
fit(epochs, self.model, self.loss_fn, self.opt, self.data, callbacks=callbacks, metrics=self.metrics, pbar=pbar)
train_ds = ImageDataset.from_folder(PATH/'train', classes=['airplane','dog'])
valid_ds = ImageDataset.from_folder(PATH/'test', classes=['airplane','dog'])
data = DataBunch.create(train_ds, valid_ds, bs=bs, train_tfm=tfms, valid_tfm=[], num_workers=4)
len(data.train_dl), len(data.valid_dl)
model = Darknet([1, 2, 2, 2, 2], num_classes=2, nf=16)
learn = Learner(data, model)
learn.fit(5,0.01)
learn.recorder.pbar.vbox