%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai import *
from fastai.torch_core import *
from fastai.vision import *
from fastai.metrics import *
from torchvision.models import resnet18
import tensorflow as tf
from io import BytesIO
path = untar_data(URLs.MNIST_TINY)
data = ImageDataBunch.from_folder(path)
# From https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514
"""Simple example on how to log scalars and images to tensorboard without tensor ops.
License: Copyleft
"""
#__author__ = "Michael Gygli"
#import tensorflow as tf
#from StringIO import StringIO
#import matplotlib.pyplot as plt
#import numpy as np
class Logger(object):
"""Logging in tensorboard without tensorflow ops."""
def __init__(self, log_dir):
"""Creates a summary writer logging to log_dir."""
self.writer = tf.summary.FileWriter(log_dir)
def log_scalar(self, tag, value, step):
"""Log a scalar variable.
Parameter
----------
tag : basestring
Name of the scalar
value
step : int
training iteration
"""
summary = tf.Summary(value=[tf.Summary.Value(tag=tag,
simple_value=value)])
self.writer.add_summary(summary, step)
def log_images(self, tag, images, step):
"""Logs a list of images."""
im_summaries = []
for nr, img in enumerate(images):
# Write the image to a string
s = StringIO()
plt.imsave(s, img, format='png')
# Create an Image object
img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),
height=img.shape[0],
width=img.shape[1])
# Create a Summary value
im_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, nr),
image=img_sum))
# Create and write Summary
summary = tf.Summary(value=im_summaries)
self.writer.add_summary(summary, step)
def log_histogram(self, tag, values, step, bins=1000):
"""Logs the histogram of a list/vector of values."""
# Convert to a numpy array
values = np.array(values)
# Create histogram using numpy
counts, bin_edges = np.histogram(values, bins=bins)
# Fill fields of histogram proto
hist = tf.HistogramProto()
hist.min = float(np.min(values))
hist.max = float(np.max(values))
hist.num = int(np.prod(values.shape))
hist.sum = float(np.sum(values))
hist.sum_squares = float(np.sum(values**2))
# Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1]
# See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto#L30
# Thus, we drop the start of the first bin
bin_edges = bin_edges[1:]
# Add bin edges and counts
for edge in bin_edges:
hist.bucket_limit.append(edge)
for c in counts:
hist.bucket.append(c)
# Create and write Summary
summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])
self.writer.add_summary(summary, step)
self.writer.flush()
"A `Callback` that saves tracked metrics into a log file for Tensorboard."
# Based on https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514
# and devforfu: https://nbviewer.jupyter.org/gist/devforfu/ea0b3fcfe194dad323c3762492b05cae
# Contribution from MicPie
#from ..torch_core import *
#from ..basic_data import DataBunch
#from ..callback import *
#from ..basic_train import Learner, LearnerCallback
#import tensorflow as tf
__all__ = ['TBLogger']
@dataclass
class TBLogger(LearnerCallback):
"A `LearnerCallback` that saves history of metrics while training `learn` into log files for Tensorboard."
log_dir:str = 'logs'
log_name:str = 'data'
log_scalar:bool = True # log scalar values for Tensorboard scalar summary
log_hist:bool = True # log values and gradients of the parameters for Tensorboard histogram summary
log_img:bool = False # log values for Tensorboard image summary
def __post_init__(self):
super().__post_init__()
#def __init__(self):
# super().__init__()
self.path = self.learn.path
(self.path/self.log_dir).mkdir(parents=True, exist_ok=True) # setup logs directory
self.Log = Logger(str(self.path/self.log_dir/self.log_name))
self.epoch = 0
self.batch = 0
self.log_grads = {}
def on_backward_end(self, **kwargs:Any):
self.batch = self.batch+1
#print('\nBatch: ',self.batch)
if self.log_hist:
for tag, value in learn.model.named_parameters():
tag_grad = tag.replace('.', '/')+'/grad'
if tag_grad in self.log_grads:
#self.log_grads[tag_grad] += value.grad.data.cpu().detach().numpy()
self.log_grads[tag_grad] = self.log_grads[tag_grad] + value.grad.data.cpu().detach().numpy() # gradients are summed up from every batch
#print('if')
else:
self.log_grads[tag_grad] = value.grad.data.cpu().detach().numpy()
#print('else')
#print(tag_grad, self.log_grads[tag_grad].sum())
return self.log_grads
#def on_step_end(self, **kwards:Any):
#print('Step end: ', self.log_grads)
def on_epoch_end(self, epoch:int, smooth_loss:Tensor, last_metrics:MetricsList, **kwargs:Any) -> bool:
last_metrics = ifnone(last_metrics, [])
tr_info = {name: stat for name, stat in zip(self.learn.recorder.names, [epoch, smooth_loss] + last_metrics)}
self.epoch = tr_info['epoch']
self.batch = 0 # reset batch count
#print('\nEpoch: ',self.epoch)
if self.log_scalar:
for tag, value in tr_info.items():
if tag == 'epoch': continue
self.Log.log_scalar(tag, value, self.epoch+1)
if self.log_hist:
for tag, value in learn.model.named_parameters():
tag = tag.replace('.', '/')
self.Log.log_histogram(tag, value.data.cpu().numpy(), self.epoch+1)
tag_grad = tag.replace('.', '/')+'/grad'
self.Log.log_histogram(tag_grad, self.log_grads[tag_grad], self.epoch+1)
#print(tag_grad, self.log_grads[tag_grad].sum())
#if self.log_img:
# for tag, value in learn.model.named_parameters():
#
# tag = tag.replace('.', '/')
# self.Log.log_images(tag, value.data.cpu().numpy(), self.epoch+1)
# If you want to save the log files in a special directory use partial (default directory is 'data'):
#TB = partial(TBLogger, log_name='name')
learn = Learner(data, simple_cnn((3, 16, 16, 2)), metrics=[accuracy, error_rate], callback_fns=[TBLogger])
learn.fit_one_cycle(10)
Total time: 00:14 epoch train_loss valid_loss accuracy error_rate 1 0.688752 0.674943 0.741059 0.258941 (00:03) 2 0.650126 0.518507 0.915594 0.084406 (00:01) 3 0.539764 0.203503 0.942775 0.057225 (00:01) 4 0.420268 0.148786 0.948498 0.051502 (00:01) 5 0.350732 0.134421 0.942775 0.057225 (00:01) 6 0.315148 0.123754 0.955651 0.044349 (00:01) 7 0.277169 0.139031 0.945637 0.054363 (00:01) 8 0.250696 0.121647 0.957082 0.042918 (00:01) 9 0.229372 0.119526 0.961373 0.038627 (00:01) 10 0.211137 0.119364 0.961373 0.038627 (00:01)
# https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/04-utils/tensorboard/logger.py ?
# https://github.com/lanpa/tensorboardX ??? ??? ???
# To start Tensorboard run the following command in the directoy of the log file folder:
# tensorboard --logdir=./logs
# Then open localhost:6006 if you are on your local machine