Image Segmentation Task 3¶
In [1]:
!pip install positional_encodings;
!pip install cityscapesscripts;
Collecting positional_encodings
Downloading positional_encodings-3.0.0-py3-none-any.whl (5.0 kB)
Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from positional_encodings) (1.18.5)
Requirement already satisfied: torch in /opt/conda/lib/python3.6/site-packages (from positional_encodings) (1.6.0a0+9907a3e)
Requirement already satisfied: future in /opt/conda/lib/python3.6/site-packages (from torch->positional_encodings) (0.18.2)
Installing collected packages: positional-encodings
Successfully installed positional-encodings-3.0.0
Collecting cityscapesscripts
Downloading cityscapesScripts-2.2.0-py3-none-any.whl (472 kB)
|████████████████████████████████| 472 kB 10.8 MB/s eta 0:00:01
Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from cityscapesscripts) (1.18.5)
Collecting pillow
Downloading Pillow-8.1.2-cp36-cp36m-manylinux1_x86_64.whl (2.2 MB)
|████████████████████████████████| 2.2 MB 11.7 MB/s eta 0:00:01
Requirement already satisfied: appdirs in /opt/conda/lib/python3.6/site-packages (from cityscapesscripts) (1.4.4)
Requirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from cityscapesscripts) (4.31.1)
Collecting pyquaternion
Downloading pyquaternion-0.9.9-py3-none-any.whl (14 kB)
Requirement already satisfied: typing in /opt/conda/lib/python3.6/site-packages (from cityscapesscripts) (3.7.4.1)
Collecting coloredlogs
Downloading coloredlogs-15.0-py2.py3-none-any.whl (45 kB)
|████████████████████████████████| 45 kB 7.0 MB/s eta 0:00:01
Requirement already satisfied: matplotlib in /opt/conda/lib/python3.6/site-packages (from cityscapesscripts) (3.2.2)
Collecting humanfriendly>=9.1
Downloading humanfriendly-9.1-py2.py3-none-any.whl (86 kB)
|████████████████████████████████| 86 kB 10.9 MB/s eta 0:00:01
Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib->cityscapesscripts) (2.8.1)
Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.6/site-packages (from matplotlib->cityscapesscripts) (0.10.0)
Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib->cityscapesscripts) (2.4.7)
Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib->cityscapesscripts) (1.2.0)
Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.6/site-packages (from python-dateutil>=2.1->matplotlib->cityscapesscripts) (1.15.0)
Installing collected packages: pillow, pyquaternion, humanfriendly, coloredlogs, cityscapesscripts
Successfully installed cityscapesscripts-2.2.0 coloredlogs-15.0 humanfriendly-9.1 pillow-8.1.2 pyquaternion-0.9.9
Imports¶
In [1]:
import os
import copy
import random
import torch
import numpy as np
from os.path import join as pjoin
from datasets.cityscapes import cityscapesDataset
Setup¶
In [2]:
# reproducibility setups
torch.manual_seed(253452757)
random.seed(253452757)
np.random.seed(253452757)
In [3]:
local_path = "./Cityscapes"
os.environ["CUDA_VISIBLE_DEVICES"]="1"
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device
Out[3]:
device(type='cuda')
Hyperparameters¶
In [4]:
bs = 5
epochs = 15
learning_rate = 0.0002
Dataset and Dataloader¶
In [5]:
training_data = cityscapesDataset(local_path, split="train")
training_data_raw = cityscapesDataset(local_path, split="train", is_transform=False)
validation_data = cityscapesDataset(local_path, split="val")
test_data = cityscapesDataset(local_path, split="test")
Annotations files processed Annotations files processed Annotations files processed Annotations files processed
In [6]:
dataset_sizes = {
'train': len(training_data),
'val': len(validation_data),
'test': len(test_data),
'total': len(training_data) + len(validation_data) + len(test_data)
}
dataset_sizes
Out[6]:
{'train': 2975, 'val': 500, 'test': 1525, 'total': 5000}
In [7]:
dataloaders = {
'train': torch.utils.data.DataLoader(training_data, batch_size=bs, shuffle=True),
'val': torch.utils.data.DataLoader(validation_data, batch_size=bs, shuffle=True),
'test': torch.utils.data.DataLoader(test_data, batch_size=bs, shuffle=True),
}
In [9]:
img, lbl = training_data[0]
In [9]:
training_data.decode_segmap(lbl.numpy(), plot=True)
Model setup¶
In [8]:
import importlib
from networks.utils import count_parameters
from networks.r2unet_hanet import R2Unet64HANet
In [9]:
loss_f = torch.nn.CrossEntropyLoss(ignore_index = 255)
R2UNet64 + 1 HANet layer¶
In [11]:
model1 = R2Unet64HANet(hanet_layers=1).to(device)
In [12]:
print("Number of trainable parameters = ", count_parameters(model1, only_trainable=True))
print("Number of total parameters = ", count_parameters(model1))
Number of trainable parameters = 78490051 Number of total parameters = 78490051
In [ ]:
optimizer1 = torch.optim.Adam(model1.parameters(), lr = learning_rate)
R2UNet64 + 2 HANet layers¶
In [9]:
model2 = R2Unet64HANet(hanet_layers=2).to(device)
In [10]:
print("Number of trainable parameters = ", count_parameters(model2, only_trainable=True))
print("Number of total parameters = ", count_parameters(model2))
Number of trainable parameters = 78494427 Number of total parameters = 78494427
In [ ]:
optimizer2 = torch.optim.Adam(model2.parameters(), lr = learning_rate)
R2UNet64 + 3 HANet layers¶
In [10]:
model3 = R2Unet64HANet(hanet_layers=3).to(device)
In [11]:
print("Number of trainable parameters = ", count_parameters(model3, only_trainable=True))
print("Number of total parameters = ", count_parameters(model3))
Number of trainable parameters = 78495559 Number of total parameters = 78495559
In [31]:
optimizer3 = torch.optim.Adam(model3.parameters(), lr = learning_rate)
Training¶
In [19]:
import training
R2UNet64 + 1 HANet layer¶
In [13]:
model_path1 = 'models2/r2unet_hanet64/1'
if not os.path.isdir(model_path1):
os.makedirs(model_path1)
In [ ]:
model = training.train(model1, dataloaders, dataset_sizes,
model_path1, loss_f, optimizer1, epochs)
R2UNet64 + 2 HANet layers¶
In [11]:
model_path2 = 'models2/r2unet_hanet64/2'
if not os.path.isdir(model_path2):
os.makedirs(model_path2)
In [ ]:
model2 = training.train(model2, dataloaders, dataset_sizes,
model_path2, loss_f, optimizer2, epochs)
R2UNet64 + 3 HANet layers¶
In [12]:
model_path3 = 'models/r2unet_hanet64/3'
if not os.path.isdir(model_path3):
os.makedirs(model_path3)
In [ ]:
model3 = training.train(model3, dataloaders, dataset_sizes,
model_path3, loss_f, optimizer3, epochs)
Evaluate the results¶
In [16]:
from evaluation import EvaluationReport
In [17]:
metrics = ["accuracy", "sensitivity", "specificity", "dice_coeff", "jaccard_sim"]
R2UNet64 + 1 HANet layer¶
In [16]:
checkpoint1 = torch.load(pjoin(model_path1, "epoch-{}.pt".format(epochs-1)))
In [17]:
model1.load_state_dict(checkpoint1['model_state_dict'])
Out[17]:
<All keys matched successfully>
In [18]:
val_report1 = EvaluationReport.from_model(dataloaders['val'], model1, labels=list(range(19)))
In [19]:
val_report1.get_metrics(metrics, average="macro")
Out[19]:
{'accuracy': 0.9336,
'sensitivity': 0.3032,
'specificity': 0.9654,
'dice_coeff': 0.2538,
'jaccard_sim': 0.171}
In [20]:
val_report1.get_metrics(metrics, average="weighted")
Out[20]:
{'accuracy': 0.8325,
'sensitivity': 0.3689,
'specificity': 0.9743,
'dice_coeff': 0.4779,
'jaccard_sim': 0.3272}
In [21]:
for i in range(19):
print("Class {} - {}".format(i, training_data.label_names()[i]))
class_metrics = val_report1.get_metrics(metrics, pos_label=i)
for k in class_metrics.keys():
print(" {} = {}".format(k, class_metrics[k]))
print()
Class 0 - road accuracy = 0.713 sensitivity = 0.3099 specificity = 0.956 dice_coeff = 0.4482 jaccard_sim = 0.2888 Class 1 - sidewalk accuracy = 0.9468 sensitivity = 0.0315 specificity = 0.9991 dice_coeff = 0.0601 jaccard_sim = 0.031 Class 2 - building accuracy = 0.8598 sensitivity = 0.4007 specificity = 0.9889 dice_coeff = 0.5564 jaccard_sim = 0.3854 Class 3 - wall accuracy = 0.9924 sensitivity = 0.0054 specificity = 0.9997 dice_coeff = 0.0104 jaccard_sim = 0.0052 Class 4 - fence accuracy = 0.9915 sensitivity = 0.088 specificity = 0.999 dice_coeff = 0.1449 jaccard_sim = 0.0781 Class 5 - pole accuracy = 0.9824 sensitivity = 0.3187 specificity = 0.9924 dice_coeff = 0.3487 jaccard_sim = 0.2112 Class 6 - traffic light accuracy = 0.9788 sensitivity = 0.2289 specificity = 0.9803 dice_coeff = 0.0409 jaccard_sim = 0.0209 Class 7 - traffic sign accuracy = 0.5932 sensitivity = 0.7122 specificity = 0.5924 dice_coeff = 0.0228 jaccard_sim = 0.0115 Class 8 - vegetation accuracy = 0.8967 sensitivity = 0.4247 specificity = 0.9956 dice_coeff = 0.5875 jaccard_sim = 0.416 Class 9 - terrain accuracy = 0.9866 sensitivity = 0.4658 specificity = 0.991 dice_coeff = 0.3671 jaccard_sim = 0.2248 Class 10 - sky accuracy = 0.9892 sensitivity = 0.7277 specificity = 0.9983 dice_coeff = 0.8188 jaccard_sim = 0.6932 Class 11 - person accuracy = 0.9849 sensitivity = 0.2573 specificity = 0.9944 dice_coeff = 0.3059 jaccard_sim = 0.1805 Class 12 - rider accuracy = 0.9933 sensitivity = 0.1792 specificity = 0.9951 dice_coeff = 0.1032 jaccard_sim = 0.0544 Class 13 - car accuracy = 0.9417 sensitivity = 0.6577 specificity = 0.9615 dice_coeff = 0.5951 jaccard_sim = 0.4236 Class 14 - truck accuracy = 0.9132 sensitivity = 0.3801 specificity = 0.9148 dice_coeff = 0.0257 jaccard_sim = 0.013 Class 15 - bus accuracy = 0.9853 sensitivity = 0.4494 specificity = 0.9874 dice_coeff = 0.1922 jaccard_sim = 0.1063 Class 16 - train accuracy = 0.9974 sensitivity = 0.0038 specificity = 0.9985 dice_coeff = 0.0033 jaccard_sim = 0.0016 Class 17 - motorcycle accuracy = 0.9984 sensitivity = 0.0324 specificity = 0.9992 dice_coeff = 0.0312 jaccard_sim = 0.0159 Class 18 - bicycle accuracy = 0.9935 sensitivity = 0.0883 specificity = 0.9999 dice_coeff = 0.1607 jaccard_sim = 0.0874
R2UNet64 + 2 HANet layers¶
In [12]:
checkpoint2 = torch.load(pjoin(model_path2, "epoch-{}.pt".format(epochs-1)))
In [13]:
model2.load_state_dict(checkpoint2['model_state_dict'])
Out[13]:
<All keys matched successfully>
In [18]:
val_report2 = EvaluationReport.from_model(dataloaders['val'], model2, labels=list(range(19)))
In [19]:
val_report2.get_metrics(metrics, average="macro")
Out[19]:
{'accuracy': 0.9663,
'sensitivity': 0.3154,
'specificity': 0.9797,
'dice_coeff': 0.265,
'jaccard_sim': 0.2002}
In [20]:
val_report2.get_metrics(metrics, average="weighted")
Out[20]:
{'accuracy': 0.9134,
'sensitivity': 0.6799,
'specificity': 0.9341,
'dice_coeff': 0.6926,
'jaccard_sim': 0.5743}
In [21]:
for i in range(19):
print("Class {} - {}".format(i, training_data.label_names()[i]))
class_metrics = val_report2.get_metrics(metrics, pos_label=i)
for k in class_metrics.keys():
print(" {} = {}".format(k, class_metrics[k]))
print()
Class 0 - road accuracy = 0.8826 sensitivity = 0.9094 specificity = 0.8664 dice_coeff = 0.8535 jaccard_sim = 0.7445 Class 1 - sidewalk accuracy = 0.9467 sensitivity = 0.0222 specificity = 0.9996 dice_coeff = 0.0431 jaccard_sim = 0.022 Class 2 - building accuracy = 0.9103 sensitivity = 0.7374 specificity = 0.9589 dice_coeff = 0.7829 jaccard_sim = 0.6433 Class 3 - wall accuracy = 0.9919 sensitivity = 0.0207 specificity = 0.9991 dice_coeff = 0.0362 jaccard_sim = 0.0184 Class 4 - fence accuracy = 0.9903 sensitivity = 0.1286 specificity = 0.9975 dice_coeff = 0.1793 jaccard_sim = 0.0985 Class 5 - pole accuracy = 0.9808 sensitivity = 0.1293 specificity = 0.9935 dice_coeff = 0.1656 jaccard_sim = 0.0903 Class 6 - traffic light accuracy = 0.9336 sensitivity = 0.6077 specificity = 0.9342 dice_coeff = 0.0349 jaccard_sim = 0.0178 Class 7 - traffic sign accuracy = 0.9295 sensitivity = 0.4574 specificity = 0.9327 dice_coeff = 0.0797 jaccard_sim = 0.0415 Class 8 - vegetation accuracy = 0.9121 sensitivity = 0.6442 specificity = 0.9682 dice_coeff = 0.7174 jaccard_sim = 0.5593 Class 9 - terrain accuracy = 0.9919 sensitivity = 0.1129 specificity = 0.9993 dice_coeff = 0.1887 jaccard_sim = 0.1042 Class 10 - sky accuracy = 0.9943 sensitivity = 0.8741 specificity = 0.9985 dice_coeff = 0.9112 jaccard_sim = 0.8369 Class 11 - person accuracy = 0.9872 sensitivity = 0.1432 specificity = 0.9983 dice_coeff = 0.225 jaccard_sim = 0.1268 Class 12 - rider accuracy = 0.9978 sensitivity = 0.0131 specificity = 0.9999 dice_coeff = 0.0251 jaccard_sim = 0.0127 Class 13 - car accuracy = 0.953 sensitivity = 0.3383 specificity = 0.9959 dice_coeff = 0.4843 jaccard_sim = 0.3195 Class 14 - truck accuracy = 0.997 sensitivity = 0.0 specificity = 1.0 dice_coeff = 0.0 jaccard_sim = 0.0 Class 15 - bus accuracy = 0.9961 sensitivity = 0.0 specificity = 1.0 dice_coeff = 0.0 jaccard_sim = 0.0 Class 16 - train accuracy = 0.9889 sensitivity = 0.3251 specificity = 0.9897 dice_coeff = 0.0617 jaccard_sim = 0.0319 Class 17 - motorcycle accuracy = 0.9826 sensitivity = 0.4015 specificity = 0.983 dice_coeff = 0.0353 jaccard_sim = 0.018 Class 18 - bicycle accuracy = 0.9932 sensitivity = 0.127 specificity = 0.9994 dice_coeff = 0.2103 jaccard_sim = 0.1175
R2UNet64 + 3 HANet layers¶
In [13]:
checkpoint3 = torch.load(pjoin(model_path3, "epoch-{}.pt".format(epochs-1)))
In [14]:
model3.load_state_dict(checkpoint3['model_state_dict'])
Out[14]:
<All keys matched successfully>
In [21]:
val_report3 = EvaluationReport.from_model(dataloaders['val'], model3, labels=list(range(19)))
In [23]:
val_report3.get_metrics(metrics, average="macro")
Out[23]:
{'accuracy': 0.9225,
'sensitivity': 0.1397,
'specificity': 0.9592,
'dice_coeff': 0.128,
'jaccard_sim': 0.0823}
In [24]:
val_report3.get_metrics(metrics, average="weighted")
Out[24]:
{'accuracy': 0.7868,
'sensitivity': 0.2633,
'specificity': 0.9609,
'dice_coeff': 0.3174,
'jaccard_sim': 0.2009}
In [28]:
for i in range(19):
print("Class {} - {}".format(i, training_data.label_names()[i]))
class_metrics = val_report3.get_metrics(metrics, pos_label=i)
for k in class_metrics.keys():
print(" {} = {}".format(k, class_metrics[k]))
print()
Class 0 - road accuracy = 0.6896 sensitivity = 0.18 specificity = 0.9969 dice_coeff = 0.3037 jaccard_sim = 0.179 Class 1 - sidewalk accuracy = 0.9459 sensitivity = 0.0005 specificity = 1.0 dice_coeff = 0.001 jaccard_sim = 0.0005 Class 2 - building accuracy = 0.808 sensitivity = 0.1772 specificity = 0.9853 dice_coeff = 0.2882 jaccard_sim = 0.1683 Class 3 - wall accuracy = 0.9923 sensitivity = 0.0009 specificity = 0.9997 dice_coeff = 0.0018 jaccard_sim = 0.0009 Class 4 - fence accuracy = 0.9918 sensitivity = 0.0 specificity = 1.0 dice_coeff = 0.0 jaccard_sim = 0.0 Class 5 - pole accuracy = 0.9826 sensitivity = 0.0687 specificity = 0.9963 dice_coeff = 0.1047 jaccard_sim = 0.0552 Class 6 - traffic light accuracy = 0.9965 sensitivity = 0.0579 specificity = 0.9983 dice_coeff = 0.0612 jaccard_sim = 0.0316 Class 7 - traffic sign accuracy = 0.9926 sensitivity = 0.1113 specificity = 0.9985 dice_coeff = 0.1671 jaccard_sim = 0.0912 Class 8 - vegetation accuracy = 0.8957 sensitivity = 0.4165 specificity = 0.9962 dice_coeff = 0.5806 jaccard_sim = 0.4091 Class 9 - terrain accuracy = 0.9918 sensitivity = 0.0107 specificity = 1.0 dice_coeff = 0.0212 jaccard_sim = 0.0107 Class 10 - sky accuracy = 0.9799 sensitivity = 0.5373 specificity = 0.9953 dice_coeff = 0.6426 jaccard_sim = 0.4734 Class 11 - person accuracy = 0.9286 sensitivity = 0.1289 specificity = 0.9391 dice_coeff = 0.0447 jaccard_sim = 0.0229 Class 12 - rider accuracy = 0.9979 sensitivity = 0.0 specificity = 1.0 dice_coeff = 0.0 jaccard_sim = 0.0 Class 13 - car accuracy = 0.5435 sensitivity = 0.9638 specificity = 0.5142 dice_coeff = 0.2158 jaccard_sim = 0.121 Class 14 - truck accuracy = 0.997 sensitivity = 0.0 specificity = 1.0 dice_coeff = 0.0 jaccard_sim = 0.0 Class 15 - bus accuracy = 0.9961 sensitivity = 0.0 specificity = 1.0 dice_coeff = 0.0 jaccard_sim = 0.0 Class 16 - train accuracy = 0.9989 sensitivity = 0.0 specificity = 1.0 dice_coeff = 0.0 jaccard_sim = 0.0 Class 17 - motorcycle accuracy = 0.9992 sensitivity = 0.0 specificity = 1.0 dice_coeff = 0.0 jaccard_sim = 0.0 Class 18 - bicycle accuracy = 0.7987 sensitivity = 0.0014 specificity = 0.8044 dice_coeff = 0.0001 jaccard_sim = 0.0001
Visualize the results¶
In [13]:
from utils import plot_seg_results
Select 5 images and predict their segmentation mask using the best weights:
In [14]:
img_size = (512,256)
init_pic = 123
n_pics = 5
In [15]:
inputs = torch.stack([training_data[i][0] for i in range(init_pic, init_pic + n_pics)])
images = [training_data_raw[i][0].resize(img_size) for i in range(init_pic, init_pic + n_pics)]
ground_truths = torch.stack([training_data[i][1] for i in range(init_pic, init_pic + n_pics)])
ground_truths = [training_data.decode_segmap(gt.numpy()) for gt in ground_truths]
R2Unet64 + 1 HANet layers¶
In [16]:
outputs = model1(inputs.to(device))
predictions = torch.argmax(outputs.squeeze().cpu(), dim = 1)
predictions = [training_data.decode_segmap(pred.numpy()) for pred in predictions]
In [17]:
plot_seg_results(images, ground_truths, predictions)
R2Unet64 + 2 HANet layers¶
In [16]:
outputs = model2(inputs.to(device))
predictions = torch.argmax(outputs.squeeze().cpu(), dim = 1)
predictions = [training_data.decode_segmap(pred.numpy()) for pred in predictions]
In [17]:
plot_seg_results(images, ground_truths, predictions)
R2Unet64 + 3 HANet layers¶
In [19]:
outputs = model3(inputs.to(device))
predictions = torch.argmax(outputs.squeeze().cpu(), dim = 1)
predictions = [training_data.decode_segmap(pred.numpy()) for pred in predictions]
In [22]:
plot_seg_results(images, ground_truths, predictions)
In [ ]: