MURA (musculoskeletal radiographs) is a large dataset of bone X-rays. Algorithms are tasked with determining whether an X-ray study is normal or abnormal.
Musculoskeletal conditions affect more than 1.7 billion people worldwide, and are the most common cause of severe, long-term pain and disability, with 30 million emergency department visits annually and increasing. We hope that our dataset can lead to significant advances in medical imaging technologies which can diagnose at the level of experts, towards improving healthcare access in parts of the world where access to skilled radiologists is limited.
MURA is one of the largest public radiographic image datasets. We're making this dataset available to the community and hosting a competition to see if your models can perform as well as radiologists on the task.
%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.vision import *
from fastai.widgets import *
import shutil
from fastai.callbacks import *
import fastai
print(f'fastai: {fastai.__version__}')
print(f'cuda: {torch.cuda.is_available()}')
fastai: 1.0.50.post1 cuda: True
import gc
import torch
Config.data_path()
PosixPath('/home/jupyter/tutorials/data')
path = Config.data_path() / 'MURA-v1.1'
path.ls()
[PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/valid_labeled_studies.csv'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/results_densenet169_112.csv'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/valid'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/results_densenet169_320.csv'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/train_image_paths.csv'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/train_labeled_studies.csv'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/valid_image_paths.csv'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/train'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/data2')]
df_train = pd.read_csv(path / 'train_image_paths.csv', header=None, names=['image'])
df_train.image[0]
'MURA-v1.1/train/XR_SHOULDER/patient00001/study1_positive/image1.png'
df_valid = pd.read_csv(path / 'valid_image_paths.csv', header=None, names=['image'])
df_valid.image[0]
'MURA-v1.1/valid/XR_WRIST/patient11185/study1_positive/image1.png'
df_train_label = pd.read_csv(path / 'train_labeled_studies.csv', header=None, names=['image', 'label'])
df_train_label.head()
image | label | |
---|---|---|
0 | MURA-v1.1/train/XR_SHOULDER/patient00001/study... | 1 |
1 | MURA-v1.1/train/XR_SHOULDER/patient00002/study... | 1 |
2 | MURA-v1.1/train/XR_SHOULDER/patient00003/study... | 1 |
3 | MURA-v1.1/train/XR_SHOULDER/patient00004/study... | 1 |
4 | MURA-v1.1/train/XR_SHOULDER/patient00005/study... | 1 |
# Create data2 with data to train our models
path_train = path / 'data2/train'
path_valid = path / 'data2/valid'
path_train.mkdir(parents=True, exist_ok=True)
path_valid.mkdir(parents=True, exist_ok=True)
path_train_neg = path_train / '0'
path_train_pos = path_train / '1'
path_train_neg.mkdir(parents=True, exist_ok=True)
path_train_pos.mkdir(parents=True, exist_ok=True)
path_valid_neg = path_valid / '0'
path_valid_pos = path_valid / '1'
path_valid_neg.mkdir(parents=True, exist_ok=True)
path_valid_pos.mkdir(parents=True, exist_ok=True)
fnames_train = get_image_files(path/'train', recurse=True)
print(len(fnames_train))
fnames_train[:5]
36808
[PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/train/XR_FINGER/patient04021/study1_negative/image1.png'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/train/XR_FINGER/patient03816/study1_negative/image1.png'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/train/XR_FINGER/patient04345/study1_negative/image2.png'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/train/XR_FINGER/patient04345/study1_negative/image3.png'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/train/XR_FINGER/patient04345/study1_negative/image1.png')]
fnames_valid = get_image_files(path/'valid', recurse=True)
print(len(fnames_valid))
fnames_valid[:5]
3197
[PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/valid/XR_FINGER/patient11855/study1_negative/image1.png'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/valid/XR_FINGER/patient11606/study1_negative/image1.png'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/valid/XR_FINGER/patient11513/study1_negative/image1.png'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/valid/XR_FINGER/patient11911/study1_positive/image2.png'), PosixPath('/home/jupyter/tutorials/data/MURA-v1.1/valid/XR_FINGER/patient11911/study1_positive/image3.png')]
pat_label = re.compile(r'/XR_([^/]+)/[^/]+/[^/]+/[^/]+.png$')
pat_patient = re.compile(r'/[^/]+/patient([^/]+)/[^/]+/[^/]+.png$')
pat_study = re.compile(r'/[^/]+/[^/]+/([^/]+)/[^/]+.png$')
# pat_study_negpos = re.compile(r'\\[^\\]+\\[^\\]+\\study\d+_([^\\]+)\\[^\\]+.png$')
%%time
# copy all train images in corresponding class folders under MURA-v1.1/data2/train
for src in fnames_train:
# get image label
label = pat_label.search(str(src))
label = label.group(1)
# get patient number
patient = pat_patient.search(str(src))
patient = patient.group(1)
# get study name
study = pat_study.search(str(src))
study = study.group(1)
# create class folder if necessary
if 'negative' in study:
path_label = path_train_neg
else:
path_label = path_train_pos
# copy image to its class folder
img_name = label + '_patient' + patient + '_' + study + '_' + src.name
dest = path_label / img_name
shutil.copy(str(src), str(dest))
CPU times: user 3.42 s, sys: 6.41 s, total: 9.83 s Wall time: 2min 22s
%%time
# copy all valid images in corresponding class folders under MURA-v1.1/data2/valid
for src in fnames_valid:
# get image label
label = pat_label.search(str(src))
label = label.group(1)
# get patient number
patient = pat_patient.search(str(src))
patient = patient.group(1)
# get study name
study = pat_study.search(str(src))
study = study.group(1)
# create class folder if necessary
if 'negative' in study:
path_label = path_valid_neg
else:
path_label = path_valid_pos
# copy image to its class folder
img_name = label + '_patient' + patient + '_' + study + '_' + src.name
dest = path_label / img_name
shutil.copy(str(src), str(dest))
CPU times: user 336 ms, sys: 564 ms, total: 900 ms Wall time: 11.4 s
pat_label = re.compile(r'/XR_([^/]+)/[^/]+/[^/]+/[^/]+.png$')
pat_patient = re.compile(r'/[^/]+/patient([^/]+)/[^/]+/[^/]+.png$')
pat_study = re.compile(r'/([^/]+)_[^/]+/[^/]+.png$')
mura = ['elbow', 'finger', 'forearm', 'hand', 'humerus', 'shoulder', 'wrist']
study_train_dict = dict()
study_valid_dict = dict()
for m in mura:
study_train_dict[m] = list()
study_valid_dict[m] = list()
for src in fnames_train:
# get image label
label = pat_label.search(str(src))
label = label.group(1)
# get patient number
patient = pat_patient.search(str(src))
patient = patient.group(1)
# get study name
study = pat_study.search(str(src))
study = study.group(1)
# add to label list
s = 'patient' + patient + '_' + study
study_train_dict[label.lower()].append(s)
for src in fnames_valid:
# get image label
label = pat_label.search(str(src))
label = label.group(1)
# get patient number
patient = pat_patient.search(str(src))
patient = patient.group(1)
# get study name
study = pat_study.search(str(src))
study = study.group(1)
# add to label list
s = 'patient' + patient + '_' + study
study_valid_dict[label.lower()].append(s)
num_train_studies = 0
num_valid_studies = 0
for m in mura:
# train
myset = set(study_train_dict[m])
num_train_studies += len(myset)
# valid
myset = set(study_valid_dict[m])
num_valid_studies += len(myset)
# 207 studies in test
num_train_studies, num_valid_studies, num_train_studies + num_valid_studies + 207
(13457, 1199, 14863)
learn = None
gc.collect()
8
torch.cuda.empty_cache()
size = 112
bs = 8
np.random.seed(42)
data = ImageDataBunch.from_folder(path/'data2', ds_tfms=get_transforms(max_rotate=30.0, max_warp=0.),
size=size, bs=bs,
).normalize(imagenet_stats)
kappa = KappaScore()
kappa.weights = "quadratic"
learn = cnn_learner(data, models.densenet169, metrics=[error_rate, accuracy, kappa], wd=0.1)
learn.lr_find()
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
lr = 1e-4
learn.fit_one_cycle(10,max_lr=lr,callbacks=[ShowGraph(learn),SaveModelCallback(learn)])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.740211 | 0.662548 | 0.353144 | 0.646856 | 0.285176 | 07:36 |
1 | 0.680526 | 0.592127 | 0.297779 | 0.702221 | 0.396014 | 07:28 |
2 | 0.562902 | 0.555230 | 0.273381 | 0.726619 | 0.443564 | 07:25 |
3 | 0.523256 | 0.498727 | 0.233344 | 0.766656 | 0.526991 | 07:23 |
4 | 0.526669 | 0.489869 | 0.228339 | 0.771661 | 0.538790 | 07:28 |
5 | 0.498140 | 0.476925 | 0.223022 | 0.776978 | 0.549446 | 07:26 |
6 | 0.493177 | 0.466605 | 0.209884 | 0.790116 | 0.576147 | 07:26 |
7 | 0.500830 | 0.484909 | 0.220206 | 0.779794 | 0.552917 | 07:32 |
8 | 0.479140 | 0.472091 | 0.210197 | 0.789803 | 0.574351 | 07:30 |
9 | 0.489796 | 0.455843 | 0.206756 | 0.793244 | 0.583546 | 07:32 |
Better model found at epoch 0 with val_loss value: 0.6625477075576782. Better model found at epoch 1 with val_loss value: 0.5921269059181213. Better model found at epoch 2 with val_loss value: 0.5552298426628113. Better model found at epoch 3 with val_loss value: 0.4987266957759857. Better model found at epoch 4 with val_loss value: 0.48986929655075073. Better model found at epoch 5 with val_loss value: 0.4769245982170105. Better model found at epoch 6 with val_loss value: 0.4666053354740143. Better model found at epoch 9 with val_loss value: 0.4558432996273041.
learn.save('densenet169-stage-1-n2')
learn.load('densenet169-stage-1-n2');
learn.lr_find()
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
lr = 1e-5
learn.fit_one_cycle(5,max_lr=lr,callbacks=[ShowGraph(learn),SaveModelCallback(learn)])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.498136 | 0.470024 | 0.213012 | 0.786988 | 0.569107 | 09:57 |
1 | 0.489590 | 0.470511 | 0.211761 | 0.788239 | 0.571498 | 10:04 |
2 | 0.498423 | 0.448467 | 0.204254 | 0.795746 | 0.587606 | 10:05 |
3 | 0.454724 | 0.443896 | 0.200500 | 0.799500 | 0.596168 | 09:56 |
4 | 0.457709 | 0.442879 | 0.196747 | 0.803253 | 0.602720 | 09:55 |
Better model found at epoch 0 with val_loss value: 0.4700239300727844. Better model found at epoch 2 with val_loss value: 0.4484669268131256. Better model found at epoch 3 with val_loss value: 0.4438956081867218. Better model found at epoch 4 with val_loss value: 0.44287851452827454.
learn.save('densenet169-stage-2-n2')
learn.load('densenet169-stage-2-n2');
lr = 1e-6
learn.fit_one_cycle(5,max_lr=lr,callbacks=[ShowGraph(learn),SaveModelCallback(learn)])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.468389 | 0.436768 | 0.195183 | 0.804817 | 0.606974 | 07:40 |
1 | 0.421702 | 0.446237 | 0.197060 | 0.802940 | 0.601840 | 07:27 |
2 | 0.437157 | 0.438054 | 0.194870 | 0.805130 | 0.607211 | 07:28 |
3 | 0.487413 | 0.444061 | 0.196121 | 0.803879 | 0.603640 | 07:33 |
4 | 0.495843 | 0.446020 | 0.195809 | 0.804191 | 0.604283 | 07:31 |
Better model found at epoch 0 with val_loss value: 0.43676796555519104.
learn.save('densenet169-stage-3-n2')
learn.purge()
learn.unfreeze()
learn.lr_find()
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
lr=4e-6
learn.fit_one_cycle(5,max_lr=slice(lr/100,lr),callbacks=[ShowGraph(learn),SaveModelCallback(learn)])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.454974 | 0.443591 | 0.199562 | 0.800438 | 0.596850 | 09:41 |
1 | 0.443154 | 0.444782 | 0.196747 | 0.803253 | 0.602849 | 09:40 |
2 | 0.450476 | 0.445256 | 0.193619 | 0.806381 | 0.608676 | 09:41 |
3 | 0.474479 | 0.439208 | 0.191742 | 0.808258 | 0.613704 | 09:44 |
4 | 0.454220 | 0.448611 | 0.199249 | 0.800751 | 0.597340 | 09:43 |
Better model found at epoch 0 with val_loss value: 0.4435909390449524. Better model found at epoch 3 with val_loss value: 0.4392077624797821.
learn.save('densenet169-stage-4-n2')
size = 112
bs = 8
np.random.seed(42)
data = ImageDataBunch.from_folder(path/'data2', ds_tfms=get_transforms(max_rotate=30.0, max_warp=0.),
size=size, bs=bs,
).normalize(imagenet_stats)
kappa = KappaScore()
kappa.weights = "quadratic"
learn = cnn_learner(data, models.densenet169, metrics=[error_rate, accuracy, kappa], wd=0.1)
learn.load('densenet169-stage-4-n2');
interp = ClassificationInterpretation.from_learner(learn)
losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs)
True
# first interpretation
interp.plot_confusion_matrix()
interp.most_confused(min_val=2)
[('1', '0', 408), ('0', '1', 205)]
interp.plot_top_losses(9, figsize=(15,11))
Source for kappa score calculation: Cohen's kappa in plain English
learn.load('densenet169-stage-4-n2');
for img_url in data.valid_ds.x.items:
print(img_url)
break
/home/jupyter/tutorials/data/MURA-v1.1/data2/valid/1/HAND_patient11538_study1_positive_image1.png
# predictions on validation
preds_val, y_val = learn.get_preds()
# file to save
file = path / 'results_densenet169_112.csv'
%%time
probs_of_images_by_studies = dict()
num_img_by_studies = dict()
labels_of_studies = dict()
num_studies_by_label = dict()
list_studies_by_label = dict()
idx_by_studies = dict()
results_by_label = dict()
acc_by_label = OrderedDict()
kappa_score = OrderedDict()
pat_label = re.compile(r'/([^/]+)_patient[^/]+.png$')
pat_study = re.compile(r'/([^/]+)_[^_]+.png$')
for idx, src in enumerate(data.valid_ds.x.items):
# get label name
label = pat_label.search(str(src))
label = label.group(1)
# get study name
study = pat_study.search(str(src))
study = study.group(1)
# sum probabilities by study
if study in probs_of_images_by_studies:
probs_of_images_by_studies[study] += preds_val[idx,:].clone()
num_img_by_studies[study] += 1
idx_by_studies[study].append(idx)
else:
probs_of_images_by_studies[study] = preds_val[idx,:].clone()
num_img_by_studies[study] = 1
labels_of_studies[study] = label
idx_by_studies[study] = [idx]
for m in mura:
num_studies_by_label[m] = sum([1 for k,v in labels_of_studies.items() if v.lower() == m])
list_studies_by_label[m] = [k for k,v in labels_of_studies.items() if v.lower() == m]
for m in mura:
list_studies = list_studies_by_label[m]
TN, FP, TP, FN = 0, 0, 0, 0
for study in list_studies:
preds_val2 = preds_val.clone()
idxs = idx_by_studies[study]
preds = preds_val2[idxs]
y_hat = np.argmax(preds, axis=1)
y = y_val[idxs]
if 'negative' in study:
TN += sum(y_hat.numpy() == y.numpy())
FP += sum(y_hat.numpy() != y.numpy())
else:
TP += sum(y_hat.numpy() == y.numpy())
FN += sum(y_hat.numpy() != y.numpy())
results_by_label[m] = [TN, FP, TP, FN]
# accuracy
acc_by_label[m] = (TN + TP) / (TN+FP+FN+TP)
# kappa
ground_truth_neg = TN+FN
ground_truth_pos = FP+TP
classifier_neg = TN+FP
classifier_pos = FN+TP
total = classifier_neg + classifier_pos
observed_accuracy = (TN + TP) / total
expected_accuracy = ((ground_truth_neg * classifier_neg / total) + (ground_truth_pos * classifier_pos / total)) / total
kappa_score[m] = (observed_accuracy - expected_accuracy) / (1 - expected_accuracy)
# overall accuracy and kappa
[TN, FP, TP, FN] = [0,0,0,0]
for m in mura:
results = results_by_label[m]
TN += results[0]
FP += results[1]
TP += results[2]
FN += results[3]
# overall accuracy
acc_by_label['overall'] = (TN + TP) / (TN+FP+FN+TP)
# overall kappa
ground_truth_neg = TN+FN
ground_truth_pos = FP+TP
classifier_neg = TN+FP
classifier_pos = FN+TP
total = classifier_neg + classifier_pos
observed_accuracy = (TN + TP) / total
expected_accuracy = ((ground_truth_neg * classifier_neg / total) + (ground_truth_pos * classifier_pos / total)) / total
kappa_score['overall'] = (observed_accuracy - expected_accuracy) / (1 - expected_accuracy)
# Create DataFrame
for k, v in acc_by_label.items():
acc_by_label[k] = round(v, 3)
for k, v in kappa_score.items():
kappa_score[k] = round(v, 3)
df = pd.DataFrame([acc_by_label, kappa_score], index =['accuracy', 'kappa'])
df.name = 'Results on validation at a study level'
df = df.transpose()
# save df
df.to_csv(file,header=True)
print(df)
accuracy kappa elbow 0.869 0.737 finger 0.764 0.530 forearm 0.814 0.628 hand 0.796 0.556 humerus 0.837 0.673 shoulder 0.773 0.546 wrist 0.821 0.632 overall 0.808 0.614 CPU times: user 544 ms, sys: 0 ns, total: 544 ms Wall time: 139 ms
# first interpretation
interp.plot_confusion_matrix()
cm = interp.confusion_matrix()
TN = cm[0,0]
FP = cm[0,1]
FN = cm[1,0]
TP = cm[1,1]
# accuracy
acc = (TN + TP) / (TN+FP+FN+TP)
# kappa
ground_truth_neg = TN+FN
ground_truth_pos = FP+TP
classifier_neg = TN+FP
classifier_pos = FN+TP
total = classifier_neg + classifier_pos
observed_accuracy = (TN + TP) / total
expected_accuracy = ((ground_truth_neg * classifier_neg / total) + (ground_truth_pos * classifier_pos / total)) / total
kappa_score = (observed_accuracy - expected_accuracy) / (1 - expected_accuracy)
round(acc,3), round(kappa_score,3)
(0.808, 0.614)
learn = None
gc.collect()
1479
torch.cuda.empty_cache()
size = 320
bs = 8
np.random.seed(42)
data = ImageDataBunch.from_folder(path/'data2', ds_tfms=get_transforms(max_rotate=30.0, max_warp=0.),
size=size, bs=bs,
).normalize(imagenet_stats)
kappa = KappaScore()
kappa.weights = "quadratic"
learn = cnn_learner(data, models.densenet169, metrics=[error_rate, accuracy, kappa], wd=0.1)
learn.load('densenet169-stage-4-n2');
# freeze the model
learn.purge()
learn.freeze()
learn.lr_find()
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
lr = 2e-4
learn.fit_one_cycle(5,max_lr=lr,callbacks=[ShowGraph(learn),SaveModelCallback(learn, monitor='kappa_score', mode='max')])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.487032 | 0.463355 | 0.206444 | 0.793556 | 0.582541 | 18:29 |
1 | 0.481628 | 0.418717 | 0.183610 | 0.816390 | 0.630148 | 18:28 |
2 | 0.468028 | 0.440058 | 0.194870 | 0.805130 | 0.605613 | 18:27 |
3 | 0.453611 | 0.423079 | 0.182358 | 0.817642 | 0.631395 | 18:26 |
4 | 0.430081 | 0.408062 | 0.177979 | 0.822021 | 0.641063 | 18:26 |
Better model found at epoch 0 with kappa_score value: 0.582541286945343. Better model found at epoch 1 with kappa_score value: 0.6301482915878296. Better model found at epoch 3 with kappa_score value: 0.631394624710083. Better model found at epoch 4 with kappa_score value: 0.6410631537437439.
learn.save('densenet169-stage-5-n2')
learn = None
gc.collect()
torch.cuda.empty_cache()
learn = cnn_learner(data, models.densenet169, metrics=[error_rate, accuracy, kappa], wd=0.1)
learn.load('densenet169-stage-5-n2');
learn.lr_find()
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
lr = 2e-4
learn.fit_one_cycle(5,max_lr=lr,callbacks=[ShowGraph(learn),SaveModelCallback(learn, monitor='kappa_score', mode='max')])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.449537 | 0.423536 | 0.176728 | 0.823272 | 0.643355 | 18:40 |
1 | 0.427389 | 0.427020 | 0.185799 | 0.814201 | 0.624796 | 18:31 |
2 | 0.418857 | 0.424010 | 0.183610 | 0.816390 | 0.629187 | 18:30 |
3 | 0.425186 | 0.413969 | 0.174226 | 0.825774 | 0.648481 | 18:31 |
4 | 0.443072 | 0.408605 | 0.173913 | 0.826087 | 0.649103 | 18:33 |
Better model found at epoch 0 with kappa_score value: 0.6433550119400024. Better model found at epoch 3 with kappa_score value: 0.648481011390686. Better model found at epoch 4 with kappa_score value: 0.6491025686264038.
learn.save('densenet169-stage-6-n2')
learn = None
gc.collect()
10077
torch.cuda.empty_cache()
size = 320
bs = 16
np.random.seed(42)
data = ImageDataBunch.from_folder(path/'data2', ds_tfms=get_transforms(max_rotate=30.0, max_warp=0.),
size=size, bs=bs,
).normalize(imagenet_stats)
kappa = KappaScore()
kappa.weights = "quadratic"
learn = cnn_learner(data, models.densenet169, metrics=[error_rate, accuracy, kappa], wd=0.1)
learn.load('densenet169-stage-6-n2');
learn.lr_find()
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
lr = 2e-5
learn.fit_one_cycle(5,max_lr=lr,callbacks=[ShowGraph(learn),SaveModelCallback(learn, monitor='kappa_score', mode='max')])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.413274 | 0.408363 | 0.173913 | 0.826087 | 0.649027 | 16:53 |
1 | 0.385019 | 0.403746 | 0.170472 | 0.829528 | 0.656648 | 16:53 |
2 | 0.399650 | 0.406853 | 0.170160 | 0.829840 | 0.656880 | 16:54 |
3 | 0.425203 | 0.404733 | 0.169221 | 0.830779 | 0.658818 | 16:54 |
4 | 0.402293 | 0.407672 | 0.169847 | 0.830153 | 0.657409 | 16:54 |
Better model found at epoch 0 with kappa_score value: 0.64902663230896. Better model found at epoch 1 with kappa_score value: 0.6566483974456787. Better model found at epoch 2 with kappa_score value: 0.6568800210952759. Better model found at epoch 3 with kappa_score value: 0.6588183641433716.
learn.save('densenet169-stage-7-n2')
lr = 2e-6
learn.fit_one_cycle(5,max_lr=lr,callbacks=[ShowGraph(learn),SaveModelCallback(learn, monitor='kappa_score', mode='max')])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.374961 | 0.406740 | 0.170472 | 0.829528 | 0.656166 | 16:54 |
1 | 0.376538 | 0.403568 | 0.168908 | 0.831092 | 0.659458 | 16:54 |
2 | 0.403122 | 0.404368 | 0.165780 | 0.834220 | 0.666071 | 16:55 |
3 | 0.392660 | 0.406049 | 0.169221 | 0.830779 | 0.658781 | 16:55 |
4 | 0.376754 | 0.409395 | 0.171098 | 0.828902 | 0.654792 | 16:56 |
Better model found at epoch 0 with kappa_score value: 0.6561656594276428. Better model found at epoch 1 with kappa_score value: 0.6594582200050354. Better model found at epoch 2 with kappa_score value: 0.6660714149475098.
learn.save('densenet169-stage-8-n2')
learn = None
gc.collect()
torch.cuda.empty_cache()
size = 320
bs = 16
np.random.seed(42)
data = ImageDataBunch.from_folder(path/'data2', ds_tfms=get_transforms(max_rotate=30.0, max_warp=0.),
size=size, bs=bs,
).normalize(imagenet_stats)
kappa = KappaScore()
kappa.weights = "quadratic"
learn = cnn_learner(data, models.densenet169, metrics=[error_rate, accuracy, kappa], wd=0.1)
learn.load('densenet169-stage-8-n2');
learn.purge()
learn.unfreeze()
learn.lr_find()
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
lr=3e-5
learn.fit_one_cycle(5,max_lr=slice(lr/100,lr),callbacks=[ShowGraph(learn),SaveModelCallback(learn, monitor='kappa_score', mode='max')])
epoch | train_loss | valid_loss | error_rate | accuracy | kappa_score | time |
---|---|---|---|---|---|---|
0 | 0.391157 | 0.416407 | 0.171723 | 0.828277 | 0.653248 | 21:42 |
1 | 0.398008 | 0.398131 | 0.163278 | 0.836722 | 0.671413 | 21:25 |
2 | 0.399078 | 0.421693 | 0.181733 | 0.818267 | 0.632440 | 21:27 |
3 | 0.396580 | 0.399806 | 0.161714 | 0.838286 | 0.674253 | 21:27 |
4 | 0.371800 | 0.408447 | 0.166406 | 0.833594 | 0.664394 | 21:27 |
Better model found at epoch 0 with kappa_score value: 0.6532483696937561. Better model found at epoch 1 with kappa_score value: 0.6714134216308594. Better model found at epoch 3 with kappa_score value: 0.6742532253265381.
learn.save('densenet169-stage-9-n2')
learn.lr_find()
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
doc(learn.loss_func)
size = 320
bs = 8
np.random.seed(42)
data = ImageDataBunch.from_folder(path/'data2', ds_tfms=get_transforms(max_rotate=30.0, max_warp=0.),
size=size, bs=bs,
).normalize(imagenet_stats)
kappa = KappaScore()
kappa.weights = "quadratic"
learn = cnn_learner(data, models.densenet169, metrics=[error_rate, accuracy, kappa], wd=0.1)
learn.load('densenet169-stage-9-n2');
interp = ClassificationInterpretation.from_learner(learn)
losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs)
True
# first interpretation
interp.plot_confusion_matrix()
interp.most_confused(min_val=2)
[('1', '0', 357), ('0', '1', 160)]
interp.plot_top_losses(9, figsize=(15,11))
Source for kappa score calculation: Cohen's kappa in plain English
learn.load('densenet169-stage-9-n2');
for img_url in data.valid_ds.x.items:
print(img_url)
break
/home/jupyter/tutorials/data/MURA-v1.1/data2/valid/1/HAND_patient11538_study1_positive_image1.png
# predictions on validation
preds_val, y_val = learn.get_preds()
# file to save df
file = path / 'results_densenet169_320.csv'
%%time
probs_of_images_by_studies = dict()
num_img_by_studies = dict()
labels_of_studies = dict()
num_studies_by_label = dict()
list_studies_by_label = dict()
idx_by_studies = dict()
results_by_label = dict()
acc_by_label = OrderedDict()
kappa_score = OrderedDict()
pat_label = re.compile(r'/([^/]+)_patient[^/]+.png$')
pat_study = re.compile(r'/([^/]+)_[^_]+.png$')
for idx, src in enumerate(data.valid_ds.x.items):
# get label name
label = pat_label.search(str(src))
label = label.group(1)
# get study name
study = pat_study.search(str(src))
study = study.group(1)
# sum probabilities by study
if study in probs_of_images_by_studies:
probs_of_images_by_studies[study] += preds_val[idx,:].clone()
num_img_by_studies[study] += 1
idx_by_studies[study].append(idx)
else:
probs_of_images_by_studies[study] = preds_val[idx,:].clone()
num_img_by_studies[study] = 1
labels_of_studies[study] = label
idx_by_studies[study] = [idx]
for m in mura:
num_studies_by_label[m] = sum([1 for k,v in labels_of_studies.items() if v.lower() == m])
list_studies_by_label[m] = [k for k,v in labels_of_studies.items() if v.lower() == m]
for m in mura:
list_studies = list_studies_by_label[m]
TN, FP, TP, FN = 0, 0, 0, 0
for study in list_studies:
preds_val2 = preds_val.clone()
idxs = idx_by_studies[study]
preds = preds_val2[idxs]
y_hat = np.argmax(preds, axis=1)
y = y_val[idxs]
if 'negative' in study:
TN += sum(y_hat.numpy() == y.numpy())
FP += sum(y_hat.numpy() != y.numpy())
else:
TP += sum(y_hat.numpy() == y.numpy())
FN += sum(y_hat.numpy() != y.numpy())
results_by_label[m] = [TN, FP, TP, FN]
# accuracy
acc_by_label[m] = (TN + TP) / (TN+FP+FN+TP)
# kappa
ground_truth_neg = TN+FN
ground_truth_pos = FP+TP
classifier_neg = TN+FP
classifier_pos = FN+TP
total = classifier_neg + classifier_pos
observed_accuracy = (TN + TP) / total
expected_accuracy = ((ground_truth_neg * classifier_neg / total) + (ground_truth_pos * classifier_pos / total)) / total
kappa_score[m] = (observed_accuracy - expected_accuracy) / (1 - expected_accuracy)
# overall accuracy and kappa
[TN, FP, TP, FN] = [0,0,0,0]
for m in mura:
results = results_by_label[m]
TN += results[0]
FP += results[1]
TP += results[2]
FN += results[3]
# overall accuracy
acc_by_label['overall'] = (TN + TP) / (TN+FP+FN+TP)
# overall kappa
ground_truth_neg = TN+FN
ground_truth_pos = FP+TP
classifier_neg = TN+FP
classifier_pos = FN+TP
total = classifier_neg + classifier_pos
observed_accuracy = (TN + TP) / total
expected_accuracy = ((ground_truth_neg * classifier_neg / total) + (ground_truth_pos * classifier_pos / total)) / total
kappa_score['overall'] = (observed_accuracy - expected_accuracy) / (1 - expected_accuracy)
# Create DataFrame
for k, v in acc_by_label.items():
acc_by_label[k] = round(v, 3)
for k, v in kappa_score.items():
kappa_score[k] = round(v, 3)
df = pd.DataFrame([acc_by_label, kappa_score], index =['accuracy', 'kappa'])
df.name = 'Results on validation at a study level'
df = df.transpose()
# save df
df.to_csv(file,header=True)
print(df)
accuracy kappa elbow 0.869 0.737 finger 0.779 0.561 forearm 0.870 0.741 hand 0.807 0.583 humerus 0.889 0.778 shoulder 0.821 0.641 wrist 0.859 0.711 overall 0.838 0.674 CPU times: user 552 ms, sys: 0 ns, total: 552 ms Wall time: 141 ms
# first interpretation
interp.plot_confusion_matrix()
cm = interp.confusion_matrix()
TN = cm[0,0]
FP = cm[0,1]
FN = cm[1,0]
TP = cm[1,1]
# accuracy
acc = (TN + TP) / (TN+FP+FN+TP)
# kappa
ground_truth_neg = TN+FN
ground_truth_pos = FP+TP
classifier_neg = TN+FP
classifier_pos = FN+TP
total = classifier_neg + classifier_pos
observed_accuracy = (TN + TP) / total
expected_accuracy = ((ground_truth_neg * classifier_neg / total) + (ground_truth_pos * classifier_pos / total)) / total
kappa_score = (observed_accuracy - expected_accuracy) / (1 - expected_accuracy)
round(acc,3), round(kappa_score,3)
(0.838, 0.674)