import pandas as pd
from fastai.text import *
import os
import numpy as np
os.getcwd()
'/home/ubuntu/Austin_endoscopy'
filestem = 'Total_ECE_Copy.xlsx'
path = os.getcwd()
df = pd.read_excel(path+'/'+filestem,sheet_name=1)
df
columnsTitles=["Entire Colon Examined","Advanced To"]
df=df.reindex(columns=columnsTitles)
df
Entire Colon Examined | Advanced To | |
---|---|---|
0 | 1 | the terminal ileum |
1 | 1 | the terminal ileum |
2 | 1 | the terminal ileum |
3 | 1 | the terminal ileum |
4 | 1 | the terminal ileum |
5 | 1 | the caecum, identified by appendiceal orifice ... |
6 | 1 | the caecum, identified by appendiceal orifice ... |
7 | 1 | the terminal ileum |
8 | 1 | the caecum, identified by appendiceal orifice ... |
9 | 1 | the caecum, identified by appendiceal orifice ... |
10 | 1 | the caecum, identified by appendiceal orifice ... |
11 | 1 | the terminal ileum |
12 | 1 | the terminal ileum |
13 | 1 | the caecum, identified by appendiceal orifice ... |
14 | 1 | the terminal ileum |
15 | 1 | the caecum, identified by appendiceal orifice ... |
16 | 1 | the caecum, identified by appendiceal orifice ... |
17 | 1 | the terminal ileum |
18 | 1 | the caecum, identified by its appearance |
19 | 1 | the terminal ileum |
20 | 1 | the ileocaecal valve |
21 | 1 | the terminal ileum |
22 | 1 | the terminal ileum |
23 | 1 | the terminal ileum |
24 | 1 | the terminal ileum |
25 | 1 | the terminal ileum |
26 | 1 | the caecum, identified by appendiceal orifice ... |
27 | 1 | the terminal ileum |
28 | 1 | the terminal ileum |
29 | 1 | the terminal ileum |
... | ... | ... |
2678 | 1 | the terminal ileum |
2679 | 1 | 20 cm into the ileum |
2680 | 1 | the terminal ileum |
2681 | 1 | the terminal ileum |
2682 | 1 | the caecum, identified by appendiceal orifice ... |
2683 | 1 | the terminal ileum |
2684 | 1 | the terminal ileum |
2685 | 1 | the terminal ileum |
2686 | 1 | the terminal ileum |
2687 | 1 | the caecum, identified by appendiceal orifice ... |
2688 | 1 | the terminal ileum, with identification of the... |
2689 | 1 | the terminal ileum, with identification of the... |
2690 | 1 | the caecum, identified by appendiceal orifice ... |
2691 | 1 | the terminal ileum, with identification of the... |
2692 | 1 | the caecum, identified by appendiceal orifice ... |
2693 | 1 | the terminal ileum |
2694 | 1 | Caecum |
2695 | 1 | the terminal ileum, with identification of the... |
2696 | 0 | the ileocolonic anastomosis |
2697 | 1 | the terminal ileum |
2698 | 1 | the caecum, identified by appendiceal orifice ... |
2699 | 1 | the terminal ileum |
2700 | 1 | the terminal ileum, with identification of the... |
2701 | 1 | the terminal ileum |
2702 | 1 | the terminal ileum, with identification of the... |
2703 | 1 | the terminal ileum |
2704 | 0 | the ileocolonic anastomosis |
2705 | 1 | the terminal ileum |
2706 | 1 | the caecum, identified by the appendiceal orifice |
2707 | 0 | the ascending colon |
2708 rows × 2 columns
msk = np.random.rand(len(df)) < 0.8
train = df[msk]
valid = df[~msk]
data_lm = TextLMDataBunch.from_df(path, train_df = train, valid_df = valid)
data_clas = TextClasDataBunch.from_df(path, train_df = train, valid_df = valid, vocab=data_lm.train_ds.vocab, bs=32)
data_lm.save('data_lm_export.pkl')
data_clas.save('data_clas_export.pkl')
learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.5)
learn.fit_one_cycle(1, 1e-2)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 8.520858 | 7.221618 | 0.116295 | 00:01 |
learn.predict("the ileocolonic")
'the ileocolonic '
learn.save_encoder('ft_enc')
learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)
learn.load_encoder('ft_enc')
data_clas.show_batch()
text | target |
---|---|
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 |
learn.fit_one_cycle(1, 1e-2)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 0.272039 | 0.256514 | 0.917466 | 00:01 |
learn.freeze_to(-2)
learn.predict("the ileocolonic anastomosis")
(Category 1, tensor(1), tensor([0.0508, 0.9492]))
txt_ci = TextClassificationInterpretation.from_learner(learn)
txt_ci.show_top_losses(50)
Text | Prediction | Actual | Loss | Probability |
---|---|---|---|---|
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 | 1 | 3.38 | 0.97 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 3.38 | 0.97 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 3.30 | 0.96 |
xxbos the descending colon | 1 | 0 | 3.30 | 0.04 |
xxbos the terminal ileum | 1 | 1 | 3.29 | 0.96 |
xxbos the splenic flexure | 1 | 0 | 3.29 | 0.04 |
xxbos the terminal ileum | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum | 1 | 1 | 2.98 | 0.95 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum | 1 | 1 | 2.98 | 0.95 |
xxbos the caecum , identified by the ileocaecal valve | 1 | 1 | 2.98 | 0.95 |
xxbos the ileocaecal valve | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum | 1 | 1 | 2.98 | 0.95 |
xxbos the ileocaecal valve | 1 | 1 | 2.98 | 0.95 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum | 1 | 1 | 2.98 | 0.95 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 2.98 | 0.95 |
xxbos 10 cm into the ileum | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum | 1 | 1 | 2.98 | 0.95 |
xxbos ileum;sigmoid colon;given | 1 | 0 | 2.98 | 0.05 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum | 1 | 1 | 2.98 | 0.95 |
xxbos the terminal ileum | 0 | 1 | 2.22 | 0.11 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 0 | 1 | 2.22 | 0.11 |
xxbos the rectum | 1 | 0 | 2.19 | 0.11 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 | 1 | 2.19 | 0.89 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 | 1 | 2.11 | 0.88 |
xxbos the caecum , identified by the ileocaecal valve | 1 | 1 | 2.11 | 0.88 |
xxbos the terminal ileum | 1 | 1 | 2.11 | 0.88 |
xxbos the terminal ileum | 1 | 1 | 2.11 | 0.88 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 2.11 | 0.88 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 2.11 | 0.88 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 2.11 | 0.88 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 1.78 | 0.83 |
xxbos the terminal ileum | 1 | 1 | 1.78 | 0.83 |
xxbos the terminal ileum | 1 | 1 | 1.64 | 0.81 |
xxbos the terminal ileum | 1 | 1 | 1.64 | 0.81 |
xxbos the terminal ileum | 0 | 1 | 1.48 | 0.23 |
xxbos the terminal ileum | 0 | 1 | 1.48 | 0.23 |
xxbos the ileocolonic anastomosis | 1 | 0 | 0.75 | 0.47 |
xxbos the terminal ileum | 0 | 1 | 0.55 | 0.42 |
xxbos the terminal ileum | 0 | 1 | 0.52 | 0.41 |
xxbos the terminal ileum | 0 | 1 | 0.46 | 0.37 |
xxbos the caecum , identified by appendiceal orifice and ileocaecal valve | 1 | 1 | 0.32 | 0.73 |
xxbos the terminal ileum | 1 | 1 | 0.27 | 0.76 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 0 | 1 | 0.26 | 0.23 |
xxbos the terminal ileum , with identification of the appendiceal orifice and xxup ic valve | 1 | 1 | 0.25 | 0.78 |
learn.predict("the terminal ileum")
(Category 1, tensor(1), tensor([3.1341e-04, 9.9969e-01]))