Please cite us if you use the software

# Example-8 (Confidence interval)¶

## Install matplotlib¶

In [1]:
import sys
!{sys.executable} -m pip -q -q install matplotlib;


## Plot function¶

In [2]:
import numpy as np
import matplotlib.pyplot as plt
import pycm

def plot_ci(cm,param,alpha=0.05,method="normal-approx"):
"""
Plot two-sided confidence interval.

:param cm: ConfusionMatrix
:type cm : pycm.ConfusionMatrix object
:param param: input parameter
:type param: str
:param alpha: type I error
:type alpha: float
:param method: binomial confidence intervals method
:type method: str
:return: None
"""
conf_str = str(round(100*(1-alpha)))
print(conf_str+"%CI :")
if param in cm.class_stat.keys():
mean = []
error = [[],[]]
data = cm.CI(param,alpha=alpha,binom_method=method)
class_names_str = list(map(str,(cm.classes)))
for class_index, class_name in enumerate(cm.classes):
print(str(class_name)+" : "+str(data[class_name][1]))
mean.append(cm.class_stat[param][class_name])
error[0].append(cm.class_stat[param][class_name]-data[class_name][1][0])
error[1].append(data[class_name][1][1]-cm.class_stat[param][class_name])
fig = plt.figure()
plt.errorbar(mean,class_names_str,xerr = error,fmt='o',capsize=5,linestyle="dotted")
plt.ylabel('Class')
fig.suptitle("Param :"+param + ", Alpha:"+str(alpha), fontsize=16)
for index,value in enumerate(mean):
down_point = data[cm.classes[index]][1][0]
up_point = data[cm.classes[index]][1][1]
plt.text(value, class_names_str[index], "%f" %value, ha="center",va="top",color="red")
plt.text(down_point, class_names_str[index], "%f" %down_point, ha="right",va="bottom",color="red")
plt.text(up_point , class_names_str[index], "%f" %up_point, ha="left",va="bottom",color="red")
else:
mean = cm.overall_stat[param]
data = cm.CI(param,alpha=alpha,binom_method=method)
print(data[1])
error = [[],[]]
up_point = data[1][1]
down_point = data[1][0]
error[0] = [cm.overall_stat[param] - down_point]
error[1] = [up_point - cm.overall_stat[param]]
fig = plt.figure()
plt.errorbar(mean,[param],xerr = error,fmt='o',capsize=5,linestyle="dotted")
fig.suptitle("Alpha:"+str(alpha), fontsize=16)
plt.text(mean, param, "%f" %mean, ha="center",va="top",color="red")
plt.text(down_point, param, "%f" %down_point, ha="right",va="bottom",color="red")
plt.text(up_point, param, "%f" %up_point, ha="left",va="bottom",color="red")

plt.show()


In [3]:
cm = pycm.ConfusionMatrix(matrix={0:{0:13,1:2,2:5},1:{0:1,1:10,2:6},2:{0:2,1:0,2:9}})


## TPR¶

In [4]:
plot_ci(cm,param="TPR",method="normal-approx")

95%CI :
0 : (0.4409588557245249, 0.8590411442754751)
1 : (0.3542806291426003, 0.8221899590926939)
2 : (0.5902508705143656, 1.0461127658492708)

In [5]:
plot_ci(cm,param="TPR",method="wilson")

95%CI :
0 : (0.43285051021946463, 0.8188104521320554)
1 : (0.36005074385515545, 0.7838921396086862)
2 : (0.5230138624217553, 0.9486333993289995)

In [6]:
plot_ci(cm,param="TPR",method="agresti-coull")

95%CI :
0 : (0.4315849969359111, 0.8200759654156089)
1 : (0.3595423027276775, 0.7844005807361641)
2 : (0.5115131538244717, 0.9601341079262828)


## FPR¶

In [7]:
plot_ci(cm,param="FPR",method="normal-approx")

95%CI :
0 : (-0.0074215352310388755, 0.22170724951675308)
1 : (-0.021966229952848706, 0.15099848801736493)
2 : (0.1500197639598834, 0.44457483063471115)

In [8]:
plot_ci(cm,param="FPR",method="wilson")

95%CI :
0 : (0.03711769205479948, 0.27196225367657073)
1 : (0.017874407775063175, 0.20718988318746445)
2 : (0.17489302346408198, 0.45783439172043583)

In [9]:
plot_ci(cm,param="FPR",method="agresti-coull")

95%CI :
0 : (0.02898740902933511, 0.28009253670203504)
1 : (0.0075968375750390255, 0.21746745338748863)
2 : (0.1737338065288983, 0.4589936086556194)


## AUC¶

In [10]:
plot_ci(cm,param="AUC")

95%CI :
0 : (0.6399211771547619, 0.902935965702381)
1 : (0.6273084086303518, 0.8964107564550372)
2 : (0.6151497954659743, 0.9057347254185467)


## PLR¶

In [11]:
plot_ci(cm,param="PLR")

95%CI :
0 : (1.986202987452899, 18.530051901514057)
1 : (2.2523561191462638, 36.90867850896665)
2 : (1.5589394210441203, 4.858346516200761)


## Overall ACC¶

In [12]:
plot_ci(cm,param="Overall ACC")

95%CI :
(0.5333055584484714, 0.8000277748848619)


## Kappa¶

In [13]:
plot_ci(cm,param="Kappa")

95%CI :
(0.31072820940081924, 0.7046564059837961)