This Jupyter Notebook studies performance (execution time) vs accuracy (top1 / top5) using the Arm Compute Library on two development platforms:
hikey962
: instr=1,clexperimental=1,softjobpatch
):$ ck run program:tool-print-opencl-devices | grep "version:"
OpenCL 2.0 v1.r6p0-01rel0.24c5f5e966f2b7f1f19b91d6f32ff53e
RAM:
BSP:
$ uname -a
Linux hikey962 4.4.74-00216-g10816f6 #3 SMP PREEMPT Thu Jul 6 14:38:42 BST 2017 aarch64 GNU/Linux
hikey_model = 'HiKey960\x00'
hikey_name = 'Linaro HiKey960'
hikey_id = 'hikey-960'
hikey_gpu = 'Mali-G71 MP8'
hikey_gpu_mhz = '807 MHz'
$ ck run program:tool-print-opencl-devices | grep "version:"
v1.r13p0-00rel0-git(a4271c9).31ba04af2d3c01618138bef3aed66c2c
$ cat /etc/lsb-release
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=16.04
DISTRIB_CODENAME=xenial
DISTRIB_DESCRIPTION="Ubuntu 16.04.4 LTS"
$ uname -a
Linux firefly 4.4.77 #554 SMP Thu Nov 30 11:30:11 HKT 2017 aarch64 aarch64 aarch64 GNU/Linux
firefly_model = 'Rockchip RK3399 Firefly Board (Linux Opensource)\x00'
firefly_name = 'Firefly RK3399'
firefly_id = 'firefly'
firefly_gpu = 'Mali-T860 MP4'
firefly_gpu_mhz = '800 MHz'
model_to_id = {
firefly_model : firefly_id,
hikey_model : hikey_id
}
id_to_name = {
firefly_id : firefly_name,
hikey_id : hikey_name
}
id_to_gpu = {
firefly_id : firefly_gpu,
hikey_id : hikey_gpu
}
id_to_gpu_mhz = {
firefly_id : firefly_gpu_mhz,
hikey_id : hikey_gpu_mhz
}
convolution_method_to_name = [
'gemm',
'direct',
'winograd'
]
The experimental data can be downloaded and registered with CK as described below.
$ wget https://www.dropbox.com/s/tm1qlom7ehfbe0w/ck-request-asplos18-mobilenets-armcl-opencl-accuracy-50000.zip
$ ck add repo --zip=ck-request-asplos18-mobilenets-armcl-opencl-accuracy-50000.zip
armcl_accuracy_50000_repo_uoa = 'ck-request-asplos18-mobilenets-armcl-opencl-accuracy-50000'
!ck list $armcl_accuracy_50000_repo_uoa:experiment:* | sort
mobilenets-accuracy-128-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-1.0-armcl-opencl-request-d8f69c13
$ wget https://www.dropbox.com/s/wqqchrhr36skm9y/ck-request-asplos18-mobilenets-armcl-opencl-accuracy-500.zip
$ ck add repo --zip=ck-request-asplos18-mobilenets-armcl-opencl-accuracy-500.zip
armcl_accuracy_500_repo_uoa = 'ck-request-asplos18-mobilenets-armcl-opencl-accuracy-500'
!ck list $armcl_accuracy_500_repo_uoa:experiment:* | sort
mobilenets-accuracy-128-0.25-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-128-0.25-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-128-0.25-armcl-opencl-18.03-e40997bb mobilenets-accuracy-128-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-0.5-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-128-0.5-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-128-0.5-armcl-opencl-18.03-e40997bb mobilenets-accuracy-128-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-0.75-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-128-0.75-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-128-0.75-armcl-opencl-18.03-e40997bb mobilenets-accuracy-128-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-1.0-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-128-1.0-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-128-1.0-armcl-opencl-18.03-e40997bb mobilenets-accuracy-128-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.25-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-160-0.25-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-160-0.25-armcl-opencl-18.03-e40997bb mobilenets-accuracy-160-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.5-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-160-0.5-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-160-0.5-armcl-opencl-18.03-e40997bb mobilenets-accuracy-160-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.75-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-160-0.75-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-160-0.75-armcl-opencl-18.03-e40997bb mobilenets-accuracy-160-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-1.0-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-160-1.0-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-160-1.0-armcl-opencl-18.03-e40997bb mobilenets-accuracy-160-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.25-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-192-0.25-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-192-0.25-armcl-opencl-18.03-e40997bb mobilenets-accuracy-192-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.5-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-192-0.5-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-192-0.5-armcl-opencl-18.03-e40997bb mobilenets-accuracy-192-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.75-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-192-0.75-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-192-0.75-armcl-opencl-18.03-e40997bb mobilenets-accuracy-192-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-1.0-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-192-1.0-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-192-1.0-armcl-opencl-18.03-e40997bb mobilenets-accuracy-192-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.25-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-224-0.25-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-224-0.25-armcl-opencl-18.03-e40997bb mobilenets-accuracy-224-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.5-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-224-0.5-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-224-0.5-armcl-opencl-18.03-e40997bb mobilenets-accuracy-224-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.75-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-224-0.75-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-224-0.75-armcl-opencl-18.03-e40997bb mobilenets-accuracy-224-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-1.0-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-224-1.0-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-224-1.0-armcl-opencl-18.03-e40997bb mobilenets-accuracy-224-1.0-armcl-opencl-request-d8f69c13
$ wget https://www.dropbox.com/s/wm3ahhm20y7g04k/ck-request-asplos18-mobilenets-armcl-opencl-performance.zip
$ ck add repo --zip=ck-request-asplos18-mobilenets-armcl-opencl-performance.zip
armcl_performance_repo_uoa = 'ck-request-asplos18-mobilenets-armcl-opencl-performance'
!ck list $armcl_performance_repo_uoa:experiment:* | sort
mobilenets-performance-128-0.25-17.12-48bc34ea mobilenets-performance-128-0.25-18.01-f45d5a9b mobilenets-performance-128-0.25-18.03-e40997bb mobilenets-performance-128-0.25-request-d8f69c13 mobilenets-performance-128-0.5-17.12-48bc34ea mobilenets-performance-128-0.5-18.01-f45d5a9b mobilenets-performance-128-0.5-18.03-e40997bb mobilenets-performance-128-0.5-request-d8f69c13 mobilenets-performance-128-0.75-17.12-48bc34ea mobilenets-performance-128-0.75-18.01-f45d5a9b mobilenets-performance-128-0.75-18.03-e40997bb mobilenets-performance-128-0.75-request-d8f69c13 mobilenets-performance-128-1.0-17.12-48bc34ea mobilenets-performance-128-1.0-18.01-f45d5a9b mobilenets-performance-128-1.0-18.03-e40997bb mobilenets-performance-128-1.0-request-d8f69c13 mobilenets-performance-160-0.25-17.12-48bc34ea mobilenets-performance-160-0.25-18.01-f45d5a9b mobilenets-performance-160-0.25-18.03-e40997bb mobilenets-performance-160-0.25-request-d8f69c13 mobilenets-performance-160-0.5-17.12-48bc34ea mobilenets-performance-160-0.5-18.01-f45d5a9b mobilenets-performance-160-0.5-18.03-e40997bb mobilenets-performance-160-0.5-request-d8f69c13 mobilenets-performance-160-0.75-17.12-48bc34ea mobilenets-performance-160-0.75-18.01-f45d5a9b mobilenets-performance-160-0.75-18.03-e40997bb mobilenets-performance-160-0.75-request-d8f69c13 mobilenets-performance-160-1.0-17.12-48bc34ea mobilenets-performance-160-1.0-18.01-f45d5a9b mobilenets-performance-160-1.0-18.03-e40997bb mobilenets-performance-160-1.0-request-d8f69c13 mobilenets-performance-192-0.25-17.12-48bc34ea mobilenets-performance-192-0.25-18.01-f45d5a9b mobilenets-performance-192-0.25-18.03-e40997bb mobilenets-performance-192-0.25-request-d8f69c13 mobilenets-performance-192-0.5-17.12-48bc34ea mobilenets-performance-192-0.5-18.01-f45d5a9b mobilenets-performance-192-0.5-18.03-e40997bb mobilenets-performance-192-0.5-request-d8f69c13 mobilenets-performance-192-0.75-17.12-48bc34ea mobilenets-performance-192-0.75-18.01-f45d5a9b mobilenets-performance-192-0.75-18.03-e40997bb mobilenets-performance-192-0.75-request-d8f69c13 mobilenets-performance-192-1.0-17.12-48bc34ea mobilenets-performance-192-1.0-18.01-f45d5a9b mobilenets-performance-192-1.0-18.03-e40997bb mobilenets-performance-192-1.0-request-d8f69c13 mobilenets-performance-224-0.25-17.12-48bc34ea mobilenets-performance-224-0.25-18.01-f45d5a9b mobilenets-performance-224-0.25-18.03-e40997bb mobilenets-performance-224-0.25-request-d8f69c13 mobilenets-performance-224-0.5-17.12-48bc34ea mobilenets-performance-224-0.5-18.01-f45d5a9b mobilenets-performance-224-0.5-18.03-e40997bb mobilenets-performance-224-0.5-request-d8f69c13 mobilenets-performance-224-0.75-17.12-48bc34ea mobilenets-performance-224-0.75-18.01-f45d5a9b mobilenets-performance-224-0.75-18.03-e40997bb mobilenets-performance-224-0.75-request-d8f69c13 mobilenets-performance-224-1.0-17.12-48bc34ea mobilenets-performance-224-1.0-18.01-f45d5a9b mobilenets-performance-224-1.0-18.03-e40997bb mobilenets-performance-224-1.0-request-d8f69c13
$ wget https://www.dropbox.com/s/ro5txjz9n396s0t/ck-request-asplos18-mobilenets-tensorflow-accuracy-50000.zip
$ ck add repo --zip=ck-request-asplos18-mobilenets-tensorflow-accuracy-50000.zip
tensorflow_accuracy_50000_repo_uoa = 'ck-request-asplos18-mobilenets-tensorflow-accuracy-50000'
!ck list $tensorflow_accuracy_50000_repo_uoa:experiment:* | sort
mobilenets-accuracy-128-0.25-tensorflow-1.7 mobilenets-accuracy-128-0.5-tensorflow-1.7 mobilenets-accuracy-128-0.75-tensorflow-1.7 mobilenets-accuracy-128-1.0-tensorflow-1.7 mobilenets-accuracy-160-0.25-tensorflow-1.7 mobilenets-accuracy-160-0.5-tensorflow-1.7 mobilenets-accuracy-160-0.75-tensorflow-1.7 mobilenets-accuracy-160-1.0-tensorflow-1.7 mobilenets-accuracy-192-0.25-tensorflow-1.7 mobilenets-accuracy-192-0.5-tensorflow-1.7 mobilenets-accuracy-192-0.75-tensorflow-1.7 mobilenets-accuracy-192-1.0-tensorflow-1.7 mobilenets-accuracy-224-0.25-tensorflow-1.7 mobilenets-accuracy-224-0.5-tensorflow-1.7 mobilenets-accuracy-224-0.75-tensorflow-1.7 mobilenets-accuracy-224-1.0-tensorflow-1.7
$ wget https://www.dropbox.com/s/k0xhhb7owwvyfgu/ck-request-asplos18-mobilenets-tensorflow-accuracy-500.zip
$ ck add repo --zip=ck-request-asplos18-mobilenets-tensorflow-accuracy-500.zip
tensorflow_accuracy_500_repo_uoa = 'ck-request-asplos18-mobilenets-tensorflow-accuracy-500'
!ck list $tensorflow_accuracy_500_repo_uoa:experiment:* | sort
mobilenets-accuracy-128-0.25-tensorflow-1.7 mobilenets-accuracy-128-0.5-tensorflow-1.7 mobilenets-accuracy-128-0.75-tensorflow-1.7 mobilenets-accuracy-128-1.0-tensorflow-1.7 mobilenets-accuracy-160-0.25-tensorflow-1.7 mobilenets-accuracy-160-0.5-tensorflow-1.7 mobilenets-accuracy-160-0.75-tensorflow-1.7 mobilenets-accuracy-160-1.0-tensorflow-1.7 mobilenets-accuracy-192-0.25-tensorflow-1.7 mobilenets-accuracy-192-0.5-tensorflow-1.7 mobilenets-accuracy-192-0.75-tensorflow-1.7 mobilenets-accuracy-192-1.0-tensorflow-1.7 mobilenets-accuracy-224-0.25-tensorflow-1.7 mobilenets-accuracy-224-0.5-tensorflow-1.7 mobilenets-accuracy-224-0.75-tensorflow-1.7 mobilenets-accuracy-224-1.0-tensorflow-1.7
$ wget https://www.dropbox.com/s/1fagdonfaqsdfou/ck-request-asplos18-mobilenets-tensorflow-performance.zip
$ ck add repo --zip=ck-request-asplos18-mobilenets-tensorflow-performance.zip
tensorflow_performance_repo_uoa = 'ck-request-asplos18-mobilenets-tensorflow-performance'
!ck list $tensorflow_performance_repo_uoa:experiment:* | sort
mobilenets-performance-128-0.25-tensorflow-1.7 mobilenets-performance-128-0.5-tensorflow-1.7 mobilenets-performance-128-0.75-tensorflow-1.7 mobilenets-performance-128-1.0-tensorflow-1.7 mobilenets-performance-160-0.25-tensorflow-1.7 mobilenets-performance-160-0.5-tensorflow-1.7 mobilenets-performance-160-0.75-tensorflow-1.7 mobilenets-performance-160-1.0-tensorflow-1.7 mobilenets-performance-192-0.25-tensorflow-1.7 mobilenets-performance-192-0.5-tensorflow-1.7 mobilenets-performance-192-0.75-tensorflow-1.7 mobilenets-performance-192-1.0-tensorflow-1.7 mobilenets-performance-224-0.25-tensorflow-1.7 mobilenets-performance-224-0.5-tensorflow-1.7 mobilenets-performance-224-0.75-tensorflow-1.7 mobilenets-performance-224-1.0-tensorflow-1.7
firefly_tensorflow_accuracy_500_repo_uoa = 'ck-request-asplos18-mobilenets-tensorflow-accuracy-500-firefly'
!ck list $firefly_tensorflow_accuracy_500_repo_uoa:experiment:* | sort
mobilenets-accuracy-128-0.25-tensorflow-1.7 mobilenets-accuracy-128-0.25-tensorflow-1.8 mobilenets-accuracy-128-0.5-tensorflow-1.7 mobilenets-accuracy-128-0.5-tensorflow-1.8 mobilenets-accuracy-128-0.75-tensorflow-1.7 mobilenets-accuracy-128-0.75-tensorflow-1.8 mobilenets-accuracy-128-1.0-tensorflow-1.7 mobilenets-accuracy-128-1.0-tensorflow-1.8 mobilenets-accuracy-160-0.25-tensorflow-1.7 mobilenets-accuracy-160-0.25-tensorflow-1.8 mobilenets-accuracy-160-0.5-tensorflow-1.7 mobilenets-accuracy-160-0.5-tensorflow-1.8 mobilenets-accuracy-160-0.75-tensorflow-1.7 mobilenets-accuracy-160-0.75-tensorflow-1.8 mobilenets-accuracy-160-1.0-tensorflow-1.7 mobilenets-accuracy-160-1.0-tensorflow-1.8 mobilenets-accuracy-192-0.25-tensorflow-1.7 mobilenets-accuracy-192-0.25-tensorflow-1.8 mobilenets-accuracy-192-0.5-tensorflow-1.7 mobilenets-accuracy-192-0.5-tensorflow-1.8 mobilenets-accuracy-192-0.75-tensorflow-1.7 mobilenets-accuracy-192-0.75-tensorflow-1.8 mobilenets-accuracy-192-1.0-tensorflow-1.7 mobilenets-accuracy-192-1.0-tensorflow-1.8 mobilenets-accuracy-224-0.25-tensorflow-1.7 mobilenets-accuracy-224-0.25-tensorflow-1.8 mobilenets-accuracy-224-0.5-tensorflow-1.7 mobilenets-accuracy-224-0.5-tensorflow-1.8 mobilenets-accuracy-224-0.75-tensorflow-1.7 mobilenets-accuracy-224-0.75-tensorflow-1.8 mobilenets-accuracy-224-1.0-tensorflow-1.7 mobilenets-accuracy-224-1.0-tensorflow-1.8
firefly_tensorflow_performance_repo_uoa = 'ck-request-asplos18-mobilenets-tensorflow-performance-firefly'
!ck list $firefly_tensorflow_performance_repo_uoa:experiment:* | sort
mobilenets-performance-128-0.25-tensorflow-1.7 mobilenets-performance-128-0.25-tensorflow-1.8 mobilenets-performance-128-0.5-tensorflow-1.7 mobilenets-performance-128-0.5-tensorflow-1.8 mobilenets-performance-128-0.75-tensorflow-1.7 mobilenets-performance-128-0.75-tensorflow-1.8 mobilenets-performance-128-1.0-tensorflow-1.7 mobilenets-performance-128-1.0-tensorflow-1.8 mobilenets-performance-160-0.25-tensorflow-1.7 mobilenets-performance-160-0.25-tensorflow-1.8 mobilenets-performance-160-0.5-tensorflow-1.7 mobilenets-performance-160-0.5-tensorflow-1.8 mobilenets-performance-160-0.75-tensorflow-1.7 mobilenets-performance-160-0.75-tensorflow-1.8 mobilenets-performance-160-1.0-tensorflow-1.7 mobilenets-performance-160-1.0-tensorflow-1.8 mobilenets-performance-192-0.25-tensorflow-1.7 mobilenets-performance-192-0.25-tensorflow-1.8 mobilenets-performance-192-0.5-tensorflow-1.7 mobilenets-performance-192-0.5-tensorflow-1.8 mobilenets-performance-192-0.75-tensorflow-1.7 mobilenets-performance-192-0.75-tensorflow-1.8 mobilenets-performance-192-1.0-tensorflow-1.7 mobilenets-performance-192-1.0-tensorflow-1.8 mobilenets-performance-224-0.25-tensorflow-1.7 mobilenets-performance-224-0.25-tensorflow-1.8 mobilenets-performance-224-0.5-tensorflow-1.7 mobilenets-performance-224-0.5-tensorflow-1.8 mobilenets-performance-224-0.75-tensorflow-1.7 mobilenets-performance-224-0.75-tensorflow-1.8 mobilenets-performance-224-1.0-tensorflow-1.7 mobilenets-performance-224-1.0-tensorflow-1.8
firefly_armcl_performance_repo_uoa = 'ck-request-asplos18-mobilenets-armcl-opencl-performance-firefly'
!ck list $firefly_armcl_performance_repo_uoa:experiment:* | sort
mobilenets-performance-128-0.25-armcl-opencl-17.12-48bc34e mobilenets-performance-128-0.25-armcl-opencl-18.01-f45d5a9 mobilenets-performance-128-0.25-armcl-opencl-18.03-e40997b mobilenets-performance-128-0.5-armcl-opencl-17.12-48bc34e mobilenets-performance-128-0.5-armcl-opencl-18.01-f45d5a9 mobilenets-performance-128-0.5-armcl-opencl-18.03-e40997b mobilenets-performance-128-0.75-armcl-opencl-17.12-48bc34e mobilenets-performance-128-0.75-armcl-opencl-18.01-f45d5a9 mobilenets-performance-128-0.75-armcl-opencl-18.03-e40997b mobilenets-performance-128-1.0-armcl-opencl-17.12-48bc34e mobilenets-performance-128-1.0-armcl-opencl-18.01-f45d5a9 mobilenets-performance-128-1.0-armcl-opencl-18.03-e40997b mobilenets-performance-160-0.25-armcl-opencl-17.12-48bc34e mobilenets-performance-160-0.25-armcl-opencl-18.01-f45d5a9 mobilenets-performance-160-0.25-armcl-opencl-18.03-e40997b mobilenets-performance-160-0.5-armcl-opencl-17.12-48bc34e mobilenets-performance-160-0.5-armcl-opencl-18.01-f45d5a9 mobilenets-performance-160-0.5-armcl-opencl-18.03-e40997b mobilenets-performance-160-0.75-armcl-opencl-17.12-48bc34e mobilenets-performance-160-0.75-armcl-opencl-18.01-f45d5a9 mobilenets-performance-160-0.75-armcl-opencl-18.03-e40997b mobilenets-performance-160-1.0-armcl-opencl-17.12-48bc34e mobilenets-performance-160-1.0-armcl-opencl-18.01-f45d5a9 mobilenets-performance-160-1.0-armcl-opencl-18.03-e40997b mobilenets-performance-192-0.25-armcl-opencl-17.12-48bc34e mobilenets-performance-192-0.25-armcl-opencl-18.01-f45d5a9 mobilenets-performance-192-0.25-armcl-opencl-18.03-e40997b mobilenets-performance-192-0.5-armcl-opencl-17.12-48bc34e mobilenets-performance-192-0.5-armcl-opencl-18.01-f45d5a9 mobilenets-performance-192-0.5-armcl-opencl-18.03-e40997b mobilenets-performance-192-0.75-armcl-opencl-17.12-48bc34e mobilenets-performance-192-0.75-armcl-opencl-18.01-f45d5a9 mobilenets-performance-192-0.75-armcl-opencl-18.03-e40997b mobilenets-performance-192-1.0-armcl-opencl-17.12-48bc34e mobilenets-performance-192-1.0-armcl-opencl-18.01-f45d5a9 mobilenets-performance-192-1.0-armcl-opencl-18.03-e40997b mobilenets-performance-224-0.25-armcl-opencl-17.12-48bc34e mobilenets-performance-224-0.25-armcl-opencl-18.01-f45d5a9 mobilenets-performance-224-0.25-armcl-opencl-18.03-e40997b mobilenets-performance-224-0.5-armcl-opencl-17.12-48bc34e mobilenets-performance-224-0.5-armcl-opencl-18.01-f45d5a9 mobilenets-performance-224-0.5-armcl-opencl-18.03-e40997b mobilenets-performance-224-0.75-armcl-opencl-17.12-48bc34e mobilenets-performance-224-0.75-armcl-opencl-18.01-f45d5a9 mobilenets-performance-224-0.75-armcl-opencl-18.03-e40997b mobilenets-performance-224-1.0-armcl-opencl-17.12-48bc34e mobilenets-performance-224-1.0-armcl-opencl-18.01-f45d5a9 mobilenets-performance-224-1.0-armcl-opencl-18.03-e40997b
firefly_armcl_accuracy_500_repo_uoa = 'ck-request-asplos18-mobilenets-armcl-opencl-accuracy-500-firefly'
!ck list $armcl_accuracy_500_repo_uoa:experiment:* | sort
mobilenets-accuracy-128-0.25-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-128-0.25-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-128-0.25-armcl-opencl-18.03-e40997bb mobilenets-accuracy-128-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-0.5-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-128-0.5-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-128-0.5-armcl-opencl-18.03-e40997bb mobilenets-accuracy-128-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-0.75-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-128-0.75-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-128-0.75-armcl-opencl-18.03-e40997bb mobilenets-accuracy-128-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-128-1.0-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-128-1.0-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-128-1.0-armcl-opencl-18.03-e40997bb mobilenets-accuracy-128-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.25-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-160-0.25-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-160-0.25-armcl-opencl-18.03-e40997bb mobilenets-accuracy-160-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.5-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-160-0.5-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-160-0.5-armcl-opencl-18.03-e40997bb mobilenets-accuracy-160-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-0.75-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-160-0.75-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-160-0.75-armcl-opencl-18.03-e40997bb mobilenets-accuracy-160-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-160-1.0-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-160-1.0-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-160-1.0-armcl-opencl-18.03-e40997bb mobilenets-accuracy-160-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.25-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-192-0.25-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-192-0.25-armcl-opencl-18.03-e40997bb mobilenets-accuracy-192-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.5-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-192-0.5-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-192-0.5-armcl-opencl-18.03-e40997bb mobilenets-accuracy-192-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-0.75-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-192-0.75-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-192-0.75-armcl-opencl-18.03-e40997bb mobilenets-accuracy-192-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-192-1.0-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-192-1.0-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-192-1.0-armcl-opencl-18.03-e40997bb mobilenets-accuracy-192-1.0-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.25-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-224-0.25-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-224-0.25-armcl-opencl-18.03-e40997bb mobilenets-accuracy-224-0.25-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.5-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-224-0.5-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-224-0.5-armcl-opencl-18.03-e40997bb mobilenets-accuracy-224-0.5-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-0.75-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-224-0.75-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-224-0.75-armcl-opencl-18.03-e40997bb mobilenets-accuracy-224-0.75-armcl-opencl-request-d8f69c13 mobilenets-accuracy-224-1.0-armcl-opencl-17.12-48bc34ea mobilenets-accuracy-224-1.0-armcl-opencl-18.01-f45d5a9b mobilenets-accuracy-224-1.0-armcl-opencl-18.03-e40997bb mobilenets-accuracy-224-1.0-armcl-opencl-request-d8f69c13
NB: Please ignore this section if you are not interested in re-running or modifying this notebook.
import os
import sys
import json
import re
If some of the scientific packages are missing, please install them using:
# pip install jupyter pandas numpy matplotlib
import IPython as ip
import pandas as pd
import numpy as np
import matplotlib as mp
import seaborn as sb
print ('IPython version: %s' % ip.__version__)
print ('Pandas version: %s' % pd.__version__)
print ('NumPy version: %s' % np.__version__)
print ('Matplotlib version: %s' % mp.__version__)
print ('Seaborn version: %s' % sb.__version__)
IPython version: 6.2.1 Pandas version: 0.21.1 NumPy version: 1.13.3 Matplotlib version: 2.1.1 Seaborn version: 0.8.1
from IPython.display import Image, display
def display_in_full(df):
pd.options.display.max_columns = len(df.columns)
pd.options.display.max_rows = len(df.index)
display(df)
import matplotlib.pyplot as plt
from matplotlib import cm
%matplotlib inline
default_colormap = cm.autumn
default_fontsize = 16
default_barwidth = 0.8
default_figwidth = 24
default_figheight = 3
default_figdpi = 200
default_figsize = [default_figwidth, default_figheight]
if mp.__version__[0]=='2': mp.style.use('classic')
mp.rcParams['figure.max_open_warning'] = 200
mp.rcParams['figure.dpi'] = default_figdpi
mp.rcParams['font.size'] = default_fontsize
mp.rcParams['legend.fontsize'] = 'medium'
If CK is not installed, please install it using:
# pip install ck
import ck.kernel as ck
print ('CK version: %s' % ck.__version__)
CK version: 1.9.4
def get_experimental_results(repo_uoa, tags='explore-mobilenets-performance', accuracy=False,
module_uoa='experiment', _library=None, _platform=None):
r = ck.access({'action':'search', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'tags':tags})
if r['return']>0:
print('Error: %s' % r['error'])
exit(1)
experiments = r['lst']
dfs = []
for experiment in experiments:
data_uoa = experiment['data_uoa']
r = ck.access({'action':'list_points', 'repo_uoa':repo_uoa, 'module_uoa':module_uoa, 'data_uoa':data_uoa})
if r['return']>0:
print('Error: %s' % r['error'])
exit(1)
# Mapping of expected library tags to reader-friendly names.
tag_to_name = {
# ArmCL tags on HiKey.
'17.12-48bc34ea' : 'armcl-17.12',
'18.01-f45d5a9b' : 'armcl-18.01',
'18.03-e40997bb' : 'armcl-18.03',
'request-d8f69c13' : 'armcl-dv/dt', # armcl-18.03+
'18.05-b3a371bc' : 'armcl-18.05',
# ArmCL tags on Firefly.
'17.12-48bc34e' : 'armcl-17.12',
'18.01-f45d5a9' : 'armcl-18.01',
'18.03-e40997b' : 'armcl-18.03',
'18.05-b3a371b' : 'armcl-18.05',
# TensorFlow tags.
'tensorflow-1.7' : 'tensorflow-1.7',
'tensorflow-1.8' : 'tensorflow-1.8',
}
# Library.
library_tags = [ tag for tag in r['dict']['tags'] if tag in tag_to_name.keys() ]
if len(library_tags)==1:
library = tag_to_name[library_tags[0]]
else:
print('[Warning] Bad library tags. Skipping experiment with tags:')
print(r['dict']['tags'])
continue
if _library and _library!=library: continue
# For each point.
for point in r['points']:
point_file_path = os.path.join(r['path'], 'ckp-%s.0001.json' % point)
with open(point_file_path) as point_file:
point_data_raw = json.load(point_file)
characteristics_list = point_data_raw['characteristics_list']
num_repetitions = len(characteristics_list)
platform = model_to_id[point_data_raw['features']['platform']['platform']['model']]
if _platform and _platform!=platform: continue
batch_size = np.int64(point_data_raw['choices']['env'].get('CK_BATCH_SIZE',-1))
batch_count = np.int64(point_data_raw['choices']['env'].get('CK_BATCH_COUNT',-1))
convolution_method = convolution_method_to_name[np.int64(point_data_raw['choices']['env'].get('CK_CONVOLUTION_METHOD_HINT',1))]
if library.startswith('tensorflow-'):
multiplier = np.float64(point_data_raw['choices']['env'].get('CK_ENV_TENSORFLOW_MODEL_MOBILENET_MULTIPLIER',-1))
resolution = np.int64(point_data_raw['choices']['env'].get('CK_ENV_TENSORFLOW_MODEL_MOBILENET_RESOLUTION',-1))
else:
multiplier = np.float64(point_data_raw['choices']['env'].get('CK_ENV_MOBILENET_WIDTH_MULTIPLIER',-1))
resolution = np.int64(point_data_raw['choices']['env'].get('CK_ENV_MOBILENET_RESOLUTION',-1))
model = 'v1-%.2f-%d' % (multiplier, resolution)
if accuracy:
data = [
{
# features
'platform': platform,
'library': library,
# choices
'model': model,
'batch_size': batch_size,
'batch_count': batch_count,
'convolution_method': convolution_method,
'resolution': resolution,
'multiplier': multiplier,
# statistical repetition
'repetition_id': repetition_id,
# runtime characteristics
'success?': characteristics['run'].get('run_success', 'n/a'),
'accuracy_top1': characteristics['run'].get('accuracy_top1', 0),
'accuracy_top5': characteristics['run'].get('accuracy_top5', 0),
'frame_predictions': characteristics['run'].get('frame_predictions', []),
# # recompute accuracy from frame_predictions (was incorrectly recorded in early experiments)
# 'accuracy_top1_': len([
# prediction for prediction in characteristics['run'].get('frame_predictions', [])
# if prediction['accuracy_top1']=='yes'
# ]) / np.float64(batch_count),
# 'accuracy_top5_': len([
# prediction for prediction in characteristics['run'].get('frame_predictions', [])
# if prediction['accuracy_top5']=='yes'
# ]) / np.float64(batch_count)
}
for (repetition_id, characteristics) in zip(range(num_repetitions), characteristics_list)
]
else: # performance
data = [
{
# features
'platform': platform,
'library': library,
# choices
'model': model,
'batch_size': batch_size,
'batch_count': batch_count,
'convolution_method': convolution_method,
'resolution': resolution,
'multiplier': multiplier,
# statistical repetition
'repetition_id': repetition_id,
# runtime characteristics
'success?': characteristics['run'].get('run_success', 'n/a'),
'time_avg_ms': characteristics['run']['prediction_time_avg_s']*1e+3,
'time_total_ms': characteristics['run']['prediction_time_total_s']*1e+3,
}
for (repetition_id, characteristics) in zip(range(num_repetitions), characteristics_list)
]
index = [
'platform', 'library', 'model', 'multiplier', 'resolution', 'batch_size', 'convolution_method', 'repetition_id'
]
# Construct a DataFrame.
df = pd.DataFrame(data)
df = df.set_index(index)
# Append to the list of similarly constructed DataFrames.
dfs.append(df)
if dfs:
# Concatenate all thus constructed DataFrames (i.e. stack on top of each other).
result = pd.concat(dfs)
result.sort_index(ascending=True, inplace=True)
else:
# Construct a dummy DataFrame the success status of which can be safely checked.
result = pd.DataFrame(columns=['success?'])
return result
# Return a new DataFrame with only the performance and accuracy metrics.
def merge_performance_accuracy(df_performance, df_accuracy,
reference_platform=None, reference_lib=None, reference_convolution_method='direct',
performance_metric='time_avg_ms', accuracy_metric='accuracy_top1'):
df = df_performance[[performance_metric]]
accuracy_list = []
for index, row in df.iterrows():
(platform, lib, model, multiplier, resolution, batch_size, convolution_method) = index
if reference_platform: platform = reference_platform
try:
accuracy = df_accuracy.loc[(platform, lib, model, multiplier, resolution, batch_size, convolution_method)][accuracy_metric]
except:
if reference_lib: lib = reference_lib
convolution_method = reference_convolution_method
accuracy = df_accuracy.loc[(platform, lib, model, multiplier, resolution, batch_size, convolution_method)][accuracy_metric]
accuracy_list.append(accuracy)
df = df.assign(accuracy_top1=accuracy_list) # FIXME: assign to the value of accuracy_metric
return df
def plot_performance(df_raw, groupby_level='convolution_method', platform_id=hikey_id,
performance_metric='time_avg_ms', title=None, figsize=None, rot=90):
df_bar = pd.DataFrame(
data=df_raw[performance_metric].values, columns=['ms'],
index=pd.MultiIndex.from_tuples(
tuples=[ (l,m[3:],c,r) for (p,l,m,_,_,_,c,r) in df_raw.index.values ],
names=[ 'library', 'model', 'convolution_method', 'repetition_id' ]
)
)
df_bar.columns.names = ['time']
if groupby_level=='convolution_method':
unstack_level = 'library'
xlabel='(Model [channel multiplier - input resolution], Convolution Method)'
colormap = cm.autumn
elif groupby_level=='library':
unstack_level = 'convolution_method'
xlabel='(Library, Model [channel multiplier - input resolution])'
colormap = cm.summer
# Set default style.
ylabel='Image recognition time (ms)'
if not title: title = '%s (GPU: %s @ %s)' % (id_to_name[platform_id], id_to_gpu[platform_id], id_to_gpu_mhz[platform_id])
if not figsize: figsize = [default_figwidth, 8]
# Plot
mean = df_bar.groupby(level=df_bar.index.names[:-1]).mean().unstack(unstack_level)
std = df_bar.groupby(level=df_bar.index.names[:-1]).std().unstack(unstack_level)
axes = mean.groupby(level=groupby_level) \
.plot(yerr=std, kind='bar', grid=True, width=0.8, rot=rot, figsize=figsize,
fontsize=default_fontsize, colormap=colormap)
for ax in axes:
# Title.
ax.set_title(title)
# X label.
ax.set_xlabel(xlabel)
# Y axis.
ax.set_ylabel(ylabel)
def plot_performance_violin(df_raw, groupby_level='convolution_method', platform_id=hikey_id,
performance_metric='time_avg_ms', title=None, figsize=None, fontscale=1.75):
df_violin = pd.DataFrame(
data=df_raw[performance_metric].values, columns=['ms'],
index=pd.MultiIndex.from_tuples(
tuples=[ (l,m[3:],c,r) for (p,l,m,_,_,_,c,r) in df_raw.index.values ],
names=[ 'library', 'model', 'convolution_method', 'repetition_id' ]
)
)
if groupby_level=='convolution_method':
df_violin = df_violin.swaplevel('convolution_method', 'library')
hue_level = 'library'
palette = 'autumn'
elif groupby_level=='library':
hue_level = 'convolution_method'
palette = 'summer'
num_model_values = len(df_violin.index.get_level_values(level='model').unique())
# Set default style.
xlabel='Model [channel multiplier - input resolution]'
ylabel='Image recognition time (ms)'
if not title: title = '%s (GPU: %s @ %s)' % (id_to_name[platform_id], id_to_gpu[platform_id], id_to_gpu_mhz[platform_id])
if not figsize: figsize = (num_model_values*1.5, 12)
sb.set_style('whitegrid')
sb.set_palette(palette)
# For each unique groupby value.
groupby_values = df_violin.index.get_level_values(level=groupby_level).unique()
for groupby_value in groupby_values:
fig = plt.figure(figsize=figsize, dpi=default_figdpi)
ax = fig.gca()
df_violin_loc = df_violin.loc[groupby_value].reset_index()
sb.violinplot(ax=ax, data=df_violin_loc, x='model', y='ms', hue=hue_level,
fontscale=fontscale, inner='point', split=False, saturation=0.8)
# Title.
groupby_title = '%s: %s=%s' % (title, groupby_level, groupby_value)
ax.set_title(groupby_title)
# X axis.
ax.set_xlabel(xlabel)
# Y axis.
ystep = 10
ymin = np.int64(df_violin_loc['ms'].min())
ymax = np.int64(df_violin_loc['ms'].max()) // ystep * ystep + ystep + 1
ax.set_ylim([ymin, ymax])
ax.set_yticks(range(0, ymax, ystep))
ax.set_ylabel(ylabel)
# Vertical lines between groups of violins.
for x in ax.get_xticks():
ax.vlines(x=x+0.5, ymin=0, ymax=ymax, linestyles='dotted', colors='purple')
def plot(df_performance_accuracy, libs=None, platform_id=hikey_id,
performance_metric='time_avg_ms', accuracy_metric='accuracy_top1',
xmin=0.0, xmax=75.1, xstep=5.0, ymin=0.4, ymax=0.751, ystep=0.05,
title=None, save_fig=False, save_fig_name='mobilenets-default'):
fig = plt.figure(figsize=(8,4), dpi=default_figdpi)
ax = fig.gca()
lib_to_color = {
'armcl-17.12' : 'red',
'armcl-18.01' : 'yellow',
'armcl-18.03' : 'orange',
'armcl-dv/dt' : 'green',
'armcl-18.05' : 'purple',
'tensorflow-1.7' : 'cyan',
'tensorflow-1.8' : 'blue',
}
multiplier_to_marker = {
'gemm' : { 1.00 : '*', 0.75 : 'D', 0.50: 'v', 0.25 : '8' },
'direct' : { 1.00 : 'p', 0.75 : 's', 0.50: '^', 0.25 : 'o' },
'winograd' : { 1.00 : 'P', 0.75 : 'X', 0.50: '<', 0.25 : '.' },
}
if libs==None: libs = df_performance_accuracy.index.levels[1].tolist()
df = df_performance_accuracy.loc[platform_id].loc[libs]
for index, row in df.iterrows():
(lib, model, multiplier, resolution, batch_size, convolution_method) = index
performance = row[performance_metric]
accuracy = row[accuracy_metric]
# Mark Pareto-optimal points.
is_on_pareto = True
for index1, row1 in df.iterrows():
is_faster = row1[performance_metric] < row[performance_metric]
is_no_less_accurate = row1[accuracy_metric] >= row[accuracy_metric]
if is_faster and is_no_less_accurate:
is_on_pareto = False
break
# GEMM-based convolution should be exactly the same in '18.03' and 'dv/dt', so plot
# the minimum execution time of '18.03' and 'dv/dt' as '18.03'.
if 'armcl-dv/dt' in libs and convolution_method=='gemm' and (lib=='armcl-dv/dt' or lib=='armcl-18.03'):
performance_dv_dt = df.loc[('armcl-dv/dt', model, multiplier, resolution, batch_size, convolution_method)][performance_metric]
performance_18_03 = df.loc[('armcl-18.03', model, multiplier, resolution, batch_size, convolution_method)][performance_metric]
if lib=='armcl-18.03':
if (performance_dv_dt < performance_18_03):
continue
if lib=='armcl-dv/dt':
if (performance_dv_dt < performance_18_03):
lib = 'armcl-18.03' # change color
else:
continue
# Select size, color and marker.
size = resolution / 16
color = lib_to_color[lib]
marker = multiplier_to_marker[convolution_method][multiplier]
# Plot.
ax.plot(performance, accuracy, marker, markerfacecolor=color, markersize=size)
# Mark Pareto-optimal points with scaled black pluses.
if is_on_pareto:
ax.plot(performance, accuracy, 'k+', markersize=size)
# Title.
if not title: title = '%s (GPU: %s @ %s)' % (id_to_name[platform_id], id_to_gpu[platform_id], id_to_gpu_mhz[platform_id])
ax.set_title(title)
# X axis.
xlabel='Image recognition time (ms)' if performance_metric=='time_avg_ms' else ''
ax.set_xlabel(xlabel)
ax.set_xlim(xmin, xmax)
ax.set_xticks(np.arange(xmin, xmax, xstep))
for xtick in ax.xaxis.get_major_ticks(): xtick.label.set_fontsize(12)
# Y axis.
ylabel='Image recognition accuracy (top %s)' % accuracy_metric[-1]
ax.set_ylabel(ylabel)
ax.set_ylim(ymin, ymax)
ax.set_yticks(np.arange(ymin, ymax, ystep))
for ytick in ax.yaxis.get_major_ticks(): ytick.label.set_fontsize(12)
# Legend.
handles = [
mp.patches.Patch(color=color, label=lib)
for (lib, color) in sorted(lib_to_color.items())
if lib in libs
]
plt.legend(title='Library', handles=handles[::-1], loc='lower right')
# Show with grid on.
plt.grid(True)
plt.show()
# Save figure.
if save_fig:
save_fig_path = os.path.join(save_fig_dir, '%s.%s' % (save_fig_name, save_fig_ext))
plt.savefig(save_fig_path, dpi=default_figdpi, bbox_inches='tight')
def get_paper_dir(module_uoa='dissemination.publication', data_uoa='08da9685582866a0'):
r = ck.access({'action':'find','module_uoa':module_uoa,'data_uoa':data_uoa})
if r['return']>0:
print('Warning: %s' % r['error'])
paper_dir = os.path.curdir
else:
paper_dir = r['path']
return paper_dir
save_fig_ext = 'pdf'
save_fig_dir = os.path.join(get_paper_dir(), 'figures')
if not os.path.exists(save_fig_dir):
os.makedirs(save_fig_dir)
save_tab = False
save_tab_ext = 'tex'
save_tab_dir = os.path.join(get_paper_dir(), 'tables')
if not os.path.exists(save_tab_dir):
os.makedirs(save_tab_dir)
df_tensorflow_performance_raw = get_experimental_results(repo_uoa=tensorflow_performance_repo_uoa,
tags='explore-mobilenets-performance', accuracy=False)
# Take the minimum execution time out of several repetitions.
df_tensorflow_performance = \
df_tensorflow_performance_raw.groupby(level=df_tensorflow_performance_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_tensorflow_performance)
batch_count | success? | time_avg_ms | time_total_ms | |||||||
---|---|---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||||
hikey-960 | tensorflow-1.7 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 2 | yes | 12.423992 | 12.423992 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 2 | yes | 16.618967 | 16.618967 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 2 | yes | 21.809101 | 21.809101 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 2 | yes | 24.497032 | 24.497032 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 2 | yes | 21.347046 | 21.347046 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 2 | yes | 24.481058 | 24.481058 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 2 | yes | 32.360792 | 32.360792 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 2 | yes | 37.299871 | 37.299871 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 2 | yes | 29.517889 | 29.517889 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 2 | yes | 36.680937 | 36.680937 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 2 | yes | 46.771049 | 46.771049 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 2 | yes | 63.611984 | 63.611984 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 2 | yes | 43.389082 | 43.389082 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 2 | yes | 49.995899 | 49.995899 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 2 | yes | 71.686029 | 71.686029 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 2 | yes | 82.866192 | 82.866192 |
plot_performance(df_tensorflow_performance_raw, platform_id=hikey_id, groupby_level='convolution_method')
plot_performance_violin(df_tensorflow_performance_raw, platform_id=hikey_id, groupby_level='convolution_method')
plot_performance(df_tensorflow_performance_raw, platform_id=hikey_id, groupby_level='library')
plot_performance_violin(df_tensorflow_performance_raw, platform_id=hikey_id, groupby_level='library')
df_tensorflow_accuracy_500_raw = get_experimental_results(repo_uoa=tensorflow_accuracy_500_repo_uoa,
tags='explore-mobilenets-accuracy', accuracy=True)
# Extract frame predictions.
df_tensorflow_predictions_500 = df_tensorflow_accuracy_500_raw[['frame_predictions']]
# Reduce the repetition_id index dimension (only 1 repetition anyway).
df_tensorflow_accuracy_500 = \
df_tensorflow_accuracy_500_raw[['accuracy_top1', 'accuracy_top5']] \
.groupby(level=df_tensorflow_accuracy_500_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_tensorflow_accuracy_500)
accuracy_top1 | accuracy_top5 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | tensorflow-1.7 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.410 | 0.672 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.446 | 0.704 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.502 | 0.762 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.508 | 0.754 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.536 | 0.796 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.584 | 0.822 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.622 | 0.818 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.648 | 0.852 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.608 | 0.826 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.644 | 0.852 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.676 | 0.896 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.680 | 0.876 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.620 | 0.860 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.666 | 0.874 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.704 | 0.882 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.722 | 0.904 |
df_tensorflow_accuracy_50000_raw = get_experimental_results(repo_uoa=tensorflow_accuracy_50000_repo_uoa,
tags='explore-mobilenets-accuracy', accuracy=True)
# Extract frame predictions.
df_tensorflow_predictions_50000 = df_tensorflow_accuracy_50000_raw[['frame_predictions']]
# Reduce the repetition_id index dimension (only 1 repetition anyway).
df_tensorflow_accuracy_50000 = \
df_tensorflow_accuracy_50000_raw[['accuracy_top1', 'accuracy_top5']] \
.groupby(level=df_tensorflow_accuracy_50000_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_tensorflow_accuracy_50000)
accuracy_top1 | accuracy_top5 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | tensorflow-1.7 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.40694 | 0.65714 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.45322 | 0.70182 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.48676 | 0.73242 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.50766 | 0.74878 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.55652 | 0.79092 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.59376 | 0.81992 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.61578 | 0.83592 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.63722 | 0.85180 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.61060 | 0.83010 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.64386 | 0.85594 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.66830 | 0.87074 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.68178 | 0.87986 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.63580 | 0.84928 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.66504 | 0.87084 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.68824 | 0.88466 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.70466 | 0.89410 |
# TensorFlow accuracy reported with the MobileNets pretrained weights shared on 2017_06_14. Copied from:
# https://github.com/tensorflow/models/blob/1630da3434974e9ad5a0b6d887ac716a97ce03d3/research/slim/nets/mobilenet_v1.md#pre-trained-models
tensorflow_accuracy_50000_table = {
'v1-1.00-224':[569, 4.24, 70.7, 89.5],
'v1-1.00-192':[418, 4.24, 69.3, 88.9],
'v1-1.00-160':[291, 4.24, 67.2, 87.5],
'v1-1.00-128':[186, 4.24, 64.1, 85.3],
'v1-0.75-224':[317, 2.59, 68.4, 88.2],
'v1-0.75-192':[233, 2.59, 67.4, 87.3],
'v1-0.75-160':[162, 2.59, 65.2, 86.1],
'v1-0.75-128':[104, 2.59, 61.8, 83.6],
'v1-0.50-224':[150, 1.34, 64.0, 85.4],
'v1-0.50-192':[110, 1.34, 62.1, 84.0],
'v1-0.50-160':[77, 1.34, 59.9, 82.5],
'v1-0.50-128':[49, 1.34, 56.2, 79.6],
'v1-0.25-224':[41, 0.47, 50.6, 75.0],
'v1-0.25-192':[34, 0.47, 49.0, 73.6],
'v1-0.25-160':[21, 0.47, 46.0, 70.7],
'v1-0.25-128':[14, 0.47, 41.3, 66.2],
}
df_tensorflow_accuracy_50000_claimed = pd.DataFrame(
index=['MACs (million)', 'Parameters (million)', 'accuracy_top1 (%)', 'accuracy_top5 (%)'],
data=tensorflow_accuracy_50000_table,
).T.sort_index()
accuracy_top1 = df_tensorflow_accuracy_50000_claimed['accuracy_top1 (%)']/100
accuracy_top5 = df_tensorflow_accuracy_50000_claimed['accuracy_top5 (%)']/100
df_tensorflow_accuracy_50000_claimed = df_tensorflow_accuracy_50000_claimed.assign(accuracy_top1=accuracy_top1)
df_tensorflow_accuracy_50000_claimed = df_tensorflow_accuracy_50000_claimed.assign(accuracy_top5=accuracy_top5)
df_tensorflow_accuracy_50000_claimed.index = df_tensorflow_accuracy_50000.index
display_in_full(df_tensorflow_accuracy_50000_claimed)
MACs (million) | Parameters (million) | accuracy_top1 (%) | accuracy_top5 (%) | accuracy_top1 | accuracy_top5 | |||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||||||
hikey-960 | tensorflow-1.7 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 14.0 | 0.47 | 41.3 | 66.2 | 0.413 | 0.662 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 21.0 | 0.47 | 46.0 | 70.7 | 0.460 | 0.707 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 34.0 | 0.47 | 49.0 | 73.6 | 0.490 | 0.736 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 41.0 | 0.47 | 50.6 | 75.0 | 0.506 | 0.750 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 49.0 | 1.34 | 56.2 | 79.6 | 0.562 | 0.796 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 77.0 | 1.34 | 59.9 | 82.5 | 0.599 | 0.825 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 110.0 | 1.34 | 62.1 | 84.0 | 0.621 | 0.840 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 150.0 | 1.34 | 64.0 | 85.4 | 0.640 | 0.854 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 104.0 | 2.59 | 61.8 | 83.6 | 0.618 | 0.836 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 162.0 | 2.59 | 65.2 | 86.1 | 0.652 | 0.861 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 233.0 | 2.59 | 67.4 | 87.3 | 0.674 | 0.873 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 317.0 | 2.59 | 68.4 | 88.2 | 0.684 | 0.882 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 186.0 | 4.24 | 64.1 | 85.3 | 0.641 | 0.853 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 291.0 | 4.24 | 67.2 | 87.5 | 0.672 | 0.875 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 418.0 | 4.24 | 69.3 | 88.9 | 0.693 | 0.889 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 569.0 | 4.24 | 70.7 | 89.5 | 0.707 | 0.895 |
# Diff measured as the fraction of correctly predicted images.
df_tensorflow_accuracy_50000_diff = \
df_tensorflow_accuracy_50000_claimed[['accuracy_top1', 'accuracy_top5']] - \
df_tensorflow_accuracy_50000[['accuracy_top1', 'accuracy_top5']]
display_in_full(df_tensorflow_accuracy_50000_diff)
accuracy_top1 | accuracy_top5 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | tensorflow-1.7 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.00606 | 0.00486 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.00678 | 0.00518 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.00324 | 0.00358 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | -0.00166 | 0.00122 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.00548 | 0.00508 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.00524 | 0.00508 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.00522 | 0.00408 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.00278 | 0.00220 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.00740 | 0.00590 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.00814 | 0.00506 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.00570 | 0.00226 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.00222 | 0.00214 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.00520 | 0.00372 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.00696 | 0.00416 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.00476 | 0.00434 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.00234 | 0.00090 |
# Diff measured as the number of mispredicted images.
df_tensorflow_accuracy_50000_diff_mispredicted = (df_tensorflow_accuracy_50000_diff) * 50000
df_tensorflow_accuracy_50000_diff_mispredicted.columns = ['mispredicted_top1', 'mispredicted_top5']
display_in_full(df_tensorflow_accuracy_50000_diff_mispredicted)
mispredicted_top1 | mispredicted_top5 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | tensorflow-1.7 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 303.0 | 243.0 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 339.0 | 259.0 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 162.0 | 179.0 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | -83.0 | 61.0 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 274.0 | 254.0 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 262.0 | 254.0 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 261.0 | 204.0 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 139.0 | 110.0 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 370.0 | 295.0 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 407.0 | 253.0 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 285.0 | 113.0 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 111.0 | 107.0 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 260.0 | 186.0 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 348.0 | 208.0 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 238.0 | 217.0 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 117.0 | 45.0 |
df_armcl_performance_raw = get_experimental_results(repo_uoa=armcl_performance_repo_uoa,
tags='explore-mobilenets-performance', accuracy=False)
# Take the minimum execution time out of several repetitions.
df_armcl_performance = df_armcl_performance_raw.groupby(level=df_armcl_performance_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_armcl_performance)
batch_count | success? | time_avg_ms | time_total_ms | |||||||
---|---|---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||||
hikey-960 | armcl-17.12 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 1 | yes | 26.351 | 26.351 |
gemm | 1 | yes | 27.241 | 27.241 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 1 | yes | 32.842 | 32.842 | ||
gemm | 1 | yes | 36.124 | 36.124 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 1 | yes | 27.684 | 27.684 | ||
gemm | 1 | yes | 35.715 | 35.715 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 1 | yes | 38.406 | 38.406 | ||
gemm | 1 | yes | 38.645 | 38.645 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 1 | yes | 36.301 | 36.301 | ||
gemm | 1 | yes | 32.069 | 32.069 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 1 | yes | 43.377 | 43.377 | ||
gemm | 1 | yes | 44.402 | 44.402 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 1 | yes | 51.574 | 51.574 | ||
gemm | 1 | yes | 50.743 | 50.743 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 1 | yes | 53.061 | 53.061 | ||
gemm | 1 | yes | 51.806 | 51.806 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 1 | yes | 48.204 | 48.204 | ||
gemm | 1 | yes | 42.519 | 42.519 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 1 | yes | 65.698 | 65.698 | ||
gemm | 1 | yes | 60.465 | 60.465 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 1 | yes | 66.093 | 66.093 | ||
gemm | 1 | yes | 66.494 | 66.494 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 1 | yes | 79.667 | 79.667 | ||
gemm | 1 | yes | 73.126 | 73.126 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 1 | yes | 59.597 | 59.597 | ||
gemm | 1 | yes | 61.465 | 61.465 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 1 | yes | 76.589 | 76.589 | ||
gemm | 1 | yes | 71.438 | 71.438 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 1 | yes | 83.530 | 83.530 | ||
gemm | 1 | yes | 84.506 | 84.506 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 1 | yes | 110.056 | 110.056 | ||
gemm | 1 | yes | 99.716 | 99.716 | ||||||
armcl-18.01 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 1 | yes | 14.905 | 14.905 | |
gemm | 1 | yes | 20.814 | 20.814 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 1 | yes | 15.242 | 15.242 | ||
gemm | 1 | yes | 21.745 | 21.745 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 1 | yes | 14.707 | 14.707 | ||
gemm | 1 | yes | 25.132 | 25.132 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 1 | yes | 21.095 | 21.095 | ||
gemm | 1 | yes | 19.667 | 19.667 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 1 | yes | 16.165 | 16.165 | ||
gemm | 1 | yes | 24.612 | 24.612 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 1 | yes | 26.134 | 26.134 | ||
gemm | 1 | yes | 29.175 | 29.175 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 1 | yes | 19.364 | 19.364 | ||
gemm | 1 | yes | 32.092 | 32.092 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 1 | yes | 29.999 | 29.999 | ||
gemm | 1 | yes | 33.439 | 33.439 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 1 | yes | 24.002 | 24.002 | ||
gemm | 1 | yes | 31.210 | 31.210 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 1 | yes | 28.477 | 28.477 | ||
gemm | 1 | yes | 33.265 | 33.265 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 1 | yes | 32.497 | 32.497 | ||
gemm | 1 | yes | 41.742 | 41.742 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 1 | yes | 40.049 | 40.049 | ||
gemm | 1 | yes | 45.219 | 45.219 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 1 | yes | 27.316 | 27.316 | ||
gemm | 1 | yes | 30.669 | 30.669 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 1 | yes | 40.225 | 40.225 | ||
gemm | 1 | yes | 43.144 | 43.144 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 1 | yes | 44.281 | 44.281 | ||
gemm | 1 | yes | 46.430 | 46.430 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 1 | yes | 59.731 | 59.731 | ||
gemm | 1 | yes | 56.551 | 56.551 | ||||||
armcl-18.03 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 1 | yes | 12.707 | 12.707 | |
gemm | 1 | yes | 16.728 | 16.728 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 1 | yes | 14.383 | 14.383 | ||
gemm | 1 | yes | 18.216 | 18.216 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 1 | yes | 15.371 | 15.371 | ||
gemm | 1 | yes | 17.185 | 17.185 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 1 | yes | 16.886 | 16.886 | ||
gemm | 1 | yes | 18.681 | 18.681 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 1 | yes | 16.783 | 16.783 | ||
gemm | 1 | yes | 22.973 | 22.973 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 1 | yes | 20.924 | 20.924 | ||
gemm | 1 | yes | 23.157 | 23.157 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 1 | yes | 20.309 | 20.309 | ||
gemm | 1 | yes | 22.660 | 22.660 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 1 | yes | 25.841 | 25.841 | ||
gemm | 1 | yes | 28.633 | 28.633 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 1 | yes | 21.033 | 21.033 | ||
gemm | 1 | yes | 28.262 | 28.262 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 1 | yes | 30.696 | 30.696 | ||
gemm | 1 | yes | 32.900 | 32.900 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 1 | yes | 32.020 | 32.020 | ||
gemm | 1 | yes | 33.825 | 33.825 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 1 | yes | 37.928 | 37.928 | ||
gemm | 1 | yes | 39.586 | 39.586 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 1 | yes | 25.773 | 25.773 | ||
gemm | 1 | yes | 31.809 | 31.809 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 1 | yes | 40.454 | 40.454 | ||
gemm | 1 | yes | 41.419 | 41.419 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 1 | yes | 43.872 | 43.872 | ||
gemm | 1 | yes | 44.584 | 44.584 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 1 | yes | 57.862 | 57.862 | ||
gemm | 1 | yes | 51.734 | 51.734 | ||||||
armcl-dv/dt | v1-0.25-128 | 0.25 | 128 | 1 | direct | 1 | yes | 9.935 | 9.935 | |
gemm | 1 | yes | 15.733 | 15.733 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 1 | yes | 11.576 | 11.576 | ||
gemm | 1 | yes | 19.484 | 19.484 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 1 | yes | 12.563 | 12.563 | ||
gemm | 1 | yes | 18.398 | 18.398 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 1 | yes | 13.907 | 13.907 | ||
gemm | 1 | yes | 18.759 | 18.759 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 1 | yes | 14.870 | 14.870 | ||
gemm | 1 | yes | 22.073 | 22.073 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 1 | yes | 17.257 | 17.257 | ||
gemm | 1 | yes | 22.623 | 22.623 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 1 | yes | 19.995 | 19.995 | ||
gemm | 1 | yes | 25.422 | 25.422 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 1 | yes | 24.318 | 24.318 | ||
gemm | 1 | yes | 27.787 | 27.787 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 1 | yes | 18.783 | 18.783 | ||
gemm | 1 | yes | 28.302 | 28.302 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 1 | yes | 27.018 | 27.018 | ||
gemm | 1 | yes | 32.536 | 32.536 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 1 | yes | 29.356 | 29.356 | ||
gemm | 1 | yes | 34.091 | 34.091 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 1 | yes | 36.176 | 36.176 | ||
gemm | 1 | yes | 40.297 | 40.297 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 1 | yes | 23.396 | 23.396 | ||
gemm | 1 | yes | 32.279 | 32.279 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 1 | yes | 35.815 | 35.815 | ||
gemm | 1 | yes | 40.727 | 40.727 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 1 | yes | 39.738 | 39.738 | ||
gemm | 1 | yes | 44.144 | 44.144 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 1 | yes | 59.403 | 59.403 | ||
gemm | 1 | yes | 48.584 | 48.584 |
plot_performance(df_armcl_performance_raw, platform_id=hikey_id, groupby_level='convolution_method')
plot_performance_violin(df_armcl_performance_raw, platform_id=hikey_id, groupby_level='convolution_method')
plot_performance(df_armcl_performance_raw, platform_id=hikey_id, groupby_level='library')
plot_performance_violin(df_armcl_performance_raw, platform_id=hikey_id, groupby_level='library')
df_armcl_accuracy_500_raw = get_experimental_results(repo_uoa=armcl_accuracy_500_repo_uoa,
tags='explore-mobilenets-accuracy', accuracy=True)
# Extract frame predictionsdf_armcl_accuracy_500_raw
df_armcl_predictions_500 = df_armcl_accuracy_500_raw[['frame_predictions']]
# Reduce the repetition_id index dimension (only 1 repetition anyway).
df_armcl_accuracy_500 = \
df_armcl_accuracy_500_raw[['accuracy_top1', 'accuracy_top5']] \
.groupby(level=df_armcl_accuracy_500_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_armcl_accuracy_500)
accuracy_top1 | accuracy_top5 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | armcl-17.12 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.410 | 0.672 |
gemm | 0.410 | 0.672 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.018 | 0.070 | ||
gemm | 0.008 | 0.068 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.158 | 0.364 | ||
gemm | 0.158 | 0.382 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.406 | 0.672 | ||
gemm | 0.404 | 0.662 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.536 | 0.798 | ||
gemm | 0.536 | 0.798 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.058 | 0.154 | ||
gemm | 0.066 | 0.156 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.276 | 0.492 | ||
gemm | 0.264 | 0.510 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.560 | 0.782 | ||
gemm | 0.560 | 0.786 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.608 | 0.822 | ||
gemm | 0.608 | 0.822 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.092 | 0.240 | ||
gemm | 0.112 | 0.276 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.312 | 0.552 | ||
gemm | 0.310 | 0.568 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.612 | 0.832 | ||
gemm | 0.620 | 0.840 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.622 | 0.860 | ||
gemm | 0.622 | 0.860 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.086 | 0.198 | ||
gemm | 0.080 | 0.186 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.390 | 0.682 | ||
gemm | 0.402 | 0.682 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.618 | 0.818 | ||
gemm | 0.642 | 0.850 | ||||||
armcl-18.01 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.410 | 0.672 | |
gemm | 0.410 | 0.672 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.450 | 0.704 | ||
gemm | 0.450 | 0.704 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.502 | 0.760 | ||
gemm | 0.502 | 0.760 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.508 | 0.756 | ||
gemm | 0.508 | 0.756 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.536 | 0.798 | ||
gemm | 0.536 | 0.798 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.586 | 0.822 | ||
gemm | 0.586 | 0.822 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.622 | 0.824 | ||
gemm | 0.622 | 0.824 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.650 | 0.852 | ||
gemm | 0.650 | 0.852 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.608 | 0.822 | ||
gemm | 0.608 | 0.822 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.642 | 0.850 | ||
gemm | 0.642 | 0.850 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.676 | 0.890 | ||
gemm | 0.676 | 0.890 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.678 | 0.878 | ||
gemm | 0.678 | 0.878 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.622 | 0.860 | ||
gemm | 0.622 | 0.860 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.666 | 0.868 | ||
gemm | 0.666 | 0.868 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.704 | 0.882 | ||
gemm | 0.704 | 0.882 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.716 | 0.906 | ||
gemm | 0.716 | 0.906 | ||||||
armcl-18.03 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.410 | 0.672 | |
gemm | 0.410 | 0.672 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.450 | 0.704 | ||
gemm | 0.450 | 0.704 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.502 | 0.760 | ||
gemm | 0.502 | 0.760 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.508 | 0.756 | ||
gemm | 0.508 | 0.756 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.536 | 0.798 | ||
gemm | 0.536 | 0.798 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.586 | 0.822 | ||
gemm | 0.586 | 0.822 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.622 | 0.824 | ||
gemm | 0.622 | 0.824 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.650 | 0.852 | ||
gemm | 0.650 | 0.852 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.608 | 0.822 | ||
gemm | 0.608 | 0.822 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.642 | 0.850 | ||
gemm | 0.642 | 0.850 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.676 | 0.890 | ||
gemm | 0.676 | 0.890 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.678 | 0.878 | ||
gemm | 0.678 | 0.878 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.622 | 0.860 | ||
gemm | 0.622 | 0.860 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.666 | 0.868 | ||
gemm | 0.666 | 0.868 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.704 | 0.882 | ||
gemm | 0.704 | 0.882 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.716 | 0.906 | ||
gemm | 0.716 | 0.906 | ||||||
armcl-dv/dt | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.410 | 0.672 | |
gemm | 0.410 | 0.672 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.450 | 0.704 | ||
gemm | 0.450 | 0.704 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.502 | 0.760 | ||
gemm | 0.502 | 0.760 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.508 | 0.756 | ||
gemm | 0.508 | 0.756 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.536 | 0.798 | ||
gemm | 0.536 | 0.798 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.586 | 0.822 | ||
gemm | 0.586 | 0.822 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.622 | 0.824 | ||
gemm | 0.622 | 0.824 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.650 | 0.852 | ||
gemm | 0.650 | 0.852 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.608 | 0.822 | ||
gemm | 0.608 | 0.822 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.642 | 0.850 | ||
gemm | 0.642 | 0.850 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.676 | 0.890 | ||
gemm | 0.676 | 0.890 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.678 | 0.878 | ||
gemm | 0.678 | 0.878 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.622 | 0.860 | ||
gemm | 0.622 | 0.860 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.666 | 0.868 | ||
gemm | 0.666 | 0.868 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.704 | 0.882 | ||
gemm | 0.704 | 0.882 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.716 | 0.906 | ||
gemm | 0.716 | 0.906 |
# Identical accuracy for "18.03" and "dv/dt".
(df_armcl_accuracy_500.loc[hikey_id,'armcl-18.03'] - df_armcl_accuracy_500.loc[hikey_id,'armcl-dv/dt'] == 0).all()
accuracy_top1 True accuracy_top5 True dtype: bool
# Identical accuracy for "18.03" and "18.01".
(df_armcl_accuracy_500.loc[hikey_id,'armcl-18.03'] - df_armcl_accuracy_500.loc[hikey_id,'armcl-18.01'] == 0).all()
accuracy_top1 True accuracy_top5 True dtype: bool
df_armcl_accuracy_500.loc[hikey_id,'armcl-18.03'] - df_armcl_accuracy_500.loc[hikey_id,'armcl-17.12']
accuracy_top1 | accuracy_top5 | |||||
---|---|---|---|---|---|---|
model | multiplier | resolution | batch_size | convolution_method | ||
v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.000 | 0.000 |
gemm | 0.000 | 0.000 | ||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.432 | 0.634 |
gemm | 0.442 | 0.636 | ||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.344 | 0.396 |
gemm | 0.344 | 0.378 | ||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.102 | 0.084 |
gemm | 0.104 | 0.094 | ||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.000 | 0.000 |
gemm | 0.000 | 0.000 | ||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.528 | 0.668 |
gemm | 0.520 | 0.666 | ||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.346 | 0.332 |
gemm | 0.358 | 0.314 | ||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.090 | 0.070 |
gemm | 0.090 | 0.066 | ||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.000 | 0.000 |
gemm | 0.000 | 0.000 | ||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.550 | 0.610 |
gemm | 0.530 | 0.574 | ||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.364 | 0.338 |
gemm | 0.366 | 0.322 | ||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.066 | 0.046 |
gemm | 0.058 | 0.038 | ||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.000 | 0.000 |
gemm | 0.000 | 0.000 | ||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.580 | 0.670 |
gemm | 0.586 | 0.682 | ||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.314 | 0.200 |
gemm | 0.302 | 0.200 | ||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.098 | 0.088 |
gemm | 0.074 | 0.056 |
# TODO: Outline into a function for comparing ArmCL and TensorFlow predictions.
df_armcl_predictions = df_armcl_predictions_500
df_tensorflow_predictions = df_tensorflow_predictions_500
tensorflow_lib = 'tensorflow-1.7'
tensorflow_convolution_method = 'direct'
for index, row in df_armcl_predictions.iterrows():
(platform, lib, model, multiplier, resolution, batch_size, convolution_method, repetition_id) = index
# For now, only check mispredictions for '18.03' and 'v1-1.00-224'.
if not lib=='armcl-18.03' or not model=='v1-1.00-224': continue
tensorflow_index = (platform, tensorflow_lib, model, multiplier, resolution, batch_size, tensorflow_convolution_method, repetition_id)
# Extract frame predictions.
armcl_predictions = row['frame_predictions']
tensorflow_predictions = df_tensorflow_predictions.loc[tensorflow_index]['frame_predictions']
# At the very minimum, the frame predictions should be of the same length.
if len(armcl_predictions) != len(tensorflow_predictions):
print('[Warning] ArmCL and TensorFlow predictions have different length! Skipping...')
continue
# Iterate over the frame predictions.
for (armcl_prediction, tensorflow_prediction) in zip(armcl_predictions, tensorflow_predictions):
if(armcl_prediction['accuracy_top1'] != tensorflow_prediction['accuracy_top1']):
print(index)
print('ArmCL: '+str(armcl_prediction))
print('TensorFlow: '+str(tensorflow_prediction))
print('')
('hikey-960', 'armcl-18.03', 'v1-1.00-224', 1.0, 224, 1, 'direct', 0) ArmCL: {'class_topmost': 492, 'file_name': 'ILSVRC2012_val_00000060.JPEG', 'accuracy_top5': 'yes', 'class_correct': 588, 'accuracy_top1': 'no'} TensorFlow: {'class_topmost': 588, 'file_name': 'ILSVRC2012_val_00000060.JPEG', 'accuracy_top5': 'yes', 'class_correct': 588, 'accuracy_top1': 'yes'} ('hikey-960', 'armcl-18.03', 'v1-1.00-224', 1.0, 224, 1, 'direct', 0) ArmCL: {'class_topmost': 926, 'file_name': 'ILSVRC2012_val_00000302.JPEG', 'accuracy_top5': 'yes', 'class_correct': 469, 'accuracy_top1': 'no'} TensorFlow: {'class_topmost': 469, 'file_name': 'ILSVRC2012_val_00000302.JPEG', 'accuracy_top5': 'yes', 'class_correct': 469, 'accuracy_top1': 'yes'} ('hikey-960', 'armcl-18.03', 'v1-1.00-224', 1.0, 224, 1, 'direct', 0) ArmCL: {'class_topmost': 984, 'file_name': 'ILSVRC2012_val_00000313.JPEG', 'accuracy_top5': 'yes', 'class_correct': 979, 'accuracy_top1': 'no'} TensorFlow: {'class_topmost': 979, 'file_name': 'ILSVRC2012_val_00000313.JPEG', 'accuracy_top5': 'yes', 'class_correct': 979, 'accuracy_top1': 'yes'} ('hikey-960', 'armcl-18.03', 'v1-1.00-224', 1.0, 224, 1, 'gemm', 0) ArmCL: {'class_topmost': 492, 'file_name': 'ILSVRC2012_val_00000060.JPEG', 'accuracy_top5': 'yes', 'class_correct': 588, 'accuracy_top1': 'no'} TensorFlow: {'class_topmost': 588, 'file_name': 'ILSVRC2012_val_00000060.JPEG', 'accuracy_top5': 'yes', 'class_correct': 588, 'accuracy_top1': 'yes'} ('hikey-960', 'armcl-18.03', 'v1-1.00-224', 1.0, 224, 1, 'gemm', 0) ArmCL: {'class_topmost': 926, 'file_name': 'ILSVRC2012_val_00000302.JPEG', 'accuracy_top5': 'yes', 'class_correct': 469, 'accuracy_top1': 'no'} TensorFlow: {'class_topmost': 469, 'file_name': 'ILSVRC2012_val_00000302.JPEG', 'accuracy_top5': 'yes', 'class_correct': 469, 'accuracy_top1': 'yes'} ('hikey-960', 'armcl-18.03', 'v1-1.00-224', 1.0, 224, 1, 'gemm', 0) ArmCL: {'class_topmost': 984, 'file_name': 'ILSVRC2012_val_00000313.JPEG', 'accuracy_top5': 'yes', 'class_correct': 979, 'accuracy_top1': 'no'} TensorFlow: {'class_topmost': 979, 'file_name': 'ILSVRC2012_val_00000313.JPEG', 'accuracy_top5': 'yes', 'class_correct': 979, 'accuracy_top1': 'yes'}
df_armcl_accuracy_50000_raw = get_experimental_results(repo_uoa=armcl_accuracy_50000_repo_uoa,
tags='explore-mobilenets-accuracy', accuracy=True)
# Extract frame predictions.
df_armcl_predictions_50000 = df_armcl_accuracy_50000_raw[['frame_predictions']]
# Reduce the repetition_id index dimension (only 1 repetition anyway).
df_armcl_accuracy_50000 = \
df_armcl_accuracy_50000_raw[['accuracy_top1', 'accuracy_top5']] \
.groupby(level=df_armcl_accuracy_50000_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_armcl_accuracy_50000)
accuracy_top1 | accuracy_top5 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | armcl-dv/dt | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.40724 | 0.65718 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.45354 | 0.70206 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.48658 | 0.73232 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.50794 | 0.74872 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.55674 | 0.79034 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.59354 | 0.82008 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.61674 | 0.83594 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.63690 | 0.85146 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.61096 | 0.83018 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.64350 | 0.85616 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.66840 | 0.87040 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.68172 | 0.88010 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.63586 | 0.84940 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.66458 | 0.87088 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.68830 | 0.88474 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.70464 | 0.89398 |
accuracy_metric = 'accuracy_top1'
df_armcl_performance_accuracy_50000 = merge_performance_accuracy(df_armcl_performance, df_armcl_accuracy_50000,
reference_lib='armcl-dv/dt',
reference_convolution_method='direct')
display_in_full(df_armcl_performance_accuracy_50000)
time_avg_ms | accuracy_top1 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | armcl-17.12 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 26.351 | 0.40724 |
gemm | 27.241 | 0.40724 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 32.842 | 0.45354 | ||
gemm | 36.124 | 0.45354 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 27.684 | 0.48658 | ||
gemm | 35.715 | 0.48658 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 38.406 | 0.50794 | ||
gemm | 38.645 | 0.50794 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 36.301 | 0.55674 | ||
gemm | 32.069 | 0.55674 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 43.377 | 0.59354 | ||
gemm | 44.402 | 0.59354 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 51.574 | 0.61674 | ||
gemm | 50.743 | 0.61674 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 53.061 | 0.63690 | ||
gemm | 51.806 | 0.63690 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 48.204 | 0.61096 | ||
gemm | 42.519 | 0.61096 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 65.698 | 0.64350 | ||
gemm | 60.465 | 0.64350 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 66.093 | 0.66840 | ||
gemm | 66.494 | 0.66840 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 79.667 | 0.68172 | ||
gemm | 73.126 | 0.68172 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 59.597 | 0.63586 | ||
gemm | 61.465 | 0.63586 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 76.589 | 0.66458 | ||
gemm | 71.438 | 0.66458 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 83.530 | 0.68830 | ||
gemm | 84.506 | 0.68830 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 110.056 | 0.70464 | ||
gemm | 99.716 | 0.70464 | ||||||
armcl-18.01 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 14.905 | 0.40724 | |
gemm | 20.814 | 0.40724 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 15.242 | 0.45354 | ||
gemm | 21.745 | 0.45354 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 14.707 | 0.48658 | ||
gemm | 25.132 | 0.48658 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 21.095 | 0.50794 | ||
gemm | 19.667 | 0.50794 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 16.165 | 0.55674 | ||
gemm | 24.612 | 0.55674 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 26.134 | 0.59354 | ||
gemm | 29.175 | 0.59354 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 19.364 | 0.61674 | ||
gemm | 32.092 | 0.61674 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 29.999 | 0.63690 | ||
gemm | 33.439 | 0.63690 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 24.002 | 0.61096 | ||
gemm | 31.210 | 0.61096 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 28.477 | 0.64350 | ||
gemm | 33.265 | 0.64350 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 32.497 | 0.66840 | ||
gemm | 41.742 | 0.66840 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 40.049 | 0.68172 | ||
gemm | 45.219 | 0.68172 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 27.316 | 0.63586 | ||
gemm | 30.669 | 0.63586 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 40.225 | 0.66458 | ||
gemm | 43.144 | 0.66458 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 44.281 | 0.68830 | ||
gemm | 46.430 | 0.68830 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 59.731 | 0.70464 | ||
gemm | 56.551 | 0.70464 | ||||||
armcl-18.03 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 12.707 | 0.40724 | |
gemm | 16.728 | 0.40724 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 14.383 | 0.45354 | ||
gemm | 18.216 | 0.45354 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 15.371 | 0.48658 | ||
gemm | 17.185 | 0.48658 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 16.886 | 0.50794 | ||
gemm | 18.681 | 0.50794 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 16.783 | 0.55674 | ||
gemm | 22.973 | 0.55674 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 20.924 | 0.59354 | ||
gemm | 23.157 | 0.59354 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 20.309 | 0.61674 | ||
gemm | 22.660 | 0.61674 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 25.841 | 0.63690 | ||
gemm | 28.633 | 0.63690 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 21.033 | 0.61096 | ||
gemm | 28.262 | 0.61096 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 30.696 | 0.64350 | ||
gemm | 32.900 | 0.64350 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 32.020 | 0.66840 | ||
gemm | 33.825 | 0.66840 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 37.928 | 0.68172 | ||
gemm | 39.586 | 0.68172 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 25.773 | 0.63586 | ||
gemm | 31.809 | 0.63586 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 40.454 | 0.66458 | ||
gemm | 41.419 | 0.66458 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 43.872 | 0.68830 | ||
gemm | 44.584 | 0.68830 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 57.862 | 0.70464 | ||
gemm | 51.734 | 0.70464 | ||||||
armcl-dv/dt | v1-0.25-128 | 0.25 | 128 | 1 | direct | 9.935 | 0.40724 | |
gemm | 15.733 | 0.40724 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 11.576 | 0.45354 | ||
gemm | 19.484 | 0.45354 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 12.563 | 0.48658 | ||
gemm | 18.398 | 0.48658 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 13.907 | 0.50794 | ||
gemm | 18.759 | 0.50794 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 14.870 | 0.55674 | ||
gemm | 22.073 | 0.55674 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 17.257 | 0.59354 | ||
gemm | 22.623 | 0.59354 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 19.995 | 0.61674 | ||
gemm | 25.422 | 0.61674 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 24.318 | 0.63690 | ||
gemm | 27.787 | 0.63690 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 18.783 | 0.61096 | ||
gemm | 28.302 | 0.61096 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 27.018 | 0.64350 | ||
gemm | 32.536 | 0.64350 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 29.356 | 0.66840 | ||
gemm | 34.091 | 0.66840 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 36.176 | 0.68172 | ||
gemm | 40.297 | 0.68172 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 23.396 | 0.63586 | ||
gemm | 32.279 | 0.63586 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 35.815 | 0.66458 | ||
gemm | 40.727 | 0.66458 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 39.738 | 0.68830 | ||
gemm | 44.144 | 0.68830 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 59.403 | 0.70464 | ||
gemm | 48.584 | 0.70464 |
plot(df_armcl_performance_accuracy_50000, libs=['armcl-18.03'], accuracy_metric=accuracy_metric,
save_fig_name='%s-%s-50000-18_03' % (hikey_id, accuracy_metric+'_'))
plot(df_armcl_performance_accuracy_50000, libs=['armcl-18.03','armcl-dv/dt'], accuracy_metric=accuracy_metric,
save_fig_name='%s-%s-50000-dv_dt__18_03' % (hikey_id, accuracy_metric+'_'))
plot(df_armcl_performance_accuracy_50000, accuracy_metric=accuracy_metric,
save_fig_name='%s-%s-50000-dv_dt__18_03__18_01__17_12' % (hikey_id, accuracy_metric+'_'))
df_armcl_performance_accuracy_500 = merge_performance_accuracy(df_armcl_performance, df_armcl_accuracy_500,
reference_lib='armcl-dv/dt',
reference_convolution_method='direct')
display_in_full(df_armcl_performance_accuracy_500)
time_avg_ms | accuracy_top1 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | armcl-17.12 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 26.351 | 0.410 |
gemm | 27.241 | 0.410 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 32.842 | 0.018 | ||
gemm | 36.124 | 0.008 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 27.684 | 0.158 | ||
gemm | 35.715 | 0.158 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 38.406 | 0.406 | ||
gemm | 38.645 | 0.404 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 36.301 | 0.536 | ||
gemm | 32.069 | 0.536 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 43.377 | 0.058 | ||
gemm | 44.402 | 0.066 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 51.574 | 0.276 | ||
gemm | 50.743 | 0.264 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 53.061 | 0.560 | ||
gemm | 51.806 | 0.560 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 48.204 | 0.608 | ||
gemm | 42.519 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 65.698 | 0.092 | ||
gemm | 60.465 | 0.112 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 66.093 | 0.312 | ||
gemm | 66.494 | 0.310 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 79.667 | 0.612 | ||
gemm | 73.126 | 0.620 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 59.597 | 0.622 | ||
gemm | 61.465 | 0.622 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 76.589 | 0.086 | ||
gemm | 71.438 | 0.080 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 83.530 | 0.390 | ||
gemm | 84.506 | 0.402 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 110.056 | 0.618 | ||
gemm | 99.716 | 0.642 | ||||||
armcl-18.01 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 14.905 | 0.410 | |
gemm | 20.814 | 0.410 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 15.242 | 0.450 | ||
gemm | 21.745 | 0.450 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 14.707 | 0.502 | ||
gemm | 25.132 | 0.502 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 21.095 | 0.508 | ||
gemm | 19.667 | 0.508 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 16.165 | 0.536 | ||
gemm | 24.612 | 0.536 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 26.134 | 0.586 | ||
gemm | 29.175 | 0.586 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 19.364 | 0.622 | ||
gemm | 32.092 | 0.622 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 29.999 | 0.650 | ||
gemm | 33.439 | 0.650 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 24.002 | 0.608 | ||
gemm | 31.210 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 28.477 | 0.642 | ||
gemm | 33.265 | 0.642 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 32.497 | 0.676 | ||
gemm | 41.742 | 0.676 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 40.049 | 0.678 | ||
gemm | 45.219 | 0.678 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 27.316 | 0.622 | ||
gemm | 30.669 | 0.622 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 40.225 | 0.666 | ||
gemm | 43.144 | 0.666 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 44.281 | 0.704 | ||
gemm | 46.430 | 0.704 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 59.731 | 0.716 | ||
gemm | 56.551 | 0.716 | ||||||
armcl-18.03 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 12.707 | 0.410 | |
gemm | 16.728 | 0.410 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 14.383 | 0.450 | ||
gemm | 18.216 | 0.450 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 15.371 | 0.502 | ||
gemm | 17.185 | 0.502 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 16.886 | 0.508 | ||
gemm | 18.681 | 0.508 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 16.783 | 0.536 | ||
gemm | 22.973 | 0.536 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 20.924 | 0.586 | ||
gemm | 23.157 | 0.586 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 20.309 | 0.622 | ||
gemm | 22.660 | 0.622 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 25.841 | 0.650 | ||
gemm | 28.633 | 0.650 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 21.033 | 0.608 | ||
gemm | 28.262 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 30.696 | 0.642 | ||
gemm | 32.900 | 0.642 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 32.020 | 0.676 | ||
gemm | 33.825 | 0.676 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 37.928 | 0.678 | ||
gemm | 39.586 | 0.678 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 25.773 | 0.622 | ||
gemm | 31.809 | 0.622 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 40.454 | 0.666 | ||
gemm | 41.419 | 0.666 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 43.872 | 0.704 | ||
gemm | 44.584 | 0.704 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 57.862 | 0.716 | ||
gemm | 51.734 | 0.716 | ||||||
armcl-dv/dt | v1-0.25-128 | 0.25 | 128 | 1 | direct | 9.935 | 0.410 | |
gemm | 15.733 | 0.410 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 11.576 | 0.450 | ||
gemm | 19.484 | 0.450 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 12.563 | 0.502 | ||
gemm | 18.398 | 0.502 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 13.907 | 0.508 | ||
gemm | 18.759 | 0.508 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 14.870 | 0.536 | ||
gemm | 22.073 | 0.536 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 17.257 | 0.586 | ||
gemm | 22.623 | 0.586 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 19.995 | 0.622 | ||
gemm | 25.422 | 0.622 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 24.318 | 0.650 | ||
gemm | 27.787 | 0.650 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 18.783 | 0.608 | ||
gemm | 28.302 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 27.018 | 0.642 | ||
gemm | 32.536 | 0.642 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 29.356 | 0.676 | ||
gemm | 34.091 | 0.676 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 36.176 | 0.678 | ||
gemm | 40.297 | 0.678 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 23.396 | 0.622 | ||
gemm | 32.279 | 0.622 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 35.815 | 0.666 | ||
gemm | 40.727 | 0.666 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 39.738 | 0.704 | ||
gemm | 44.144 | 0.704 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 59.403 | 0.716 | ||
gemm | 48.584 | 0.716 |
plot(df_armcl_performance_accuracy_500, accuracy_metric=accuracy_metric,
save_fig_name='%s-%s-500-dv_dt__18_03__18_01__17_12' % (hikey_id, accuracy_metric+'_'))
df_armcl_performance_accuracy_500 = merge_performance_accuracy(df_armcl_performance, df_armcl_accuracy_500)
display_in_full(df_armcl_performance_accuracy_500)
time_avg_ms | accuracy_top1 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
hikey-960 | armcl-17.12 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 26.351 | 0.410 |
gemm | 27.241 | 0.410 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 32.842 | 0.018 | ||
gemm | 36.124 | 0.008 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 27.684 | 0.158 | ||
gemm | 35.715 | 0.158 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 38.406 | 0.406 | ||
gemm | 38.645 | 0.404 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 36.301 | 0.536 | ||
gemm | 32.069 | 0.536 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 43.377 | 0.058 | ||
gemm | 44.402 | 0.066 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 51.574 | 0.276 | ||
gemm | 50.743 | 0.264 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 53.061 | 0.560 | ||
gemm | 51.806 | 0.560 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 48.204 | 0.608 | ||
gemm | 42.519 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 65.698 | 0.092 | ||
gemm | 60.465 | 0.112 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 66.093 | 0.312 | ||
gemm | 66.494 | 0.310 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 79.667 | 0.612 | ||
gemm | 73.126 | 0.620 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 59.597 | 0.622 | ||
gemm | 61.465 | 0.622 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 76.589 | 0.086 | ||
gemm | 71.438 | 0.080 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 83.530 | 0.390 | ||
gemm | 84.506 | 0.402 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 110.056 | 0.618 | ||
gemm | 99.716 | 0.642 | ||||||
armcl-18.01 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 14.905 | 0.410 | |
gemm | 20.814 | 0.410 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 15.242 | 0.450 | ||
gemm | 21.745 | 0.450 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 14.707 | 0.502 | ||
gemm | 25.132 | 0.502 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 21.095 | 0.508 | ||
gemm | 19.667 | 0.508 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 16.165 | 0.536 | ||
gemm | 24.612 | 0.536 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 26.134 | 0.586 | ||
gemm | 29.175 | 0.586 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 19.364 | 0.622 | ||
gemm | 32.092 | 0.622 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 29.999 | 0.650 | ||
gemm | 33.439 | 0.650 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 24.002 | 0.608 | ||
gemm | 31.210 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 28.477 | 0.642 | ||
gemm | 33.265 | 0.642 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 32.497 | 0.676 | ||
gemm | 41.742 | 0.676 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 40.049 | 0.678 | ||
gemm | 45.219 | 0.678 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 27.316 | 0.622 | ||
gemm | 30.669 | 0.622 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 40.225 | 0.666 | ||
gemm | 43.144 | 0.666 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 44.281 | 0.704 | ||
gemm | 46.430 | 0.704 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 59.731 | 0.716 | ||
gemm | 56.551 | 0.716 | ||||||
armcl-18.03 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 12.707 | 0.410 | |
gemm | 16.728 | 0.410 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 14.383 | 0.450 | ||
gemm | 18.216 | 0.450 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 15.371 | 0.502 | ||
gemm | 17.185 | 0.502 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 16.886 | 0.508 | ||
gemm | 18.681 | 0.508 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 16.783 | 0.536 | ||
gemm | 22.973 | 0.536 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 20.924 | 0.586 | ||
gemm | 23.157 | 0.586 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 20.309 | 0.622 | ||
gemm | 22.660 | 0.622 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 25.841 | 0.650 | ||
gemm | 28.633 | 0.650 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 21.033 | 0.608 | ||
gemm | 28.262 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 30.696 | 0.642 | ||
gemm | 32.900 | 0.642 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 32.020 | 0.676 | ||
gemm | 33.825 | 0.676 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 37.928 | 0.678 | ||
gemm | 39.586 | 0.678 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 25.773 | 0.622 | ||
gemm | 31.809 | 0.622 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 40.454 | 0.666 | ||
gemm | 41.419 | 0.666 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 43.872 | 0.704 | ||
gemm | 44.584 | 0.704 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 57.862 | 0.716 | ||
gemm | 51.734 | 0.716 | ||||||
armcl-dv/dt | v1-0.25-128 | 0.25 | 128 | 1 | direct | 9.935 | 0.410 | |
gemm | 15.733 | 0.410 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 11.576 | 0.450 | ||
gemm | 19.484 | 0.450 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 12.563 | 0.502 | ||
gemm | 18.398 | 0.502 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 13.907 | 0.508 | ||
gemm | 18.759 | 0.508 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 14.870 | 0.536 | ||
gemm | 22.073 | 0.536 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 17.257 | 0.586 | ||
gemm | 22.623 | 0.586 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 19.995 | 0.622 | ||
gemm | 25.422 | 0.622 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 24.318 | 0.650 | ||
gemm | 27.787 | 0.650 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 18.783 | 0.608 | ||
gemm | 28.302 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 27.018 | 0.642 | ||
gemm | 32.536 | 0.642 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 29.356 | 0.676 | ||
gemm | 34.091 | 0.676 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 36.176 | 0.678 | ||
gemm | 40.297 | 0.678 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 23.396 | 0.622 | ||
gemm | 32.279 | 0.622 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 35.815 | 0.666 | ||
gemm | 40.727 | 0.666 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 39.738 | 0.704 | ||
gemm | 44.144 | 0.704 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 59.403 | 0.716 | ||
gemm | 48.584 | 0.716 |
plot(df_armcl_performance_accuracy_500, accuracy_metric=accuracy_metric,
save_fig_name='%s-%s-500-dv_dt__18_03__18_01__17_12' % (hikey_id, accuracy_metric))
df_accuracy_50000 = pd.DataFrame(
data=[
df_armcl_accuracy_50000['accuracy_top1'].values,
df_tensorflow_accuracy_50000['accuracy_top1'].values,
df_tensorflow_accuracy_50000_claimed['accuracy_top1'].values,
],
index=[
'ArmCL 18.03 (measured)',
'TensorFlow 1.7 (measured)',
'TensorFlow 1.x (claimed)',
],
columns=df_tensorflow_accuracy_50000_claimed.index.get_level_values(level='model').values
).T.sort_index(ascending=False)
# df_accuracy_50000.index.name = 'model'
if save_tab:
save_tab_name = 'accuracy_top1-50000'
save_tab_path = os.path.join(save_tab_dir, '%s.%s' % (save_tab_name, save_tab_ext))
with open(save_tab_path, 'w') as f: f.write(df_accuracy_50000.to_latex())
display_in_full(df_accuracy_50000)
ArmCL 18.03 (measured) | TensorFlow 1.7 (measured) | TensorFlow 1.x (claimed) | |
---|---|---|---|
v1-1.00-224 | 0.70464 | 0.70466 | 0.707 |
v1-1.00-192 | 0.68830 | 0.68824 | 0.693 |
v1-1.00-160 | 0.66458 | 0.66504 | 0.672 |
v1-1.00-128 | 0.63586 | 0.63580 | 0.641 |
v1-0.75-224 | 0.68172 | 0.68178 | 0.684 |
v1-0.75-192 | 0.66840 | 0.66830 | 0.674 |
v1-0.75-160 | 0.64350 | 0.64386 | 0.652 |
v1-0.75-128 | 0.61096 | 0.61060 | 0.618 |
v1-0.50-224 | 0.63690 | 0.63722 | 0.640 |
v1-0.50-192 | 0.61674 | 0.61578 | 0.621 |
v1-0.50-160 | 0.59354 | 0.59376 | 0.599 |
v1-0.50-128 | 0.55674 | 0.55652 | 0.562 |
v1-0.25-224 | 0.50794 | 0.50766 | 0.506 |
v1-0.25-192 | 0.48658 | 0.48676 | 0.490 |
v1-0.25-160 | 0.45354 | 0.45322 | 0.460 |
v1-0.25-128 | 0.40724 | 0.40694 | 0.413 |
df_performance = pd.concat([df_armcl_performance, df_tensorflow_performance])
df_accuracy_500 = pd.concat([df_armcl_accuracy_500, df_tensorflow_accuracy_500])
df_performance_accuracy_500 = merge_performance_accuracy(df_performance, df_accuracy_500)
plot(df_performance_accuracy_500, accuracy_metric=accuracy_metric, save_fig=True,
save_fig_name='%s-%s-500-dv_dt__18_03__18_01__17_12__tf' % (hikey_id, accuracy_metric))
<matplotlib.figure.Figure at 0x7f02e43e5160>
df_accuracy_50000 = pd.concat([df_armcl_accuracy_50000, df_tensorflow_accuracy_50000])
df_performance_accuracy_50000 = merge_performance_accuracy(df_performance, df_accuracy_50000,
reference_lib='armcl-dv/dt',
reference_convolution_method='direct')
plot(df_performance_accuracy_50000, accuracy_metric=accuracy_metric, save_fig=True,
save_fig_name='%s-%s-50000-dv_dt__18_03__18_01__17_12__tf' % (hikey_id, accuracy_metric+'_'))
<matplotlib.figure.Figure at 0x7f02e3efc860>
plot(df_performance_accuracy_50000, libs=['armcl-17.12','armcl-18.03','armcl-dv/dt','tensorflow-1.7'],
accuracy_metric=accuracy_metric,
save_fig_name='%s-%s-50000-dv_dt__18_03__17_12__tf' % (hikey_id, accuracy_metric))
df_firefly_tensorflow_accuracy_500_raw = get_experimental_results(repo_uoa=firefly_tensorflow_accuracy_500_repo_uoa,
tags='explore-mobilenets-accuracy', accuracy=True)
# Extract frame predictions.
df_firefly_tensorflow_predictions_500 = df_firefly_tensorflow_accuracy_500_raw[['frame_predictions']]
# Reduce the repetition_id index dimension (only 1 repetition anyway).
df_firefly_tensorflow_accuracy_500 = \
df_firefly_tensorflow_accuracy_500_raw[['accuracy_top1', 'accuracy_top5']] \
.groupby(level=df_firefly_tensorflow_accuracy_500_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_firefly_tensorflow_accuracy_500)
accuracy_top1 | accuracy_top5 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
firefly | tensorflow-1.7 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.406 | 0.670 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.448 | 0.702 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.496 | 0.760 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.508 | 0.752 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.556 | 0.794 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.606 | 0.832 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.620 | 0.826 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.646 | 0.844 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.612 | 0.826 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.650 | 0.856 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.680 | 0.886 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.680 | 0.886 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.634 | 0.870 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.664 | 0.880 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.684 | 0.880 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.732 | 0.896 | ||
tensorflow-1.8 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.406 | 0.670 | |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.448 | 0.702 | ||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.496 | 0.760 | ||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.508 | 0.752 | ||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.556 | 0.794 | ||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.606 | 0.832 | ||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.620 | 0.826 | ||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.646 | 0.844 | ||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.612 | 0.826 | ||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.650 | 0.856 | ||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.680 | 0.886 | ||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.680 | 0.886 | ||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.634 | 0.870 | ||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.664 | 0.880 | ||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.684 | 0.880 | ||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.732 | 0.896 |
# Check whether TensorFlow accuracy on Firefly is the same as on HiKey. (It's not!)
df_firefly_tensorflow_accuracy_500.loc[firefly_id].loc['tensorflow-1.7'] - \
df_tensorflow_accuracy_500.loc[hikey_id].loc['tensorflow-1.7']
accuracy_top1 | accuracy_top5 | |||||
---|---|---|---|---|---|---|
model | multiplier | resolution | batch_size | convolution_method | ||
v1-0.25-128 | 0.25 | 128 | 1 | direct | -0.004 | -0.002 |
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.002 | -0.002 |
v1-0.25-192 | 0.25 | 192 | 1 | direct | -0.006 | -0.002 |
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.000 | -0.002 |
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.020 | -0.002 |
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.022 | 0.010 |
v1-0.50-192 | 0.50 | 192 | 1 | direct | -0.002 | 0.008 |
v1-0.50-224 | 0.50 | 224 | 1 | direct | -0.002 | -0.008 |
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.004 | 0.000 |
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.006 | 0.004 |
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.004 | -0.010 |
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.000 | 0.010 |
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.014 | 0.010 |
v1-1.00-160 | 1.00 | 160 | 1 | direct | -0.002 | 0.006 |
v1-1.00-192 | 1.00 | 192 | 1 | direct | -0.020 | -0.002 |
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.010 | -0.008 |
df_firefly_tensorflow_performance_raw = get_experimental_results(repo_uoa=firefly_tensorflow_performance_repo_uoa,
tags='explore-mobilenets-performance', accuracy=False)
# Take the minimum execution time out of several repetitions.
df_firefly_tensorflow_performance_min = \
df_firefly_tensorflow_performance_raw.groupby(level=df_firefly_tensorflow_performance_raw.index.names[:-1]).min()
# Display all rows and columns.
# display_in_full(df_firefly_tensorflow_performance_min)
# Take the maximum execution time out of several repetitions.
df_firefly_tensorflow_performance_max = \
df_firefly_tensorflow_performance_raw.groupby(level=df_firefly_tensorflow_performance_raw.index.names[:-1]).max()
# Set 'convolution_method' to 'gemm' for all rows to reuse the available plotting functionality.
df_firefly_tensorflow_performance_max.index = \
df_firefly_tensorflow_performance_max.index \
.set_levels(levels=['gemm']*df_firefly_tensorflow_performance_max.index.size, level='convolution_method')
# Display all rows and columns.
# display_in_full(df_firefly_tensorflow_performance_max)
df_firefly_tensorflow_performance = pd.concat([df_firefly_tensorflow_performance_min, df_firefly_tensorflow_performance_max])
# TODO: Use df_firefly_armcl_accuracy_500 and df_firefly_tensorflow_accuracy_500.
df_accuracy_500 = pd.concat([df_armcl_accuracy_500, df_tensorflow_accuracy_500])
df_firefly_performance_accuracy_500 = merge_performance_accuracy(df_firefly_tensorflow_performance, df_accuracy_500,
reference_platform=hikey_id,
reference_lib='tensorflow-1.7')
plot(df_firefly_performance_accuracy_500, accuracy_metric=accuracy_metric, platform_id=firefly_id,
xmin=0, xmax=150.1, xstep=10, save_fig_name='%s-%s-500-tf-min_max' % (firefly_id, accuracy_metric))
# plot(df_firefly_performance_accuracy_500, platform_id=firefly_id, title=firefly_name,
# xmin=10., xmax=190.1, xstep=10, ymin=0.35, ymax=.801,
# accuracy_metric=accuracy_metric, save_fig_name='%s-%s-500-tf-min_max-complete' % (firefly_id, accuracy_metric))
df_firefly_performance = df_firefly_tensorflow_performance_min
df_firefly_accuracy_500 = df_firefly_tensorflow_accuracy_500
df_firefly_performance_accuracy_500 = merge_performance_accuracy(df_firefly_performance, df_firefly_accuracy_500)
plot(df_firefly_performance_accuracy_500, platform_id=firefly_id, title=firefly_name, xmin=0, xmax=150.1, xstep=10,
accuracy_metric=accuracy_metric, save_fig_name='%s-%s-500-tf' % (firefly_id, accuracy_metric))
plot_performance(df_firefly_tensorflow_performance_raw, platform_id=firefly_id, groupby_level='convolution_method')
plot_performance_violin(df_firefly_tensorflow_performance_raw, platform_id=firefly_id, groupby_level='convolution_method')
plot_performance(df_firefly_tensorflow_performance_raw, platform_id=firefly_id, groupby_level='library')
plot_performance_violin(df_firefly_tensorflow_performance_raw, platform_id=firefly_id, groupby_level='library')
df_firefly_armcl_performance_raw = get_experimental_results(repo_uoa=firefly_armcl_performance_repo_uoa,
tags='explore-mobilenets-performance', accuracy=False)
# Take the minimum execution time out of several repetitions.
df_firefly_armcl_performance = \
df_firefly_armcl_performance_raw .groupby(level=df_firefly_armcl_performance_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_firefly_armcl_performance)
batch_count | success? | time_avg_ms | time_total_ms | |||||||
---|---|---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||||
firefly | armcl-17.12 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 1 | yes | 22.246 | 22.246 |
gemm | 1 | yes | 23.447 | 23.447 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 1 | yes | 27.279 | 27.279 | ||
gemm | 1 | yes | 28.194 | 28.194 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 1 | yes | 31.506 | 31.506 | ||
gemm | 1 | yes | 32.579 | 32.579 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 1 | yes | 39.173 | 39.173 | ||
gemm | 1 | yes | 40.517 | 40.517 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 1 | yes | 34.582 | 34.582 | ||
gemm | 1 | yes | 36.129 | 36.129 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 1 | yes | 48.316 | 48.316 | ||
gemm | 1 | yes | 49.344 | 49.344 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 1 | yes | 62.199 | 62.199 | ||
gemm | 1 | yes | 63.763 | 63.763 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 1 | yes | 82.348 | 82.348 | ||
gemm | 1 | yes | 83.736 | 83.736 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 1 | yes | 59.090 | 59.090 | ||
gemm | 1 | yes | 63.768 | 63.768 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 1 | yes | 82.623 | 82.623 | ||
gemm | 1 | yes | 87.209 | 87.209 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 1 | yes | 103.751 | 103.751 | ||
gemm | 1 | yes | 108.426 | 108.426 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 1 | yes | 135.358 | 135.358 | ||
gemm | 1 | yes | 140.016 | 140.016 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 1 | yes | 95.590 | 95.590 | ||
gemm | 1 | yes | 103.318 | 103.318 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 1 | yes | 131.516 | 131.516 | ||
gemm | 1 | yes | 139.289 | 139.289 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 1 | yes | 162.850 | 162.850 | ||
gemm | 1 | yes | 172.009 | 172.009 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 1 | yes | 210.589 | 210.589 | ||
gemm | 1 | yes | 218.269 | 218.269 | ||||||
armcl-18.01 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 1 | yes | 10.883 | 10.883 | |
gemm | 1 | yes | 17.157 | 17.157 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 1 | yes | 15.352 | 15.352 | ||
gemm | 1 | yes | 20.162 | 20.162 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 1 | yes | 18.298 | 18.298 | ||
gemm | 1 | yes | 22.599 | 22.599 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 1 | yes | 22.870 | 22.870 | ||
gemm | 1 | yes | 26.141 | 26.141 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 1 | yes | 23.622 | 23.622 | ||
gemm | 1 | yes | 25.954 | 25.954 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 1 | yes | 40.517 | 40.517 | ||
gemm | 1 | yes | 34.828 | 34.828 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 1 | yes | 50.848 | 50.848 | ||
gemm | 1 | yes | 44.506 | 44.506 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 1 | yes | 65.739 | 65.739 | ||
gemm | 1 | yes | 55.469 | 55.469 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 1 | yes | 45.102 | 45.102 | ||
gemm | 1 | yes | 48.755 | 48.755 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 1 | yes | 82.196 | 82.196 | ||
gemm | 1 | yes | 66.217 | 66.217 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 1 | yes | 103.078 | 103.078 | ||
gemm | 1 | yes | 80.218 | 80.218 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 1 | yes | 135.367 | 135.367 | ||
gemm | 1 | yes | 99.223 | 99.223 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 1 | yes | 74.354 | 74.354 | ||
gemm | 1 | yes | 80.451 | 80.451 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 1 | yes | 137.860 | 137.860 | ||
gemm | 1 | yes | 110.485 | 110.485 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 1 | yes | 175.016 | 175.016 | ||
gemm | 1 | yes | 133.170 | 133.170 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 1 | yes | 243.377 | 243.377 | ||
gemm | 1 | yes | 162.652 | 162.652 | ||||||
armcl-18.03 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 1 | yes | 10.226 | 10.226 | |
gemm | 1 | yes | 16.144 | 16.144 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 1 | yes | 15.154 | 15.154 | ||
gemm | 1 | yes | 18.131 | 18.131 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 1 | yes | 18.093 | 18.093 | ||
gemm | 1 | yes | 19.487 | 19.487 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 1 | yes | 22.735 | 22.735 | ||
gemm | 1 | yes | 21.317 | 21.317 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 1 | yes | 23.360 | 23.360 | ||
gemm | 1 | yes | 23.784 | 23.784 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 1 | yes | 40.456 | 40.456 | ||
gemm | 1 | yes | 30.185 | 30.185 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 1 | yes | 51.069 | 51.069 | ||
gemm | 1 | yes | 35.830 | 35.830 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 1 | yes | 65.842 | 65.842 | ||
gemm | 1 | yes | 43.699 | 43.699 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 1 | yes | 44.798 | 44.798 | ||
gemm | 1 | yes | 46.029 | 46.029 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 1 | yes | 82.306 | 82.306 | ||
gemm | 1 | yes | 59.286 | 59.286 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 1 | yes | 103.804 | 103.804 | ||
gemm | 1 | yes | 67.599 | 67.599 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 1 | yes | 139.403 | 139.403 | ||
gemm | 1 | yes | 82.881 | 82.881 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 1 | yes | 73.852 | 73.852 | ||
gemm | 1 | yes | 77.928 | 77.928 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 1 | yes | 138.565 | 138.565 | ||
gemm | 1 | yes | 100.118 | 100.118 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 1 | yes | 175.420 | 175.420 | ||
gemm | 1 | yes | 115.956 | 115.956 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 1 | yes | 243.607 | 243.607 | ||
gemm | 1 | yes | 140.257 | 140.257 |
plot_performance(df_firefly_armcl_performance_raw, platform_id=firefly_id, groupby_level='convolution_method')
plot_performance_violin(df_firefly_armcl_performance_raw, platform_id=firefly_id, groupby_level='convolution_method')
plot_performance(df_firefly_armcl_performance_raw, platform_id=firefly_id, groupby_level='library')
plot_performance_violin(df_firefly_armcl_performance_raw, platform_id=firefly_id, groupby_level='library')
df_firefly_armcl_accuracy_500_raw = get_experimental_results(repo_uoa=firefly_armcl_accuracy_500_repo_uoa,
tags='explore-mobilenets-accuracy', accuracy=True)
# Extract frame predictions.
# df_firefly_armcl_predictions_500 = df_firefly_armcl_accuracy_500_raw[['frame_predictions']]
# Reduce the repetition_id index dimension (only 1 repetition anyway).
df_firefly_armcl_accuracy_500 = \
df_firefly_armcl_accuracy_500_raw[['accuracy_top1', 'accuracy_top5']] \
.groupby(level=df_firefly_armcl_accuracy_500_raw.index.names[:-1]).min()
# Display all rows and columns.
display_in_full(df_firefly_armcl_accuracy_500)
accuracy_top1 | accuracy_top5 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
firefly | armcl-17.12 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.424 | 0.666 |
gemm | 0.424 | 0.666 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.020 | 0.066 | ||
gemm | 0.016 | 0.062 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.226 | 0.496 | ||
gemm | 0.236 | 0.502 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.394 | 0.672 | ||
gemm | 0.418 | 0.680 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.566 | 0.800 | ||
gemm | 0.566 | 0.800 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.050 | 0.178 | ||
gemm | 0.066 | 0.172 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.328 | 0.582 | ||
gemm | 0.344 | 0.602 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.550 | 0.760 | ||
gemm | 0.536 | 0.754 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.608 | 0.830 | ||
gemm | 0.608 | 0.830 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.080 | 0.254 | ||
gemm | 0.110 | 0.268 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.394 | 0.670 | ||
gemm | 0.410 | 0.664 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.576 | 0.836 | ||
gemm | 0.604 | 0.834 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.640 | 0.864 | ||
gemm | 0.640 | 0.864 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.072 | 0.172 | ||
gemm | 0.062 | 0.190 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.510 | 0.760 | ||
gemm | 0.494 | 0.756 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.634 | 0.846 | ||
gemm | 0.632 | 0.854 | ||||||
armcl-18.01 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.424 | 0.666 | |
gemm | 0.424 | 0.666 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.468 | 0.702 | ||
gemm | 0.468 | 0.702 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.498 | 0.768 | ||
gemm | 0.498 | 0.768 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.504 | 0.752 | ||
gemm | 0.504 | 0.752 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.566 | 0.800 | ||
gemm | 0.566 | 0.800 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.606 | 0.838 | ||
gemm | 0.606 | 0.838 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.624 | 0.822 | ||
gemm | 0.624 | 0.822 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.654 | 0.858 | ||
gemm | 0.654 | 0.858 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.608 | 0.830 | ||
gemm | 0.608 | 0.830 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.654 | 0.868 | ||
gemm | 0.654 | 0.868 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.682 | 0.888 | ||
gemm | 0.682 | 0.888 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.676 | 0.882 | ||
gemm | 0.676 | 0.882 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.640 | 0.864 | ||
gemm | 0.640 | 0.864 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.656 | 0.878 | ||
gemm | 0.656 | 0.878 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.682 | 0.878 | ||
gemm | 0.682 | 0.878 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.722 | 0.896 | ||
gemm | 0.722 | 0.896 | ||||||
armcl-18.03 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.424 | 0.666 | |
gemm | 0.424 | 0.666 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.468 | 0.702 | ||
gemm | 0.468 | 0.702 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.498 | 0.768 | ||
gemm | 0.498 | 0.768 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.504 | 0.752 | ||
gemm | 0.504 | 0.752 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.566 | 0.800 | ||
gemm | 0.566 | 0.800 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.606 | 0.838 | ||
gemm | 0.606 | 0.838 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.624 | 0.822 | ||
gemm | 0.624 | 0.822 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.654 | 0.858 | ||
gemm | 0.654 | 0.858 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.608 | 0.830 | ||
gemm | 0.608 | 0.830 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.654 | 0.868 | ||
gemm | 0.654 | 0.868 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.682 | 0.888 | ||
gemm | 0.682 | 0.888 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.676 | 0.882 | ||
gemm | 0.676 | 0.882 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.640 | 0.864 | ||
gemm | 0.640 | 0.864 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.656 | 0.878 | ||
gemm | 0.656 | 0.878 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.682 | 0.878 | ||
gemm | 0.682 | 0.878 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.722 | 0.896 | ||
gemm | 0.722 | 0.896 |
# Identical accuracy for "18.03" and "18.01".
(df_firefly_armcl_accuracy_500.loc[firefly_id,'armcl-18.03'] - df_firefly_armcl_accuracy_500.loc[firefly_id,'armcl-18.01'] == 0).all()
accuracy_top1 True accuracy_top5 True dtype: bool
# Non-identical accuracy for "18.03" and "17.12".
df_firefly_armcl_accuracy_500.loc[firefly_id,'armcl-18.03'] - df_firefly_armcl_accuracy_500.loc[firefly_id,'armcl-17.12']
accuracy_top1 | accuracy_top5 | |||||
---|---|---|---|---|---|---|
model | multiplier | resolution | batch_size | convolution_method | ||
v1-0.25-128 | 0.25 | 128 | 1 | direct | 0.000 | 0.000 |
gemm | 0.000 | 0.000 | ||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 0.448 | 0.636 |
gemm | 0.452 | 0.640 | ||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.272 | 0.272 |
gemm | 0.262 | 0.266 | ||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.110 | 0.080 |
gemm | 0.086 | 0.072 | ||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 0.000 | 0.000 |
gemm | 0.000 | 0.000 | ||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 0.556 | 0.660 |
gemm | 0.540 | 0.666 | ||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 0.296 | 0.240 |
gemm | 0.280 | 0.220 | ||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 0.104 | 0.098 |
gemm | 0.118 | 0.104 | ||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.000 | 0.000 |
gemm | 0.000 | 0.000 | ||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 0.574 | 0.614 |
gemm | 0.544 | 0.600 | ||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 0.288 | 0.218 |
gemm | 0.272 | 0.224 | ||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.100 | 0.046 |
gemm | 0.072 | 0.048 | ||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 0.000 | 0.000 |
gemm | 0.000 | 0.000 | ||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.584 | 0.706 |
gemm | 0.594 | 0.688 | ||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.172 | 0.118 |
gemm | 0.188 | 0.122 | ||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 0.088 | 0.050 |
gemm | 0.090 | 0.042 |
# Non-identical accuracy for "18.03" on HiKey and "18.03" on Firefly.
df_armcl_accuracy_500.loc[hikey_id,'armcl-18.03'] - df_firefly_armcl_accuracy_500.loc[firefly_id,'armcl-18.03']
accuracy_top1 | accuracy_top5 | |||||
---|---|---|---|---|---|---|
model | multiplier | resolution | batch_size | convolution_method | ||
v1-0.25-128 | 0.25 | 128 | 1 | direct | -0.014 | 0.006 |
gemm | -0.014 | 0.006 | ||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | -0.018 | 0.002 |
gemm | -0.018 | 0.002 | ||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 0.004 | -0.008 |
gemm | 0.004 | -0.008 | ||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 0.004 | 0.004 |
gemm | 0.004 | 0.004 | ||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | -0.030 | -0.002 |
gemm | -0.030 | -0.002 | ||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | -0.020 | -0.016 |
gemm | -0.020 | -0.016 | ||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | -0.002 | 0.002 |
gemm | -0.002 | 0.002 | ||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | -0.004 | -0.006 |
gemm | -0.004 | -0.006 | ||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 0.000 | -0.008 |
gemm | 0.000 | -0.008 | ||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | -0.012 | -0.018 |
gemm | -0.012 | -0.018 | ||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | -0.006 | 0.002 |
gemm | -0.006 | 0.002 | ||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 0.002 | -0.004 |
gemm | 0.002 | -0.004 | ||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | -0.018 | -0.004 |
gemm | -0.018 | -0.004 | ||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 0.010 | -0.010 |
gemm | 0.010 | -0.010 | ||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 0.022 | 0.004 |
gemm | 0.022 | 0.004 | ||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | -0.006 | 0.010 |
gemm | -0.006 | 0.010 |
df_firefly_armcl_performance_accuracy_500 = merge_performance_accuracy(df_firefly_armcl_performance, df_firefly_armcl_accuracy_500)
display_in_full(df_firefly_armcl_performance_accuracy_500)
time_avg_ms | accuracy_top1 | |||||||
---|---|---|---|---|---|---|---|---|
platform | library | model | multiplier | resolution | batch_size | convolution_method | ||
firefly | armcl-17.12 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 22.246 | 0.424 |
gemm | 23.447 | 0.424 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 27.279 | 0.020 | ||
gemm | 28.194 | 0.016 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 31.506 | 0.226 | ||
gemm | 32.579 | 0.236 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 39.173 | 0.394 | ||
gemm | 40.517 | 0.418 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 34.582 | 0.566 | ||
gemm | 36.129 | 0.566 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 48.316 | 0.050 | ||
gemm | 49.344 | 0.066 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 62.199 | 0.328 | ||
gemm | 63.763 | 0.344 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 82.348 | 0.550 | ||
gemm | 83.736 | 0.536 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 59.090 | 0.608 | ||
gemm | 63.768 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 82.623 | 0.080 | ||
gemm | 87.209 | 0.110 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 103.751 | 0.394 | ||
gemm | 108.426 | 0.410 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 135.358 | 0.576 | ||
gemm | 140.016 | 0.604 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 95.590 | 0.640 | ||
gemm | 103.318 | 0.640 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 131.516 | 0.072 | ||
gemm | 139.289 | 0.062 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 162.850 | 0.510 | ||
gemm | 172.009 | 0.494 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 210.589 | 0.634 | ||
gemm | 218.269 | 0.632 | ||||||
armcl-18.01 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 10.883 | 0.424 | |
gemm | 17.157 | 0.424 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 15.352 | 0.468 | ||
gemm | 20.162 | 0.468 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 18.298 | 0.498 | ||
gemm | 22.599 | 0.498 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 22.870 | 0.504 | ||
gemm | 26.141 | 0.504 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 23.622 | 0.566 | ||
gemm | 25.954 | 0.566 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 40.517 | 0.606 | ||
gemm | 34.828 | 0.606 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 50.848 | 0.624 | ||
gemm | 44.506 | 0.624 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 65.739 | 0.654 | ||
gemm | 55.469 | 0.654 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 45.102 | 0.608 | ||
gemm | 48.755 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 82.196 | 0.654 | ||
gemm | 66.217 | 0.654 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 103.078 | 0.682 | ||
gemm | 80.218 | 0.682 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 135.367 | 0.676 | ||
gemm | 99.223 | 0.676 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 74.354 | 0.640 | ||
gemm | 80.451 | 0.640 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 137.860 | 0.656 | ||
gemm | 110.485 | 0.656 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 175.016 | 0.682 | ||
gemm | 133.170 | 0.682 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 243.377 | 0.722 | ||
gemm | 162.652 | 0.722 | ||||||
armcl-18.03 | v1-0.25-128 | 0.25 | 128 | 1 | direct | 10.226 | 0.424 | |
gemm | 16.144 | 0.424 | ||||||
v1-0.25-160 | 0.25 | 160 | 1 | direct | 15.154 | 0.468 | ||
gemm | 18.131 | 0.468 | ||||||
v1-0.25-192 | 0.25 | 192 | 1 | direct | 18.093 | 0.498 | ||
gemm | 19.487 | 0.498 | ||||||
v1-0.25-224 | 0.25 | 224 | 1 | direct | 22.735 | 0.504 | ||
gemm | 21.317 | 0.504 | ||||||
v1-0.50-128 | 0.50 | 128 | 1 | direct | 23.360 | 0.566 | ||
gemm | 23.784 | 0.566 | ||||||
v1-0.50-160 | 0.50 | 160 | 1 | direct | 40.456 | 0.606 | ||
gemm | 30.185 | 0.606 | ||||||
v1-0.50-192 | 0.50 | 192 | 1 | direct | 51.069 | 0.624 | ||
gemm | 35.830 | 0.624 | ||||||
v1-0.50-224 | 0.50 | 224 | 1 | direct | 65.842 | 0.654 | ||
gemm | 43.699 | 0.654 | ||||||
v1-0.75-128 | 0.75 | 128 | 1 | direct | 44.798 | 0.608 | ||
gemm | 46.029 | 0.608 | ||||||
v1-0.75-160 | 0.75 | 160 | 1 | direct | 82.306 | 0.654 | ||
gemm | 59.286 | 0.654 | ||||||
v1-0.75-192 | 0.75 | 192 | 1 | direct | 103.804 | 0.682 | ||
gemm | 67.599 | 0.682 | ||||||
v1-0.75-224 | 0.75 | 224 | 1 | direct | 139.403 | 0.676 | ||
gemm | 82.881 | 0.676 | ||||||
v1-1.00-128 | 1.00 | 128 | 1 | direct | 73.852 | 0.640 | ||
gemm | 77.928 | 0.640 | ||||||
v1-1.00-160 | 1.00 | 160 | 1 | direct | 138.565 | 0.656 | ||
gemm | 100.118 | 0.656 | ||||||
v1-1.00-192 | 1.00 | 192 | 1 | direct | 175.420 | 0.682 | ||
gemm | 115.956 | 0.682 | ||||||
v1-1.00-224 | 1.00 | 224 | 1 | direct | 243.607 | 0.722 | ||
gemm | 140.257 | 0.722 |
plot(df_firefly_armcl_performance_accuracy_500, accuracy_metric=accuracy_metric, platform_id=firefly_id,
xmin=0, xmax=150.1, xstep=10, save_fig_name='%s-%s-500-18_03__18_01__17_12' % (firefly_id, accuracy_metric))
df_firefly_tensorflow_performance = df_firefly_tensorflow_performance_min.loc[firefly_id,['tensorflow-1.7'],:]
df_firefly_accuracy_500 = pd.concat([df_firefly_armcl_accuracy_500, df_firefly_tensorflow_accuracy_500])
df_firefly_performance = pd.concat([df_firefly_armcl_performance, df_firefly_tensorflow_performance])
df_firefly_performance_accuracy_500 = merge_performance_accuracy(df_firefly_performance, df_firefly_accuracy_500)
plot(df_firefly_performance_accuracy_500, accuracy_metric=accuracy_metric, platform_id=firefly_id, save_fig=True,
xmin=0, xmax=150.1, xstep=10, save_fig_name='%s-%s-500-18_03__18_01__17_12__tf' % (firefly_id, accuracy_metric))
<matplotlib.figure.Figure at 0x7f02e2dbbcf8>