Hodhocení prací probíhá pomocí následujícího kódu. Práce jsou automaticky staženy z adres zadaných v "solution_list.ipynb". Pak jsou spuštěny a všechny otestovány na shodné náhodné podskupině testovacího datasetu. Na výsledky je pak možno navázat ve skrpitu "ZDO2014tabulka.ipynb"
V případě opakovaného spouštění testování je z důvodu opakovaných importů modulů se stejnými jmnény nezbytné restartovat kernel. To lze udělat pomocí Kernel -> Restart
Výchozí adresář s testovacími daty je na staven na následující cestu
/home/mjirik/data/zdo2014/znacky-testing/
import signal
from contextlib import contextmanager
import skimage
import skimage.io
import numpy as np
import urllib
import glob
import zipfile
import shutil
import sys
import os
import os.path
import traceback
import datetime
import pickle
import random
#print dir(urllib)
#import urllib.request
# ziskani seznamu
import solutions_list
%run solutions_list
# cesta k trenovacim datum
test_data_path = '/home/mjirik/data/zdo2014/zdo2014-znacky-testing/'
# Nastaveni logovaciho souboru
import logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.basicConfig(filename='znacky.log', level=logging.DEBUG, format='%(asctime)s %(levelname)s:%(message)s')
def readImageDir(path):
"""
Načítání dat z adresářové struktury. Funkce vrací seznam souborů a seznam labelů.
Ty jsou získávány z názvů jednotlivých složek.
"""
dirs = glob.glob(os.path.join(os.path.normpath(path) ,'*'))
labels = []
files = []
for onedir in dirs:
#print onedir
base, lab = os.path.split(onedir)
if os.path.isdir(onedir):
filesInDir = glob.glob(os.path.join(onedir, '*'))
for onefile in filesInDir:
labels.append(lab)
files.append(onefile)
return files, labels
class TimeoutException(Exception): pass
@contextmanager
def time_limit(seconds):
def signal_handler(signum, frame):
raise TimeoutException, "Timed out!"
signal.signal(signal.SIGALRM, signal_handler)
signal.alarm(seconds)
try:
yield
finally:
signal.alarm(0)
def kontrola(ukazatel, obrazky, reseni):
"""
Kontrola jednoho studentského řešení
"""
studentske_reseni = ukazatel() # tim je zavolán váš konstruktor __init__
vysledky = []
for i in range(0, len(obrazky)):
im = skimage.io.imread(obrazky[i])
# Zde je volána vaše funkce rozpoznejZnacku
try:
with time_limit(1):
result = studentske_reseni.rozpoznejZnacku(im)
except TimeoutException, msg:
print "Timed out!"
result = None
vysledky.append(result)
hodnoceni = np.array(reseni) == np.array(vysledky)
skore = np.sum(hodnoceni.astype(np.int)) / np.float(len(reseni))
return skore, vysledky
def my_import(name):
__import__(name)
return sys.modules[name]
def downloadAll(soldir):
try:
shutil.rmtree(soldir)
except:
print "Problem with rmtree"
traceback.print_exc()
try:
os.mkdir(soldir)
open(os.path.join(soldir, "__init__.py"), 'a').close()
except:
print "Problem with mkdir"
traceback.print_exc()
# download all solutions
for one in solutions_list:
zip_path = os.path.join(soldir, one[1] + '.zip')
urllib.urlretrieve(one[0], zip_path)
zf = zipfile.ZipFile(zip_path, "r")
zf.extractall(soldir)
teamsoldir = zip_path = os.path.join(soldir, one[1])
# create __init__.py
open(os.path.join(teamsoldir, "__init__.py"), 'a').close()
# remove '-' from package path
teamsoldirnew = teamsoldir.replace('-', '') + one[4]
os.rename(teamsoldir, teamsoldirnew)
def kontrolaVse():
"""
Kontrola všech studentských řešení
"""
soldir = 'ZDO2014students'
downloadAll(soldir)
scores = []
teams = []
stamps = []
classifs = []
filesall, labelsall = readImageDir(test_data_path)
#sklearn.cross_validation.train_test_split(
init_seed = int(random.random()*5.0)
obrazky = filesall[init_seed::6]
reseni = labelsall[init_seed::6]
classifs.append(reseni)
print "Total image number: ", len(obrazky)
logger.debug("Total image number: " + str(len(obrazky)))
# evaluate solutions
for one in solutions_list:
logger.debug("Team: " + one[3])
scoreone = 0
vysledky = []
try:
imp1 = my_import('ZDO2014students.' + one[1].replace('-', '') + one[4])
imp2 = my_import('ZDO2014students.' + one[1].replace('-', '') + one[4]
+ '.' + one[2])
#print imp2
pointer = imp2.Znacky
#import ZDO2014sample_solution
#pointer = ZDO2014sample_solution.Znacky
scoreone, vysledky = kontrola(pointer, obrazky, reseni)
except:
print "Problem with: " + one[3]
exc = traceback.format_exc()
print exc
logger.debug("Problem with: " + one[3])
logger.debug(exc)
print "------------------------"
vysledky = reseni
teams.append(one[3])
scores.append(scoreone)
classifs.append(vysledky)
#this is whole timestamp. I want only date
dat = datetime.datetime.now()
#nw = datetime.datetime.now()
#dat = datetime.date(nw.year, nw.month, nw.day)
stamps.append(dat)
#print teams
#print scores
return teams, scores, stamps, classifs
def saveAndMergeEvaluation(teams, scores, stamps,
filename="ZDO2014evaluation.csv"):
# write to csv
import pandas as pd
try:
orig_data = pd.read_csv(filename)
#print 'orig data'
#print orig_data
except:
print "Cannot read stored data"
dates = []
times = []
for stamp in stamps:
dat = stamp.date()
tm = stamp.time()
dates.append(dat)
times.append(tm)
npdata = np.array([teams, scores, dates, times]).T
df = pd.DataFrame(npdata, columns=['team', 'score', 'date', 'time'])
try:
# there could be wrong format
df = orig_data.append(df, ignore_index=True)
except:
print "wrong format of input csv, rewriting"
df.to_csv(filename, index=False)
return df
def saveClassifs(teams, classifs, filename='ZDO2014classifs.csv'):
"""
Zápis veškerých klasifikací do csv souboru
"""
import pandas as pd
cl = teams[:]
cl.insert(0, 'reference')
# vytvoř vhodný formát pro uložení
aa = {cl[i]:classifs[i] for i in range(0, len(cl))}
df = pd.DataFrame(aa)
df.to_csv(filename, index=False)
if __name__ == "__main__":
teams, scores, stamps, classifs = kontrolaVse()
tdf = saveAndMergeEvaluation(teams, scores, stamps)
saveClassifs(teams, classifs)
#printEvaluation(teams, scores, stamps)
DEBUG:root:Total image number: 103 DEBUG:root:Team: sample M. Jirik DEBUG:ZDO2014students.ZDO2014sample_solutionmaster.ZDO2014sample_solution:Ukazka loggovani, nacteni klasifikatoru ok DEBUG:root:Team: cervenym DEBUG:root:Team: nedvedj /usr/lib/python2.7/dist-packages/skimage/exposure/exposure.py:51: UserWarning: This might be a color image. The histogram will be computed on the flattened image. You can instead apply this function to each color channel. warnings.warn("This might be a color image. The histogram will be " DEBUG:root:Team: RakHejdovaJuna DEBUG:ZDO2014students.Semestralkamaster1.ZDO2014juna_rak_hejdova_hog:Ukazka loggovani, nacteni klasifikatoru ok
Total image number: 103
print tdf.tail(50)
team score date time 122 RakHejdovaJuna2 0.7692308 2014-06-02 11:02:35.514358 123 nedvedj2 0.009615385 2014-06-02 11:02:40.192876 124 sample M. Jirik 0.2718447 2014-06-02 11:05:16.284897 125 cervenym 0.6019417 2014-06-02 11:05:25.691898 126 nedvedj 0 2014-06-02 11:05:25.967018 127 RakHejdovaJuna 0 2014-06-02 11:05:26.084831 128 RakHejdovaJuna2 0.7378641 2014-06-02 11:05:31.869983 129 nedvedj2 0.01941748 2014-06-02 11:05:36.611275 130 sample M. Jirik 0 2014-06-10 10:24:30.635871 131 cervenym 0.6172616 2014-06-10 10:26:27.343067 132 nedvedj 0.3239946 2014-06-10 10:27:20.666130 133 RakHejdovaJuna 0.9408043 2014-06-10 10:28:26.399551 134 RakHejdovaJuna2 0 2014-06-10 10:28:26.485093 135 nedvedj2 0 2014-06-10 10:28:26.662503 136 sample M. Jirik 0 2014-06-10 10:30:10.339569 137 cervenym 0.6184448 2014-06-10 10:32:06.897540 138 nedvedj 0.3449367 2014-06-10 10:32:59.784266 139 RakHejdovaJuna 0.9398734 2014-06-10 10:34:05.136954 140 RakHejdovaJuna2 0 2014-06-10 10:34:05.223635 141 nedvedj2 0 2014-06-10 10:34:05.401491 142 sample M. Jirik 0 2014-06-10 10:40:05.238037 143 cervenym 0.6292948 2014-06-10 10:42:01.262574 144 nedvedj 0.3227848 2014-06-10 10:42:54.287133 145 RakHejdovaJuna 0.9407776 2014-06-10 10:43:58.248363 146 RakHejdovaJuna2 0 2014-06-10 10:43:58.333504 147 nedvedj2 0 2014-06-10 10:43:58.510197 148 sample M. Jirik 0.3004971 2014-06-10 10:53:00.830229 149 cervenym 0.6172616 2014-06-10 10:54:55.937289 150 nedvedj 0.3239946 2014-06-10 10:55:49.244489 151 RakHejdovaJuna 0.9408043 2014-06-10 10:56:52.098387 152 sample M. Jirik 0.2983725 2014-06-20 09:47:57.243949 153 cervenym 0.6184448 2014-06-20 09:49:51.402369 154 nedvedj 0 2014-06-20 09:49:51.403685 155 RakHejdovaJuna 0.9398734 2014-06-20 09:50:55.445074 156 sample M. Jirik 0.25 2014-06-23 09:26:35.657323 157 cervenym 0.6730769 2014-06-23 09:26:42.847864 158 nedvedj 0.6730769 2014-06-23 09:26:44.600640 159 RakHejdovaJuna 0.7596154 2014-06-23 09:26:49.038464 160 sample M. Jirik 0.223301 2014-06-23 09:50:27.159108 161 cervenym 0.592233 2014-06-23 09:50:33.911572 162 nedvedj 0.6990291 2014-06-23 09:50:35.571370 163 RakHejdovaJuna 0.7184466 2014-06-23 09:50:39.678260 164 sample M. Jirik 0.223301 2014-06-23 09:55:08.823339 165 cervenym 0.592233 2014-06-23 09:55:15.655003 166 nedvedj 0.6990291 2014-06-23 09:55:17.301895 167 RakHejdovaJuna 0.7184466 2014-06-23 09:55:21.398045 168 sample M. Jirik 0.1747573 2014-06-23 09:57:23.405620 169 cervenym 0.5436893 2014-06-23 09:57:30.615811 170 nedvedj 0.6893204 2014-06-23 09:57:32.268525 171 RakHejdovaJuna 0.8543689 2014-06-23 09:57:36.362442 [50 rows x 4 columns]