#!/usr/bin/env python # coding: utf-8 # ## Sacred # - 모든 실험은 신성하다 # - 모든 실험은 훌륭함 # - 만약 실험이 낭비되면 신이 매우 화를 낼 것 # # - Sacred는 실험을 구성, 기록 복제하는 도구 # - 최소한의 오버 헤드를 도입해 모듈화, 실험 구성을 도움 # - 기능 # - 실험의 모든 파라미터 추적 # - 여러 설정에서 실험을 쉽게 할 수 있음 # - 파일이나 데이터베이스에 실행 설정을 저장 # - 결과 재현 # # - 메인 메커니즘 # - Config Scope는 `@ex.config` 데코레이터 함수를 사용해 쉽게 설정 가능 # - dependency injection을 통해 캡쳐된 기능을 사용할 수 있음. 시스템이 파라미터를 전달해 쉽게 config value를 사용할 수 있음 # - Command line interface가 쉽게 파라미터를 바꿀 수 있게 함 # - 실험 및 구성에 대한 모든 정보를 기록해 실험을 추적 # - Automatic seed는 무작위성을 제어해 재현성을 유지할 때 도움이 됨 # # In[28]: from numpy.random import permutation from sklearn import svm, datasets from sacred import Experiment ex = Experiment('iris_rbf_svm', interactive=True) # jupyter notebook일 경우 interactive=True, python 스크립트라면 없어도 됨 @ex.config def cfg(): C = 1.0 gamma = 0.7 # ex.automain은 python 스크립트일 때 사용 @ex.main def run(C, gamma): iris = datasets.load_iris() per = permutation(iris.target.size) iris.data = iris.data[per] iris.target = iris.target[per] clf = svm.SVC(C, 'rbf', gamma=gamma) clf.fit(iris.data[:90], iris.target[:90]) return clf.score(iris.data[90:], iris.target[90:]) # In[12]: ex.run() # In[13]: from sacred import Experiment ex = Experiment('my_experiment', interactive=True) @ex.config def my_config(): foo = 42 bar = 'baz' @ex.capture def some_function(a, foo, bar=10): print(a, foo, bar) @ex.main def my_main(): some_function(1, 2, 3) # 1 2 3 some_function(1) # 1 42 'baz' some_function(1, bar=12) # 1 42 12 some_function() # TypeError: missing value for 'a' # In[14]: ex.run() # ## Observing an Experiment # - The main one is the Mongo Observer which stores all information in a MongoDB. # - The File Storage Observer stores the run information as files in a given directory and will therefore only work locally. # - The TinyDB Observer provides another local way of observing experiments by using tinydb to store run information in a JSON file. # - The SQL Observer connects to any SQL database and will store the relevant information there. # In[17]: from sacred.observers import MongoObserver ex.observers.append(MongoObserver.create()) # In[18]: from pymongo import MongoClient # In[19]: import urllib.parse # - 몽고DB 설치 # - [홈페이지](https://docs.mongodb.com/manual/tutorial/install-mongodb-on-os-x/) # In[20]: username = urllib.parse.quote_plus('user') # In[21]: password = urllib.parse.quote_plus('password') # In[23]: from sacred.observers import MongoObserver ex.observers.append(MongoObserver.create( url='mongodb://user:password@example.com/the_database?authMechanism=SCRAM-SHA-1', db_name='MY_DB')) # ### Filt Storage Observer # In[27]: from sacred.observers import FileStorageObserver # In[43]: ex = Experiment('iris_rbf_svm', interactive=True) # jupyter notebook일 경우 interactive=True, python 스크립트라면 없어도 됨 @ex.config def cfg(): C = 1.0 gamma = 0.7 # ex.automain은 python 스크립트일 때 사용 @ex.main def run(C, gamma): iris = datasets.load_iris() per = permutation(iris.target.size) iris.data = iris.data[per] iris.target = iris.target[per] clf = svm.SVC(C, 'rbf', gamma=gamma) clf.fit(iris.data[:90], iris.target[:90]) return clf.score(iris.data[90:], iris.target[90:]) ex.observers.append(FileStorageObserver.create('my_runs')) # In[44]: ex.run() # ### Slack Observer # - [링크](https://sacred.readthedocs.io/en/latest/observers.html#slack-observer) # ![image](https://sacred.readthedocs.io/en/latest/_images/slack_observer.png) # ### 실험과 logging을 합치고 싶은 경우 # - `_log` 사용 # In[46]: @ex.capture def some_function(_log): _log.warning('My warning message!') # In[48]: #!/usr/bin/env python # coding=utf-8 """ An example showcasing the logging system of Sacred.""" from __future__ import division, print_function, unicode_literals import logging from sacred import Experiment ex = Experiment('log_example', interactive=True) # set up a custom logger logger = logging.getLogger('mylogger') logger.handlers = [] ch = logging.StreamHandler() formatter = logging.Formatter('[%(levelname).1s] %(name)s >> "%(message)s"') ch.setFormatter(formatter) logger.addHandler(ch) logger.setLevel('INFO') # attach it to the experiment ex.logger = logger @ex.config def cfg(): number = 2 got_gizmo = False @ex.capture def transmogrify(got_gizmo, number, _log): if got_gizmo: _log.debug("Got gizmo. Performing transmogrification...") return number * 42 else: _log.warning("No gizmo. Can't transmogrify!") return 0 @ex.main def main(number, _log): _log.info('Attempting to transmogrify %d...', number) result = transmogrify() _log.info('Transmogrification complete: %d', result) return result # In[49]: ex.run() # In[50]: from __future__ import division, print_function, unicode_literals from sacred import Experiment ex = Experiment('hello_config', interactive=True) @ex.named_config def rude(): """A rude named config""" recipient = "bastard" message = "Fuck off you {}!".format(recipient) @ex.config def cfg(): recipient = "world" message = "Hello {}!".format(recipient) @ex.main def main(message): print(__name__) print(message) # In[51]: ex.run() # ### Basic Example # In[ ]: from __future__ import division, print_function, unicode_literals from sacred import Experiment, Ingredient # ============== Ingredient 0: settings ================= s = Ingredient("settings") @s.config def cfg1(): verbose = True # ============== Ingredient 1: dataset.paths ================= data_paths = Ingredient("dataset.paths", ingredients=[s]) @data_paths.config def cfg2(settings): v = not settings['verbose'] base = '/home/sacred/' # ============== Ingredient 2: dataset ======================= data = Ingredient("dataset", ingredients=[data_paths, s]) @data.config def cfg3(paths): basepath = paths['base'] + 'datasets/' filename = "foo.hdf5" @data.capture def foo(basepath, filename, paths, settings): print(paths) print(settings) return basepath + filename # ============== Experiment ============================== ex = Experiment('modular_example', ingredients=[data, data_paths]) @ex.config def cfg(dataset): a = 10 b = 17 c = a + b out_base = dataset['paths']['base'] + 'outputs/' out_filename = dataset['filename'].replace('.hdf5', '.out') @ex.automain def main(a, b, c, out_base, out_filename, dataset): print('a =', a) print('b =', b) print('c =', c) print('out_base =', out_base, out_filename) # print("dataset", dataset) # print("dataset.paths", dataset['paths']) print("foo()", foo())