This macro provides a simple example for the training and testing of the TMVA multiclass classification
Author: Andreas Hoecker
This notebook tutorial was automatically generated with ROOTBOOK-izer from the macro found in the ROOT repository on Wednesday, April 17, 2024 at 11:05 AM.
%%cpp -d
#include <cstdlib>
#include <iostream>
#include <map>
#include <string>
#include "TFile.h"
#include "TTree.h"
#include "TString.h"
#include "TSystem.h"
#include "TROOT.h"
#include "TMVA/Tools.h"
#include "TMVA/Factory.h"
#include "TMVA/DataLoader.h"
#include "TMVA/TMVAMultiClassGui.h"
using namespace TMVA;
Arguments are defined.
TString myMethodList = "";
This loads the library
TMVA::Tools::Instance();
to get access to the GUI and all tmva macros
TString tmva_dir(TString(gRootDir) + "/tmva"); if(gSystem->Getenv("TMVASYS")) tmva_dir = TString(gSystem->Getenv("TMVASYS")); gROOT->SetMacroPath(tmva_dir + "/test/:" + gROOT->GetMacroPath() ); gROOT->ProcessLine(".L TMVAMultiClassGui.C");
Default MVA methods to be trained + tested
std::map<std::string,int> Use;
Use["MLP"] = 1;
Use["BDTG"] = 1;
#ifdef R__HAS_TMVAGPU
Use["DL_CPU"] = 1;
Use["DL_GPU"] = 1;
#else
Use["DL_CPU"] = 1;
Use["DL_GPU"] = 0;
#endif
Use["FDA_GA"] = 0;
Use["PDEFoam"] = 1;
std::cout << std::endl;
std::cout << "==> Start TMVAMulticlass" << std::endl;
if (myMethodList != "") {
for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) it->second = 0;
std::vector<TString> mlist = TMVA::gTools().SplitString( myMethodList, ',' );
for (UInt_t i=0; i<mlist.size(); i++) {
std::string regMethod(mlist[i]);
if (Use.find(regMethod) == Use.end()) {
std::cout << "Method \"" << regMethod << "\" not known in TMVA under this name. Choose among the following:" << std::endl;
for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) std::cout << it->first << " ";
std::cout << std::endl;
return;
}
Use[regMethod] = 1;
}
}
==> Start TMVAMulticlass
Create a new root output file.
TString outfileName = "TMVAMulticlass.root";
TFile* outputFile = TFile::Open( outfileName, "RECREATE" );
TMVA::Factory *factory = new TMVA::Factory( "TMVAMulticlass", outputFile,
"!V:!Silent:Color:!DrawProgressBar:Transformations=I;D;P;G,D:AnalysisType=multiclass" );
TMVA::DataLoader *dataloader=new TMVA::DataLoader("dataset");
dataloader->AddVariable( "var1", 'F' );
dataloader->AddVariable( "var2", "Variable 2", "", 'F' );
dataloader->AddVariable( "var3", "Variable 3", "units", 'F' );
dataloader->AddVariable( "var4", "Variable 4", "units", 'F' );
TFile *input(0);
TString fname = "./tmva_example_multiclass.root";
if (!gSystem->AccessPathName( fname )) {
input = TFile::Open( fname ); // check if file in local directory exists
}
else {
TFile::SetCacheFileDir(".");
input = TFile::Open("http://root.cern/files/tmva_multiclass_example.root", "CACHEREAD");
}
if (!input) {
std::cout << "ERROR: could not open data file" << std::endl;
exit(1);
}
std::cout << "--- TMVAMulticlass: Using input file: " << input->GetName() << std::endl;
TTree *signalTree = (TTree*)input->Get("TreeS");
TTree *background0 = (TTree*)input->Get("TreeB0");
TTree *background1 = (TTree*)input->Get("TreeB1");
TTree *background2 = (TTree*)input->Get("TreeB2");
gROOT->cd( outfileName+TString(":/") );
dataloader->AddTree (signalTree,"Signal");
dataloader->AddTree (background0,"bg0");
dataloader->AddTree (background1,"bg1");
dataloader->AddTree (background2,"bg2");
dataloader->PrepareTrainingAndTestTree( "", "SplitMode=Random:NormMode=NumEvents:!V" );
if (Use["BDTG"]) // gradient boosted decision trees
factory->BookMethod( dataloader, TMVA::Types::kBDT, "BDTG", "!H:!V:NTrees=1000:BoostType=Grad:Shrinkage=0.10:UseBaggedBoost:BaggedSampleFraction=0.50:nCuts=20:MaxDepth=2");
if (Use["MLP"]) // neural network
factory->BookMethod( dataloader, TMVA::Types::kMLP, "MLP", "!H:!V:NeuronType=tanh:NCycles=1000:HiddenLayers=N+5,5:TestRate=5:EstimatorType=MSE");
if (Use["FDA_GA"]) // functional discriminant with GA minimizer
factory->BookMethod( dataloader, TMVA::Types::kFDA, "FDA_GA", "H:!V:Formula=(0)+(1)*x0+(2)*x1+(3)*x2+(4)*x3:ParRanges=(-1,1);(-10,10);(-10,10);(-10,10);(-10,10):FitMethod=GA:PopSize=300:Cycles=3:Steps=20:Trim=True:SaveBestGen=1" );
if (Use["PDEFoam"]) // PDE-Foam approach
factory->BookMethod( dataloader, TMVA::Types::kPDEFoam, "PDEFoam", "!H:!V:TailCut=0.001:VolFrac=0.0666:nActiveCells=500:nSampl=2000:nBin=5:Nmin=100:Kernel=None:Compress=T" );
if (Use["DL_CPU"]) {
TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
"TestRepetitions=1,ConvergenceSteps=10,BatchSize=100,MaxEpochs=20");
TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
"WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
nnOptions.Append(":");
nnOptions.Append(layoutString);
nnOptions.Append(":");
nnOptions.Append(trainingStrategyString);
factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_CPU", nnOptions);
}
if (Use["DL_GPU"]) {
TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
"TestRepetitions=1,ConvergenceSteps=10,BatchSize=100,MaxEpochs=20");
TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
"WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
nnOptions.Append(":");
nnOptions.Append(layoutString);
nnOptions.Append(":");
nnOptions.Append(trainingStrategyString);
factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_GPU", nnOptions);
}
--- TMVAMulticlass: Using input file: ./files/tmva_multiclass_example.root DataSetInfo : [dataset] : Added class "Signal" : Add Tree TreeS of type Signal with 2000 events DataSetInfo : [dataset] : Added class "bg0" : Add Tree TreeB0 of type bg0 with 2000 events DataSetInfo : [dataset] : Added class "bg1" : Add Tree TreeB1 of type bg1 with 2000 events DataSetInfo : [dataset] : Added class "bg2" : Add Tree TreeB2 of type bg2 with 2000 events : Dataset[dataset] : Class index : 0 name : Signal : Dataset[dataset] : Class index : 1 name : bg0 : Dataset[dataset] : Class index : 2 name : bg1 : Dataset[dataset] : Class index : 3 name : bg2 Factory : Booking method: BDTG : : the option NegWeightTreatment=InverseBoostNegWeights does not exist for BoostType=Grad : --> change to new default NegWeightTreatment=Pray : Rebuilding Dataset dataset : Building event vectors for type 2 Signal : Dataset[dataset] : create input formulas for tree TreeS : Building event vectors for type 2 bg0 : Dataset[dataset] : create input formulas for tree TreeB0 : Building event vectors for type 2 bg1 : Dataset[dataset] : create input formulas for tree TreeB1 : Building event vectors for type 2 bg2 : Dataset[dataset] : create input formulas for tree TreeB2 DataSetFactory : [dataset] : Number of events in input trees : : : : : Number of training and testing events : --------------------------------------------------------------------------- : Signal -- training events : 1000 : Signal -- testing events : 1000 : Signal -- training and testing events: 2000 : bg0 -- training events : 1000 : bg0 -- testing events : 1000 : bg0 -- training and testing events: 2000 : bg1 -- training events : 1000 : bg1 -- testing events : 1000 : bg1 -- training and testing events: 2000 : bg2 -- training events : 1000 : bg2 -- testing events : 1000 : bg2 -- training and testing events: 2000 : DataSetInfo : Correlation matrix (Signal): : ---------------------------------------- : var1 var2 var3 var4 : var1: +1.000 +0.397 +0.623 +0.832 : var2: +0.397 +1.000 +0.716 +0.737 : var3: +0.623 +0.716 +1.000 +0.859 : var4: +0.832 +0.737 +0.859 +1.000 : ---------------------------------------- DataSetInfo : Correlation matrix (bg0): : ---------------------------------------- : var1 var2 var3 var4 : var1: +1.000 +0.365 +0.592 +0.811 : var2: +0.365 +1.000 +0.708 +0.740 : var3: +0.592 +0.708 +1.000 +0.859 : var4: +0.811 +0.740 +0.859 +1.000 : ---------------------------------------- DataSetInfo : Correlation matrix (bg1): : ---------------------------------------- : var1 var2 var3 var4 : var1: +1.000 +0.407 +0.610 +0.834 : var2: +0.407 +1.000 +0.710 +0.741 : var3: +0.610 +0.710 +1.000 +0.851 : var4: +0.834 +0.741 +0.851 +1.000 : ---------------------------------------- DataSetInfo : Correlation matrix (bg2): : ---------------------------------------- : var1 var2 var3 var4 : var1: +1.000 -0.647 -0.016 -0.013 : var2: -0.647 +1.000 +0.015 +0.002 : var3: -0.016 +0.015 +1.000 -0.024 : var4: -0.013 +0.002 -0.024 +1.000 : ---------------------------------------- DataSetFactory : [dataset] : : Factory : Booking method: MLP : MLP : Building Network. : Initializing weights Factory : Booking method: PDEFoam : Factory : Booking method: DL_CPU : : Parsing option string: : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100,MaxEpochs=20" : The following options are set: : - By User: : <none> : - Default: : Boost_num: "0" [Number of times the classifier will be boosted] : Parsing option string: : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100,MaxEpochs=20" : The following options are set: : - By User: : V: "True" [Verbose output (short form of "VerbosityLevel" below - overrides the latter one)] : VarTransform: "N" [List of variable transformations performed before training, e.g., "D_Background,P_Signal,G,N_AllClasses" for: "Decorrelation, PCA-transformation, Gaussianisation, Normalisation, each for the given class of events ('AllClasses' denotes all events of all classes, if no class indication is given, 'All' is assumed)"] : H: "False" [Print method-specific help message] : Layout: "TANH|100,TANH|50,TANH|10,LINEAR" [Layout of the network.] : ErrorStrategy: "CROSSENTROPY" [Loss function: Mean squared error (regression) or cross entropy (binary classification).] : WeightInitialization: "XAVIERUNIFORM" [Weight initialization strategy] : Architecture: "GPU" [Which architecture to perform the training on.] : TrainingStrategy: "Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100,MaxEpochs=20" [Defines the training strategies.] : - Default: : VerbosityLevel: "Default" [Verbosity level] : CreateMVAPdfs: "False" [Create PDFs for classifier outputs (signal and background)] : IgnoreNegWeightsInTraining: "False" [Events with negative weights are ignored in the training (but are included for testing and performance evaluation)] : InputLayout: "0|0|0" [The Layout of the input] : BatchLayout: "0|0|0" [The Layout of the batch] : RandomSeed: "0" [Random seed used for weight initialization and batch shuffling] : ValidationSize: "20%" [Part of the training data to use for validation. Specify as 0.2 or 20% to use a fifth of the data set as validation set. Specify as 100 to use exactly 100 events. (Default: 20%)] DL_CPU : [dataset] : Create Transformation "N" with events from all classes. : : Transformation, Variable selection : : Input : variable 'var1' <---> Output : variable 'var1' : Input : variable 'var2' <---> Output : variable 'var2' : Input : variable 'var3' <---> Output : variable 'var3' : Input : variable 'var4' <---> Output : variable 'var4' <ERROR> : CUDA backend not enabled. Please make sure you have CUDA installed and it was successfully detected by CMAKE by using -Dtmva-gpu=On : Will now use instead the CPU architecture ! : Will now use the CPU architecture with BLAS and IMT support !
[TFile::Cp] Total 0.12 MB |====================| 100.00 % [56.2 MB/s] Info in <TFile::OpenFromCache>: using local cache copy of http://root.cern/files/tmva_multiclass_example.root [./files/tmva_multiclass_example.root]
Train MVAs using the set of training events
factory->TrainAllMethods();
Factory : Train all methods Factory : [dataset] : Create Transformation "I" with events from all classes. : : Transformation, Variable selection : : Input : variable 'var1' <---> Output : variable 'var1' : Input : variable 'var2' <---> Output : variable 'var2' : Input : variable 'var3' <---> Output : variable 'var3' : Input : variable 'var4' <---> Output : variable 'var4' Factory : [dataset] : Create Transformation "D" with events from all classes. : : Transformation, Variable selection : : Input : variable 'var1' <---> Output : variable 'var1' : Input : variable 'var2' <---> Output : variable 'var2' : Input : variable 'var3' <---> Output : variable 'var3' : Input : variable 'var4' <---> Output : variable 'var4' Factory : [dataset] : Create Transformation "P" with events from all classes. : : Transformation, Variable selection : : Input : variable 'var1' <---> Output : variable 'var1' : Input : variable 'var2' <---> Output : variable 'var2' : Input : variable 'var3' <---> Output : variable 'var3' : Input : variable 'var4' <---> Output : variable 'var4' Factory : [dataset] : Create Transformation "G" with events from all classes. : : Transformation, Variable selection : : Input : variable 'var1' <---> Output : variable 'var1' : Input : variable 'var2' <---> Output : variable 'var2' : Input : variable 'var3' <---> Output : variable 'var3' : Input : variable 'var4' <---> Output : variable 'var4' Factory : [dataset] : Create Transformation "D" with events from all classes. : : Transformation, Variable selection : : Input : variable 'var1' <---> Output : variable 'var1' : Input : variable 'var2' <---> Output : variable 'var2' : Input : variable 'var3' <---> Output : variable 'var3' : Input : variable 'var4' <---> Output : variable 'var4' TFHandler_Factory : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.047647 1.0025 [ -3.6592 3.2645 ] : var2: 0.32647 1.0646 [ -3.6891 3.7877 ] : var3: 0.11493 1.1230 [ -4.5727 4.5640 ] : var4: -0.076531 1.2652 [ -4.8486 5.0412 ] : ----------------------------------------------------------- : Preparing the Decorrelation transformation... TFHandler_Factory : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.082544 1.0000 [ -3.6274 3.1017 ] : var2: 0.36715 1.0000 [ -3.3020 3.4950 ] : var3: 0.066865 1.0000 [ -2.9882 3.3086 ] : var4: -0.20593 1.0000 [ -3.3088 2.8423 ] : ----------------------------------------------------------- : Preparing the Principle Component (PCA) transformation... TFHandler_Factory : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 5.7502e-10 1.8064 [ -8.0344 7.8312 ] : var2:-1.6078e-11 0.90130 [ -2.6765 2.7523 ] : var3: 3.0841e-10 0.73386 [ -2.6572 2.2255 ] : var4:-2.6886e-10 0.62168 [ -1.7384 2.2297 ] : ----------------------------------------------------------- : Preparing the Gaussian transformation... : Preparing the Decorrelation transformation... TFHandler_Factory : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.013510 1.0000 [ -2.6520 6.2074 ] : var2: 0.0096839 1.0000 [ -2.8402 6.3073 ] : var3: 0.010397 1.0000 [ -3.0251 5.8860 ] : var4: 0.0053980 1.0000 [ -3.0998 5.7078 ] : ----------------------------------------------------------- : Ranking input variables (method unspecific)... Factory : Train method: BDTG for Multiclass classification : : Training 1000 Decision Trees ... patience please : Elapsed time for training with 4000 events: 5.56 sec : Dataset[dataset] : Create results for training : Dataset[dataset] : Multiclass evaluation of BDTG on training sample : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 1.9 sec : Creating multiclass response histograms... : Creating multiclass performance histograms... : Creating xml weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml : Creating standalone class: dataset/weights/TMVAMulticlass_BDTG.class.C : TMVAMulticlass.root:/dataset/Method_BDT/BDTG Factory : Training finished : Factory : Train method: MLP for Multiclass classification : : Training Network : : Elapsed time for training with 4000 events: 25.3 sec : Dataset[dataset] : Create results for training : Dataset[dataset] : Multiclass evaluation of MLP on training sample : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.00922 sec : Creating multiclass response histograms... : Creating multiclass performance histograms... : Creating xml weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml : Creating standalone class: dataset/weights/TMVAMulticlass_MLP.class.C : Write special histos to file: TMVAMulticlass.root:/dataset/Method_MLP/MLP Factory : Training finished : Factory : Train method: PDEFoam for Multiclass classification : : Build up multiclass foam 0 : Elapsed time: 0.66 sec : Build up multiclass foam 1 : Elapsed time: 0.673 sec : Build up multiclass foam 2 : Elapsed time: 0.689 sec : Build up multiclass foam 3 : Elapsed time: 0.486 sec : Elapsed time for training with 4000 events: 2.69 sec : Dataset[dataset] : Create results for training : Dataset[dataset] : Multiclass evaluation of PDEFoam on training sample : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.129 sec : Creating multiclass response histograms... : Creating multiclass performance histograms... : Creating xml weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml : writing foam MultiClassFoam0 to file : writing foam MultiClassFoam1 to file : writing foam MultiClassFoam2 to file : writing foam MultiClassFoam3 to file : Foams written to file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root : Creating standalone class: dataset/weights/TMVAMulticlass_PDEFoam.class.C Factory : Training finished : Factory : Train method: DL_CPU for Multiclass classification : TFHandler_DL_CPU : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.070769 0.28960 [ -1.0000 1.0000 ] : var2: 0.074130 0.28477 [ -1.0000 1.0000 ] : var3: 0.026106 0.24582 [ -1.0000 1.0000 ] : var4: -0.034951 0.25587 [ -1.0000 1.0000 ] : ----------------------------------------------------------- : Start of deep neural network training on CPU using MT, nthreads = 1 : TFHandler_DL_CPU : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.070769 0.28960 [ -1.0000 1.0000 ] : var2: 0.074130 0.28477 [ -1.0000 1.0000 ] : var3: 0.026106 0.24582 [ -1.0000 1.0000 ] : var4: -0.034951 0.25587 [ -1.0000 1.0000 ] : ----------------------------------------------------------- : ***** Deep Learning Network ***** DEEP NEURAL NETWORK: Depth = 4 Input = ( 1, 1, 4 ) Batch size = 100 Loss function = C Layer 0 DENSE Layer: ( Input = 4 , Width = 100 ) Output = ( 1 , 100 , 100 ) Activation Function = Tanh Layer 1 DENSE Layer: ( Input = 100 , Width = 50 ) Output = ( 1 , 100 , 50 ) Activation Function = Tanh Layer 2 DENSE Layer: ( Input = 50 , Width = 10 ) Output = ( 1 , 100 , 10 ) Activation Function = Tanh Layer 3 DENSE Layer: ( Input = 10 , Width = 4 ) Output = ( 1 , 100 , 4 ) Activation Function = Identity : Using 3200 events for training and 800 for testing : Compute initial loss on the validation data : Training phase 1 of 1: Optimizer ADAM (beta1=0.9,beta2=0.999,eps=1e-07) Learning rate = 0.001 regularization 0 minimum error = 0.703441 : -------------------------------------------------------------- : Epoch | Train Err. Val. Err. t(s)/epoch t(s)/Loss nEvents/s Conv. Steps : -------------------------------------------------------------- : Start epoch iteration ... : 1 Minimum Test error found - save the configuration : 1 | 0.620621 0.543743 0.0766189 0.00666508 45744.5 0 : 2 Minimum Test error found - save the configuration : 2 | 0.506261 0.47456 0.0774523 0.00663554 45187.1 0 : 3 Minimum Test error found - save the configuration : 3 | 0.45991 0.438479 0.0784336 0.00677715 44657.5 0 : 4 Minimum Test error found - save the configuration : 4 | 0.432622 0.41486 0.0784973 0.0067172 44580.6 0 : 5 Minimum Test error found - save the configuration : 5 | 0.413484 0.398984 0.0789418 0.00674887 44325.6 0 : 6 Minimum Test error found - save the configuration : 6 | 0.399891 0.387611 0.0793529 0.00678584 44097.1 0 : 7 Minimum Test error found - save the configuration : 7 | 0.389139 0.378073 0.0794202 0.00681757 44075.6 0 : 8 Minimum Test error found - save the configuration : 8 | 0.379376 0.369616 0.0801537 0.00687988 43671.8 0 : 9 Minimum Test error found - save the configuration : 9 | 0.372404 0.363373 0.0807042 0.00707495 43461 0 : 10 Minimum Test error found - save the configuration : 10 | 0.364765 0.356029 0.0829412 0.00714861 42220.5 0 : 11 Minimum Test error found - save the configuration : 11 | 0.358467 0.347816 0.0833505 0.00712739 41982 0 : 12 Minimum Test error found - save the configuration : 12 | 0.352723 0.342359 0.0832843 0.00706128 41982.1 0 : 13 Minimum Test error found - save the configuration : 13 | 0.347158 0.336689 0.0804567 0.00693152 43522.5 0 : 14 Minimum Test error found - save the configuration : 14 | 0.341732 0.329405 0.0809448 0.00692756 43233.2 0 : 15 Minimum Test error found - save the configuration : 15 | 0.335982 0.326309 0.0812397 0.00700177 43104.6 0 : 16 Minimum Test error found - save the configuration : 16 | 0.330871 0.318934 0.0828382 0.00699921 42194.7 0 : 17 Minimum Test error found - save the configuration : 17 | 0.326005 0.315708 0.0807883 0.00695833 43342.8 0 : 18 Minimum Test error found - save the configuration : 18 | 0.320842 0.309877 0.0812016 0.00705541 43158 0 : 19 Minimum Test error found - save the configuration : 19 | 0.316612 0.303529 0.0813382 0.0071296 43121.7 0 : 20 Minimum Test error found - save the configuration : 20 | 0.310762 0.300611 0.0817526 0.00702616 42822.9 0 : : Elapsed time for training with 4000 events: 1.64 sec : Dataset[dataset] : Create results for training : Dataset[dataset] : Multiclass evaluation of DL_CPU on training sample : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.104 sec : Creating multiclass response histograms... : Creating multiclass performance histograms... : Creating xml weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml : Creating standalone class: dataset/weights/TMVAMulticlass_DL_CPU.class.C Factory : Training finished : : Ranking input variables (method specific)... BDTG : Ranking result (top variable is best ranked) : -------------------------------------- : Rank : Variable : Variable Importance : -------------------------------------- : 1 : var4 : 3.117e-01 : 2 : var1 : 2.504e-01 : 3 : var2 : 2.430e-01 : 4 : var3 : 1.949e-01 : -------------------------------------- MLP : Ranking result (top variable is best ranked) : ----------------------------- : Rank : Variable : Importance : ----------------------------- : 1 : var4 : 6.076e+01 : 2 : var2 : 4.824e+01 : 3 : var1 : 2.116e+01 : 4 : var3 : 1.692e+01 : ----------------------------- PDEFoam : Ranking result (top variable is best ranked) : -------------------------------------- : Rank : Variable : Variable Importance : -------------------------------------- : 1 : var4 : 2.991e-01 : 2 : var1 : 2.930e-01 : 3 : var3 : 2.365e-01 : 4 : var2 : 1.714e-01 : -------------------------------------- : No variable ranking supplied by classifier: DL_CPU TH1.Print Name = TrainingHistory_DL_CPU_trainingError, Entries= 0, Total sum= 7.67963 TH1.Print Name = TrainingHistory_DL_CPU_valError, Entries= 0, Total sum= 7.35656 Factory : === Destroy and recreate all methods via weight files for testing === : : Reading weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml : Reading weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml MLP : Building Network. : Initializing weights : Reading weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml : Read foams from file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root : Reading weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml
Evaluate all MVAs using the set of test events
factory->TestAllMethods();
Factory : Test all methods
Factory : Test method: BDTG for Multiclass classification performance
:
: Dataset[dataset] : Create results for testing
: Dataset[dataset] : Multiclass evaluation of BDTG on testing sample
: Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.962 sec
: Creating multiclass response histograms...
: Creating multiclass performance histograms...
Factory : Test method: MLP for Multiclass classification performance
:
: Dataset[dataset] : Create results for testing
: Dataset[dataset] : Multiclass evaluation of MLP on testing sample
: Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.00983 sec
: Creating multiclass response histograms...
: Creating multiclass performance histograms...
Factory : Test method: PDEFoam for Multiclass classification performance
:
: Dataset[dataset] : Create results for testing
: Dataset[dataset] : Multiclass evaluation of PDEFoam on testing sample
: Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.127 sec
: Creating multiclass response histograms...
: Creating multiclass performance histograms...
Factory : Test method: DL_CPU for Multiclass classification performance
:
: Dataset[dataset] : Create results for testing
: Dataset[dataset] : Multiclass evaluation of DL_CPU on testing sample
: Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.103 sec
: Creating multiclass response histograms...
: Creating multiclass performance histograms...
Evaluate and compare performance of all configured MVAs
factory->EvaluateAllMethods();
Factory : Evaluate all methods : Evaluate multiclass classification method: BDTG : Creating multiclass response histograms... : Creating multiclass performance histograms... : Creating multiclass response histograms... : Creating multiclass performance histograms... TFHandler_BDTG : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.070153 1.0224 [ -4.0592 3.5808 ] : var2: 0.30372 1.0460 [ -3.6952 3.7877 ] : var3: 0.12152 1.1222 [ -3.6800 3.9200 ] : var4: -0.072602 1.2766 [ -4.8486 4.2221 ] : ----------------------------------------------------------- : Evaluate multiclass classification method: MLP : Creating multiclass response histograms... : Creating multiclass performance histograms... : Creating multiclass response histograms... : Creating multiclass performance histograms... TFHandler_MLP : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.070153 1.0224 [ -4.0592 3.5808 ] : var2: 0.30372 1.0460 [ -3.6952 3.7877 ] : var3: 0.12152 1.1222 [ -3.6800 3.9200 ] : var4: -0.072602 1.2766 [ -4.8486 4.2221 ] : ----------------------------------------------------------- : Evaluate multiclass classification method: PDEFoam : Creating multiclass response histograms... : Creating multiclass performance histograms... : Creating multiclass response histograms... : Creating multiclass performance histograms... TFHandler_PDEFoam : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.070153 1.0224 [ -4.0592 3.5808 ] : var2: 0.30372 1.0460 [ -3.6952 3.7877 ] : var3: 0.12152 1.1222 [ -3.6800 3.9200 ] : var4: -0.072602 1.2766 [ -4.8486 4.2221 ] : ----------------------------------------------------------- : Evaluate multiclass classification method: DL_CPU : Creating multiclass response histograms... : Creating multiclass performance histograms... : Creating multiclass response histograms... : Creating multiclass performance histograms... TFHandler_DL_CPU : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.077270 0.29534 [ -1.1155 1.0914 ] : var2: 0.068045 0.27981 [ -1.0016 1.0000 ] : var3: 0.027548 0.24565 [ -0.80459 0.85902 ] : var4: -0.034157 0.25816 [ -1.0000 0.83435 ] : ----------------------------------------------------------- TFHandler_DL_CPU : Variable Mean RMS [ Min Max ] : ----------------------------------------------------------- : var1: 0.077270 0.29534 [ -1.1155 1.0914 ] : var2: 0.068045 0.27981 [ -1.0016 1.0000 ] : var3: 0.027548 0.24565 [ -0.80459 0.85902 ] : var4: -0.034157 0.25816 [ -1.0000 0.83435 ] : ----------------------------------------------------------- : : 1-vs-rest performance metrics per class : ------------------------------------------------------------------------------------------------------- : : Considers the listed class as signal and the other classes : as background, reporting the resulting binary performance. : A score of 0.820 (0.850) means 0.820 was acheived on the : test set and 0.850 on the training set. : : Dataset MVA Method ROC AUC Sig eff@B=0.01 Sig eff@B=0.10 Sig eff@B=0.30 : Name: / Class: test (train) test (train) test (train) test (train) : : dataset BDTG : ------------------------------ : Signal 0.968 (0.978) 0.508 (0.605) 0.914 (0.945) 0.990 (0.996) : bg0 0.910 (0.931) 0.256 (0.288) 0.737 (0.791) 0.922 (0.956) : bg1 0.947 (0.954) 0.437 (0.511) 0.833 (0.856) 0.971 (0.971) : bg2 0.978 (0.982) 0.585 (0.678) 0.951 (0.956) 0.999 (0.996) : : dataset MLP : ------------------------------ : Signal 0.970 (0.975) 0.596 (0.632) 0.933 (0.938) 0.988 (0.993) : bg0 0.929 (0.934) 0.303 (0.298) 0.787 (0.793) 0.949 (0.961) : bg1 0.962 (0.967) 0.467 (0.553) 0.881 (0.906) 0.985 (0.992) : bg2 0.975 (0.979) 0.629 (0.699) 0.929 (0.940) 0.998 (0.998) : : dataset PDEFoam : ------------------------------ : Signal 0.916 (0.928) 0.294 (0.382) 0.744 (0.782) 0.932 (0.952) : bg0 0.837 (0.848) 0.109 (0.147) 0.519 (0.543) 0.833 (0.851) : bg1 0.890 (0.902) 0.190 (0.226) 0.606 (0.646) 0.923 (0.929) : bg2 0.967 (0.972) 0.510 (0.527) 0.900 (0.926) 0.993 (0.998) : : dataset DL_CPU : ------------------------------ : Signal 0.961 (0.960) 0.358 (0.385) 0.927 (0.916) 0.990 (0.989) : bg0 0.914 (0.913) 0.283 (0.316) 0.767 (0.759) 0.925 (0.926) : bg1 0.946 (0.944) 0.268 (0.276) 0.858 (0.865) 0.975 (0.973) : bg2 0.907 (0.909) 0.552 (0.508) 0.748 (0.751) 0.865 (0.873) : : ------------------------------------------------------------------------------------------------------- : : : Confusion matrices for all methods : ------------------------------------------------------------------------------------------------------- : : Does a binary comparison between the two classes given by a : particular row-column combination. In each case, the class : given by the row is considered signal while the class given : by the column index is considered background. : : === Showing confusion matrix for method : BDTG : (Signal Efficiency for Background Efficiency 0.01%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.497 (0.373) 0.710 (0.693) 0.680 (0.574) : bg0 0.271 (0.184) - 0.239 (0.145) 0.705 (0.667) : bg1 0.855 (0.766) 0.369 (0.222) - 0.587 (0.578) : bg2 0.714 (0.585) 0.705 (0.581) 0.648 (0.601) - : : (Signal Efficiency for Background Efficiency 0.10%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.911 (0.853) 0.991 (0.981) 0.945 (0.913) : bg0 0.833 (0.774) - 0.654 (0.582) 0.930 (0.901) : bg1 0.971 (0.980) 0.716 (0.681) - 0.871 (0.862) : bg2 0.976 (0.951) 0.971 (0.973) 0.936 (0.941) - : : (Signal Efficiency for Background Efficiency 0.30%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.978 (0.957) 0.999 (1.000) 0.998 (0.997) : bg0 0.965 (0.926) - 0.874 (0.835) 0.991 (0.976) : bg1 1.000 (0.999) 0.916 (0.894) - 0.988 (0.985) : bg2 0.999 (0.999) 0.997 (0.999) 0.996 (0.997) - : : === Showing confusion matrix for method : MLP : (Signal Efficiency for Background Efficiency 0.01%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.465 (0.490) 0.974 (0.953) 0.632 (0.498) : bg0 0.320 (0.269) - 0.224 (0.250) 0.655 (0.627) : bg1 0.943 (0.920) 0.341 (0.275) - 0.632 (0.687) : bg2 0.665 (0.642) 0.697 (0.680) 0.706 (0.598) - : : (Signal Efficiency for Background Efficiency 0.10%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.865 (0.854) 0.996 (0.994) 0.908 (0.907) : bg0 0.784 (0.776) - 0.666 (0.655) 0.919 (0.895) : bg1 0.998 (0.998) 0.791 (0.785) - 0.912 (0.902) : bg2 0.943 (0.903) 0.946 (0.939) 0.924 (0.928) - : : (Signal Efficiency for Background Efficiency 0.30%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.978 (0.964) 0.997 (0.997) 0.993 (0.986) : bg0 0.952 (0.924) - 0.936 (0.928) 0.992 (0.990) : bg1 1.000 (1.000) 0.945 (0.936) - 0.998 (0.995) : bg2 0.994 (0.985) 0.998 (0.998) 0.998 (0.998) - : : === Showing confusion matrix for method : PDEFoam : (Signal Efficiency for Background Efficiency 0.01%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.289 (0.233) 0.467 (0.436) 0.421 (0.332) : bg0 0.100 (0.045) - 0.132 (0.116) 0.540 (0.313) : bg1 0.209 (0.434) 0.153 (0.092) - 0.347 (0.323) : bg2 0.560 (0.552) 0.445 (0.424) 0.501 (0.506) - : : (Signal Efficiency for Background Efficiency 0.10%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.665 (0.640) 0.854 (0.822) 0.807 (0.790) : bg0 0.538 (0.520) - 0.415 (0.374) 0.843 (0.833) : bg1 0.885 (0.886) 0.542 (0.491) - 0.728 (0.646) : bg2 0.928 (0.890) 0.956 (0.959) 0.847 (0.895) - : : (Signal Efficiency for Background Efficiency 0.30%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.898 (0.878) 0.971 (0.950) 0.982 (0.975) : bg0 0.828 (0.810) - 0.696 (0.676) 0.954 (0.951) : bg1 0.951 (0.966) 0.803 (0.745) - 0.958 (0.966) : bg2 0.998 (0.991) 0.998 (0.996) 0.998 (0.993) - : : === Showing confusion matrix for method : DL_CPU : (Signal Efficiency for Background Efficiency 0.01%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.423 (0.525) 0.929 (0.951) 0.256 (0.081) : bg0 0.288 (0.217) - 0.217 (0.233) 0.472 (0.478) : bg1 0.910 (0.890) 0.267 (0.159) - 0.133 (0.173) : bg2 0.655 (0.601) 0.485 (0.524) 0.414 (0.471) - : : (Signal Efficiency for Background Efficiency 0.10%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.898 (0.911) 0.989 (0.991) 0.782 (0.741) : bg0 0.740 (0.762) - 0.723 (0.762) 0.788 (0.780) : bg1 0.998 (0.998) 0.721 (0.718) - 0.793 (0.823) : bg2 0.832 (0.812) 0.743 (0.748) 0.703 (0.704) - : : (Signal Efficiency for Background Efficiency 0.30%) : --------------------------------------------------- : Signal bg0 bg1 bg2 : test (train) test (train) test (train) test (train) : Signal - 0.979 (0.982) 1.000 (1.000) 0.952 (0.959) : bg0 0.921 (0.925) - 0.904 (0.911) 0.957 (0.953) : bg1 1.000 (0.999) 0.956 (0.951) - 0.930 (0.937) : bg2 0.925 (0.908) 0.865 (0.854) 0.831 (0.832) - : : ------------------------------------------------------------------------------------------------------- : Dataset:dataset : Created tree 'TestTree' with 4000 events : Dataset:dataset : Created tree 'TrainTree' with 4000 events : Factory : Thank you for using TMVA! : For citation information, please visit: http://tmva.sf.net/citeTMVA.html
Save the output
outputFile->Close();
std::cout << "==> Wrote root file: " << outputFile->GetName() << std::endl;
std::cout << "==> TMVAMulticlass is done!" << std::endl;
delete factory;
delete dataloader;
==> Wrote root file: TMVAMulticlass.root ==> TMVAMulticlass is done!
Launch the GUI for the root macros
if (!gROOT->IsBatch()) TMVAMultiClassGui( outfileName );