T M V A Multiclass

This macro provides a simple example for the training and testing of the TMVA multiclass classification

  • Project : TMVA - a Root-integrated toolkit for multivariate data analysis
  • Package : TMVA
  • Root Macro: TMVAMulticlass

Author: Andreas Hoecker
This notebook tutorial was automatically generated with ROOTBOOK-izer from the macro found in the ROOT repository on Thursday, June 17, 2021 at 06:06 PM.

In [1]:
%%cpp -d
#include <cstdlib>
#include <iostream>
#include <map>
#include <string>

#include "TFile.h"
#include "TTree.h"
#include "TString.h"
#include "TSystem.h"
#include "TROOT.h"


#include "TMVA/Tools.h"
#include "TMVA/Factory.h"
#include "TMVA/DataLoader.h"
#include "TMVA/TMVAMultiClassGui.h"


using namespace TMVA;

Arguments are defined.

In [2]:
TString myMethodList = "";

This loads the library

In [3]:
TMVA::Tools::Instance();

To get access to the gui and all tmva macros

 TString tmva_dir(TString(gRootDir) + "/tmva");
 if(gSystem->Getenv("TMVASYS"))
    tmva_dir = TString(gSystem->Getenv("TMVASYS"));
 gROOT->SetMacroPath(tmva_dir + "/test/:" + gROOT->GetMacroPath() );
 gROOT->ProcessLine(".L TMVAMultiClassGui.C");

Default MVA methods to be trained + tested

In [4]:
std::map<std::string,int> Use;
Use["MLP"]             = 1;
Use["BDTG"]            = 1;
#ifdef R__HAS_TMVAGPU
Use["DL_CPU"]          = 1;
Use["DL_GPU"]          = 1;
#else
Use["DL_CPU"]          = 1;
Use["DL_GPU"]          = 0;
#endif
Use["FDA_GA"]          = 0;
Use["PDEFoam"]         = 1;

In [5]:
std::cout << std::endl;
std::cout << "==> Start TMVAMulticlass" << std::endl;

if (myMethodList != "") {
   for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) it->second = 0;

   std::vector<TString> mlist = TMVA::gTools().SplitString( myMethodList, ',' );
   for (UInt_t i=0; i<mlist.size(); i++) {
      std::string regMethod(mlist[i]);

      if (Use.find(regMethod) == Use.end()) {
         std::cout << "Method \"" << regMethod << "\" not known in TMVA under this name. Choose among the following:" << std::endl;
         for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) std::cout << it->first << " ";
         std::cout << std::endl;
         return;
      }
      Use[regMethod] = 1;
   }
}
==> Start TMVAMulticlass

Create a new root output file.

In [6]:
TString outfileName = "TMVAMulticlass.root";
TFile* outputFile = TFile::Open( outfileName, "RECREATE" );

TMVA::Factory *factory = new TMVA::Factory( "TMVAMulticlass", outputFile,
                                            "!V:!Silent:Color:!DrawProgressBar:Transformations=I;D;P;G,D:AnalysisType=multiclass" );
TMVA::DataLoader *dataloader=new TMVA::DataLoader("dataset");

dataloader->AddVariable( "var1", 'F' );
dataloader->AddVariable( "var2", "Variable 2", "", 'F' );
dataloader->AddVariable( "var3", "Variable 3", "units", 'F' );
dataloader->AddVariable( "var4", "Variable 4", "units", 'F' );

TFile *input(0);
TString fname = "./tmva_example_multiclass.root";
if (!gSystem->AccessPathName( fname )) {
   input = TFile::Open( fname ); // check if file in local directory exists
}
else {
   TFile::SetCacheFileDir(".");
   input = TFile::Open("http://root.cern.ch/files/tmva_multiclass_example.root", "CACHEREAD");
}
if (!input) {
   std::cout << "ERROR: could not open data file" << std::endl;
   exit(1);
}
std::cout << "--- TMVAMulticlass: Using input file: " << input->GetName() << std::endl;

TTree *signalTree  = (TTree*)input->Get("TreeS");
TTree *background0 = (TTree*)input->Get("TreeB0");
TTree *background1 = (TTree*)input->Get("TreeB1");
TTree *background2 = (TTree*)input->Get("TreeB2");

gROOT->cd( outfileName+TString(":/") );
dataloader->AddTree    (signalTree,"Signal");
dataloader->AddTree    (background0,"bg0");
dataloader->AddTree    (background1,"bg1");
dataloader->AddTree    (background2,"bg2");

dataloader->PrepareTrainingAndTestTree( "", "SplitMode=Random:NormMode=NumEvents:!V" );

if (Use["BDTG"]) // gradient boosted decision trees
   factory->BookMethod( dataloader,  TMVA::Types::kBDT, "BDTG", "!H:!V:NTrees=1000:BoostType=Grad:Shrinkage=0.10:UseBaggedBoost:BaggedSampleFraction=0.50:nCuts=20:MaxDepth=2");
if (Use["MLP"]) // neural network
   factory->BookMethod( dataloader,  TMVA::Types::kMLP, "MLP", "!H:!V:NeuronType=tanh:NCycles=1000:HiddenLayers=N+5,5:TestRate=5:EstimatorType=MSE");
if (Use["FDA_GA"]) // functional discriminant with GA minimizer
   factory->BookMethod( dataloader,  TMVA::Types::kFDA, "FDA_GA", "H:!V:Formula=(0)+(1)*x0+(2)*x1+(3)*x2+(4)*x3:ParRanges=(-1,1);(-10,10);(-10,10);(-10,10);(-10,10):FitMethod=GA:PopSize=300:Cycles=3:Steps=20:Trim=True:SaveBestGen=1" );
if (Use["PDEFoam"]) // PDE-Foam approach
   factory->BookMethod( dataloader,  TMVA::Types::kPDEFoam, "PDEFoam", "!H:!V:TailCut=0.001:VolFrac=0.0666:nActiveCells=500:nSampl=2000:nBin=5:Nmin=100:Kernel=None:Compress=T" );


if (Use["DL_CPU"]) {
   TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
   TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
                                  "TestRepetitions=1,ConvergenceSteps=10,BatchSize=100");
   TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
                     "WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
   nnOptions.Append(":");
   nnOptions.Append(layoutString);
   nnOptions.Append(":");
   nnOptions.Append(trainingStrategyString);
   factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_CPU", nnOptions);
}
if (Use["DL_GPU"]) {
   TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
   TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
                                  "TestRepetitions=1,ConvergenceSteps=10,BatchSize=100");
   TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
                     "WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
   nnOptions.Append(":");
   nnOptions.Append(layoutString);
   nnOptions.Append(":");
   nnOptions.Append(trainingStrategyString);
   factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_GPU", nnOptions);
}
--- TMVAMulticlass: Using input file: ./files/tmva_multiclass_example.root
DataSetInfo              : [dataset] : Added class "Signal"
                         : Add Tree TreeS of type Signal with 2000 events
DataSetInfo              : [dataset] : Added class "bg0"
                         : Add Tree TreeB0 of type bg0 with 2000 events
DataSetInfo              : [dataset] : Added class "bg1"
                         : Add Tree TreeB1 of type bg1 with 2000 events
DataSetInfo              : [dataset] : Added class "bg2"
                         : Add Tree TreeB2 of type bg2 with 2000 events
                         : Dataset[dataset] : Class index : 0  name : Signal
                         : Dataset[dataset] : Class index : 1  name : bg0
                         : Dataset[dataset] : Class index : 2  name : bg1
                         : Dataset[dataset] : Class index : 3  name : bg2
Factory                  : Booking method: BDTG
                         : 
                         : the option NegWeightTreatment=InverseBoostNegWeights does not exist for BoostType=Grad
                         : --> change to new default NegWeightTreatment=Pray
                         : Rebuilding Dataset dataset
                         : Building event vectors for type 2 Signal
                         : Dataset[dataset] :  create input formulas for tree TreeS
                         : Building event vectors for type 2 bg0
                         : Dataset[dataset] :  create input formulas for tree TreeB0
                         : Building event vectors for type 2 bg1
                         : Dataset[dataset] :  create input formulas for tree TreeB1
                         : Building event vectors for type 2 bg2
                         : Dataset[dataset] :  create input formulas for tree TreeB2
DataSetFactory           : [dataset] : Number of events in input trees
                         : 
                         : 
                         : 
                         : 
                         : Number of training and testing events
                         : ---------------------------------------------------------------------------
                         : Signal -- training events            : 1000
                         : Signal -- testing events             : 1000
                         : Signal -- training and testing events: 2000
                         : bg0    -- training events            : 1000
                         : bg0    -- testing events             : 1000
                         : bg0    -- training and testing events: 2000
                         : bg1    -- training events            : 1000
                         : bg1    -- testing events             : 1000
                         : bg1    -- training and testing events: 2000
                         : bg2    -- training events            : 1000
                         : bg2    -- testing events             : 1000
                         : bg2    -- training and testing events: 2000
                         : 
DataSetInfo              : Correlation matrix (Signal):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.397  +0.623  +0.832
                         :    var2:  +0.397  +1.000  +0.716  +0.737
                         :    var3:  +0.623  +0.716  +1.000  +0.859
                         :    var4:  +0.832  +0.737  +0.859  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg0):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.365  +0.592  +0.811
                         :    var2:  +0.365  +1.000  +0.708  +0.740
                         :    var3:  +0.592  +0.708  +1.000  +0.859
                         :    var4:  +0.811  +0.740  +0.859  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg1):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.407  +0.610  +0.834
                         :    var2:  +0.407  +1.000  +0.710  +0.741
                         :    var3:  +0.610  +0.710  +1.000  +0.851
                         :    var4:  +0.834  +0.741  +0.851  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg2):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  -0.647  -0.016  -0.013
                         :    var2:  -0.647  +1.000  +0.015  +0.002
                         :    var3:  -0.016  +0.015  +1.000  -0.024
                         :    var4:  -0.013  +0.002  -0.024  +1.000
                         : ----------------------------------------
DataSetFactory           : [dataset] :  
                         : 
Factory                  : Booking method: MLP
                         : 
MLP                      : Building Network. 
                         : Initializing weights
Factory                  : Booking method: PDEFoam
                         : 
Factory                  : Booking method: DL_CPU
                         : 
                         : Parsing option string: 
                         : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100"
                         : The following options are set:
                         : - By User:
                         :     <none>
                         : - Default:
                         :     Boost_num: "0" [Number of times the classifier will be boosted]
                         : Parsing option string: 
                         : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100"
                         : The following options are set:
                         : - By User:
                         :     V: "True" [Verbose output (short form of "VerbosityLevel" below - overrides the latter one)]
                         :     VarTransform: "N" [List of variable transformations performed before training, e.g., "D_Background,P_Signal,G,N_AllClasses" for: "Decorrelation, PCA-transformation, Gaussianisation, Normalisation, each for the given class of events ('AllClasses' denotes all events of all classes, if no class indication is given, 'All' is assumed)"]
                         :     H: "False" [Print method-specific help message]
                         :     Layout: "TANH|100,TANH|50,TANH|10,LINEAR" [Layout of the network.]
                         :     ErrorStrategy: "CROSSENTROPY" [Loss function: Mean squared error (regression) or cross entropy (binary classification).]
                         :     WeightInitialization: "XAVIERUNIFORM" [Weight initialization strategy]
                         :     Architecture: "GPU" [Which architecture to perform the training on.]
                         :     TrainingStrategy: "Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100" [Defines the training strategies.]
                         : - Default:
                         :     VerbosityLevel: "Default" [Verbosity level]
                         :     CreateMVAPdfs: "False" [Create PDFs for classifier outputs (signal and background)]
                         :     IgnoreNegWeightsInTraining: "False" [Events with negative weights are ignored in the training (but are included for testing and performance evaluation)]
                         :     InputLayout: "0|0|0" [The Layout of the input]
                         :     BatchLayout: "0|0|0" [The Layout of the batch]
                         :     RandomSeed: "0" [Random seed used for weight initialization and batch shuffling]
                         :     ValidationSize: "20%" [Part of the training data to use for validation. Specify as 0.2 or 20% to use a fifth of the data set as validation set. Specify as 100 to use exactly 100 events. (Default: 20%)]
DL_CPU                   : [dataset] : Create Transformation "N" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
<ERROR>                         : CUDA backend not enabled. Please make sure you have CUDA installed and it was successfully detected by CMAKE by using -Dtmva-gpu=On  
                         : Will now use instead the CPU architecture !
                         : Will now use the CPU architecture with BLAS and IMT support !
[TFile::Cp] Total 0.12 MB	|====================| 100.00 % [57.4 MB/s]
Info in <TFile::OpenFromCache>: using local cache copy of http://root.cern.ch/files/tmva_multiclass_example.root [./files/tmva_multiclass_example.root]

Train mvas using the set of training events

In [7]:
factory->TrainAllMethods();
Factory                  : Train all methods
Factory                  : [dataset] : Create Transformation "I" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "D" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "P" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "G" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "D" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.047647     1.0025   [    -3.6592     3.2645 ]
                         :     var2:    0.32647     1.0646   [    -3.6891     3.7877 ]
                         :     var3:    0.11493     1.1230   [    -4.5727     4.5640 ]
                         :     var4:  -0.076531     1.2652   [    -4.8486     5.0412 ]
                         : -----------------------------------------------------------
                         : Preparing the Decorrelation transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.082544     1.0000   [    -3.6274     3.1017 ]
                         :     var2:    0.36715     1.0000   [    -3.3020     3.4950 ]
                         :     var3:   0.066865     1.0000   [    -2.9882     3.3086 ]
                         :     var4:   -0.20593     1.0000   [    -3.3088     2.8423 ]
                         : -----------------------------------------------------------
                         : Preparing the Principle Component (PCA) transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1: 5.7502e-10     1.8064   [    -8.0344     7.8312 ]
                         :     var2:-1.6078e-11    0.90130   [    -2.6765     2.7523 ]
                         :     var3: 3.0841e-10    0.73386   [    -2.6572     2.2255 ]
                         :     var4:-2.6886e-10    0.62168   [    -1.7384     2.2297 ]
                         : -----------------------------------------------------------
                         : Preparing the Gaussian transformation...
                         : Preparing the Decorrelation transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.013510     1.0000   [    -2.6520     6.2074 ]
                         :     var2:  0.0096839     1.0000   [    -2.8402     6.3073 ]
                         :     var3:   0.010397     1.0000   [    -3.0251     5.8860 ]
                         :     var4:  0.0053980     1.0000   [    -3.0998     5.7078 ]
                         : -----------------------------------------------------------
                         : Ranking input variables (method unspecific)...
Factory                  : Train method: BDTG for Multiclass classification
                         : 
                         : Training 1000 Decision Trees ... patience please
                         : Elapsed time for training with 4000 events: 5.49 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of BDTG on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 1.78 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_BDTG.class.C
                         : TMVAMulticlass.root:/dataset/Method_BDT/BDTG
Factory                  : Training finished
                         : 
Factory                  : Train method: MLP for Multiclass classification
                         : 
                         : Training Network
                         : 
                         : Elapsed time for training with 4000 events: 25.5 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of MLP on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.00922 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_MLP.class.C
                         : Write special histos to file: TMVAMulticlass.root:/dataset/Method_MLP/MLP
Factory                  : Training finished
                         : 
Factory                  : Train method: PDEFoam for Multiclass classification
                         : 
                         : Build up multiclass foam 0
                         : Elapsed time: 0.653 sec                                 
                         : Build up multiclass foam 1
                         : Elapsed time: 0.656 sec                                 
                         : Build up multiclass foam 2
                         : Elapsed time: 0.669 sec                                 
                         : Build up multiclass foam 3
                         : Elapsed time: 0.464 sec                                 
                         : Elapsed time for training with 4000 events: 2.61 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of PDEFoam on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.121 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml
                         : writing foam MultiClassFoam0 to file
                         : writing foam MultiClassFoam1 to file
                         : writing foam MultiClassFoam2 to file
                         : writing foam MultiClassFoam3 to file
                         : Foams written to file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root
                         : Creating standalone class: dataset/weights/TMVAMulticlass_PDEFoam.class.C
Factory                  : Training finished
                         : 
Factory                  : Train method: DL_CPU for Multiclass classification
                         : 
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070769    0.28960   [    -1.0000     1.0000 ]
                         :     var2:   0.074130    0.28477   [    -1.0000     1.0000 ]
                         :     var3:   0.026106    0.24582   [    -1.0000     1.0000 ]
                         :     var4:  -0.034951    0.25587   [    -1.0000     1.0000 ]
                         : -----------------------------------------------------------
                         : Start of deep neural network training on CPU using MT,  nthreads = 1
                         : 
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070769    0.28960   [    -1.0000     1.0000 ]
                         :     var2:   0.074130    0.28477   [    -1.0000     1.0000 ]
                         :     var3:   0.026106    0.24582   [    -1.0000     1.0000 ]
                         :     var4:  -0.034951    0.25587   [    -1.0000     1.0000 ]
                         : -----------------------------------------------------------
                         : *****   Deep Learning Network *****
DEEP NEURAL NETWORK:   Depth = 4  Input = ( 1, 1, 4 )  Batch size = 100  Loss function = C
	Layer 0	 DENSE Layer: 	 ( Input =     4 , Width =   100 ) 	Output = (  1 ,   100 ,   100 ) 	 Activation Function = Tanh
	Layer 1	 DENSE Layer: 	 ( Input =   100 , Width =    50 ) 	Output = (  1 ,   100 ,    50 ) 	 Activation Function = Tanh
	Layer 2	 DENSE Layer: 	 ( Input =    50 , Width =    10 ) 	Output = (  1 ,   100 ,    10 ) 	 Activation Function = Tanh
	Layer 3	 DENSE Layer: 	 ( Input =    10 , Width =     4 ) 	Output = (  1 ,   100 ,     4 ) 	 Activation Function = Identity
                         : Using 3200 events for training and 800 for testing
                         : Compute initial loss  on the validation data 
                         : Training phase 1 of 1:  Optimizer ADAM (beta1=0.9,beta2=0.999,eps=1e-07) Learning rate = 0.001 regularization 0 minimum error = 0.699491
                         : --------------------------------------------------------------
                         :      Epoch |   Train Err.   Val. Err.  t(s)/epoch   t(s)/Loss   nEvents/s Conv. Steps
                         : --------------------------------------------------------------
                         :    Start epoch iteration ...
                         :          1 Minimum Test error found - save the configuration 
                         :          1 |     0.634414    0.570296   0.0764078  0.00659792     45838.8           0
                         :          2 Minimum Test error found - save the configuration 
                         :          2 |      0.53133    0.495082   0.0771698  0.00659872     45344.3           0
                         :          3 Minimum Test error found - save the configuration 
                         :          3 |     0.480944    0.458699   0.0779577  0.00669923     44906.9           0
                         :          4 Minimum Test error found - save the configuration 
                         :          4 |     0.453168    0.433573   0.0784348  0.00672058     44621.5           0
                         :          5 Minimum Test error found - save the configuration 
                         :          5 |     0.432017    0.413574   0.0788115  0.00676534       44416           0
                         :          6 Minimum Test error found - save the configuration 
                         :          6 |       0.4143    0.400325   0.0790618  0.00678395     44273.6           0
                         :          7 Minimum Test error found - save the configuration 
                         :          7 |     0.400472    0.384302   0.0792982  0.00680986       44145           0
                         :          8 Minimum Test error found - save the configuration 
                         :          8 |     0.387946    0.375005   0.0795118  0.00684376     44035.8           0
                         :          9 Minimum Test error found - save the configuration 
                         :          9 |     0.377972    0.366612   0.0796336  0.00683797     43958.7           0
                         :         10 Minimum Test error found - save the configuration 
                         :         10 |     0.369406    0.356647   0.0797672  0.00685595       43889           0
                         :         11 Minimum Test error found - save the configuration 
                         :         11 |     0.361704    0.349831   0.0799348  0.00687428     43799.3           0
                         :         12 Minimum Test error found - save the configuration 
                         :         12 |     0.355527    0.342873    0.080102   0.0068803     43702.9           0
                         :         13 Minimum Test error found - save the configuration 
                         :         13 |     0.347573    0.335248   0.0802536  0.00690155     43625.2           0
                         :         14 Minimum Test error found - save the configuration 
                         :         14 |     0.341717    0.329667   0.0804189  0.00693975     43549.8           0
                         :         15 Minimum Test error found - save the configuration 
                         :         15 |     0.333984     0.32499   0.0805808   0.0069459     43457.7           0
                         :         16 Minimum Test error found - save the configuration 
                         :         16 |     0.327741    0.319055   0.0806396  0.00696248     43432.8           0
                         :         17 Minimum Test error found - save the configuration 
                         :         17 |     0.322056    0.312946   0.0807721  0.00698926     43370.5           0
                         :         18 Minimum Test error found - save the configuration 
                         :         18 |     0.317285    0.309453   0.0808854  0.00697627     43296.4           0
                         :         19 |     0.313079    0.310913   0.0809198   0.0069337     43251.4           1
                         :         20 Minimum Test error found - save the configuration 
                         :         20 |      0.30871    0.300948   0.0810397  0.00699925     43219.6           0
                         :         21 Minimum Test error found - save the configuration 
                         :         21 |     0.304136    0.295065   0.0810913  0.00700356     43192.1           0
                         :         22 Minimum Test error found - save the configuration 
                         :         22 |     0.301058    0.290675   0.0811658  0.00700534     43149.7           0
                         :         23 Minimum Test error found - save the configuration 
                         :         23 |     0.297713    0.287124   0.0812594  0.00704665     43119.3           0
                         :         24 Minimum Test error found - save the configuration 
                         :         24 |     0.294601    0.286989    0.081303  0.00702779       43083           0
                         :         25 Minimum Test error found - save the configuration 
                         :         25 |      0.29151    0.281638   0.0813486   0.0070255     43055.3           0
                         :         26 Minimum Test error found - save the configuration 
                         :         26 |      0.28835    0.279108   0.0814913  0.00704984     42986.8           0
                         :         27 Minimum Test error found - save the configuration 
                         :         27 |     0.286686    0.276396   0.0818454  0.00734482     42952.7           0
                         :         28 Minimum Test error found - save the configuration 
                         :         28 |     0.283722    0.275799   0.0822307  0.00715534     42623.8           0
                         :         29 Minimum Test error found - save the configuration 
                         :         29 |     0.281451    0.270827   0.0822681  0.00716281     42606.8           0
                         :         30 Minimum Test error found - save the configuration 
                         :         30 |     0.279603     0.26961   0.0824473  0.00716116     42504.5           0
                         :         31 |      0.27759     0.27014   0.0826239  0.00706297     42349.9           1
                         :         32 Minimum Test error found - save the configuration 
                         :         32 |       0.2758    0.266633   0.0826328  0.00717603     42408.4           0
                         :         33 Minimum Test error found - save the configuration 
                         :         33 |      0.27497    0.264143   0.0826286  0.00717588     42410.7           0
                         :         34 Minimum Test error found - save the configuration 
                         :         34 |     0.272958    0.262094    0.082678  0.00715631     42371.9           0
                         :         35 Minimum Test error found - save the configuration 
                         :         35 |     0.271775    0.258868   0.0825843  0.00717422     42434.6           0
                         :         36 |     0.270302    0.260003   0.0826824  0.00708291     42328.3           1
                         :         37 Minimum Test error found - save the configuration 
                         :         37 |     0.269463    0.258534   0.0855926    0.010101     42388.8           0
                         :         38 Minimum Test error found - save the configuration 
                         :         38 |     0.267563    0.254389   0.0821053  0.00709506     42660.8           0
                         :         39 |     0.265961    0.256969   0.0818956  0.00702657     42741.3           1
                         :         40 Minimum Test error found - save the configuration 
                         :         40 |     0.264446     0.25339   0.0820339  0.00711042     42710.2           0
                         :         41 |     0.263038    0.254898   0.0820435  0.00703831     42663.7           1
                         :         42 |     0.262783    0.256036   0.0820636  0.00705022     42659.1           2
                         :         43 Minimum Test error found - save the configuration 
                         :         43 |     0.260513     0.25215   0.0821086  0.00710046       42662           0
                         :         44 |     0.260391    0.252525   0.0820499  0.00705082     42667.2           1
                         :         45 |     0.259871    0.253937   0.0820803  0.00705166     42650.4           2
                         :         46 Minimum Test error found - save the configuration 
                         :         46 |     0.257912    0.248483    0.082169  0.00711562     42636.3           0
                         :         47 Minimum Test error found - save the configuration 
                         :         47 |     0.256847    0.244842   0.0821824  0.00711688     42629.4           0
                         :         48 |     0.255985    0.249636   0.0821772  0.00706708     42604.1           1
                         :         49 |     0.254212    0.248114   0.0821925  0.00706592     42594.8           2
                         :         50 |     0.253513    0.246898   0.0821692   0.0070718     42611.3           3
                         :         51 |     0.252772    0.246036   0.0823277  0.00707872     42525.5           4
                         :         52 |     0.250397    0.246617   0.0824194  0.00708051     42474.8           5
                         :         53 Minimum Test error found - save the configuration 
                         :         53 |     0.250259     0.24446     0.08244  0.00714771       42501           0
                         :         54 Minimum Test error found - save the configuration 
                         :         54 |      0.24868    0.239647   0.0823688  0.00714083     42537.4           0
                         :         55 |     0.245701    0.239675   0.0823607  0.00709528     42516.2           1
                         :         56 |     0.244046      0.2399   0.0823978  0.00708923     42491.8           2
                         :         57 Minimum Test error found - save the configuration 
                         :         57 |     0.242932    0.239544   0.0824244  0.00713838     42504.6           0
                         :         58 Minimum Test error found - save the configuration 
                         :         58 |     0.242418    0.234808    0.082485  0.00714518     42474.2           0
                         :         59 |     0.240088    0.236823   0.0824433  0.00709218     42467.8           1
                         :         60 |     0.239725    0.235897   0.0824571  0.00709505     42461.7           2
                         :         61 |     0.237835    0.235993   0.0825055  0.00709983     42437.1           3
                         :         62 |     0.236465    0.235199   0.0824939  0.00709697     42442.1           4
                         :         63 |     0.235713    0.239833    0.082555  0.00710536     42412.4           5
                         :         64 |     0.235419    0.238649   0.0825858  0.00711736     42401.8           6
                         :         65 Minimum Test error found - save the configuration 
                         :         65 |     0.234538    0.232212   0.0828331  0.00720253     42310.9           0
                         :         66 |     0.232846    0.233947   0.0827733  0.00712328       42300           1
                         :         67 |     0.231685    0.236438   0.0828402  0.00713138     42267.2           2
                         :         68 Minimum Test error found - save the configuration 
                         :         68 |     0.231685    0.231324   0.0829028  0.00720479     42273.3           0
                         :         69 Minimum Test error found - save the configuration 
                         :         69 |     0.230485    0.229175   0.0829024  0.00719959     42270.5           0
                         :         70 Minimum Test error found - save the configuration 
                         :         70 |     0.230266    0.226807   0.0829172  0.00720766     42266.8           0
                         :         71 |     0.228958    0.228991   0.0828494  0.00713721     42265.3           1
                         :         72 |     0.228257    0.227469   0.0828526  0.00712987     42259.4           2
                         :         73 |     0.227867    0.233008   0.0828337  0.00712915     42269.6           3
                         :         74 Minimum Test error found - save the configuration 
                         :         74 |     0.226271     0.22588    0.082926  0.00721044     42263.5           0
                         :         75 |     0.226168     0.23209   0.0828893  0.00713502     42241.8           1
                         :         76 Minimum Test error found - save the configuration 
                         :         76 |     0.227109    0.224454   0.0829797  0.00721074     42233.6           0
                         :         77 Minimum Test error found - save the configuration 
                         :         77 |     0.226127    0.223369   0.0830313  0.00721932     42209.7           0
                         :         78 |     0.224352    0.225416   0.0829512  0.00714279     42211.7           1
                         :         79 |     0.223508    0.223445   0.0829824  0.00714369     42194.8           2
                         :         80 Minimum Test error found - save the configuration 
                         :         80 |     0.222584    0.221377   0.0830352  0.00721669       42206           0
                         :         81 Minimum Test error found - save the configuration 
                         :         81 |     0.222576    0.220426   0.0830665   0.0072235     42192.5           0
                         :         82 |     0.222136    0.222861   0.0830083   0.0071572     42187.9           1
                         :         83 |     0.221239    0.221388   0.0829933  0.00715426     42194.6           2
                         :         84 |     0.221562     0.22279   0.0829996  0.00714717     42187.2           3
                         :         85 Minimum Test error found - save the configuration 
                         :         85 |     0.221745    0.217594   0.0831053  0.00722165     42169.8           0
                         :         86 |     0.220238    0.222261    0.083032  0.00714864       42170           1
                         :         87 |     0.219695    0.220617   0.0830487   0.0071486     42160.7           2
                         :         88 |     0.219196    0.220032   0.0830709  0.00715208     42150.3           3
                         :         89 |     0.218415    0.223763   0.0829959  0.00713299     42181.4           4
                         :         90 |     0.219251    0.220836   0.0829028  0.00713957     42236.9           5
                         :         91 |      0.21768    0.224697    0.082934  0.00714589       42223           6
                         :         92 |     0.218079    0.218518   0.0829843  0.00714332     42193.5           7
                         :         93 |     0.217562    0.221728   0.0829846  0.00713893     42190.9           8
                         :         94 |     0.217022    0.219535   0.0829531  0.00715018     42214.8           9
                         :         95 Minimum Test error found - save the configuration 
                         :         95 |     0.216532     0.21571   0.0830785  0.00721915     42183.3           0
                         :         96 |     0.216123    0.219457   0.0829555  0.00714993     42213.3           1
                         :         97 |      0.21768    0.219175   0.0830382  0.00714695     42165.6           2
                         :         98 |     0.215634    0.219671   0.0831436  0.00717266     42121.4           3
                         :         99 |     0.215948    0.217179    0.083102  0.00716041     42137.6           4
                         :        100 |     0.214562    0.219249   0.0833833  0.00718809     41997.4           5
                         :        101 |     0.214424    0.222368   0.0831111  0.00717163     42138.8           6
                         :        102 |     0.214229    0.218401    0.083162  0.00715833     42103.2           7
                         :        103 Minimum Test error found - save the configuration 
                         :        103 |      0.21343    0.215508   0.0832821  0.00725921     42092.6           0
                         :        104 |     0.214148    0.217528   0.0831982  0.00716541     42087.1           1
                         :        105 |     0.213384     0.22128   0.0832018  0.00716546     42085.2           2
                         :        106 |     0.212704    0.219175   0.0832037  0.00716257     42082.5           3
                         :        107 |     0.212866    0.215878   0.0831667  0.00715801     42100.5           4
                         :        108 |     0.211734    0.220798   0.0831272  0.00716357     42125.4           5
                         :        109 |     0.213727    0.218616   0.0831192  0.00715988     42127.8           6
                         :        110 |     0.213498    0.217522   0.0831603  0.00716451     42107.6           7
                         :        111 |     0.212377    0.218079   0.0831951  0.00716542     42088.8           8
                         :        112 |     0.212311    0.221537   0.0831958  0.00716026     42085.6           9
                         :        113 |     0.211469    0.215935   0.0831902  0.00717316     42095.8          10
                         :        114 |     0.211842    0.217605   0.0832007  0.00716814     42087.2          11
                         : 
                         : Elapsed time for training with 4000 events: 9.39 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of DL_CPU on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.107 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_DL_CPU.class.C
Factory                  : Training finished
                         : 
                         : Ranking input variables (method specific)...
BDTG                     : Ranking result (top variable is best ranked)
                         : --------------------------------------
                         : Rank : Variable  : Variable Importance
                         : --------------------------------------
                         :    1 : var4      : 3.117e-01
                         :    2 : var1      : 2.504e-01
                         :    3 : var2      : 2.430e-01
                         :    4 : var3      : 1.949e-01
                         : --------------------------------------
MLP                      : Ranking result (top variable is best ranked)
                         : -----------------------------
                         : Rank : Variable  : Importance
                         : -----------------------------
                         :    1 : var4      : 6.076e+01
                         :    2 : var2      : 4.824e+01
                         :    3 : var1      : 2.116e+01
                         :    4 : var3      : 1.692e+01
                         : -----------------------------
PDEFoam                  : Ranking result (top variable is best ranked)
                         : --------------------------------------
                         : Rank : Variable  : Variable Importance
                         : --------------------------------------
                         :    1 : var4      : 2.991e-01
                         :    2 : var1      : 2.930e-01
                         :    3 : var3      : 2.365e-01
                         :    4 : var2      : 1.714e-01
                         : --------------------------------------
                         : No variable ranking supplied by classifier: DL_CPU
TH1.Print Name  = TrainingHistory_DL_CPU_trainingError, Entries= 0, Total sum= 30.4262
TH1.Print Name  = TrainingHistory_DL_CPU_valError, Entries= 0, Total sum= 29.8922
Factory                  : === Destroy and recreate all methods via weight files for testing ===
                         : 
                         : Reading weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml
                         : Reading weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml
MLP                      : Building Network. 
                         : Initializing weights
                         : Reading weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml
                         : Read foams from file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root
                         : Reading weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml

Evaluate all mvas using the set of test events

In [8]:
factory->TestAllMethods();
Factory                  : Test all methods
Factory                  : Test method: BDTG for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of BDTG on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 1.01 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: MLP for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of MLP on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.00953 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: PDEFoam for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of PDEFoam on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.123 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: DL_CPU for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of DL_CPU on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.106 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...

Evaluate and compare performance of all configured mvas

In [9]:
factory->EvaluateAllMethods();
Factory                  : Evaluate all methods
                         : Evaluate multiclass classification method: BDTG
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_BDTG           : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: MLP
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_MLP            : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: PDEFoam
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_PDEFoam        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: DL_CPU
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.077270    0.29534   [    -1.1155     1.0914 ]
                         :     var2:   0.068045    0.27981   [    -1.0016     1.0000 ]
                         :     var3:   0.027548    0.24565   [   -0.80459    0.85902 ]
                         :     var4:  -0.034157    0.25816   [    -1.0000    0.83435 ]
                         : -----------------------------------------------------------
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.077270    0.29534   [    -1.1155     1.0914 ]
                         :     var2:   0.068045    0.27981   [    -1.0016     1.0000 ]
                         :     var3:   0.027548    0.24565   [   -0.80459    0.85902 ]
                         :     var4:  -0.034157    0.25816   [    -1.0000    0.83435 ]
                         : -----------------------------------------------------------
                         : 
                         : 1-vs-rest performance metrics per class
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : Considers the listed class as signal and the other classes
                         : as background, reporting the resulting binary performance.
                         : A score of 0.820 (0.850) means 0.820 was acheived on the
                         : test set and 0.850 on the training set.
                         : 
                         : Dataset        MVA Method     ROC AUC        Sig [email protected]=0.01 Sig [email protected]=0.10 Sig [email protected]=0.30 
                         : Name:          / Class:       test  (train)  test  (train)  test  (train)  test  (train)  
                         : 
                         : dataset        BDTG           
                         : ------------------------------
                         :                Signal         0.968 (0.978)  0.508 (0.605)  0.914 (0.945)  0.990 (0.996)  
                         :                bg0            0.910 (0.931)  0.256 (0.288)  0.737 (0.791)  0.922 (0.956)  
                         :                bg1            0.947 (0.954)  0.437 (0.511)  0.833 (0.856)  0.971 (0.971)  
                         :                bg2            0.978 (0.982)  0.585 (0.678)  0.951 (0.956)  0.999 (0.996)  
                         : 
                         : dataset        MLP            
                         : ------------------------------
                         :                Signal         0.970 (0.975)  0.596 (0.632)  0.933 (0.938)  0.988 (0.993)  
                         :                bg0            0.929 (0.934)  0.303 (0.298)  0.787 (0.793)  0.949 (0.961)  
                         :                bg1            0.962 (0.967)  0.467 (0.553)  0.881 (0.906)  0.985 (0.992)  
                         :                bg2            0.975 (0.979)  0.629 (0.699)  0.929 (0.940)  0.998 (0.998)  
                         : 
                         : dataset        PDEFoam        
                         : ------------------------------
                         :                Signal         0.916 (0.928)  0.294 (0.382)  0.744 (0.782)  0.932 (0.952)  
                         :                bg0            0.837 (0.848)  0.109 (0.147)  0.519 (0.543)  0.833 (0.851)  
                         :                bg1            0.890 (0.902)  0.190 (0.226)  0.606 (0.646)  0.923 (0.929)  
                         :                bg2            0.967 (0.972)  0.510 (0.527)  0.900 (0.926)  0.993 (0.998)  
                         : 
                         : dataset        DL_CPU         
                         : ------------------------------
                         :                Signal         0.976 (0.976)  0.618 (0.625)  0.940 (0.940)  0.994 (0.997)  
                         :                bg0            0.928 (0.931)  0.310 (0.307)  0.777 (0.796)  0.948 (0.951)  
                         :                bg1            0.964 (0.965)  0.511 (0.541)  0.878 (0.893)  0.991 (0.994)  
                         :                bg2            0.975 (0.976)  0.666 (0.684)  0.911 (0.907)  0.999 (0.999)  
                         : 
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : 
                         : Confusion matrices for all methods
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : Does a binary comparison between the two classes given by a 
                         : particular row-column combination. In each case, the class 
                         : given by the row is considered signal while the class given 
                         : by the column index is considered background.
                         : 
                         : === Showing confusion matrix for method : BDTG           
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.497 (0.373)  0.710 (0.693)  0.680 (0.574) 
                         :  bg0            0.271 (0.184)  -              0.239 (0.145)  0.705 (0.667) 
                         :  bg1            0.855 (0.766)  0.369 (0.222)  -              0.587 (0.578) 
                         :  bg2            0.714 (0.585)  0.705 (0.581)  0.648 (0.601)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.911 (0.853)  0.991 (0.981)  0.945 (0.913) 
                         :  bg0            0.833 (0.774)  -              0.654 (0.582)  0.930 (0.901) 
                         :  bg1            0.971 (0.980)  0.716 (0.681)  -              0.871 (0.862) 
                         :  bg2            0.976 (0.951)  0.971 (0.973)  0.936 (0.941)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.957)  0.999 (1.000)  0.998 (0.997) 
                         :  bg0            0.965 (0.926)  -              0.874 (0.835)  0.991 (0.976) 
                         :  bg1            1.000 (0.999)  0.916 (0.894)  -              0.988 (0.985) 
                         :  bg2            0.999 (0.999)  0.997 (0.999)  0.996 (0.997)  -             
                         : 
                         : === Showing confusion matrix for method : MLP            
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.465 (0.490)  0.974 (0.953)  0.632 (0.498) 
                         :  bg0            0.320 (0.269)  -              0.224 (0.250)  0.655 (0.627) 
                         :  bg1            0.943 (0.920)  0.341 (0.275)  -              0.632 (0.687) 
                         :  bg2            0.665 (0.642)  0.697 (0.680)  0.706 (0.598)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.865 (0.854)  0.996 (0.994)  0.908 (0.907) 
                         :  bg0            0.784 (0.776)  -              0.666 (0.655)  0.919 (0.895) 
                         :  bg1            0.998 (0.998)  0.791 (0.785)  -              0.912 (0.902) 
                         :  bg2            0.943 (0.903)  0.946 (0.939)  0.924 (0.928)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.964)  0.997 (0.997)  0.993 (0.986) 
                         :  bg0            0.952 (0.924)  -              0.936 (0.928)  0.992 (0.990) 
                         :  bg1            1.000 (1.000)  0.945 (0.936)  -              0.998 (0.995) 
                         :  bg2            0.994 (0.985)  0.998 (0.998)  0.998 (0.998)  -             
                         : 
                         : === Showing confusion matrix for method : PDEFoam        
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.289 (0.233)  0.467 (0.436)  0.421 (0.332) 
                         :  bg0            0.100 (0.045)  -              0.132 (0.116)  0.540 (0.313) 
                         :  bg1            0.209 (0.434)  0.153 (0.092)  -              0.347 (0.323) 
                         :  bg2            0.560 (0.552)  0.445 (0.424)  0.501 (0.506)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.665 (0.640)  0.854 (0.822)  0.807 (0.790) 
                         :  bg0            0.538 (0.520)  -              0.415 (0.374)  0.843 (0.833) 
                         :  bg1            0.885 (0.886)  0.542 (0.491)  -              0.728 (0.646) 
                         :  bg2            0.928 (0.890)  0.956 (0.959)  0.847 (0.895)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.898 (0.878)  0.971 (0.950)  0.982 (0.975) 
                         :  bg0            0.828 (0.810)  -              0.696 (0.676)  0.954 (0.951) 
                         :  bg1            0.951 (0.966)  0.803 (0.745)  -              0.958 (0.966) 
                         :  bg2            0.998 (0.991)  0.998 (0.996)  0.998 (0.993)  -             
                         : 
                         : === Showing confusion matrix for method : DL_CPU         
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.459 (0.462)  0.903 (0.923)  0.669 (0.649) 
                         :  bg0            0.317 (0.311)  -              0.201 (0.175)  0.502 (0.511) 
                         :  bg1            0.918 (0.911)  0.345 (0.290)  -              0.610 (0.650) 
                         :  bg2            0.638 (0.625)  0.695 (0.667)  0.708 (0.703)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.862 (0.874)  0.995 (0.994)  0.941 (0.934) 
                         :  bg0            0.796 (0.778)  -              0.700 (0.710)  0.874 (0.856) 
                         :  bg1            0.997 (0.994)  0.813 (0.773)  -              0.859 (0.865) 
                         :  bg2            0.917 (0.881)  0.911 (0.920)  0.900 (0.915)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.972 (0.971)  0.999 (1.000)  0.998 (1.000) 
                         :  bg0            0.966 (0.951)  -              0.915 (0.911)  0.975 (0.976) 
                         :  bg1            1.000 (0.999)  0.950 (0.951)  -              0.998 (0.995) 
                         :  bg2            0.999 (0.998)  0.999 (0.999)  0.999 (0.999)  -             
                         : 
                         : -------------------------------------------------------------------------------------------------------
                         : 
Dataset:dataset          : Created tree 'TestTree' with 4000 events
                         : 
Dataset:dataset          : Created tree 'TrainTree' with 4000 events
                         : 
Factory                  : Thank you for using TMVA!
                         : For citation information, please visit: http://tmva.sf.net/citeTMVA.html

Save the output

In [10]:
outputFile->Close();

std::cout << "==> Wrote root file: " << outputFile->GetName() << std::endl;
std::cout << "==> TMVAMulticlass is done!" << std::endl;

delete factory;
delete dataloader;
==> Wrote root file: TMVAMulticlass.root
==> TMVAMulticlass is done!

Launch the gui for the root macros

In [11]:
if (!gROOT->IsBatch()) TMVAMultiClassGui( outfileName );