T M V A Multiclass

This macro provides a simple example for the training and testing of the TMVA multiclass classification

  • Project : TMVA - a Root-integrated toolkit for multivariate data analysis
  • Package : TMVA
  • Root Macro: TMVAMulticlass

Author: Andreas Hoecker
This notebook tutorial was automatically generated with ROOTBOOK-izer from the macro found in the ROOT repository on Saturday, September 18, 2021 at 09:47 AM.

In [1]:
%%cpp -d
#include <cstdlib>
#include <iostream>
#include <map>
#include <string>

#include "TFile.h"
#include "TTree.h"
#include "TString.h"
#include "TSystem.h"
#include "TROOT.h"


#include "TMVA/Tools.h"
#include "TMVA/Factory.h"
#include "TMVA/DataLoader.h"
#include "TMVA/TMVAMultiClassGui.h"


using namespace TMVA;

Arguments are defined.

In [2]:
TString myMethodList = "";

This loads the library

In [3]:
TMVA::Tools::Instance();

To get access to the gui and all tmva macros

 TString tmva_dir(TString(gRootDir) + "/tmva");
 if(gSystem->Getenv("TMVASYS"))
    tmva_dir = TString(gSystem->Getenv("TMVASYS"));
 gROOT->SetMacroPath(tmva_dir + "/test/:" + gROOT->GetMacroPath() );
 gROOT->ProcessLine(".L TMVAMultiClassGui.C");

Default MVA methods to be trained + tested

In [4]:
std::map<std::string,int> Use;
Use["MLP"]             = 1;
Use["BDTG"]            = 1;
#ifdef R__HAS_TMVAGPU
Use["DL_CPU"]          = 1;
Use["DL_GPU"]          = 1;
#else
Use["DL_CPU"]          = 1;
Use["DL_GPU"]          = 0;
#endif
Use["FDA_GA"]          = 0;
Use["PDEFoam"]         = 1;

In [5]:
std::cout << std::endl;
std::cout << "==> Start TMVAMulticlass" << std::endl;

if (myMethodList != "") {
   for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) it->second = 0;

   std::vector<TString> mlist = TMVA::gTools().SplitString( myMethodList, ',' );
   for (UInt_t i=0; i<mlist.size(); i++) {
      std::string regMethod(mlist[i]);

      if (Use.find(regMethod) == Use.end()) {
         std::cout << "Method \"" << regMethod << "\" not known in TMVA under this name. Choose among the following:" << std::endl;
         for (std::map<std::string,int>::iterator it = Use.begin(); it != Use.end(); it++) std::cout << it->first << " ";
         std::cout << std::endl;
         return;
      }
      Use[regMethod] = 1;
   }
}
==> Start TMVAMulticlass

Create a new root output file.

In [6]:
TString outfileName = "TMVAMulticlass.root";
TFile* outputFile = TFile::Open( outfileName, "RECREATE" );

TMVA::Factory *factory = new TMVA::Factory( "TMVAMulticlass", outputFile,
                                            "!V:!Silent:Color:!DrawProgressBar:Transformations=I;D;P;G,D:AnalysisType=multiclass" );
TMVA::DataLoader *dataloader=new TMVA::DataLoader("dataset");

dataloader->AddVariable( "var1", 'F' );
dataloader->AddVariable( "var2", "Variable 2", "", 'F' );
dataloader->AddVariable( "var3", "Variable 3", "units", 'F' );
dataloader->AddVariable( "var4", "Variable 4", "units", 'F' );

TFile *input(0);
TString fname = "./tmva_example_multiclass.root";
if (!gSystem->AccessPathName( fname )) {
   input = TFile::Open( fname ); // check if file in local directory exists
}
else {
   TFile::SetCacheFileDir(".");
   input = TFile::Open("http://root.cern.ch/files/tmva_multiclass_example.root", "CACHEREAD");
}
if (!input) {
   std::cout << "ERROR: could not open data file" << std::endl;
   exit(1);
}
std::cout << "--- TMVAMulticlass: Using input file: " << input->GetName() << std::endl;

TTree *signalTree  = (TTree*)input->Get("TreeS");
TTree *background0 = (TTree*)input->Get("TreeB0");
TTree *background1 = (TTree*)input->Get("TreeB1");
TTree *background2 = (TTree*)input->Get("TreeB2");

gROOT->cd( outfileName+TString(":/") );
dataloader->AddTree    (signalTree,"Signal");
dataloader->AddTree    (background0,"bg0");
dataloader->AddTree    (background1,"bg1");
dataloader->AddTree    (background2,"bg2");

dataloader->PrepareTrainingAndTestTree( "", "SplitMode=Random:NormMode=NumEvents:!V" );

if (Use["BDTG"]) // gradient boosted decision trees
   factory->BookMethod( dataloader,  TMVA::Types::kBDT, "BDTG", "!H:!V:NTrees=1000:BoostType=Grad:Shrinkage=0.10:UseBaggedBoost:BaggedSampleFraction=0.50:nCuts=20:MaxDepth=2");
if (Use["MLP"]) // neural network
   factory->BookMethod( dataloader,  TMVA::Types::kMLP, "MLP", "!H:!V:NeuronType=tanh:NCycles=1000:HiddenLayers=N+5,5:TestRate=5:EstimatorType=MSE");
if (Use["FDA_GA"]) // functional discriminant with GA minimizer
   factory->BookMethod( dataloader,  TMVA::Types::kFDA, "FDA_GA", "H:!V:Formula=(0)+(1)*x0+(2)*x1+(3)*x2+(4)*x3:ParRanges=(-1,1);(-10,10);(-10,10);(-10,10);(-10,10):FitMethod=GA:PopSize=300:Cycles=3:Steps=20:Trim=True:SaveBestGen=1" );
if (Use["PDEFoam"]) // PDE-Foam approach
   factory->BookMethod( dataloader,  TMVA::Types::kPDEFoam, "PDEFoam", "!H:!V:TailCut=0.001:VolFrac=0.0666:nActiveCells=500:nSampl=2000:nBin=5:Nmin=100:Kernel=None:Compress=T" );


if (Use["DL_CPU"]) {
   TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
   TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
                                  "TestRepetitions=1,ConvergenceSteps=10,BatchSize=100");
   TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
                     "WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
   nnOptions.Append(":");
   nnOptions.Append(layoutString);
   nnOptions.Append(":");
   nnOptions.Append(trainingStrategyString);
   factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_CPU", nnOptions);
}
if (Use["DL_GPU"]) {
   TString layoutString("Layout=TANH|100,TANH|50,TANH|10,LINEAR");
   TString trainingStrategyString("TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,"
                                  "TestRepetitions=1,ConvergenceSteps=10,BatchSize=100");
   TString nnOptions("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:"
                     "WeightInitialization=XAVIERUNIFORM:Architecture=GPU");
   nnOptions.Append(":");
   nnOptions.Append(layoutString);
   nnOptions.Append(":");
   nnOptions.Append(trainingStrategyString);
   factory->BookMethod(dataloader, TMVA::Types::kDL, "DL_GPU", nnOptions);
}
--- TMVAMulticlass: Using input file: ./files/tmva_multiclass_example.root
DataSetInfo              : [dataset] : Added class "Signal"
                         : Add Tree TreeS of type Signal with 2000 events
DataSetInfo              : [dataset] : Added class "bg0"
                         : Add Tree TreeB0 of type bg0 with 2000 events
DataSetInfo              : [dataset] : Added class "bg1"
                         : Add Tree TreeB1 of type bg1 with 2000 events
DataSetInfo              : [dataset] : Added class "bg2"
                         : Add Tree TreeB2 of type bg2 with 2000 events
                         : Dataset[dataset] : Class index : 0  name : Signal
                         : Dataset[dataset] : Class index : 1  name : bg0
                         : Dataset[dataset] : Class index : 2  name : bg1
                         : Dataset[dataset] : Class index : 3  name : bg2
Factory                  : Booking method: BDTG
                         : 
                         : the option NegWeightTreatment=InverseBoostNegWeights does not exist for BoostType=Grad
                         : --> change to new default NegWeightTreatment=Pray
                         : Rebuilding Dataset dataset
                         : Building event vectors for type 2 Signal
                         : Dataset[dataset] :  create input formulas for tree TreeS
                         : Building event vectors for type 2 bg0
                         : Dataset[dataset] :  create input formulas for tree TreeB0
                         : Building event vectors for type 2 bg1
                         : Dataset[dataset] :  create input formulas for tree TreeB1
                         : Building event vectors for type 2 bg2
                         : Dataset[dataset] :  create input formulas for tree TreeB2
DataSetFactory           : [dataset] : Number of events in input trees
                         : 
                         : 
                         : 
                         : 
                         : Number of training and testing events
                         : ---------------------------------------------------------------------------
                         : Signal -- training events            : 1000
                         : Signal -- testing events             : 1000
                         : Signal -- training and testing events: 2000
                         : bg0    -- training events            : 1000
                         : bg0    -- testing events             : 1000
                         : bg0    -- training and testing events: 2000
                         : bg1    -- training events            : 1000
                         : bg1    -- testing events             : 1000
                         : bg1    -- training and testing events: 2000
                         : bg2    -- training events            : 1000
                         : bg2    -- testing events             : 1000
                         : bg2    -- training and testing events: 2000
                         : 
DataSetInfo              : Correlation matrix (Signal):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.397  +0.623  +0.832
                         :    var2:  +0.397  +1.000  +0.716  +0.737
                         :    var3:  +0.623  +0.716  +1.000  +0.859
                         :    var4:  +0.832  +0.737  +0.859  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg0):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.365  +0.592  +0.811
                         :    var2:  +0.365  +1.000  +0.708  +0.740
                         :    var3:  +0.592  +0.708  +1.000  +0.859
                         :    var4:  +0.811  +0.740  +0.859  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg1):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  +0.407  +0.610  +0.834
                         :    var2:  +0.407  +1.000  +0.710  +0.741
                         :    var3:  +0.610  +0.710  +1.000  +0.851
                         :    var4:  +0.834  +0.741  +0.851  +1.000
                         : ----------------------------------------
DataSetInfo              : Correlation matrix (bg2):
                         : ----------------------------------------
                         :             var1    var2    var3    var4
                         :    var1:  +1.000  -0.647  -0.016  -0.013
                         :    var2:  -0.647  +1.000  +0.015  +0.002
                         :    var3:  -0.016  +0.015  +1.000  -0.024
                         :    var4:  -0.013  +0.002  -0.024  +1.000
                         : ----------------------------------------
DataSetFactory           : [dataset] :  
                         : 
Factory                  : Booking method: MLP
                         : 
MLP                      : Building Network. 
                         : Initializing weights
Factory                  : Booking method: PDEFoam
                         : 
Factory                  : Booking method: DL_CPU
                         : 
                         : Parsing option string: 
                         : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100"
                         : The following options are set:
                         : - By User:
                         :     <none>
                         : - Default:
                         :     Boost_num: "0" [Number of times the classifier will be boosted]
                         : Parsing option string: 
                         : ... "!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=N:WeightInitialization=XAVIERUNIFORM:Architecture=GPU:Layout=TANH|100,TANH|50,TANH|10,LINEAR:TrainingStrategy=Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100"
                         : The following options are set:
                         : - By User:
                         :     V: "True" [Verbose output (short form of "VerbosityLevel" below - overrides the latter one)]
                         :     VarTransform: "N" [List of variable transformations performed before training, e.g., "D_Background,P_Signal,G,N_AllClasses" for: "Decorrelation, PCA-transformation, Gaussianisation, Normalisation, each for the given class of events ('AllClasses' denotes all events of all classes, if no class indication is given, 'All' is assumed)"]
                         :     H: "False" [Print method-specific help message]
                         :     Layout: "TANH|100,TANH|50,TANH|10,LINEAR" [Layout of the network.]
                         :     ErrorStrategy: "CROSSENTROPY" [Loss function: Mean squared error (regression) or cross entropy (binary classification).]
                         :     WeightInitialization: "XAVIERUNIFORM" [Weight initialization strategy]
                         :     Architecture: "GPU" [Which architecture to perform the training on.]
                         :     TrainingStrategy: "Optimizer=ADAM,LearningRate=1e-3,TestRepetitions=1,ConvergenceSteps=10,BatchSize=100" [Defines the training strategies.]
                         : - Default:
                         :     VerbosityLevel: "Default" [Verbosity level]
                         :     CreateMVAPdfs: "False" [Create PDFs for classifier outputs (signal and background)]
                         :     IgnoreNegWeightsInTraining: "False" [Events with negative weights are ignored in the training (but are included for testing and performance evaluation)]
                         :     InputLayout: "0|0|0" [The Layout of the input]
                         :     BatchLayout: "0|0|0" [The Layout of the batch]
                         :     RandomSeed: "0" [Random seed used for weight initialization and batch shuffling]
                         :     ValidationSize: "20%" [Part of the training data to use for validation. Specify as 0.2 or 20% to use a fifth of the data set as validation set. Specify as 100 to use exactly 100 events. (Default: 20%)]
DL_CPU                   : [dataset] : Create Transformation "N" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
<ERROR>                         : CUDA backend not enabled. Please make sure you have CUDA installed and it was successfully detected by CMAKE by using -Dtmva-gpu=On  
                         : Will now use instead the CPU architecture !
                         : Will now use the CPU architecture with BLAS and IMT support !
[TFile::Cp] Total 0.12 MB	|====================| 100.00 % [48.2 MB/s]
Info in <TFile::OpenFromCache>: using local cache copy of http://root.cern.ch/files/tmva_multiclass_example.root [./files/tmva_multiclass_example.root]

Train mvas using the set of training events

In [7]:
factory->TrainAllMethods();
Factory                  : Train all methods
Factory                  : [dataset] : Create Transformation "I" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "D" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "P" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "G" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
Factory                  : [dataset] : Create Transformation "D" with events from all classes.
                         : 
                         : Transformation, Variable selection : 
                         : Input : variable 'var1' <---> Output : variable 'var1'
                         : Input : variable 'var2' <---> Output : variable 'var2'
                         : Input : variable 'var3' <---> Output : variable 'var3'
                         : Input : variable 'var4' <---> Output : variable 'var4'
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.047647     1.0025   [    -3.6592     3.2645 ]
                         :     var2:    0.32647     1.0646   [    -3.6891     3.7877 ]
                         :     var3:    0.11493     1.1230   [    -4.5727     4.5640 ]
                         :     var4:  -0.076531     1.2652   [    -4.8486     5.0412 ]
                         : -----------------------------------------------------------
                         : Preparing the Decorrelation transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.082544     1.0000   [    -3.6274     3.1017 ]
                         :     var2:    0.36715     1.0000   [    -3.3020     3.4950 ]
                         :     var3:   0.066865     1.0000   [    -2.9882     3.3086 ]
                         :     var4:   -0.20593     1.0000   [    -3.3088     2.8423 ]
                         : -----------------------------------------------------------
                         : Preparing the Principle Component (PCA) transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1: 5.7502e-10     1.8064   [    -8.0344     7.8312 ]
                         :     var2:-1.6078e-11    0.90130   [    -2.6765     2.7523 ]
                         :     var3: 3.0841e-10    0.73386   [    -2.6572     2.2255 ]
                         :     var4:-2.6886e-10    0.62168   [    -1.7384     2.2297 ]
                         : -----------------------------------------------------------
                         : Preparing the Gaussian transformation...
                         : Preparing the Decorrelation transformation...
TFHandler_Factory        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.013510     1.0000   [    -2.6520     6.2074 ]
                         :     var2:  0.0096839     1.0000   [    -2.8402     6.3073 ]
                         :     var3:   0.010397     1.0000   [    -3.0251     5.8860 ]
                         :     var4:  0.0053980     1.0000   [    -3.0998     5.7078 ]
                         : -----------------------------------------------------------
                         : Ranking input variables (method unspecific)...
Factory                  : Train method: BDTG for Multiclass classification
                         : 
                         : Training 1000 Decision Trees ... patience please
                         : Elapsed time for training with 4000 events: 5.45 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of BDTG on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 1.9 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_BDTG.class.C
                         : TMVAMulticlass.root:/dataset/Method_BDT/BDTG
Factory                  : Training finished
                         : 
Factory                  : Train method: MLP for Multiclass classification
                         : 
                         : Training Network
                         : 
                         : Elapsed time for training with 4000 events: 26.2 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of MLP on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.00948 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_MLP.class.C
                         : Write special histos to file: TMVAMulticlass.root:/dataset/Method_MLP/MLP
Factory                  : Training finished
                         : 
Factory                  : Train method: PDEFoam for Multiclass classification
                         : 
                         : Build up multiclass foam 0
                         : Elapsed time: 0.665 sec                                 
                         : Build up multiclass foam 1
                         : Elapsed time: 0.667 sec                                 
                         : Build up multiclass foam 2
                         : Elapsed time: 0.679 sec                                 
                         : Build up multiclass foam 3
                         : Elapsed time: 0.474 sec                                 
                         : Elapsed time for training with 4000 events: 2.66 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of PDEFoam on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.12 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml
                         : writing foam MultiClassFoam0 to file
                         : writing foam MultiClassFoam1 to file
                         : writing foam MultiClassFoam2 to file
                         : writing foam MultiClassFoam3 to file
                         : Foams written to file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root
                         : Creating standalone class: dataset/weights/TMVAMulticlass_PDEFoam.class.C
Factory                  : Training finished
                         : 
Factory                  : Train method: DL_CPU for Multiclass classification
                         : 
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070769    0.28960   [    -1.0000     1.0000 ]
                         :     var2:   0.074130    0.28477   [    -1.0000     1.0000 ]
                         :     var3:   0.026106    0.24582   [    -1.0000     1.0000 ]
                         :     var4:  -0.034951    0.25587   [    -1.0000     1.0000 ]
                         : -----------------------------------------------------------
                         : Start of deep neural network training on CPU using MT,  nthreads = 1
                         : 
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070769    0.28960   [    -1.0000     1.0000 ]
                         :     var2:   0.074130    0.28477   [    -1.0000     1.0000 ]
                         :     var3:   0.026106    0.24582   [    -1.0000     1.0000 ]
                         :     var4:  -0.034951    0.25587   [    -1.0000     1.0000 ]
                         : -----------------------------------------------------------
                         : *****   Deep Learning Network *****
DEEP NEURAL NETWORK:   Depth = 4  Input = ( 1, 1, 4 )  Batch size = 100  Loss function = C
	Layer 0	 DENSE Layer: 	 ( Input =     4 , Width =   100 ) 	Output = (  1 ,   100 ,   100 ) 	 Activation Function = Tanh
	Layer 1	 DENSE Layer: 	 ( Input =   100 , Width =    50 ) 	Output = (  1 ,   100 ,    50 ) 	 Activation Function = Tanh
	Layer 2	 DENSE Layer: 	 ( Input =    50 , Width =    10 ) 	Output = (  1 ,   100 ,    10 ) 	 Activation Function = Tanh
	Layer 3	 DENSE Layer: 	 ( Input =    10 , Width =     4 ) 	Output = (  1 ,   100 ,     4 ) 	 Activation Function = Identity
                         : Using 3200 events for training and 800 for testing
                         : Compute initial loss  on the validation data 
                         : Training phase 1 of 1:  Optimizer ADAM (beta1=0.9,beta2=0.999,eps=1e-07) Learning rate = 0.001 regularization 0 minimum error = 0.703165
                         : --------------------------------------------------------------
                         :      Epoch |   Train Err.   Val. Err.  t(s)/epoch   t(s)/Loss   nEvents/s Conv. Steps
                         : --------------------------------------------------------------
                         :    Start epoch iteration ...
                         :          1 Minimum Test error found - save the configuration 
                         :          1 |     0.616101    0.542549   0.0766319  0.00665605     45730.1           0
                         :          2 Minimum Test error found - save the configuration 
                         :          2 |     0.500584      0.4736   0.0773254  0.00660486     45248.5           0
                         :          3 Minimum Test error found - save the configuration 
                         :          3 |     0.456553     0.43675   0.0780791  0.00668169     44819.5           0
                         :          4 Minimum Test error found - save the configuration 
                         :          4 |     0.428175    0.411964   0.0785608  0.00683203     44612.5           0
                         :          5 Minimum Test error found - save the configuration 
                         :          5 |     0.407071    0.393645   0.0789985  0.00679391     44318.5           0
                         :          6 Minimum Test error found - save the configuration 
                         :          6 |     0.392449    0.378976   0.0852716  0.00682557     40792.4           0
                         :          7 Minimum Test error found - save the configuration 
                         :          7 |     0.381351     0.36887    0.079342   0.0068103     44118.6           0
                         :          8 Minimum Test error found - save the configuration 
                         :          8 |      0.37229    0.361132   0.0795946  0.00685362     43991.7           0
                         :          9 Minimum Test error found - save the configuration 
                         :          9 |     0.364402    0.352145   0.0799276  0.00685969     43794.9           0
                         :         10 Minimum Test error found - save the configuration 
                         :         10 |     0.358202    0.344168   0.0798473  0.00686514     43846.3           0
                         :         11 Minimum Test error found - save the configuration 
                         :         11 |      0.35149    0.340518    0.080141  0.00691172     43698.4           0
                         :         12 Minimum Test error found - save the configuration 
                         :         12 |     0.345869    0.331876   0.0803243  0.00691031     43588.4           0
                         :         13 Minimum Test error found - save the configuration 
                         :         13 |     0.340532    0.325867   0.0802619  0.00691144     43626.2           0
                         :         14 Minimum Test error found - save the configuration 
                         :         14 |     0.335169    0.321031   0.0804125  0.00692398     43544.2           0
                         :         15 Minimum Test error found - save the configuration 
                         :         15 |     0.329328    0.318407   0.0866737  0.00698499     40156.3           0
                         :         16 Minimum Test error found - save the configuration 
                         :         16 |     0.324636    0.313244   0.0806444  0.00694738       43421           0
                         :         17 Minimum Test error found - save the configuration 
                         :         17 |     0.319159    0.307254    0.080817  0.00696066     43327.3           0
                         :         18 Minimum Test error found - save the configuration 
                         :         18 |     0.314565    0.300984   0.0809585  0.00699001     43261.6           0
                         :         19 Minimum Test error found - save the configuration 
                         :         19 |     0.310225    0.298472   0.0810976     0.00702     43197.9           0
                         :         20 Minimum Test error found - save the configuration 
                         :         20 |     0.306105    0.295746   0.0810989  0.00700787     43190.1           0
                         :         21 Minimum Test error found - save the configuration 
                         :         21 |     0.301994    0.293544   0.0812579  0.00700438     43095.6           0
                         :         22 Minimum Test error found - save the configuration 
                         :         22 |     0.299356    0.284483   0.0812169  0.00701374     43124.8           0
                         :         23 |     0.295604     0.28587   0.0812559  0.00695805     43069.9           1
                         :         24 Minimum Test error found - save the configuration 
                         :         24 |     0.292636    0.282722   0.0814458   0.0070302     43001.7           0
                         :         25 Minimum Test error found - save the configuration 
                         :         25 |     0.289934    0.279995   0.0814538  0.00703226     42998.3           0
                         :         26 Minimum Test error found - save the configuration 
                         :         26 |     0.287411    0.277728   0.0814772  0.00705007       42995           0
                         :         27 Minimum Test error found - save the configuration 
                         :         27 |     0.284857    0.273324   0.0876383  0.00709921     39732.2           0
                         :         28 |     0.282196    0.275698   0.0816781  0.00701364     42858.4           1
                         :         29 Minimum Test error found - save the configuration 
                         :         29 |     0.280641    0.270038   0.0818648  0.00710884       42806           0
                         :         30 Minimum Test error found - save the configuration 
                         :         30 |     0.279766    0.267812   0.0818584  0.00707635       42791           0
                         :         31 |     0.277212    0.269036   0.0817145  0.00701093       42836           1
                         :         32 Minimum Test error found - save the configuration 
                         :         32 |     0.277195    0.266062   0.0818292  0.00707838     42808.9           0
                         :         33 Minimum Test error found - save the configuration 
                         :         33 |     0.274009    0.262349   0.0819952  0.00708886       42720           0
                         :         34 |      0.27296     0.26661   0.0818167  0.00701069     42777.3           1
                         :         35 Minimum Test error found - save the configuration 
                         :         35 |     0.270253    0.262178   0.0819794  0.00710191     42736.5           0
                         :         36 Minimum Test error found - save the configuration 
                         :         36 |     0.268804    0.258282   0.0881453  0.00716953       39518           0
                         :         37 Minimum Test error found - save the configuration 
                         :         37 |     0.267008    0.257236   0.0821145  0.00709496     42655.5           0
                         :         38 |     0.264929     0.25986   0.0820114  0.00703634     42680.8           1
                         :         39 Minimum Test error found - save the configuration 
                         :         39 |     0.264678    0.256982   0.0821484  0.00710621     42642.7           0
                         :         40 |     0.261698    0.259833   0.0821771  0.00705714     42598.5           1
                         :         41 Minimum Test error found - save the configuration 
                         :         41 |     0.260182    0.251149   0.0823948  0.00714203     42523.3           0
                         :         42 |     0.260257    0.251812   0.0822079  0.00706573     42585.9           1
                         :         43 Minimum Test error found - save the configuration 
                         :         43 |     0.257733    0.247039   0.0822462  0.00711781     42593.7           0
                         :         44 |     0.254675    0.251884   0.0822004   0.0070632     42588.8           1
                         :         45 Minimum Test error found - save the configuration 
                         :         45 |     0.252664    0.243869   0.0884329  0.00717432     39380.5           0
                         :         46 |     0.250834     0.24573   0.0824038  0.00707727     42481.7           1
                         :         47 |     0.249732    0.245142   0.0823965  0.00707468     42484.4           2
                         :         48 Minimum Test error found - save the configuration 
                         :         48 |     0.247989    0.241788    0.082539  0.00715732     42450.6           0
                         :         49 Minimum Test error found - save the configuration 
                         :         49 |     0.246703    0.239859   0.0824602  0.00713952       42485           0
                         :         50 Minimum Test error found - save the configuration 
                         :         50 |     0.245366    0.239847    0.082484  0.00713706     42470.2           0
                         :         51 |     0.244161    0.240237   0.0824888  0.00708003     42435.4           1
                         :         52 Minimum Test error found - save the configuration 
                         :         52 |     0.242212     0.23967    0.082578  0.00715935     42429.8           0
                         :         53 Minimum Test error found - save the configuration 
                         :         53 |     0.240339    0.235266   0.0826858  0.00715192     42365.1           0
                         :         54 |     0.239478    0.236351   0.0886598  0.00711076     39240.2           1
                         :         55 |     0.237927    0.236864   0.0826792  0.00710462     42342.3           2
                         :         56 Minimum Test error found - save the configuration 
                         :         56 |     0.238093    0.233225    0.082837  0.00719294     42303.4           0
                         :         57 |     0.236188    0.233498    0.082872  0.00710498     42234.8           1
                         :         58 Minimum Test error found - save the configuration 
                         :         58 |     0.234997    0.232466   0.0829031  0.00718576     42262.5           0
                         :         59 |     0.235229    0.233153   0.0828025  0.00711429     42278.7           1
                         :         60 |       0.2342    0.234874   0.0828503  0.00710894       42249           2
                         :         61 |     0.234474    0.233242   0.0828025  0.00711543     42279.4           3
                         :         62 Minimum Test error found - save the configuration 
                         :         62 |     0.233158    0.230328   0.0828746  0.00718542     42278.2           0
                         :         63 Minimum Test error found - save the configuration 
                         :         63 |      0.23198    0.229548   0.0829513  0.00721131     42249.8           0
                         :         64 Minimum Test error found - save the configuration 
                         :         64 |     0.230482    0.228591   0.0830565  0.00721408     42192.7           0
                         :         65 |     0.230247    0.230726   0.0830545   0.0071501     42158.3           1
                         :         66 Minimum Test error found - save the configuration 
                         :         66 |      0.22877    0.226316   0.0830599  0.00720802     42187.5           0
                         :         67 |     0.228464    0.227464    0.082942  0.00712678     42207.9           1
                         :         68 |     0.227593    0.226991   0.0829932  0.00713101     42181.7           2
                         :         69 |     0.227521    0.226491   0.0830854   0.0071364     42133.5           3
                         :         70 Minimum Test error found - save the configuration 
                         :         70 |     0.226925    0.225402   0.0831152  0.00721069     42158.2           0
                         :         71 |     0.225685    0.229203   0.0830614  0.00713666       42147           1
                         :         72 |     0.224815    0.227542   0.0891417  0.00713719     39022.3           2
                         :         73 |     0.224105    0.226646   0.0830665  0.00715025     42151.7           3
                         :         74 Minimum Test error found - save the configuration 
                         :         74 |     0.223743    0.222961   0.0831367  0.00724121     42163.3           0
                         :         75 |     0.223077    0.224041   0.0831982  0.00713913     42072.6           1
                         :         76 |     0.221996    0.225469   0.0831803   0.0071391     42082.4           2
                         :         77 Minimum Test error found - save the configuration 
                         :         77 |      0.22193    0.222626   0.0833484  0.00724908     42050.3           0
                         :         78 Minimum Test error found - save the configuration 
                         :         78 |     0.221202    0.222228   0.0833171  0.00722916     42056.6           0
                         :         79 Minimum Test error found - save the configuration 
                         :         79 |     0.220937    0.220358   0.0831994   0.0072266     42120.3           0
                         :         80 |     0.220565    0.227102   0.0832941  0.00717257     42038.1           1
                         :         81 |     0.220514    0.220818   0.0892962  0.00715909     38959.2           2
                         :         82 |     0.220064    0.226196    0.083191  0.00715549     42085.6           3
                         :         83 |     0.219966    0.221113    0.083138  0.00714411     42108.6           4
                         :         84 |     0.218075    0.224965   0.0831664  0.00716075     42102.1           5
                         :         85 |     0.217052    0.221316   0.0831304  0.00714056     42110.9           6
                         :         86 |     0.216797    0.222569   0.0832018  0.00715533     42079.5           7
                         :         87 |     0.216677    0.221457   0.0832367  0.00715688     42061.1           8
                         :         88 Minimum Test error found - save the configuration 
                         :         88 |     0.215797    0.215595   0.0833323  0.00726327       42067           0
                         :         89 |     0.215435    0.223603   0.0833594  0.00716736     41999.2           1
                         :         90 |     0.215096    0.217502   0.0893177  0.00717179     38955.1           2
                         :         91 |     0.214772     0.21805   0.0832508  0.00718062     42066.4           3
                         :         92 |      0.21432     0.21789   0.0832655  0.00717112       42053           4
                         :         93 |     0.214519    0.219549   0.0833121  0.00715717     42019.6           5
                         :         94 |     0.214393    0.222582   0.0832643  0.00715488     42044.7           6
                         :         95 Minimum Test error found - save the configuration 
                         :         95 |     0.213337    0.212509   0.0833585  0.00725895     42050.2           0
                         :         96 |     0.213277    0.216264   0.0833313  0.00716126     42011.3           1
                         :         97 |     0.211938    0.222299   0.0832218  0.00716177       42072           2
                         :         98 |     0.213369    0.218315   0.0833966  0.00723985     42018.6           3
                         :         99 |     0.211386    0.218399   0.0832081  0.00718758     42093.9           4
                         :        100 |     0.211315    0.216526   0.0832447  0.00715059     42053.2           5
                         :        101 |     0.212014    0.216287   0.0893172   0.0131129     41992.4           6
                         :        102 |     0.210518    0.215744    0.083333  0.00716686     42013.4           7
                         :        103 |     0.210506    0.214345   0.0833348  0.00716399     42010.9           8
                         :        104 |     0.208919    0.221022   0.0834827  0.00715969     41927.1           9
                         :        105 |     0.210389    0.220527   0.0833609  0.00718333     42007.1          10
                         :        106 |     0.209846    0.215958   0.0833936  0.00717473     41984.4          11
                         : 
                         : Elapsed time for training with 4000 events: 8.8 sec         
                         : Dataset[dataset] : Create results for training
                         : Dataset[dataset] : Multiclass evaluation of DL_CPU on training sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.108 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating xml weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml
                         : Creating standalone class: dataset/weights/TMVAMulticlass_DL_CPU.class.C
Factory                  : Training finished
                         : 
                         : Ranking input variables (method specific)...
BDTG                     : Ranking result (top variable is best ranked)
                         : --------------------------------------
                         : Rank : Variable  : Variable Importance
                         : --------------------------------------
                         :    1 : var4      : 3.117e-01
                         :    2 : var1      : 2.504e-01
                         :    3 : var2      : 2.430e-01
                         :    4 : var3      : 1.949e-01
                         : --------------------------------------
MLP                      : Ranking result (top variable is best ranked)
                         : -----------------------------
                         : Rank : Variable  : Importance
                         : -----------------------------
                         :    1 : var4      : 6.076e+01
                         :    2 : var2      : 4.824e+01
                         :    3 : var1      : 2.116e+01
                         :    4 : var3      : 1.692e+01
                         : -----------------------------
PDEFoam                  : Ranking result (top variable is best ranked)
                         : --------------------------------------
                         : Rank : Variable  : Variable Importance
                         : --------------------------------------
                         :    1 : var4      : 2.991e-01
                         :    2 : var1      : 2.930e-01
                         :    3 : var3      : 2.365e-01
                         :    4 : var2      : 1.714e-01
                         : --------------------------------------
                         : No variable ranking supplied by classifier: DL_CPU
TH1.Print Name  = TrainingHistory_DL_CPU_trainingError, Entries= 0, Total sum= 28.1543
TH1.Print Name  = TrainingHistory_DL_CPU_valError, Entries= 0, Total sum= 27.6752
Factory                  : === Destroy and recreate all methods via weight files for testing ===
                         : 
                         : Reading weight file: dataset/weights/TMVAMulticlass_BDTG.weights.xml
                         : Reading weight file: dataset/weights/TMVAMulticlass_MLP.weights.xml
MLP                      : Building Network. 
                         : Initializing weights
                         : Reading weight file: dataset/weights/TMVAMulticlass_PDEFoam.weights.xml
                         : Read foams from file: dataset/weights/TMVAMulticlass_PDEFoam.weights_foams.root
                         : Reading weight file: dataset/weights/TMVAMulticlass_DL_CPU.weights.xml

Evaluate all mvas using the set of test events

In [8]:
factory->TestAllMethods();
Factory                  : Test all methods
Factory                  : Test method: BDTG for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of BDTG on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.995 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: MLP for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of MLP on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.00952 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: PDEFoam for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of PDEFoam on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.123 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
Factory                  : Test method: DL_CPU for Multiclass classification performance
                         : 
                         : Dataset[dataset] : Create results for testing
                         : Dataset[dataset] : Multiclass evaluation of DL_CPU on testing sample
                         : Dataset[dataset] : Elapsed time for evaluation of 4000 events: 0.107 sec       
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...

Evaluate and compare performance of all configured mvas

In [9]:
factory->EvaluateAllMethods();
Factory                  : Evaluate all methods
                         : Evaluate multiclass classification method: BDTG
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_BDTG           : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: MLP
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_MLP            : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: PDEFoam
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_PDEFoam        : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.070153     1.0224   [    -4.0592     3.5808 ]
                         :     var2:    0.30372     1.0460   [    -3.6952     3.7877 ]
                         :     var3:    0.12152     1.1222   [    -3.6800     3.9200 ]
                         :     var4:  -0.072602     1.2766   [    -4.8486     4.2221 ]
                         : -----------------------------------------------------------
                         : Evaluate multiclass classification method: DL_CPU
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
                         : Creating multiclass response histograms...
                         : Creating multiclass performance histograms...
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.077270    0.29534   [    -1.1155     1.0914 ]
                         :     var2:   0.068045    0.27981   [    -1.0016     1.0000 ]
                         :     var3:   0.027548    0.24565   [   -0.80459    0.85902 ]
                         :     var4:  -0.034157    0.25816   [    -1.0000    0.83435 ]
                         : -----------------------------------------------------------
TFHandler_DL_CPU         : Variable        Mean        RMS   [        Min        Max ]
                         : -----------------------------------------------------------
                         :     var1:   0.077270    0.29534   [    -1.1155     1.0914 ]
                         :     var2:   0.068045    0.27981   [    -1.0016     1.0000 ]
                         :     var3:   0.027548    0.24565   [   -0.80459    0.85902 ]
                         :     var4:  -0.034157    0.25816   [    -1.0000    0.83435 ]
                         : -----------------------------------------------------------
                         : 
                         : 1-vs-rest performance metrics per class
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : Considers the listed class as signal and the other classes
                         : as background, reporting the resulting binary performance.
                         : A score of 0.820 (0.850) means 0.820 was acheived on the
                         : test set and 0.850 on the training set.
                         : 
                         : Dataset        MVA Method     ROC AUC        Sig [email protected]=0.01 Sig [email protected]=0.10 Sig [email protected]=0.30 
                         : Name:          / Class:       test  (train)  test  (train)  test  (train)  test  (train)  
                         : 
                         : dataset        BDTG           
                         : ------------------------------
                         :                Signal         0.968 (0.978)  0.508 (0.605)  0.914 (0.945)  0.990 (0.996)  
                         :                bg0            0.910 (0.931)  0.256 (0.288)  0.737 (0.791)  0.922 (0.956)  
                         :                bg1            0.947 (0.954)  0.437 (0.511)  0.833 (0.856)  0.971 (0.971)  
                         :                bg2            0.978 (0.982)  0.585 (0.678)  0.951 (0.956)  0.999 (0.996)  
                         : 
                         : dataset        MLP            
                         : ------------------------------
                         :                Signal         0.970 (0.975)  0.596 (0.632)  0.933 (0.938)  0.988 (0.993)  
                         :                bg0            0.929 (0.934)  0.303 (0.298)  0.787 (0.793)  0.949 (0.961)  
                         :                bg1            0.962 (0.967)  0.467 (0.553)  0.881 (0.906)  0.985 (0.992)  
                         :                bg2            0.975 (0.979)  0.629 (0.699)  0.929 (0.940)  0.998 (0.998)  
                         : 
                         : dataset        PDEFoam        
                         : ------------------------------
                         :                Signal         0.916 (0.928)  0.294 (0.382)  0.744 (0.782)  0.932 (0.952)  
                         :                bg0            0.837 (0.848)  0.109 (0.147)  0.519 (0.543)  0.833 (0.851)  
                         :                bg1            0.890 (0.902)  0.190 (0.226)  0.606 (0.646)  0.923 (0.929)  
                         :                bg2            0.967 (0.972)  0.510 (0.527)  0.900 (0.926)  0.993 (0.998)  
                         : 
                         : dataset        DL_CPU         
                         : ------------------------------
                         :                Signal         0.974 (0.976)  0.595 (0.630)  0.930 (0.940)  0.992 (0.997)  
                         :                bg0            0.928 (0.931)  0.318 (0.320)  0.784 (0.796)  0.945 (0.951)  
                         :                bg1            0.966 (0.966)  0.491 (0.502)  0.899 (0.896)  0.994 (0.992)  
                         :                bg2            0.980 (0.979)  0.677 (0.673)  0.935 (0.933)  0.999 (0.999)  
                         : 
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : 
                         : Confusion matrices for all methods
                         : -------------------------------------------------------------------------------------------------------
                         : 
                         : Does a binary comparison between the two classes given by a 
                         : particular row-column combination. In each case, the class 
                         : given by the row is considered signal while the class given 
                         : by the column index is considered background.
                         : 
                         : === Showing confusion matrix for method : BDTG           
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.497 (0.373)  0.710 (0.693)  0.680 (0.574) 
                         :  bg0            0.271 (0.184)  -              0.239 (0.145)  0.705 (0.667) 
                         :  bg1            0.855 (0.766)  0.369 (0.222)  -              0.587 (0.578) 
                         :  bg2            0.714 (0.585)  0.705 (0.581)  0.648 (0.601)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.911 (0.853)  0.991 (0.981)  0.945 (0.913) 
                         :  bg0            0.833 (0.774)  -              0.654 (0.582)  0.930 (0.901) 
                         :  bg1            0.971 (0.980)  0.716 (0.681)  -              0.871 (0.862) 
                         :  bg2            0.976 (0.951)  0.971 (0.973)  0.936 (0.941)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.957)  0.999 (1.000)  0.998 (0.997) 
                         :  bg0            0.965 (0.926)  -              0.874 (0.835)  0.991 (0.976) 
                         :  bg1            1.000 (0.999)  0.916 (0.894)  -              0.988 (0.985) 
                         :  bg2            0.999 (0.999)  0.997 (0.999)  0.996 (0.997)  -             
                         : 
                         : === Showing confusion matrix for method : MLP            
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.465 (0.490)  0.974 (0.953)  0.632 (0.498) 
                         :  bg0            0.320 (0.269)  -              0.224 (0.250)  0.655 (0.627) 
                         :  bg1            0.943 (0.920)  0.341 (0.275)  -              0.632 (0.687) 
                         :  bg2            0.665 (0.642)  0.697 (0.680)  0.706 (0.598)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.865 (0.854)  0.996 (0.994)  0.908 (0.907) 
                         :  bg0            0.784 (0.776)  -              0.666 (0.655)  0.919 (0.895) 
                         :  bg1            0.998 (0.998)  0.791 (0.785)  -              0.912 (0.902) 
                         :  bg2            0.943 (0.903)  0.946 (0.939)  0.924 (0.928)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.978 (0.964)  0.997 (0.997)  0.993 (0.986) 
                         :  bg0            0.952 (0.924)  -              0.936 (0.928)  0.992 (0.990) 
                         :  bg1            1.000 (1.000)  0.945 (0.936)  -              0.998 (0.995) 
                         :  bg2            0.994 (0.985)  0.998 (0.998)  0.998 (0.998)  -             
                         : 
                         : === Showing confusion matrix for method : PDEFoam        
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.289 (0.233)  0.467 (0.436)  0.421 (0.332) 
                         :  bg0            0.100 (0.045)  -              0.132 (0.116)  0.540 (0.313) 
                         :  bg1            0.209 (0.434)  0.153 (0.092)  -              0.347 (0.323) 
                         :  bg2            0.560 (0.552)  0.445 (0.424)  0.501 (0.506)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.665 (0.640)  0.854 (0.822)  0.807 (0.790) 
                         :  bg0            0.538 (0.520)  -              0.415 (0.374)  0.843 (0.833) 
                         :  bg1            0.885 (0.886)  0.542 (0.491)  -              0.728 (0.646) 
                         :  bg2            0.928 (0.890)  0.956 (0.959)  0.847 (0.895)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.898 (0.878)  0.971 (0.950)  0.982 (0.975) 
                         :  bg0            0.828 (0.810)  -              0.696 (0.676)  0.954 (0.951) 
                         :  bg1            0.951 (0.966)  0.803 (0.745)  -              0.958 (0.966) 
                         :  bg2            0.998 (0.991)  0.998 (0.996)  0.998 (0.993)  -             
                         : 
                         : === Showing confusion matrix for method : DL_CPU         
                         : (Signal Efficiency for Background Efficiency 0.01%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.417 (0.438)  0.922 (0.924)  0.693 (0.674) 
                         :  bg0            0.385 (0.354)  -              0.189 (0.234)  0.560 (0.523) 
                         :  bg1            0.935 (0.922)  0.304 (0.306)  -              0.665 (0.719) 
                         :  bg2            0.662 (0.630)  0.704 (0.690)  0.669 (0.674)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.10%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.874 (0.879)  0.995 (0.992)  0.925 (0.914) 
                         :  bg0            0.805 (0.769)  -              0.696 (0.732)  0.861 (0.837) 
                         :  bg1            0.998 (0.996)  0.785 (0.768)  -              0.884 (0.897) 
                         :  bg2            0.946 (0.933)  0.947 (0.951)  0.909 (0.919)  -             
                         : 
                         : (Signal Efficiency for Background Efficiency 0.30%)
                         : ---------------------------------------------------
                         :                 Signal         bg0            bg1            bg2           
                         :                  test (train)   test (train)   test (train)   test (train) 
                         :  Signal         -              0.980 (0.969)  0.998 (1.000)  0.998 (0.999) 
                         :  bg0            0.961 (0.951)  -              0.912 (0.908)  0.984 (0.983) 
                         :  bg1            1.000 (0.998)  0.951 (0.951)  -              1.000 (0.997) 
                         :  bg2            0.999 (0.997)  0.999 (0.999)  0.996 (0.999)  -             
                         : 
                         : -------------------------------------------------------------------------------------------------------
                         : 
Dataset:dataset          : Created tree 'TestTree' with 4000 events
                         : 
Dataset:dataset          : Created tree 'TrainTree' with 4000 events
                         : 
Factory                  : Thank you for using TMVA!
                         : For citation information, please visit: http://tmva.sf.net/citeTMVA.html

Save the output

In [10]:
outputFile->Close();

std::cout << "==> Wrote root file: " << outputFile->GetName() << std::endl;
std::cout << "==> TMVAMulticlass is done!" << std::endl;

delete factory;
delete dataloader;
==> Wrote root file: TMVAMulticlass.root
==> TMVAMulticlass is done!

Launch the gui for the root macros

In [11]:
if (!gROOT->IsBatch()) TMVAMultiClassGui( outfileName );