Source Code for Module QSTK.qstklearn.mldiagnostics

 1  #Author : Vishal Shekhar 
 2  #Email : mailvishalshekhar@gmail.com 
 3   
 4  import copy 
 5  import numpy as np 
 6  import matplotlib.pyplot as plt 
 7  from pylab import * 
 8   
 9 -class MLDiagnostics: 
10          """ 
11          This class can be used to produce learning curves. 
12          These are plots of evolution of Training Error and Cross Validation Error across lambda(in general a control param for model complexity). 
13          This plot can help diagnose if the ML algorithmic model has high bias or a high variance problem and can 
14          thus help decide the next course of action. 
15          In general, ML Algorithm is of the form, 
16                  Y=f(t,X) + lambdaVal*|t| 
17                  where Y is the output, t is the model parameter vector, lambdaVal is the regularization parameter. 
18                  |t| is the size of model parameter vector. 
19          """ 
20 -        def __init__(self,learner,Xtrain,Ytrain,Xcv,Ycv,lambdaArray): 
21                  self.learner = learner 
22                  self.Xtrain = Xtrain 
23                  self.Ytrain = Ytrain 
24                  self.Xcv = Xcv 
25                  self.Ycv = Ycv 
26                  self.lambdaArray = lambdaArray 
27                  self.ErrTrain = np.zeros((len(lambdaArray),1)) 
28                  self.ErrCV = copy.copy(self.ErrTrain) 
29   
30 -        def avgsqerror(self,Y,Ypred): 
31                  return np.sum((Y-Ypred)**2)/len(Y) 
32           
33 -        def plotCurves(self,filename): 
34                  Xrange = [i*self.step for i in range(1,len(self.ErrTrain)+1)] 
35                  plt.plot(Xrange,self.ErrTrain,label = "Train Error") 
36                  plt.plot(Xrange,self.ErrCV,label="CV Error") 
37                  plt.title('Learning Curves') 
38                  plt.xlabel('# of Training Examples') 
39                  plt.ylabel('Average Error') 
40                  plt.draw() 
41                  savefig(filename,format='pdf') 
42   
43 -        def runDiagnostics(self,filename): 
44                  for i,lambdaVal in zip(range(len(self.lambdaArray)),self.lambdaArray): 
45                          learner = copy.copy(self.learner())# is deep copy required 
46                          # setLambda needs to be a supported function for all ML strategies. 
47                          learner.setLambda(lambdaVal) 
48                          learner.addEvidence(self.Xtrain,self.Ytrain) 
49                          YtrPred = learner.query(self.Xtrain) 
50                          self.ErrTrain[i] = self.avgsqerror(self.Ytrain,YtrPred) 
51                          YcvPred = learner.query(self.Xcv) 
52                          self.ErrCV[i] = self.avgsqerror(self.Ycv,YcvPred) 
53                  self.plotCurves(filename) 
54