Source Code for Module QSTK.quicksim.quickSim

  1  # 
 
  2  # quickSim.py 
 
  3  # 
 
  4  # A module that contains simulators that quickly produce a fund history.
 
  5  # 
 
  6  #
 
  7  
 
  8  #python imports
 
  9  from pylab import * 
 10  from pandas import * 
 11  import os 
 12  import matplotlib.pyplot as plt 
 13  import time 
 14  import cPickle 
 15  import sys 
 16  import calendar 
 17  import datetime as dt 
 18  
 
 19  #qstk imports
 
 20  from qstkutil import tsutil as tsu 
 21  from qstkutil import dateutil as du 
 22  from qstkutil import DataAccess as da 
 23  
 
 24 -def quickSim(alloc,historic,start_cash): 
 25          """
 
 26          @summary Quickly back tests an allocation for certain historical data, using a starting fund value
 
 27          @param alloc: DataMatrix containing timestamps to test as indices and Symbols to test as columns, with _CASH symbol as the last column
 
 28          @param start_cash: integer specifing initial fund value
 
 29          @return funds: TimeSeries with fund values for each day in the back test
 
 30          @rtype TimeSeries
 
 31          """ 
 32          
 
 33          #original quick simulator
 
 34          #not designed to handle shorts
 
 35          
 
 36          #check each row in alloc
 
 37          for row in range(0,len(alloc.values[:,0])): 
 38                  if(abs(alloc.values[row,:].sum()-1)>.1): 
 39                          #print alloc.values[row,:]
 
 40                          #print alloc.values[row,:].sum()
 
 41                          #print "warning, alloc row "+str(row)+" does not sum to one, rebalancing"
 
 42                          #if no allocation, all in cash
 
 43                          if(alloc.values[row,:].sum()==0): 
 44                                  alloc.values[row,-1]=1 
 45                          else: 
 46                                  alloc.values[row,:]=alloc.values[row,:]/alloc.values[row,:].sum() 
 47          
 
 48          #fix invalid days
 
 49          historic=historic.fillna(method='backfill') 
 50          
 
 51          #add cash column
 
 52      #print historic.columns
 
 53          historic['_CASH'] = 1 
 54  
 
 55          #print historic.columns
 
 56          
 
 57          closest=historic[historic.index<=alloc.index[0]] 
 58          fund_ts=Series([start_cash], index=[closest.index[-1]]) 
 59          shares=alloc.values[0,:]*fund_ts.values[-1]/closest.values[-1,:] 
 60          cash_values=DataMatrix([shares*closest.values[-1,:]],index=[closest.index[-1]]) 
 61          
 
 62          #compute all trade
 
 63          for i in range(1,len(alloc.values[:,0])): 
 64                  #get closest date(previous date)
 
 65                  closest=historic[historic.index<=alloc.index[i]] 
 66                  #for loop to calculate fund daily (without rebalancing)
 
 67                  for date in closest[closest.index>fund_ts.index[-1]].index: 
 68                          #compute and record total fund value (Sum(closest close * stocks))
 
 69                          fund_ts=fund_ts.append(Series([(closest.xs(date)*shares).sum()],index=[date])) 
 70                          cash_values=cash_values.append(DataMatrix([shares*closest.xs(date)],index=[date])) 
 71                  #distribute fund in accordance with alloc
 
 72                  shares=alloc.values[i,:]*fund_ts.values[-1]/closest.xs(closest.index[-1]) 
 73          
 
 74          #compute fund value for rest of historic data with final share distribution
 
 75          for date in historic[historic.index>alloc.index[-1]].index: 
 76                  if date in closest.index : 
 77                          fund_ts=fund_ts.append(Series([(closest.xs(date)*shares).sum()],index=[date]))   
 78          #return fund record
 
 79          return fund_ts 
 80  
 
 81 -def computeShort(arr): 
 82          tally=0 
 83          for i in range(0,len(arr)-1): 
 84                  if arr[i]<0: 
 85                          tally=tally+arr[i] 
 86          return abs(tally) 
 87  
 
 88 -def computeLong(arr,fundval): 
 89          tally=0 
 90          for i in range(0,len(arr)-1): 
 91                  if arr[i]>0: 
 92                          tally=tally+arr[i] 
 93          return tally 
 94          
 
 95 -def computeLeverage(arr,fundval): 
 96          if fundval==0: 
 97                  return 0 
 98          return (computeLong(arr,fundval)-computeShort(arr))/fundval 
 99          
 
100 -def shortingQuickSim(alloc,historic,start_cash,leverage): 
101      #shortingQuickSim
 
102      #
 
103      #designed to handle shorts
 
104      #keeps track of leverage keeping it within paramaterized value
 
105      #
 
106          #ignore alloc cash column
 
107          del alloc['_CASH'] 
108          #fix invalid days
 
109          historic=historic.fillna(method='backfill')  
110      
 
111      #compute first trade
 
112          closest=historic[historic.index<=alloc.index[0]] 
113          fund_ts=Series([start_cash], index=[closest.index[-1]]) 
114          shares=alloc.values[0,:]*fund_ts.values[-1]/closest.values[-1,:] 
115          cash_values=DataMatrix([shares*closest.values[-1,:]],index=[closest.index[-1]]) 
116      
 
117      #compute all trades
 
118          for i in range(1,len(alloc.values[:,0])): 
119                  #check leverage
 
120                  this_leverage=computeLeverage(alloc.values[0,:]) 
121                  if this_leverage>leverage: 
122                          print 'Warning, leverage of ',this_leverage,' reached, exceeds leverage limit of ',leverage,'\n' 
123                  #get closest date(previous date)
 
124                  closest=historic[historic.index<=alloc.index[i]] 
125                  #for loop to calculate fund daily (without rebalancing)
 
126                  for date in closest[closest.index>fund_ts.index[-1]].index: 
127                  #compute and record total fund value (Sum(closest close * stocks))
 
128                          fund_ts=fund_ts.append(Series([(closest.xs(date)*shares).sum()],index=[date])) 
129                          cash_values=cash_values.append(DataMatrix([shares*closest.xs(date)],index=[date])) 
130          #distribute fund in accordance with alloc
 
131                  shares=alloc.values[i,:]*fund_ts.values[-1]/closest.xs(closest.index[-1]) 
132  
 
133      #compute fund value for rest of historic data with final share distribution
 
134          for date in historic[historic.index>alloc.index[-1]].index: 
135                  if date in closest.index : 
136                          fund_ts=fund_ts.append(Series([(closest.xs(date)*shares).sum()],index=[date]))   
137  
 
138      #return fund record
 
139          return fund_ts 
140  
 
141 -def alloc_backtest(alloc,start): 
142          """
 
143          @summary: Back tests an allocation from a pickle file. Uses a starting portfolio value of start.
 
144          @param alloc: Name of allocation pickle file. Pickle file contains a DataMatrix with timestamps as indexes
 
145          and stock symbols as columns, with the last column being the _CASH symbol, indicating how much
 
146          of the allocation is in cash.
 
147          @param start: integer specifying the starting value of the portfolio
 
148          @return funds: List of fund values indicating the value of the portfolio throughout the back test.
 
149          @rtype timeseries
 
150          """ 
151          
 
152          #read in alloc table from command line arguements
 
153          alloc_input_file=open(alloc,"r") 
154          alloc=cPickle.load(alloc_input_file) 
155          
 
156          # Get the data from the data store
 
157          dataobj=da.DataAccess('Norgate') 
158          historic = dataobj.get_data(list(alloc.index), list(alloc.columns[0:-1]), "close") 
159          
 
160          #backtest
 
161          funds=quickSim(alloc,historic,int(start)) 
162          
 
163          return funds 
164  
 
165 -def strat_backtest1(strat,start,end,num,diff,startval): 
166          """
 
167          @summary: Back tests a strategy defined in a python script that takes in a start
 
168          and end date along with a starting value a set number of times. 
 
169          @param strat: filename of python script strategy
 
170          @param start: starting date in a datetime object
 
171          @param end: ending date in a datetime object
 
172          @param num: number of tests to perform
 
173          @param diff: offset in days of the tests
 
174          @param startval: starting value of fund during back tests
 
175          @return fundsmatrix: Datamatrix of fund values returned from each test
 
176          @rtype datamatrix
 
177          """ 
178          fundsmatrix=[] 
179          startdates=du.getNextNNYSEdays(start,num*diff,dt.timedelta(hours=16)) 
180          enddates=du.getNextNNYSEdays(end,num*diff,dt.timedelta(hours=16)) 
181          for i in range(0,num): 
182                  os.system('python '+strat+' '+startdates[i].strftime("%m-%d-%Y")+' '+enddates[i].strftime("%m-%d-%Y")+' temp_alloc.pkl') 
183                  funds=alloc_backtest('temp_alloc.pkl',startval) 
184                  fundsmatrix.append(funds) 
185          return fundsmatrix 
186          
 
187 -def strat_backtest2(strat,start,end,diff,dur,startval): 
188          """
 
189          @summary: Back tests a strategy defined in a python script that takes in a start
 
190          and end date along with a starting value over a given period. 
 
191          @param strat: filename of python script strategy
 
192          @param start: starting date in a datetime object
 
193          @param end: ending date in a datetime object
 
194          @param diff: offset in days of the tests
 
195          @param dur: length of a test
 
196          @param startval: starting value of fund during back tests
 
197          @return fundsmatrix: Datamatrix of fund values returned from each test
 
198          @rtype datamatrix
 
199          """ 
200          fundsmatrix=[] 
201          startdates=du.getNYSEdays(start,end,dt.timedelta(hours=16)) 
202          for i in range(0,len(startdates),diff): 
203                  if(i+dur>=len(startdates)): 
204                          enddate=startdates[-1] 
205                  else: 
206                          enddate=startdates[i+dur] 
207                  os.system('python '+strat+' '+startdates[i].strftime("%m-%d-%Y")+' '+enddate.strftime("%m-%d-%Y")+' temp_alloc.pkl') 
208                  funds=alloc_backtest('temp_alloc.pkl',startval) 
209                  fundsmatrix.append(funds) 
210          return fundsmatrix 
211      
 
212  if __name__ == "__main__": 
213      #
 
214          # CmdlnQuickSim
 
215          #
 
216          # A function which runs a quick sim on an allocation provided via command line,
 
217          # along with a starting cash value
 
218          # 
 
219          # Allocation Backtest:
 
220          # python quickSim.py -a allocation_fiel start_value output_file
 
221          # python quickSim.py -a 'alloc_file.pkl' 1000 'fund_output.pkl'
 
222          #
 
223          # Strategy backtest:
 
224          # python quickSim.py -s strategy start end start_value output_file
 
225          # python quickSim.py -s 'strategy.py' '2/2/2007' '2/2/2009' 1000 'fund_output.pkl' 
 
226          #
 
227          # Robust backtest:
 
228          # python quickSim.py -r strategy start end days_between duration start_value output
 
229          # python quickSim.py -r 'strategy.py' '1-1-2004' '1-1-2007' 7 28 10000 'out.pkl'
 
230          # Drew Bratcher
 
231          #
 
232  
 
233          if(sys.argv[1]=='-a'): 
234                  funds=alloc_backtest(sys.argv[2],sys.argv[3]) 
235                  output=open(sys.argv[4],"w") 
236                  cPickle.dump(funds,output) 
237          elif(sys.argv[1]=='-s'): 
238                  t = map(int, sys.argv[3].split('-')) 
239                  startday= dt.datetime(t[2],t[0],t[1]) 
240                  t = map(int, sys.argv[4].split('-')) 
241                  endday = dt.datetime(t[2],t[0],t[1])   
242                  fundsmatrix=strat_backtest1(sys.argv[2],startday,endday,1,0,int(sys.argv[5])) 
243                  output=open(sys.argv[6],"w") 
244                  cPickle.dump(fundsmatrix,output)  
245          elif(sys.argv[1]=='-r'): 
246                  t = map(int, sys.argv[3].split('-')) 
247                  startday= dt.datetime(t[2],t[0],t[1]) 
248                  t = map(int, sys.argv[4].split('-')) 
249                  endday = dt.datetime(t[2],t[0],t[1]) 
250                  fundsmatrix=strat_backtest2(sys.argv[2],startday,endday,int(sys.argv[5]),int(sys.argv[6]),int(sys.argv[7])) 
251                  output=open(sys.argv[8],"w") 
252                  cPickle.dump(fundsmatrix,output) 
253          else: 
254                  print 'invalid command line call' 
255                  print 'use python quickSim.py -a alloc_pkl start_value output_pkl' 
256                  print 'or python quickSim.py -s strategy start_date end_date start_value output_pkl' 
257                  print 'or python quickSim.py -r strategy start_date end_date test_offset_in_days duration start_value output_pkl' 
258