1 '''
2 (c) 2011, 2012 Georgia Tech Research Corporation
3 This source code is released under the New BSD license. Please see
4 http://wiki.quantsoftware.org/index.php?title=QSTK_License
5 for license details.
6
7 Created on Nov 7, 2011
8
9 @author: John Cornwell
10 @contact: JohnWCornwellV@gmail.com
11 @summary: Contains utility functions to interact with feature functions in features.py
12 '''
13
14 ''' Python imports '''
15 import math
16 import pickle
17 import datetime as dt
18 from dateutil.relativedelta import relativedelta
19
20 ''' 3rd Party Imports '''
21 import numpy as np
22 import matplotlib.pyplot as plt
23
24 ''' Our Imports '''
25 import qstklearn.kdtknn as kdt
26 from qstkutil import DataAccess as da
27 from qstkutil import dateutil as du
28 from qstkutil import tsutil as tsu
29
30 from qstkfeat.features import *
31 from qstkfeat.classes import classFutRet
32
33
34
36 '''
37 @summary: Calculates market relative data.
38 @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
39 @param sRel - Stock ticker to make the data relative to, $SPX is default.
40 @return: Dictionary of market relative values
41 '''
42
43 if sRel not in dData['close'].columns:
44 raise KeyError( 'Market relative stock %s not found in getMR()'%sRel )
45
46
47 dRet = {}
48
49 ''' Make all data market relative, except for volume '''
50 for sKey in dData.keys():
51
52 ''' Don't calculate market relative volume, but still copy it over '''
53 if sKey == 'volume':
54 dRet['volume'] = dData['volume']
55 continue
56
57 dfAbsolute = dData[sKey]
58 dfRelative = pand.DataFrame( index=dfAbsolute.index, columns=dfAbsolute.columns, data=np.zeros(dfAbsolute.shape) )
59
60 ''' Get returns and strip off the market returns '''
61 naRets = dfAbsolute.values.copy()
62 tsu.returnize0( naRets )
63
64 naMarkRets = naRets[:, list(dfAbsolute.columns).index(sRel) ]
65
66 for i, sStock in enumerate(dfAbsolute.columns):
67 ''' Don't change the 'market' stock '''
68 if sStock == sRel:
69 dfRelative.values[:,i] = dfAbsolute.values[:,i]
70 continue
71
72 naMarkRel = (naRets[:,i] - naMarkRets) + 1.0
73
74 ''' Find the first non-nan value and start the price at 100 '''
75 for j in range(0, dfAbsolute.values.shape[0]):
76 if pand.isnull( dfAbsolute.values[j][i] ):
77 dfRelative.values[j][i] = float('nan')
78 continue
79 dfRelative.values[j][i] = 100
80 break
81
82 ''' Now fill prices out using market relative returns '''
83 for j in range(j+1, dfAbsolute.values.shape[0]):
84 dfRelative.values[j][i] = dfRelative.values[j-1][i] * naMarkRel[j]
85
86 ''' Add dataFrame to dictionary to return, move to next key '''
87 dRet[sKey] = dfRelative
88
89
90 return dRet
91
92
93
94 -def applyFeatures( dData, lfcFeatures, ldArgs, sMarketRel=None, sLog=None ):
95 '''
96 @summary: Calculates the feature values using a list of feature functions and arguments.
97 @param dData - Dictionary containing data to be used, requires specific naming: open/high/low/close/volume
98 @param lfcFeatures: List of feature functions, most likely coming from features.py
99 @param ldArgs: List of dictionaries containing arguments, passed as **kwargs
100 There is a special argument 'MR', if it exists, the data will be made market relative
101 @param sMarketRel: If not none, the data will all be made relative to the symbol provided
102 @param sLog: If not None, will be filename to log all of the features to
103 @return: list of dataframes containing values
104 '''
105
106
107
108
109 ldfRet = []
110
111 ''' Calculate market relative data '''
112 if sMarketRel != None:
113 dDataRelative = getMarketRel( dData, sRel=sMarketRel )
114
115
116 ''' Loop though feature functions, pass each data dictionary and arguments '''
117 for i, fcFeature in enumerate(lfcFeatures):
118
119 ''' Check for special arguments '''
120 if 'MR' in ldArgs[i]:
121
122 if ldArgs[i]['MR'] == False:
123 print 'Warning, setting MR to false will still be Market Relative',\
124 'simply do not include MR key in args'
125
126 if sMarketRel == None:
127 raise AssertionError('Functions require market relative stock but sMarketRel=None')
128 del ldArgs[i]['MR']
129 ldfRet.append( fcFeature( dDataRelative, **ldArgs[i] ) )
130 else:
131 ldfRet.append( fcFeature( dData, **ldArgs[i] ) )
132
133
134 if not sLog == None:
135 with open( sLog, 'wb' ) as fFile:
136 pickle.dump( ldfRet, fFile, -1 )
137
138 return ldfRet
139
141 '''
142 @summary: Loads cached features.
143 @param sLog: Filename of features.
144 @return: Numpy array containing values
145 '''
146
147 ldfRet = []
148
149 if not sLog == None:
150 with open( sLog, 'rb' ) as fFile:
151 ldfRet = pickle.load( fFile )
152
153 return ldfRet
154
155
156 -def stackSyms( ldfFeatures, dtStart=None, dtEnd=None, lsSym=None, bDelNan=True, bShowRemoved=False ):
157 '''
158 @summary: Remove symbols from the dataframes, effectively stacking all stocks on top of each other.
159 @param ldfFeatures: List of data frames of features.
160 @param dtStart: Start time, if None, uses all
161 @param dtEnd: End time, if None uses all
162 @param lsSym: List of symbols to use, if None, all are used.
163 @param bDelNan: Optional, default is true: delete all rows with a NaN in it
164 @return: Numpy array containing all features as columns and all
165 '''
166
167 if dtStart == None:
168 dtStart = ldfFeatures[0].index[0]
169 if dtEnd == None:
170 dtEnd = ldfFeatures[0].index[-1]
171
172 naRet = None
173 ''' Stack stocks vertically '''
174 for sStock in ldfFeatures[0].columns:
175
176 if lsSym != None and sStock not in lsSym:
177 continue
178
179 naStkData = None
180 ''' Loop through all features, stacking columns horizontally '''
181 for dfFeat in ldfFeatures:
182
183 dfFeat = dfFeat.ix[dtStart:dtEnd]
184
185 if naStkData == None:
186 naStkData = np.array( dfFeat[sStock].values.reshape(-1,1) )
187 else:
188 naStkData = np.hstack( (naStkData, dfFeat[sStock].values.reshape(-1,1)) )
189
190 ''' Remove nan rows possibly'''
191 if True == bDelNan:
192 llValidRows = []
193 for i in range(naStkData.shape[0]):
194 if not math.isnan( np.sum(naStkData[i,:]) ):
195 llValidRows.append(i)
196 elif bShowRemoved:
197 print 'Removed', sStock, naStkData[i,:]
198
199 naStkData = naStkData[llValidRows,:]
200
201
202 ''' Now stack each block of stock data vertically '''
203 if naRet == None:
204 naRet = naStkData
205 else:
206 naRet = np.vstack( (naRet, naStkData) )
207
208 return naRet
209
210
211 -def normFeatures( naFeatures, fMin, fMax, bAbsolute, bIgnoreLast=True ):
212 '''
213 @summary: Normalizes the featurespace.
214 @param naFeatures: Numpy array of features,
215 @param fMin: Data frame containing the price information for all of the stocks.
216 @param fMax: List of feature functions, most likely coming from features.py
217 @param bAbsolute: If true, min value will be scaled to fMin, max to fMax, if false,
218 +-1 standard deviations will be scaled to fit between fMin and fMax, i.e. ~69% of the values
219 @param bIgnoreLast: If true, last column is ignored (assumed to be classification)
220 @return: list of (weights, shifts) to be used to normalize the query points
221 '''
222
223 fNewRange = fMax - fMin
224
225 lUseCols = naFeatures.shape[1]
226 if bIgnoreLast:
227 lUseCols -= 1
228
229 ltRet = []
230 ''' Loop through all features '''
231 for i in range(lUseCols):
232
233 ''' If absolutely scaled use exact min and max '''
234 if bAbsolute:
235 fFeatMin = np.min( naFeatures[:,i] )
236 fFeatMax = np.max( naFeatures[:,i] )
237 else:
238 ''' Otherwise use mean +-1 std deviations for min/max (~94% of data) '''
239 fMean = np.average( naFeatures[:,i] )
240 fStd = np.std( naFeatures[:,i] )
241 fFeatMin = fMean - fStd
242 fFeatMax = fMean + fStd
243
244 ''' Calculate multiplier and shift variable so that new data fits in specified range '''
245 fRange = fFeatMax - fFeatMin
246 fMult = fNewRange / fRange
247 fShift = fMin - (fFeatMin * fMult)
248
249 ''' scale and shift, save in return array '''
250 naFeatures[:,i] *= fMult
251 naFeatures[:,i] += fShift
252 ltRet.append( (fMult, fShift) )
253
254 return ltRet
255
257 '''
258 @summary: Normalizes the queries using the given normalization parameters generated from training data.
259 @param naQueries: Numpy array of queries
260 @param ltWeightShift: List of weights and shift amounts to be applied to each query.
261 @return: None, modifies naQueries
262 '''
263
264 assert naQueries.shape[1] == len(ltWeightShift)
265
266 for i in range(naQueries.shape[1]):
267
268 ''' scale and shift, save in return array '''
269 naQueries[:,i] *= ltWeightShift[i][0]
270 naQueries[:,i] += ltWeightShift[i][1]
271
273 '''
274 @summary: Creates a quick KNN learner
275 @param naFeatures: Numpy array of features,
276 @param fMin: Data frame containing the price information for all of the stocks.
277 @param fMax: List of feature functions, most likely coming from features.py
278 @param bAbsolute: If true, min value will be scaled to fMin, max to fMax, if false,
279 +-1 standard deviations will be scaled to fit between fMin and fMax, i.e. ~69% of the values
280 @param bIgnoreLast: If true, last column is ignored (assumed to be classification)
281 @return: None, data is modified in place
282 '''
283 cLearner = kdt.kdtknn( k=lKnn, method='mean', leafsize=leafsize)
284
285 cLearner.addEvidence( naFeatures )
286
287 return cLearner
288
290 '''
291 @summary: Loads cached features.
292 @param sLog: Filename of features.
293 @return: Nothing, logs features to desired location
294 '''
295
296
297 lsSym = ['A', 'AA', 'AAPL', 'ABC', 'ABT', 'ACE', 'ACN', 'ADBE', 'ADI', 'ADM', 'ADP', 'ADSK', 'AEE', 'AEP', 'AES', 'AET', 'AFL', 'AGN', 'AIG', 'AIV', 'AIZ', 'AKAM', 'AKS', 'ALL', 'ALTR', 'AMAT', 'AMD', 'AMGN', 'AMP', 'AMT', 'AMZN', 'AN', 'ANF', 'ANR', 'AON', 'APA', 'APC', 'APD', 'APH', 'APOL', 'ARG', 'ATI', 'AVB', 'AVP', 'AVY', 'AXP', 'AZO', 'BA', 'BAC', 'BAX', 'BBBY', 'BBT', 'BBY', 'BCR', 'BDX', 'BEN', 'BF.B', 'BHI', 'BIG', 'BIIB', 'BK', 'BLK', 'BLL', 'BMC', 'BMS', 'BMY', 'BRCM', 'BRK.B', 'BSX', 'BTU', 'BXP', 'C', 'CA', 'CAG', 'CAH', 'CAM', 'CAT', 'CB', 'CBG', 'CBS', 'CCE', 'CCL', 'CEG', 'CELG', 'CERN', 'CF', 'CFN', 'CHK', 'CHRW', 'CI', 'CINF', 'CL', 'CLF', 'CLX', 'CMA', 'CMCSA', 'CME', 'CMG', 'CMI', 'CMS', 'CNP', 'CNX', 'COF', 'COG', 'COH', 'COL', 'COP', 'COST', 'COV', 'CPB', 'CPWR', 'CRM', 'CSC', 'CSCO', 'CSX', 'CTAS', 'CTL', 'CTSH', 'CTXS', 'CVC', 'CVH', 'CVS', 'CVX', 'D', 'DD', 'DE', 'DELL', 'DF', 'DFS', 'DGX', 'DHI', 'DHR', 'DIS', 'DISCA', 'DNB', 'DNR', 'DO', 'DOV', 'DOW', 'DPS', 'DRI', 'DTE', 'DTV', 'DUK', 'DV', 'DVA', 'DVN', 'EBAY', 'ECL', 'ED', 'EFX', 'EIX', 'EL', 'EMC', 'EMN', 'EMR', 'EOG', 'EP', 'EQR', 'EQT', 'ERTS', 'ESRX', 'ETFC', 'ETN', 'ETR', 'EW', 'EXC', 'EXPD', 'EXPE', 'F', 'FAST', 'FCX', 'FDO', 'FDX', 'FE', 'FFIV', 'FHN', 'FII', 'FIS', 'FISV', 'FITB', 'FLIR', 'FLR', 'FLS', 'FMC', 'FO', 'FRX', 'FSLR', 'FTI', 'FTR', 'GAS', 'GCI', 'GD', 'GE', 'GILD', 'GIS', 'GLW', 'GME', 'GNW', 'GOOG', 'GPC', 'GPS', 'GR', 'GS', 'GT', 'GWW', 'HAL', 'HAR', 'HAS', 'HBAN', 'HCBK', 'HCN', 'HCP', 'HD', 'HES', 'HIG', 'HNZ', 'HOG', 'HON', 'HOT', 'HP', 'HPQ', 'HRB', 'HRL', 'HRS', 'HSP', 'HST', 'HSY', 'HUM', 'IBM', 'ICE', 'IFF', 'IGT', 'INTC', 'INTU', 'IP', 'IPG', 'IR', 'IRM', 'ISRG', 'ITT', 'ITW', 'IVZ', 'JBL', 'JCI', 'JCP', 'JDSU', 'JEC', 'JNJ', 'JNPR', 'JNS', 'JOYG', 'JPM', 'JWN', 'K', 'KEY', 'KFT', 'KIM', 'KLAC', 'KMB', 'KMX', 'KO', 'KR', 'KSS', 'L', 'LEG', 'LEN', 'LH', 'LIFE', 'LLL', 'LLTC', 'LLY', 'LM', 'LMT', 'LNC', 'LO', 'LOW', 'LSI', 'LTD', 'LUK', 'LUV', 'LXK', 'M', 'MA', 'MAR', 'MAS', 'MAT', 'MCD', 'MCHP', 'MCK', 'MCO', 'MDT', 'MET', 'MHP', 'MHS', 'MJN', 'MKC', 'MMC', 'MMI', 'MMM', 'MO', 'MOLX', 'MON', 'MOS', 'MPC', 'MRK', 'MRO', 'MS', 'MSFT', 'MSI', 'MTB', 'MU', 'MUR', 'MWV', 'MWW', 'MYL', 'NBL', 'NBR', 'NDAQ', 'NE', 'NEE', 'NEM', 'NFLX', 'NFX', 'NI', 'NKE', 'NOC', 'NOV', 'NRG', 'NSC', 'NTAP', 'NTRS', 'NU', 'NUE', 'NVDA', 'NVLS', 'NWL', 'NWSA', 'NYX', 'OI', 'OKE', 'OMC', 'ORCL', 'ORLY', 'OXY', 'PAYX', 'PBCT', 'PBI', 'PCAR', 'PCG', 'PCL', 'PCLN', 'PCP', 'PCS', 'PDCO', 'PEG', 'PEP', 'PFE', 'PFG', 'PG', 'PGN', 'PGR', 'PH', 'PHM', 'PKI', 'PLD', 'PLL', 'PM', 'PNC', 'PNW', 'POM', 'PPG', 'PPL', 'PRU', 'PSA', 'PWR', 'PX', 'PXD', 'QCOM', 'QEP', 'R', 'RAI', 'RDC', 'RF', 'RHI', 'RHT', 'RL', 'ROK', 'ROP', 'ROST', 'RRC', 'RRD', 'RSG', 'RTN', 'S', 'SAI', 'SBUX', 'SCG', 'SCHW', 'SE', 'SEE', 'SHLD', 'SHW', 'SIAL', 'SJM', 'SLB', 'SLE', 'SLM', 'SNA', 'SNDK', 'SNI', 'SO', 'SPG', 'SPLS', 'SRCL', 'SRE', 'STI', 'STJ', 'STT', 'STZ', 'SUN', 'SVU', 'SWK', 'SWN', 'SWY', 'SYK', 'SYMC', 'SYY', 'T', 'TAP', 'TDC', 'TE', 'TEG', 'TEL', 'TER', 'TGT', 'THC', 'TIE', 'TIF', 'TJX', 'TLAB', 'TMK', 'TMO', 'TROW', 'TRV', 'TSN', 'TSO', 'TSS', 'TWC', 'TWX', 'TXN', 'TXT', 'TYC', 'UNH', 'UNM', 'UNP', 'UPS', 'URBN', 'USB', 'UTX', 'V', 'VAR', 'VFC', 'VIA.B', 'VLO', 'VMC', 'VNO', 'VRSN', 'VTR', 'VZ', 'WAG', 'WAT', 'WDC', 'WEC', 'WFC', 'WFM', 'WFR', 'WHR', 'WIN', 'WLP', 'WM', 'WMB', 'WMT', 'WPI', 'WPO', 'WU', 'WY', 'WYN', 'WYNN', 'X', 'XEL', 'XL', 'XLNX', 'XOM', 'XRAY', 'XRX', 'YHOO', 'YUM', 'ZION', 'ZMH']
298 lsSym.append('$SPX')
299 lsSym.sort()
300
301
302 ''' Max lookback is 6 months '''
303 dtEnd = dt.datetime.now()
304 dtEnd = dtEnd.replace(hour=16, minute=0, second=0, microsecond=0)
305 dtStart = dtEnd - relativedelta(months=6)
306
307
308 ''' Pull in current data '''
309 norObj = da.DataAccess('Norgate')
310 ''' Get 2 extra months for moving averages and future returns '''
311 ldtTimestamps = du.getNYSEdays( dtStart - relativedelta(months=2), \
312 dtEnd + relativedelta(months=2), dt.timedelta(hours=16) )
313
314 dfPrice = norObj.get_data( ldtTimestamps, lsSym, 'close' )
315 dfVolume = norObj.get_data( ldtTimestamps, lsSym, 'volume' )
316
317 ''' Imported functions from qstkfeat.features, NOTE: last function is classification '''
318 lfcFeatures, ldArgs, lsNames = getFeatureFuncs()
319
320 ''' Generate a list of DataFrames, one for each feature, with the same index/column structure as price data '''
321 applyFeatures( dfPrice, dfVolume, lfcFeatures, ldArgs, sLog=sLog )
322
323
325 '''
326 @summary: Gets feature functions supported by the website.
327 @return: Tuple containing (list of functions, list of arguments, list of names)
328 '''
329
330 lfcFeatures = [ featMA, featMA, featRSI, featDrawDown, featRunUp, featVolumeDelta, classFutRet ]
331 lsNames = ['Moving Average', 'Relative Moving Average', 'RSI', 'Draw Down', 'Run Up', 'Volume Delta', 'Future Returns']
332
333 ''' Custom Arguments '''
334 ldArgs = [ {'lLookback':30},\
335 {'lLookback':30, 'bRel':True},\
336 {},\
337 {},\
338 {},\
339 {},\
340 {'sRel':'$SPX'}]
341
342 return lfcFeatures, ldArgs, lsNames
343
344
346 '''
347 @summary: Quick function to run a feature on some data and plot it to see if it works.
348 @param fcFeature: Feature function to test
349 @param dArgs: Arguments to pass into feature function
350 @return: Void
351 '''
352
353 ''' Get Train data for 2009-2010 '''
354 dtStart = dt.datetime(2009, 1, 1)
355 dtEnd = dt.datetime(2009, 5, 1)
356
357 ''' Pull in current training data and test data '''
358 norObj = da.DataAccess('Norgate')
359 ''' Get 2 extra months for moving averages and future returns '''
360 ldtTimestamps = du.getNYSEdays( dtStart, dtEnd, dt.timedelta(hours=16) )
361
362 lsSym = ['GOOG']
363 lsSym.append('WMT')
364 lsSym.append('$SPX')
365 lsSym.sort()
366
367 lsKeys = ['open', 'high', 'low', 'close', 'volume']
368 ldfData = norObj.get_data( ldtTimestamps, lsSym, lsKeys )
369 dData = dict(zip(lsKeys, ldfData))
370 dfPrice = dData['close']
371
372
373
374 ''' Generate a list of DataFrames, one for each feature, with the same index/column structure as price data '''
375 ldfFeatures = applyFeatures( dData, [fcFeature], [dArgs], sMarketRel='$SPX' )
376
377 ''' Use last 3 months of index, to avoid lookback nans '''
378
379
380 for sSym in lsSym:
381 plt.subplot( 211 )
382 plt.plot( ldfFeatures[0].index[-60:], dfPrice[sSym].values[-60:] )
383 plt.plot( ldfFeatures[0].index[-60:], dfPrice['$SPX'].values[-60:] * dfPrice[sSym].values[-60] / dfPrice['$SPX'].values[-60] )
384 plt.legend((sSym, '$SPX'))
385 plt.title(sSym)
386 plt.subplot( 212 )
387 plt.plot( ldfFeatures[0].index[-60:], ldfFeatures[0][sSym].values[-60:] )
388 plt.title( fcFeature.__name__)
389 plt.show()
390
391 if __name__ == '__main__':
392 pass
393