1 '''
2 Created on Nov 7, 2011
3
4 @author: John Cornwell
5 @contact: JohnWCornwellV@gmail.com
6 @summary: File containing various feature functions
7
8 '''
9
10 ''' Python imports '''
11 import random
12
13 ''' 3rd Party Imports '''
14 import pandas as pand
15 import numpy as np
16
17 ''' QSTK Imports '''
18 import qstkutil.tsutil as tsu
19
20
21 -def featMA( dData, lLookback=30, bRel=True ):
22 '''
23 @summary: Calculate moving average
24 @param dData: Dictionary of data to use
25 @param lLookback: Number of days to look in the past
26 @return: DataFrame array containing values
27 '''
28
29 dfPrice = dData['close']
30
31 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
32 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
33
34 ''' Loop through stocks '''
35 for sStock in dfPrice.columns:
36
37 tsPrice = dfPrice[sStock]
38 tsRet = dfRet[sStock]
39 lSum = 0
40
41 ''' Loop over time '''
42 for i in range(len(tsPrice.index)):
43
44 if pand.isnull(tsPrice[i]):
45 continue
46
47
48 if pand.notnull( tsPrice[i] ):
49 lSum += tsPrice[i]
50
51 if i < lLookback - 1:
52 tsRet[i] = float('nan')
53 continue
54
55 ''' If we have the bare min, take the avg, else remove the last and take the avg '''
56 tsRet[i] = np.sum( tsPrice[i-(lLookback-1):i+1]) / lLookback
57
58 ''' See if we should make this relative moving average '''
59 if bRel:
60 tsRet[i] /= tsPrice[i]
61
62 return dfRet
63
64
66 '''
67 @summary: Calculate RSI
68 @param dData: Dictionary of data to use
69 @param lLookback: Number of days to look in the past, 14 is standard
70 @return: DataFrame array containing values
71 '''
72
73 dfPrice = dData['close']
74
75 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
76 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
77 fLookback = float(lLookback)
78
79 ''' Loop through stocks '''
80 for sStock in dfPrice.columns:
81 tsPrice = dfPrice[sStock]
82 tsRet = dfRet[sStock]
83
84 fGain = 0.0
85 fLoss = 0.0
86
87 lNonNull=0
88 ''' Loop over time '''
89 for i in range(len(tsPrice.index)):
90
91 if pand.isnull( tsPrice[i] ):
92 continue
93 else:
94 lNonNull += 1
95
96 ''' Once we have the proper number of periods we smooth the totals '''
97 if lNonNull > fLookback:
98 fGain *= (fLookback - 1) / fLookback
99 fLoss *= (fLookback - 1) / fLookback
100
101 ''' Calculate gain or loss and add to total '''
102 if lNonNull > 1:
103 fDelta = tsPrice[i] - tsPrice[i-1]
104 if fDelta > 0.0:
105 fGain += fDelta / fLookback
106 else:
107 fLoss += fDelta / fLookback
108
109 ''' Calculate RS and then RSI '''
110 if i > fLookback - 1:
111 if fLoss == 0.0:
112 tsRet[i] = 100.0
113 elif fGain == 0.0:
114 tsRet[i] = 0.0
115 else:
116 fRS = fGain / fLoss
117 tsRet[i] = 100 - 100 / (1-fRS)
118
119
120 return dfRet
121
122
124 '''
125 @summary: Calculate Drawdown for the stock
126 @param dData: Dictionary of data to use
127 @param lLookback: Days to look back
128 @return: DataFrame array containing values
129 @warning: Drawdown and RunUp can depend heavily on sample period
130 '''
131
132 dfPrice = dData['close']
133
134 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
135 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
136
137 ''' Loop through stocks '''
138 for sStock in dfPrice.columns:
139 tsPrice = dfPrice[sStock]
140 tsRet = dfRet[sStock]
141
142 ''' Loop over time '''
143 for i in range(len(tsPrice.index)):
144
145 ''' Get starting and stopping indexes '''
146 if i != len(tsPrice.index):
147 lStop = i + 1
148 else:
149 lStop = None
150
151 lStart = max( 0, i - (lLookback - 1) )
152
153 ''' Calculate peak value, and subsequent drawdown '''
154 fPeak = np.max( tsPrice.values[ lStart:lStop ] )
155
156 tsRet[i] = tsPrice[i] / fPeak
157
158 return dfRet
159
161 '''
162 @summary: CalculateRunup for the stock
163 @param dData: Dictionary of data to use
164 @param lLookback: Number of days to calculate min over
165 @return: DataFrame array containing feature values
166 @warning: Drawdown and RunUp can depend heavily on when the sample starts
167 '''
168
169 dfPrice = dData['open']
170
171 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
172 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
173
174 ''' Loop through stocks '''
175 for sStock in dfPrice.columns:
176 tsPrice = dfPrice[sStock]
177 tsRet = dfRet[sStock]
178
179 ''' Loop over time '''
180 for i in range(len(tsPrice.index)):
181
182 ''' Get starting and stopping indexes '''
183 if i != len(tsPrice.index):
184 lStop = i + 1
185 else:
186 lStop = None
187
188 lStart = max( 0, i - (lLookback - 1) )
189
190 ''' Calculate trough value, and subsequent drawdown '''
191 fTrough = np.min( tsPrice.values[ lStart:lStop ] )
192
193 tsRet[i] = tsPrice[i] / fTrough
194 if tsPrice[i] < fTrough or pand.isnull(fTrough):
195 fTrough = tsPrice[i]
196
197 tsRet[i] = tsPrice[i] / fTrough
198
199 return dfRet
200
201
203 '''
204 @summary: Calculate moving average
205 @param dData: Dictionary of data to use
206 @param lLookback: Number of days to use for MA
207 @return: DataFrame array containing values
208 '''
209
210 dfVolume = dData['volume']
211
212 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
213 dfRet = pand.DataFrame( index=dfVolume.index, columns=dfVolume.columns, data=np.zeros(dfVolume.shape) )
214
215 ''' Loop through stocks '''
216 for sStock in dfVolume.columns:
217
218 tsVol = dfVolume[sStock]
219 tsRet = dfRet[sStock]
220 lSum = 0
221
222 ''' Loop over time '''
223 for i in range(len(tsVol.index)):
224
225 if pand.notnull( tsVol[i] ):
226 lSum += tsVol[i]
227
228 if i < lLookback - 1:
229 tsRet[i] = float('nan')
230 continue
231
232 ''' If we have the bare min, take the avg, else remove the last and take the avg '''
233 tsRet[i] = np.sum( tsVol[i-(lLookback-1):i+1]) / lLookback
234
235 ''' Make this relative to the MA of volume '''
236 tsRet[i] /= tsVol[i]
237
238 return dfRet
239
240 -def featAroon( dData, bDown=False, lLookback=25 ):
241 '''
242 @summary: Calculate Aroon - indicator indicating days since a 25-day high/low, weighted between 0 and 100
243 @param dData: Dictionary of data to use
244 @param bDown: If false, calculates aroonUp (high), else aroonDown (lows)
245 @param lLookback: Days to lookback to calculate high/low from
246 @return: DataFrame array containing feature values
247 '''
248
249 dfPrice = dData['close']
250
251 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
252 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
253
254 ''' Loop through stocks '''
255 for sStock in dfPrice.columns:
256 tsPrice = dfPrice[sStock]
257 tsRet = dfRet[sStock]
258
259 ''' Peaks will be a sorted, descending list of highs and indexes '''
260 lfPeaks = []
261
262 ''' Loop over time '''
263 for i in range(len(tsPrice.index)):
264 j = 0
265 while j < (len(lfPeaks)):
266 if bDown:
267 ''' If down, use troughts '''
268 if tsPrice[i] < lfPeaks[j][0]:
269 break
270 else:
271 ''' If up, use peaks '''
272 if tsPrice[i] > lfPeaks[j][0]:
273 break
274 j+=1
275
276 ''' Insert into sorted list, remove all lesser, older peaks '''
277 lfPeaks.insert( j, (tsPrice[i], i) )
278 lfPeaks = lfPeaks[:j+1]
279
280 ''' Remove all outdated peaks '''
281 j = 0
282 while j < (len(lfPeaks)):
283 if i - lfPeaks[j][1] > lLookback:
284 lfPeaks.pop(j)
285 continue
286
287 j += 1
288
289
290
291 tsRet[i] = ((lLookback - (i - lfPeaks[0][1])) / float(lLookback)) * 100.0
292
293 ''' perturb value '''
294 random.seed(i)
295 tsRet[i] += random.uniform( -0.0001, 0.0001 )
296
297 return dfRet
298
299
301 '''
302 @summary: Calculate stochastic oscillator - indicates what range of recent low-high spread we are in.
303 @param dData: Dictionary of data to use
304 @param bFast: If false, do slow stochastics, 3 day MA, if not use fast, no MA
305 @return: DataFrame array containing feature values
306 '''
307
308 dfLow = dData['low']
309 dfHigh = dData['high']
310 dfPrice = dData['close']
311
312
313 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
314 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
315
316 ''' Loop through stocks '''
317 for sStock in dfPrice.columns:
318 tsPrice = dfPrice[sStock]
319 tsHigh = dfHigh[sStock]
320 tsLow = dfLow[sStock]
321 tsRet = dfRet[sStock]
322
323 ''' For slow stochastic oscillator we need to remember 3 past values '''
324 lfPastStoch = []
325
326 ''' Loop over time '''
327 for i in range(len(tsPrice.index)):
328
329 ''' NaN if not enough data to do lookback '''
330 if i < lLookback - 1:
331 tsRet[i] = float('nan')
332 continue
333
334 fLow = 1E300
335 fHigh = -1E300
336 ''' Find highest high and lowest low '''
337 for j in range(lLookback):
338
339 lInd = i-j
340
341 if tsHigh[lInd] > fHigh:
342 fHigh = tsHigh[lInd]
343 if tsLow[lInd] < fLow:
344 fLow = tsLow[lInd]
345
346 fStoch = (tsPrice[i] - fLow) / (fHigh - fLow)
347
348 ''' For fast we just take the stochastic value, slow we need 3 day MA '''
349 if bFast:
350 tsRet[i] = fStoch
351 else:
352 if len(lfPastStoch) < lMA:
353 lfPastStoch.append(fStoch)
354 continue
355
356 lfPastStoch.append(fStoch)
357 lfPastStoch.pop(0)
358
359 tsRet[i] = sum(lfPastStoch) / float(len(lfPastStoch))
360
361
362
363 return dfRet
364
365 -def featBeta( dData, lLookback=14, sMarket='$SPX' ):
366 '''
367 @summary: Calculate beta relative to a given stock/index.
368 @param dData: Dictionary of data to use
369 @param sStock: Stock to calculate beta relative to
370 @return: DataFrame array containing feature values
371 '''
372
373 dfPrice = dData['close']
374
375 ''' Calculate returns '''
376 naRets = dfPrice.values.copy()
377 tsu.returnize1(naRets)
378 dfHistReturns = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=naRets )
379
380 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
381 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
382
383 ''' Loop through stocks '''
384 for sStock in dfHistReturns.columns:
385 tsHistReturns = dfHistReturns[sStock]
386 tsMarket = dfHistReturns[sMarket]
387 tsRet = dfRet[sStock]
388
389 ''' Loop over time '''
390 for i in range(len(tsRet.index)):
391
392 ''' NaN if not enough data to do lookback '''
393 if i < lLookback - 1:
394 tsRet[i] = float('nan')
395 continue
396
397 naStock = tsHistReturns[ i - (lLookback - 1): i+1 ]
398 naMarket = tsMarket[ i - (lLookback - 1): i+1 ]
399
400 ''' Beta is the slope the line, with market returns on X, stock returns on Y '''
401 try:
402 fBeta, unused = np.polyfit( naMarket, naStock, 1)
403 tsRet[i] = fBeta
404 except:
405 'Numpy Error featBeta'
406 tsRet[i] = float('NaN')
407
408 return dfRet
409
411 '''
412 @summary: Calculate bollinger position as a function of std deviations.
413 @param dData: Dictionary of data to use
414 @param lLookback: Number of days to calculate moving average over
415 @return: DataFrame array containing feature values
416 '''
417
418 dfPrice = dData['close']
419
420 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
421 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
422
423 ''' Loop through stocks '''
424 for sStock in dfPrice.columns:
425 tsPrice = dfPrice[sStock]
426 tsRet = dfRet[sStock]
427
428 ''' Loop over time '''
429 for i in range(len(tsPrice.index)):
430
431 ''' NaN if not enough data to do lookback '''
432 if i < lLookback - 1:
433 tsRet[i] = float('nan')
434 continue
435
436 fAvg = np.average( tsPrice[ i-(lLookback-1):i+1 ] )
437 fStd = np.std( tsPrice[ i-(lLookback-1):i+1 ] )
438
439 tsRet[i] = (tsPrice[i] - fAvg) / fStd
440
441 return dfRet
442
443
445 '''
446 @summary: Calculate correlation of two stocks.
447 @param dData: Dictionary of data to use
448 @param lLookback: Number of days to calculate moving average over
449 @return: DataFrame array containing feature values
450 '''
451
452 dfPrice = dData['close']
453
454 if sRel not in dfPrice.columns:
455 raise KeyError( "%s not found in data provided to featCorrelation"%sRel )
456
457 ''' Calculate returns '''
458 naRets = dfPrice.values.copy()
459 tsu.returnize1(naRets)
460 dfHistReturns = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=naRets )
461
462 ''' Feature DataFrame will be 1:1, we can use the price as a template '''
463 dfRet = pand.DataFrame( index=dfPrice.index, columns=dfPrice.columns, data=np.zeros(dfPrice.shape) )
464
465 ''' Loop through stocks '''
466 for sStock in dfHistReturns.columns:
467 tsHistReturns = dfHistReturns[sStock]
468 tsRelativeReturns = dfHistReturns[sRel]
469 tsRet = dfRet[sStock]
470
471 ''' Loop over time '''
472 for i in range(len(tsHistReturns.index)):
473
474 ''' NaN if not enough data to do lookback '''
475 if i < lLookback - 1:
476 tsRet[i] = float('nan')
477 continue
478
479 naCorr = np.corrcoef( tsHistReturns[ i-(lLookback-1):i+1 ], tsRelativeReturns[ i-(lLookback-1):i+1 ] )
480
481 tsRet[i] = naCorr[0,1]
482
483 return dfRet
484
485
486 if __name__ == '__main__':
487 pass
488