Home       Trees       Indices       Help   
QSTK
Package QSTK :: Package csvconverter :: Module norgate_csv_to_pkl
[hide private]
[frames] | no frames]

Source Code for Module QSTK.csvconverter.norgate_csv_to_pkl

  1  ''' 
  2  Created on Feb 25, 2011 
  3   
  4  @note: This assumes that all the CSV files will have exactly 7 columns. It ignores the first row in the csv files because it it the heading. 
  5          All the data is converted from csv to pkl 
  6  @author: Shreyas Joshi 
  7  @contact: shreyasj@gatech.edu 
  8  @summary: This is used to convert CSV files from norgate to pkl files- which are used when the simulator runs. 
  9  ''' 
 10   
 11  import numpy as np 
 12  import pickle as pkl 
 13  import qstkutil.utils as utils 
 14  import os 
 15  import dircache 
 16  import time 
 17  import sys 
 18   

 19 -def main (): 

 20       
 21      print "Starting..."+ str(time.strftime("%H:%M:%S")) 
 22       
 23      try: 
 24          rootdir = os.environ['QSDATA'] 
 25      except KeyError: 
 26          #rootdir = "/hzr71/research/QSData" 
 27          print "Please be sure to set the value for QSDATA in config.sh or local.sh\n"     
 28       
 29      fileExtensionToRemove = ".csv" 
 30       
 31      listOfInputPaths= list() 
 32   
 33       
 34  #For Gekko 
 35      rootindir = rootdir + "/Raw/Norgate/Stocks/CSV/US" 
 36      listOfInputPaths.append (rootindir + "/AMEX/") 
 37      listOfInputPaths.append (rootindir + "/Delisted Securities/") 
 38      listOfInputPaths.append (rootindir + "/NASDAQ/") 
 39      listOfInputPaths.append (rootindir + "/NYSE/") 
 40      listOfInputPaths.append (rootindir + "/NYSE Arca/") 
 41      listOfInputPaths.append (rootindir + "/OTC/") 
 42      listOfInputPaths.append (rootindir + "/Indices/ADRs/") 
 43      listOfInputPaths.append (rootindir + "/Indices/AMEX/") 
 44      listOfInputPaths.append (rootindir + "/Indices/CBOE/") 
 45      listOfInputPaths.append (rootindir + "/Indices/Dow Jones Americas/") 
 46      listOfInputPaths.append (rootindir + "/Indices/Dow Jones Averages/") 
 47      listOfInputPaths.append (rootindir + "/Indices/Dow Jones Broad Market/") 
 48      listOfInputPaths.append (rootindir + "/Indices/Dow Jones Misc/") 
 49      listOfInputPaths.append (rootindir + "/Indices/Dow Jones Style/") 
 50      listOfInputPaths.append (rootindir + "/Indices/Dow Jones US Industries/") 
 51      listOfInputPaths.append (rootindir + "/Indices/Dow Jones US Sectors/") 
 52      listOfInputPaths.append (rootindir + "/Indices/Dow Jones US Subsectors/") 
 53      listOfInputPaths.append (rootindir + "/Indices/Dow Jones US Supersectors/") 
 54      listOfInputPaths.append (rootindir + "/Indices/ISE/") 
 55      listOfInputPaths.append (rootindir + "/Indices/Merrill Lynch/") 
 56      listOfInputPaths.append (rootindir + "/Indices/Misc/") 
 57      listOfInputPaths.append (rootindir + "/Indices/Morgan Stanley/") 
 58      listOfInputPaths.append (rootindir + "/Indices/NASDAQ/") 
 59      listOfInputPaths.append (rootindir + "/Indices/NYSE/") 
 60      listOfInputPaths.append (rootindir + "/Indices/NYSE Arca/") 
 61      listOfInputPaths.append (rootindir + "/Indices/PHLX/") 
 62      listOfInputPaths.append (rootindir + "/Indices/Russell/") 
 63      listOfInputPaths.append (rootindir + "/Indices/S&P/") 
 64      listOfInputPaths.append (rootindir + "/Indices/S&P 500 Industries/") 
 65      listOfInputPaths.append (rootindir + "/Indices/S&P 500 Industry Groups/") 
 66      listOfInputPaths.append (rootindir + "/Indices/S&P 500 Sectors/") 
 67      listOfInputPaths.append (rootindir + "/Indices/S&P 500 Sub-Industries/") 
 68      listOfInputPaths.append (rootindir + "/Indices/S&P Technology/") 
 69      listOfInputPaths.append (rootindir + "/Indices/S&P TSX/") 
 70      listOfInputPaths.append (rootindir + "/Indices/Wilshire/") 
 71   
 72      rootoutdir = rootdir + "/Processed/Norgate/Stocks/US" 
 73      listOfOutputPaths= list() 
 74      listOfOutputPaths.append(rootoutdir + "/AMEX/") 
 75      listOfOutputPaths.append(rootoutdir + "/Delisted Securities/") 
 76      listOfOutputPaths.append(rootoutdir + "/NASDAQ/") 
 77      listOfOutputPaths.append(rootoutdir + "/NYSE/") 
 78      listOfOutputPaths.append(rootoutdir + "/NYSE Arca/") 
 79      listOfOutputPaths.append(rootoutdir + "/OTC/")     
 80      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 81      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 82      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 83      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 84      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 85      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 86      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 87      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 88      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 89      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 90      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 91      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 92      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 93      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 94      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 95      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 96      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 97      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 98      listOfOutputPaths.append(rootoutdir + "/Indices/") 
 99      listOfOutputPaths.append(rootoutdir + "/Indices/") 
100      listOfOutputPaths.append(rootoutdir + "/Indices/") 
101      listOfOutputPaths.append(rootoutdir + "/Indices/") 
102      listOfOutputPaths.append(rootoutdir + "/Indices/") 
103      listOfOutputPaths.append(rootoutdir + "/Indices/") 
104      listOfOutputPaths.append(rootoutdir + "/Indices/") 
105      listOfOutputPaths.append(rootoutdir + "/Indices/") 
106      listOfOutputPaths.append(rootoutdir + "/Indices/") 
107      listOfOutputPaths.append(rootoutdir + "/Indices/") 
108      listOfOutputPaths.append(rootoutdir + "/Indices/") 
109       
110      #If the output paths don't exist, then create them... 
111      for path in listOfOutputPaths: 
112          if not (os.access(path, os.F_OK)): 
113              #Path does not exist, so create it 
114              os.makedirs(path) 
115      #done making all output paths! 
116       
117      #In case there are already some files there- remove them. This will remove all the pkl fils from the previous run 
118      utils.clean_paths (listOfOutputPaths) 
119       
120      if (len(listOfInputPaths)!= len(listOfOutputPaths)): 
121          print "No. of input paths not equal to the number of output paths.. quitting" 
122          sys.exit("FAILURE") 
123          #if ends 
124       
125      path_ctr = -1; 
126      use_cols = range (1, 7 + 1) # will now use cols 1 to 7 
127      for path in listOfInputPaths: 
128          path_ctr =  path_ctr + 1; 
129          stocks_at_this_path = dircache.listdir(str(path)) 
130          #Next, throw away everything that is not a .csv And these are our stocks! Example: this should throw away the '$' folder in the NYSE folder 
131          filtered_names= filter (lambda x:(str(x).find(str(fileExtensionToRemove)) > -1), stocks_at_this_path) 
132          #Now, we remove the .csv to get the name of the stock 
133          filtered_names = map(lambda x:(x.partition(str(fileExtensionToRemove))[0]),filtered_names) 
134          stock_ctr = -1 
135          for stock in filtered_names: 
136              stock_ctr = stock_ctr + 1 
137              print "csv_to_pkl: processing: " + str (path + stock) 
138              #read in the stock date from the CSV file 
139              stock_data= np.loadtxt (path + stock+".csv", np.float, None, ",", None, 1, use_cols) 
140              stock_data_shape = stock_data.shape 
141              #print "stock_data_shape is: " + str(stock_data_shape) 
142              f = open (listOfOutputPaths[path_ctr] + filtered_names[stock_ctr] + ".pkl", "wb" ) 
143              pkl.dump (stock_data, f, -1) 
144              f.close() 
145          #for stock in stocks_at_this_path ends 
146      #for path in listOfInputPaths ends 
147      print "Finished..."+ str(time.strftime("%H:%M:%S")) 

148       
149      #main ends 
150       
151       
152  if __name__ == '__main__': 
153      main() 
154

   Home       Trees       Indices       Help   
QSTK
Generated by Epydoc 3.0.1 on Mon Mar 5 00:49:16 2012 http://epydoc.sourceforge.net