Source Code for Module QSTK.csvconverter.yahoo_data_getter

  1  '''
 
  2  Created on Apr 8, 2011
 
  3  
 
  4  @author: sjoshi42
 
  5  @summary: This module get stock data from yahoo finance
 
  6  ''' 
  7  
 
  8  import qstkutil.DataAccess as da 
  9  import qstkutil.utils as utils 
 10  import urllib2 
 11  import urllib 
 12  import datetime 
 13  import sys 
 14  import os 
 15  
 
 16 -def adjust (_str): 
 17      _list= list() 
 18  #    print str ("Before" + _str)
 
 19  #    symbol= _str.partition (",")[0]
 
 20  #    rest= _str.partition (",")[2] #Everything but symbol
 
 21      
 
 22      _date= _str.partition(",")[0] 
 23      rest= _str.partition(",")[2] #Everything but symbol and date 
 24  #    print str ("DT: " + _date)
 
 25  
 
 26      _list.append(rest.partition(",")[0]) #open 
 27      rest= rest.partition (",")[2] #Removed open 
 28  #    print str ("OP: " + _list[-1])
 
 29      
 
 30      _list.append(rest.partition(",")[0]) #high 
 31      rest= rest.partition (",")[2] #Removed high 
 32  #    print str ("HI: " + _list[-1])
 
 33      
 
 34      _list.append (rest.partition(",")[0]) #low 
 35      rest= rest.partition(",")[2] #Removed low 
 36  #    print str ("LO: " + _list[-1])
 
 37      
 
 38      _list.append (rest.partition(",")[0]) #close 
 39      rest= rest.partition (",")[2] #Removed close 
 40  #    print str ("CL: " + _list[-1])
 
 41      
 
 42      
 
 43      vol= rest.partition(",")[0] #volume 
 44  #    print str ("VOL: " + vol)
 
 45      
 
 46      adj_close= float (rest.partition (",")[2]) #Removed volume, and what we have left is adjusted close.. 
 47  #    print str ("ADJ_CL: " + str(adj_close))
 
 48      
 
 49      _list = map (lambda x: (float(x)), _list) #convert strings to floats 
 50      
 
 51      try: 
 52          ratio= adj_close/ _list[-1] #Adjusted close / close 
 53          _list= map (lambda x: (x * ratio), _list) #Updating the values 
 54          _list= map (lambda x: (str(x)), _list) 
 55          _list_concat= ",".join(_list) #Concat all the elements in the list using `,' as the separator 
 56          _str= _date + "," + _list_concat + "," + vol + "," + str (adj_close) + "\n" 
 57      except ZeroDivisionError: 
 58          print ("Warning: Close value is Zero.") 
 59          
 
 60      
 
 61      return _str 
 62      #adjust ends 
 
 63  
 
 64 -def main(): 
 65      #Getting path
 
 66      path= os.environ['QSDATA'] 
 67      
 
 68      get_data_for_exchange("NASDAQ", path) 
 69      get_data_for_exchange("NYSE", path) 
 70      get_data_for_exchange("AMEX", path) 
 71      
 
 72      #main ends
 
 73 -def get_data_for_exchange (exchange, data_path): 
 74      
 
 75      #data_access= da.DataAccess('norgate')
 
 76      #symbol_list= data_access.get_all_symbols()
 
 77      data_path= data_path + "/Raw/Yahoo/US/" + str (exchange) + "/" 
 78      
 
 79      #Create path if it doesn't exist
 
 80      if not (os.access(data_path, os.F_OK)): 
 81          os.makedirs(data_path) 
 82          
 
 83      utils.clean_paths(data_path)     
 84      
 
 85      symbol_list= list() 
 86      
 
 87      print "Getting list of stocks.." 
 88      
 
 89      try: 
 90          nasdaq_params= urllib.urlencode ({'exchange':str(exchange), 'render':'download'}) 
 91          nasdaq_get= urllib2.urlopen ('http://www.nasdaq.com/screening/companies-by-name.aspx?%s' % nasdaq_params) 
 92          symbol_list.append (nasdaq_get.readline()) #Now we have all the data in a list- but we need only the symbols so we remove the rest 
 93          while (len (symbol_list[-1]) > 0): 
 94              symbol_list.append (nasdaq_get.readline()) 
 95              #while ends
 
 96          symbol_list.pop(0) #This is just the word "symbol" and not a symbol itself     
 97          symbol_list.pop(-1) #Remove the last element because its going to be blank anyway     
 98          #symbol_list = map(lambda x:(x.partition(str(","))[0]),symbol_list) #Get the stuff before the first comma- which is the symbol
 
 99          
 
100          #Unfortunately this symbol is in quotes. So we have to remove them now
 
101          symbol_list = map(lambda x:(x.partition(str("\""))[2]),symbol_list) #Keep the stuff only after the first " 
102          symbol_list = map(lambda x:(x.partition(str("\""))[0]),symbol_list) #Keep the stuff before the second " 
103          
 
104      except urllib2.HTTPError: 
105          print "Unable to get list of stocks from server. Please check your internet connection and retry." 
106      except: 
107          print"Unknown error occoured when getting list of stocks from server." 
108      
 
109      print "Got " + str (len(symbol_list)) + " symbols. Now getting symbol data..." 
110      
 
111      _now =datetime.datetime.now(); 
112      miss_ctr=0; #Counts how many symbols we could get 
113      
 
114      for symbol in symbol_list: 
115          symbol_data=list() 
116          print "Getting " + str (symbol) 
117          
 
118          try: 
119              params= urllib.urlencode ({'a':03, 'b':12, 'c':2000, 'd':_now.month, 'e':_now.day, 'f':_now.year, 's': str(symbol)}) 
120              url_get= urllib2.urlopen("http://ichart.finance.yahoo.com/table.csv?%s" % params) 
121              
 
122              header= url_get.readline() 
123              symbol_data.append (url_get.readline()) 
124              while (len(symbol_data[-1]) > 0): 
125                  symbol_data.append(url_get.readline()) 
126  #                print str(symbol_data[-1])
 
127              
 
128              symbol_data.pop(-1) #The last element is going to be the string of length zero. We don't want to write that to file. 
129  
 
130              #To change adjusted close so that Yahoo data is same as Norgate data
 
131              symbol_data= map (adjust, symbol_data) 
132              
 
133              #Following changes so that the data looks like Norgate data and the change to cav_to_pkl.csv is minimized
 
134              symbol_data = map(lambda x:(x.replace("-", "")),symbol_data)  
135              symbol_data = map(lambda x:(str(symbol) + "," + str(x)) ,symbol_data) #This means that the header is wrong but since it is ignored later anyways- this will work 
136              
 
137              #now writing data to file
 
138              f= open (data_path + symbol + ".csv", 'w') 
139              
 
140              #Writing the header
 
141              f.write (header) 
142              
 
143              while (len(symbol_data) > 0): 
144                  f.write (symbol_data.pop()) 
145               
 
146              f.close();     
147  #            print url_get.readline()
 
148  #            f= open (data_path + symbol + ".csv", 'w')
 
149  #            f.write (url_get.read())
 
150  #            f.close()
 
151                          
 
152          except urllib2.HTTPError: 
153              miss_ctr= miss_ctr+1 
154              print "Unable to fetch data for stock: " + str (symbol) 
155          except urllib2.URLError: 
156              print "URL Error for stock: " + str (symbol) 
157              
 
158  #        except:
 
159  #            print "Some error occurred"
 
160              #except ends 
 
161              
 
162          
 
163          #for ends
 
164      print "All done. Got " + str (len(symbol_list) - miss_ctr) + " stocks. Could not get " + str (miss_ctr) + " stocks."    
165      #main ends
 
166  
 
167  
 
168  if __name__ == '__main__': 
169      main() 
170