@@ -222,6 +222,21 @@ def from_dict(cls, dct):
222222 obj .result_dict = dct ['result_dict' ]
223223
224224
225+ def _storage_exception_msg (func , storage , exc ):
226+ func_id = str (get_uuid (func ))
227+ if func .is_named :
228+ func_id += " (" + func .name + ")"
229+ try :
230+ storage_id = " in " + str (storage .identifier ) + "."
231+ except :
232+ storage_id = "."
233+
234+ msg = ("A problem occured while loading disk-cached results. "
235+ "Function " + func_id + storage_id + "\n " + str (type (exc ))
236+ + str (exc ))
237+ return msg
238+
239+
225240class StorableFunction (StorableNamedObject ):
226241 """Function wrapper, providing result caching and storage to disk.
227242
@@ -254,18 +269,18 @@ def __init__(self, func, func_config=None, store_source=None, **kwargs):
254269
255270 self .local_cache = None # set correctly by self.mode setter
256271 self ._disk_cache = True
257- self ._handler = None
272+ self ._handlers = set ([])
258273 self .mode = 'analysis'
259274
260- def set_handler (self , storage , override = False ):
261- if not override and self ._handler is not None :
262- raise RuntimeError ( "Handler for this StorableFunction has "
263- + "already been set:" + str (self . _handler ))
264- self ._handler = storage
275+ def add_handler (self , storage , override = False ):
276+ self ._handlers . add ( storage )
277+
278+ def remove_handler (self , handler ):
279+ self ._handlers . discard ( handler )
265280
266281 @property
267282 def has_handler (self ):
268- return self ._handler is not None
283+ return len ( self ._handlers ) > 0
269284
270285 @property
271286 def disk_cache (self ):
@@ -314,11 +329,14 @@ def from_dict(cls, dct):
314329
315330 def preload_cache (self , storage = None ):
316331 if storage is None :
317- storage = self ._handler .storage
332+ storages = [h .storage for h in self ._handlers ]
333+ else :
334+ storages = [storage ]
318335
319336 uuid = get_uuid (self )
320- cache_values = storage .backend .load_storable_function_table (uuid )
321- self .local_cache .cache_results (cache_values )
337+ for storage in storages :
338+ cache = storage .backend .load_storable_function_table (uuid )
339+ self .local_cache .cache_results (cache )
322340
323341 def is_scalar (self , item ):
324342 """Determine whether the input needs to be wrapped in a list.
@@ -360,10 +378,27 @@ def _get_cached(self, uuid_items):
360378 return self .local_cache .get_results_as_dict (uuid_items )
361379
362380 def _get_storage (self , uuid_items ):
363- if not self ._handler :
381+ if not self .has_handler :
364382 return {}, uuid_items
365383
366- return self ._handler .get_function_results (get_uuid (self ), uuid_items )
384+ missing = uuid_items
385+ found = {}
386+ my_uuid = get_uuid (self )
387+ for handler in self ._handlers :
388+ try :
389+ uuid_map , missing = handler .get_function_results (
390+ my_uuid , missing
391+ )
392+ except Exception as e :
393+ # for any error, we just warn -- can be correctly calculated
394+ # in eval
395+ msg = _storage_exception_msg (self , handler .storage , e )
396+ warnings .warn (msg )
397+ uuid_map = {}
398+
399+ found .update (uuid_map )
400+
401+ return found , missing
367402
368403 def __call__ (self , items ):
369404 # important: implementation is that we always try to take an
@@ -475,9 +510,13 @@ def register_function(self, func, example_result=None):
475510 if not is_registered :
476511 self .all_functions [func_uuid ].append (func )
477512
478- # set handler; only if the table exists
479- if not func .has_handler and (add_table or not needs_table ):
480- func .set_handler (self )
513+ if add_table or not needs_table :
514+ func .add_handler (self )
515+
516+ def close (self ):
517+ for uuid , copies in self .all_functions .items ():
518+ for func in copies :
519+ func .remove_handler (self )
481520
482521 def clear_non_canonical (self ):
483522 self .all_functions = collections .defaultdict (list )
0 commit comments