Source Code for Module instant.cache

  1  """This module contains helper functions for working with the module cache. 
  2   
  3  Example operations: 
  4    - modulename = modulename_from_checksum(checksum) 
  5    - modulename = modulename_from_checksum(compute_checksum(signature)) 
  6    - module = import_module_directly(path, modulename) 
  7    - module = import_module(modulename) 
  8    - module = import_module(checksum) 
  9    - module = import_module(compute_checksum(signature)) 
 10    - modules = cached_modules() 
 11    - modules = cached_modules(cache_dir) 
 12  """ 
 13   
 14  import os, sys, re 
 15  from .output import instant_warning, instant_assert, instant_debug 
 16  from .paths import get_default_cache_dir, validate_cache_dir 
 17  from .signatures import compute_checksum 
 18  from .locking import get_lock, release_lock 
 19   
 20  # TODO: We could make this an argument, but it's used indirectly several places so take care. 
 21  _modulename_prefix = "instant_module_" 
 22 -def modulename_from_checksum(checksum): 
 23      "Construct a module name from a checksum for use in cache." 
 24      return _modulename_prefix + checksum 
 25   
 26   
 27 -def checksum_from_modulename(modulename): 
 28      "Construct a module name from a checksum for use in cache." 
 29      return modulename.remove(_modulename_prefix) 
 30   
 31   
 32 -def import_module_directly(path, modulename): 
 33      "Import a module with the given module name that resides in the given path." 
 34      sys.path.insert(0, path) 
 35      e = None 
 36      try: 
 37          module = __import__(modulename) 
 38      except Exception as e: 
 39          instant_warning("In instant.import_module_directly: Failed to import module '%s' from '%s';\n%s:%s;" % (modulename, path, type(e).__name__, e)) 
 40          module = None 
 41      finally: 
 42          sys.path.pop(0) 
 43      return module, e 
 44   
 45   
 46  _memory_cache = {} 
 47 -def memory_cached_module(moduleid): 
 48      "Returns the cached module if found." 
 49      import sys 
 50      module = _memory_cache.get(moduleid, None) 
 51      instant_debug("Found '%s' in memory cache with key '%r'." % (module, moduleid)) 
 52      return module 
 53   
 54   
 55 -def place_module_in_memory_cache(moduleid, module): 
 56      "Place a compiled module in cache with given id." 
 57      _memory_cache[moduleid] = module 
 58      instant_debug("Added module '%s' to cache with key '%r'." % (module, moduleid)) 
 59   
 60   
 61 -def is_valid_module_name(name): 
 62      NAMELENGTHLIMIT = 100 
 63      return len(name) < NAMELENGTHLIMIT and bool(re.search(r"^[a-zA-Z_][\w]*$", name)) 
 64   
 65   
 66 -def import_and_cache_module(path, modulename, moduleids): 
 67      module, e = import_module_directly(path, modulename) 
 68      instant_assert(module is not None, "Failed to import module found in cache. Modulename: '%s';\nPath: '%s';\n%s:%s;" % (modulename, path, type(e).__name__, e)) 
 69      for moduleid in moduleids: 
 70          place_module_in_memory_cache(moduleid, module) 
 71      return module 
 72   
 73   
 74 -def check_memory_cache(moduleid): 
 75      # Check memory cache first with the given moduleid 
 76      moduleids = [moduleid] 
 77      module = memory_cached_module(moduleid) 
 78      if module: return module, moduleids 
 79       
 80      # Get signature from moduleid if it isn't a string, 
 81      # and check memory cache again 
 82      if hasattr(moduleid, "signature"): 
 83          moduleid = moduleid.signature() 
 84          instant_debug("In instant.check_memory_cache: Got signature "\ 
 85                        "'%s' from moduleid.signature()." % moduleid) 
 86          module = memory_cached_module(moduleid) 
 87          if module: 
 88              # FIXME (GNW): I haved commented this out since it can continually  
 89              #              insert indentical objects into the cache. This 
 90              #              function checks the cache, so why should it also  
 91              #              insert? 
 92              #for moduleid in moduleids: 
 93              #    place_module_in_memory_cache(moduleid, module) 
 94              return module, moduleids 
 95          moduleids.append(moduleid) 
 96       
 97      # Construct a filename from the checksum of moduleid if it 
 98      # isn't already a valid name, and check memory cache again 
 99      if not is_valid_module_name(moduleid): 
100          moduleid = modulename_from_checksum(compute_checksum(moduleid)) 
101          instant_debug("In instant.check_memory_cache: Constructed module name "\ 
102                        "'%s' from moduleid '%s'." % (moduleid, moduleids[-1])) 
103          module = memory_cached_module(moduleid) 
104          if module: return module, moduleids 
105          moduleids.append(moduleid) 
106       
107      instant_debug("In instant.check_memory_cache: Failed to find module.") 
108      return None, moduleids 
109   
110   
111 -def check_disk_cache(modulename, cache_dir, moduleids): 
112      # Get file lock to avoid race conditions in cache 
113      lock = get_lock(cache_dir, modulename) 
114       
115      # Ensure a valid cache_dir 
116      cache_dir = validate_cache_dir(cache_dir) 
117       
118      # Check on disk, in current directory and cache directory 
119      for path in (os.getcwd(), cache_dir): 
120          if os.path.isdir(os.path.join(path, modulename)): 
121              # Found existing directory, try to import and place in memory cache 
122              module = import_and_cache_module(path, modulename, moduleids) 
123              if module: 
124                  instant_debug("In instant.check_disk_cache: Imported module "\ 
125                                "'%s' from '%s'." % (modulename, path)) 
126                  release_lock(lock) 
127                  return module 
128              else: 
129                  instant_debug("In instant.check_disk_cache: Failed to imported "\ 
130                                "module '%s' from '%s'." % (modulename, path)) 
131       
132      # All attempts failed 
133      instant_debug("In instant.check_disk_cache: Can't import module with modulename "\ 
134                    "%r using cache directory %r." % (modulename, cache_dir)) 
135      release_lock(lock) 
136      return None 
137   
138   
139 -def import_module(moduleid, cache_dir=None): 
140      """Import module from cache given its moduleid and an optional cache directory. 
141       
142      The moduleid can be either 
143        - the module name 
144        - a signature string, of which a checksum is taken to look up in the cache 
145        - a checksum string, which is used directly to look up in the cache 
146        - a hashable non-string object with a function moduleid.signature() which is used to get a signature string 
147      The hashable object is used to look up in the memory cache before signature() is called. 
148      If the module is found on disk, it is placed in the memory cache. 
149      """ 
150      # Look for module in memory cache 
151      module, moduleids = check_memory_cache(moduleid) 
152      if module: return module 
153       
154      # Look for module in disk cache 
155      modulename = moduleids[-1] 
156      return check_disk_cache(modulename, cache_dir, moduleids) 
157   
158   
159 -def cached_modules(cache_dir=None): 
160      "Return a list with the names of all cached modules." 
161      cache_dir = validate_cache_dir(cache_dir) 
162      return os.listdir(cache_dir) 
163