Avoid race condition in the matrix cache handling. (#1280)

* Improved fix for matrix cache handling.

During the MatrixDataCacheItem acquire function there was a
time when the write lock was released and the read lock was taken.
During that time, the dataset could have been deleted, later
result in the MatrixDataCacheManageri data adaptor returning None.

The solution is to demote the writer lock to a reader lock instead
of unlocking and relocking.

Also, when a the cache needs to delete an entry, the delete
is done outside the MatrixDataCacheManager lock.   This operation
only requires the write lock for the MatrixDataCacheItem.

Fixes #1255

server/data_common/matrix_loader.py

@@ -19,32 +19,33 @@ class MatrixDataCacheItem(object):
         self.data_adaptor = None
         self.data_lock = RWLock()
 
-    def acquire(self, app_config):
-        """returns the data_adaptor if cached.  opens the data_adaptor if not.
-        In either case, the a reader lock is taken.  Must call release when
-        the data_adaptor is no longer needed"""
-
+    def acquire_existing(self):
+        """If the data_adaptor exists, take a read lock and return it, else return None"""
         self.data_lock.r_acquire()
         if self.data_adaptor:
             return self.data_adaptor
 
         self.data_lock.r_release()
-        try:
-            with self.data_lock.w_locked():
-                # the data may have been loaded while waiting on the lock
-                if not self.data_adaptor:
-                    self.loader.pre_load_validation()
-                    self.data_adaptor = self.loader.open(app_config)
-
-        except Exception:
-            # necessary to acquire after an exception, since the release will occur when
-            # the context exits
-            self.data_lock.r_acquire()
-            raise
+        return None
 
+    def acquire_and_open(self, app_config):
+        """returns the data_adaptor if cached.  opens the data_adaptor if not.
+        In either case, the a reader lock is taken.  Must call release when
+        the data_adaptor is no longer needed"""
         self.data_lock.r_acquire()
         if self.data_adaptor:
             return self.data_adaptor
+        self.data_lock.r_release()
+
+        self.data_lock.w_acquire()
+        # the data may have been loaded while waiting on the lock
+        if not self.data_adaptor:
+            self.loader.pre_load_validation()
+            self.data_adaptor = self.loader.open(app_config)
+
+        # demote the write lock to a read lock.
+        self.data_lock.w_demote()
+        return self.data_adaptor
 
     def release(self):
         """Release the reader lock"""
@@ -65,7 +66,7 @@ class MatrixDataCacheManager(object):
     when the context ends.  This class currently implements a simple least recently used cache,
     which can delete a dataset from the cache to make room for a new oneo
 
-    This is the indended usage pattern:
+    This is the intended usage pattern:
 
            m = MatrixDataCacheManager()
            with m.data_adaptor(location, app_config) as data_adaptor:
@@ -77,7 +78,11 @@ class MatrixDataCacheManager(object):
     #  TODO:  This is very simple.  This can be improved by taking into account how much space is actually
     #         taken by each dataset, instead of arbitrarily picking a max datasets to cache.
     #         Also, this should be controlled by a configuration parameter.
-    MAX_CACHED = 3
+    MAX_CACHED = 5
+
+    @staticmethod
+    def set_max_datasets(max_cached):
+        MatrixDataCacheManager.MAX_CACHED = max_cached
 
     # FIXME:   If the number of active datasets exceeds the MAX_CACHED, then each request could
     # lead to a dataset being deleted and a new only being opened: the cache will get thrashed.
@@ -97,33 +102,43 @@ class MatrixDataCacheManager(object):
     @contextmanager
     def data_adaptor(self, location, app_config):
         # create a loader for to this location if it does not already exist
+
+        delete_adaptor = None
+        data_adaptor = None
+
         with self.lock:
             value = self.datasets.get(location)
             if value is not None:
                 cache_item = value[0]
                 last_accessed = time.time()
                 self.datasets[location] = (cache_item, last_accessed)
-            else:
+                data_adaptor = cache_item.acquire_existing()
+
+            if data_adaptor is None:
                 while True:
                     # find the last access times for each loader
-                    items = list(self.datasets.items())
-                    sorted(items, key=lambda x: x[1][1])
-                    if len(items) < self.MAX_CACHED:
+                    if len(self.datasets) < self.MAX_CACHED:
                         break
 
+                    items = list(self.datasets.items())
+                    sorted(items, key=lambda x: x[1][1])
                     # close the least recently used loader
                     oldest = items[0]
                     oldest_cache = oldest[1][0]
                     oldest_key = oldest[0]
-                    oldest_cache.delete()
                     del self.datasets[oldest_key]
+                    delete_adaptor = oldest_cache
 
                 last_accessed = time.time()
                 loader = MatrixDataLoader(location, app_config=app_config)
                 cache_item = MatrixDataCacheItem(loader)
                 self.datasets[location] = (cache_item, last_accessed)
+
         try:
-            data_adaptor = cache_item.acquire(app_config)
+            if delete_adaptor:
+                delete_adaptor.delete()
+            if data_adaptor is None:
+                data_adaptor = cache_item.acquire_and_open(app_config)
             yield data_adaptor
         finally:
             cache_item.release()