Avoid race condition in the matrix cache handling. (#1280)

* Improved fix for matrix cache handling.

During the MatrixDataCacheItem acquire function there was a
time when the write lock was released and the read lock was taken.
During that time, the dataset could have been deleted, later
result in the MatrixDataCacheManageri data adaptor returning None.

The solution is to demote the writer lock to a reader lock instead
of unlocking and relocking.

Also, when a the cache needs to delete an entry, the delete
is done outside the MatrixDataCacheManager lock.   This operation
only requires the write lock for the MatrixDataCacheItem.

Fixes #1255

server/common/app_config.py

@@ -56,6 +56,7 @@ class AppConfig(object):
             self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
             self.multi_dataset__index = dc["multi_dataset"]["index"]
             self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
+            self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"]
             self.single_dataset__datapath = dc["single_dataset"]["datapath"]
             self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
             self.single_dataset__var_names = dc["single_dataset"]["var_names"]
@@ -244,7 +245,8 @@ class AppConfig(object):
     def handle_multi_dataset(self, context):
         self.__check_attr("multi_dataset__dataroot", (type(None), str))
         self.__check_attr("multi_dataset__index", (type(None), bool, str))
-        self.__check_attr("multi_dataset__allowed_matrix_types", (list))
+        self.__check_attr("multi_dataset__allowed_matrix_types", (tuple, list))
+        self.__check_attr("multi_dataset__matrix_cache__max_datasets", int)
 
         if self.multi_dataset__dataroot is None:
             return
@@ -256,6 +258,9 @@ class AppConfig(object):
             except ValueError:
                 raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
 
+        # matrix cache
+        MatrixDataCacheManager.set_max_datasets(self.multi_dataset__matrix_cache__max_datasets)
+
     def handle_user_annotations(self, context):
         self.__check_attr("user_annotations__enable", bool)
         self.__check_attr("user_annotations__type", str)

server/common/default_config.py

@@ -29,6 +29,11 @@ multi_dataset:
   # A list of allowed matrix types.  If an empty list, then all matrix types are allowed
   allowed_matrix_types: []
 
+  matrix_cache:
+    # The maximum number of datasets that may be opened at one time.  The least recently used dataset
+    # is evicted from the cache first.
+    max_datasets: 5
+
 single_dataset:
   datapath: null
   obs_names: null
@@ -63,6 +68,7 @@ adaptor:
 
   anndata_adaptor:
       backed: false
+
 """