fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-27 00:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improvements to the matrix cache (#1340)

Browse Source 
* Improvements to the matrix cache

- Add a timelimit for the matrix in the cache.
Once the timelimit is reached, the matrix can be removed.

- If a DatasetAccessError occurs, then remove the dataset
from the matrix cache.

Fixes #1322
This commit is contained in:
bmccandless
2020-04-02 13:44:11 -07:00
committed by GitHub
parent fa0164fdf6
commit 308ee64f30
9 changed files with 260 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/data_common/matrix_loader.py
+90 -35
View File
@@ -47,7 +47,7 @@ class MatrixDataCacheItem(object):
# necessary to hold the reader lock after an exception, since
# the release will occur when the context exits.
self.data_lock.w_demote()
raise e
raise DatasetAccessError(str(e))
# demote the write lock to a read lock.
self.data_lock.w_demote()
@@ -64,90 +64,145 @@ class MatrixDataCacheItem(object):
self.data_adaptor.cleanup()
self.data_adaptor = None
def attempt_delete(self):
"""Delete, but only if the write lock can be immediately locked. Return True if the delete happened"""
if self.data_lock.w_acquire_non_blocking():
if self.data_adaptor:
try:
self.data_adaptor.cleanup()
self.data_adaptor = None
except Exception:
# catch all exceptions to ensure the lock is released
pass
self.data_lock.w_release()
return True
else:
return False
class MatrixDataCacheInfo(object):
def __init__(self, cache_item, timestamp):
# The MatrixDataCacheItem in the cache
self.cache_item = cache_item
# The last time the cache_item was accessed
self.last_access = timestamp
# The number of times the cache_item was accessed (used for testing)
self.num_access = 1
class MatrixDataCacheManager(object):
"""A class to manage the cached datasets. This is intended to be used as a context manager
for handling api requests. When the context is created, the data_adator is either loaded or
retrieved from a cache. In either case, the reader lock is taken during this time, and release
when the context ends. This class currently implements a simple least recently used cache,
which can delete a dataset from the cache to make room for a new oneo
which can delete a dataset from the cache to make room for a new one.
This is the intended usage pattern:
m = MatrixDataCacheManager()
m = MatrixDataCacheManager(max_cached=..., timelimmit_s = ...)
with m.data_adaptor(location, app_config) as data_adaptor:
# use the data_adaptor for some operation
"""
# The number of datasets to cache. When MAX_CACHED is reached, the least recently used
# cache is replaced with the newly requested one.
# TODO: This is very simple. This can be improved by taking into account how much space is actually
# taken by each dataset, instead of arbitrarily picking a max datasets to cache.
# Also, this should be controlled by a configuration parameter.
MAX_CACHED = 5
@staticmethod
def set_max_datasets(max_cached):
MatrixDataCacheManager.MAX_CACHED = max_cached
# FIXME: If the number of active datasets exceeds the MAX_CACHED, then each request could
# FIXME: If the number of active datasets exceeds the max_cached, then each request could
# lead to a dataset being deleted and a new only being opened: the cache will get thrashed.
# In this case, we may need to send back a 503 (Server Unavailable), or some other error message.
# FIXME: If the actual dataset is changed. E.g. a new set of datafiles replaces an existing set,
# then the cache will not react to this. Ideally this would invalidate the cache. One solution is
# to keep a small metadata file associated with each dataset, which contains versioning information.
# When the dataset is accessed, the current version can be compared with the cached version, and if
# there is a mismatch, then the cache can be refreshed.
# NOTE: If the actual dataset is changed. E.g. a new set of datafiles replaces an existing set,
# then the cache will not react to this, however once the cache time limit is reached, the dataset
# will automatically be refreshed.
def __init__(self):
# key is location, value is tuple of (MatrixDataCacheItem, last_accessed)
def __init__(self, max_cached, timelimit_s=None):
# key is location, value is a MatrixDataCacheInfo
self.datasets = {}
# lock to protect the datasets
self.lock = threading.Lock()
# The number of datasets to cache. When max_cached is reached, the least recently used
# cache is replaced with the newly requested one.
# TODO: This is very simple. This can be improved by taking into account how much space is actually
# taken by each dataset, instead of arbitrarily picking a max datasets to cache.
self.max_cached = max_cached
# items are automatically removed from the cache once this time limit is reached
self.timelimit_s = timelimit_s
@contextmanager
def data_adaptor(self, location, app_config):
# create a loader for to this location if it does not already exist
delete_adaptor = None
data_adaptor = None
cache_item = None
with self.lock:
value = self.datasets.get(location)
if value is not None:
cache_item = value[0]
last_accessed = time.time()
self.datasets[location] = (cache_item, last_accessed)
data_adaptor = cache_item.acquire_existing()
self.evict_old_datasets()
info = self.datasets.get(location)
if info is not None:
info.last_access = time.time()
info.num_access += 1
self.datasets[location] = info
data_adaptor = info.cache_item.acquire_existing()
cache_item = info.cache_item
if data_adaptor is None:
while True:
# find the last access times for each loader
if len(self.datasets) < self.MAX_CACHED:
if len(self.datasets) < self.max_cached:
break
items = list(self.datasets.items())
sorted(items, key=lambda x: x[1][1])
items = sorted(items, key=lambda x: x[1].last_access)
# close the least recently used loader
oldest = items[0]
oldest_cache = oldest[1][0]
oldest_cache = oldest[1].cache_item
oldest_key = oldest[0]
del self.datasets[oldest_key]
delete_adaptor = oldest_cache
last_accessed = time.time()
loader = MatrixDataLoader(location, app_config=app_config)
cache_item = MatrixDataCacheItem(loader)
self.datasets[location] = (cache_item, last_accessed)
item = MatrixDataCacheInfo(cache_item, time.time())
self.datasets[location] = item
try:
assert(cache_item)
if delete_adaptor:
delete_adaptor.delete()
if data_adaptor is None:
data_adaptor = cache_item.acquire_and_open(app_config)
yield data_adaptor
finally:
except DatasetAccessError:
cache_item.release()
with self.lock:
del self.datasets[location]
cache_item.delete()
cache_item = None
raise
finally:
if cache_item:
cache_item.release()
def evict_old_datasets(self):
# must be called with the lock held
if self.timelimit_s is None:
return
now = time.time()
to_del = []
for key, info in self.datasets.items():
if (now - info.last_access) > self.timelimit_s:
# remove the data_cache when if it has been in the cache too long
to_del.append((key, info))
for key, info in to_del:
# try and get the write_lock for the dataset.
# if this returns false, it means the dataset is being used, and should
# not be removed.
if info.cache_item.attempt_delete():
del self.datasets[key]
class MatrixDataType(Enum):