mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-27 07:28:12 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -1,70 +0,0 @@
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
import fastobo
|
||||
import fsspec
|
||||
|
||||
from server.common.errors import OntologyLoadFailure
|
||||
from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
|
||||
|
||||
|
||||
class Annotations(metaclass=ABCMeta):
|
||||
""" baseclass for annotations, including ontologies"""
|
||||
|
||||
""" our default ontology is the PURL for the Cell Ontology.
|
||||
See http://www.obofoundry.org/ontology/cl.html """
|
||||
DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
|
||||
|
||||
def __init__(self):
|
||||
self.ontology_data = None
|
||||
|
||||
def load_ontology(self, path):
|
||||
"""Load and parse ontologies - currently support OBO files only."""
|
||||
if path is None:
|
||||
path = self.DefaultOnotology
|
||||
|
||||
try:
|
||||
with fsspec.open(path) as f:
|
||||
obo = fastobo.iter(f)
|
||||
terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
|
||||
names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
|
||||
self.ontology_data = names
|
||||
|
||||
except FileNotFoundError as e:
|
||||
raise OntologyLoadFailure("Unable to find OBO ontology path") from e
|
||||
|
||||
except SyntaxError as e:
|
||||
raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e
|
||||
|
||||
except Exception as e:
|
||||
raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e
|
||||
|
||||
def get_schema(self, data_adaptor):
|
||||
schema = []
|
||||
labels = self.read_labels(data_adaptor)
|
||||
if labels is not None and not labels.empty:
|
||||
for col in labels.columns:
|
||||
col_schema = dict(name=col, writable=True)
|
||||
col_schema.update(get_schema_type_hint_of_array(labels[col]))
|
||||
schema.append(col_schema)
|
||||
|
||||
return schema
|
||||
|
||||
@abstractmethod
|
||||
def set_collection(self, name):
|
||||
"""set or create a new annotation collection"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def read_labels(self, data_adaptor):
|
||||
"""Return the labels as a pandas.DataFrame"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def write_labels(self, df, data_adaptor):
|
||||
"""Write the labels (df) to a persistent storage such that it can later be read"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def update_parameters(self, parameters, data_adaptor):
|
||||
"""Update configuration parameters that describe information about the annotations feature"""
|
||||
pass
|
||||
@@ -1,171 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import pandas as pd
|
||||
import tiledb
|
||||
from flask import current_app
|
||||
|
||||
from server.common.annotations.annotations import Annotations
|
||||
from server.common.errors import AnnotationCategoryNameError
|
||||
from server.common.utils.sanitization_utils import sanitize_values_in_list
|
||||
from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
|
||||
from server.db.cellxgene_orm import Annotation
|
||||
|
||||
|
||||
class AnnotationsHostedTileDB(Annotations):
|
||||
CXG_ANNO_COLLECTION = "cxg_anno_collection"
|
||||
|
||||
def __init__(self, directory_path, db):
|
||||
super().__init__()
|
||||
self.db = db
|
||||
if directory_path[-1] == "/":
|
||||
self.directory_path = directory_path
|
||||
else:
|
||||
self.directory_path = directory_path + "/"
|
||||
|
||||
def check_category_names(self, df):
|
||||
original_category_names = df.keys().to_list()
|
||||
sanitized_category_names = set(sanitize_values_in_list(original_category_names).values())
|
||||
unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
|
||||
if unsanitary_original_category_names:
|
||||
raise AnnotationCategoryNameError(
|
||||
f"{unsanitary_original_category_names} are not valid category names, please resubmit"
|
||||
)
|
||||
|
||||
def get_user_name(self):
|
||||
return current_app.auth.get_user_name()
|
||||
|
||||
def get_user_id(self):
|
||||
return current_app.auth.get_user_id()
|
||||
|
||||
def is_safe_collection_name(self, name):
|
||||
"""
|
||||
return true if this is a safe collection name
|
||||
this is ultra conservative. If we want to allow full legal file name syntax,
|
||||
we could look at modules like `pathvalidate`
|
||||
"""
|
||||
if name is None:
|
||||
return False
|
||||
return re.match(r"^[\w\-]+$", name) is not None
|
||||
|
||||
def set_collection(self, name):
|
||||
self.CXG_ANNO_COLLECTION = name
|
||||
|
||||
def read_labels(self, data_adaptor):
|
||||
user_id = self.get_user_id()
|
||||
if user_id is None:
|
||||
return
|
||||
dataset_name = data_adaptor.get_location()
|
||||
dataset_id = self.db.get_or_create_dataset(dataset_name)
|
||||
|
||||
annotation_object = self.db.query_for_most_recent(
|
||||
Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
|
||||
)
|
||||
if annotation_object:
|
||||
if annotation_object.tiledb_uri == "":
|
||||
# this mean the user has removed all the categories.
|
||||
return None
|
||||
try:
|
||||
df = tiledb.open(annotation_object.tiledb_uri)
|
||||
except tiledb.TileDBError:
|
||||
# don't crash if the annotations file is missing or can't be read.
|
||||
current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}")
|
||||
return None
|
||||
pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
|
||||
return pandas_df
|
||||
else:
|
||||
return None
|
||||
|
||||
def convert_to_pandas_df(self, tileDBArray, schema_hints):
|
||||
repr_meta = None
|
||||
index_dims = None
|
||||
schema_hints = json.loads(schema_hints)
|
||||
|
||||
if "__pandas_attribute_repr" in tileDBArray.meta:
|
||||
# backwards compatibility... unsure if necessary at this point
|
||||
repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
|
||||
if "__pandas_index_dims" in tileDBArray.meta:
|
||||
index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
|
||||
|
||||
data = tileDBArray[:]
|
||||
indexes = list()
|
||||
|
||||
for col_name, col_val in data.items():
|
||||
# If the column values are byte literals, decode them
|
||||
if isinstance(col_val[0], bytes):
|
||||
col_val = [value.decode("utf-8") for value in col_val]
|
||||
|
||||
if schema_hints and col_name in schema_hints:
|
||||
type = schema_hints.get(col_name).get("type")
|
||||
if type and type == "categorical":
|
||||
new_col = pd.Series(col_val, dtype="category")
|
||||
data[col_name] = new_col
|
||||
elif repr_meta and col_name in repr_meta:
|
||||
new_col = pd.Series(col_val, dtype=repr_meta[col_name])
|
||||
data[col_name] = new_col
|
||||
elif index_dims and col_name in index_dims:
|
||||
new_col = pd.Series(col_val, dtype=index_dims[col_name])
|
||||
data[col_name] = new_col
|
||||
indexes.append(col_name)
|
||||
|
||||
new_df = pd.DataFrame.from_dict(data)
|
||||
if len(indexes) > 0:
|
||||
new_df.set_index(indexes, inplace=True)
|
||||
|
||||
return new_df
|
||||
|
||||
def write_labels(self, df, data_adaptor):
|
||||
auth_user_id = self.get_user_id()
|
||||
user_name = self.get_user_name()
|
||||
timestamp = time.time()
|
||||
dataset_location = data_adaptor.get_location()
|
||||
dataset_id = self.db.get_or_create_dataset(dataset_location)
|
||||
dataset_name = data_adaptor.get_title()
|
||||
user_id = self.db.get_or_create_user(auth_user_id)
|
||||
"""
|
||||
NOTE: The uri contains the dataset name, user name and a timestamp as a convenience for debugging purposes.
|
||||
People may have the same name and time.time() can be server dependent.
|
||||
See - https://docs.python.org/2/library/time.html#time.time
|
||||
|
||||
The annotations objects in the database should be used as the source of truth about who an annotation belongs
|
||||
to (for authorization purposes) and what time it was created (for garbage collection).
|
||||
"""
|
||||
uri = f"{self.directory_path}{dataset_name}/{user_name}/{timestamp}"
|
||||
if uri.startswith("s3://"):
|
||||
pass
|
||||
else:
|
||||
os.makedirs(uri, exist_ok=True)
|
||||
_, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
|
||||
if not df.empty:
|
||||
self.check_category_names(df)
|
||||
# convert to tiledb datatypes
|
||||
|
||||
for col in df:
|
||||
df[col] = df[col].astype(get_dtype_of_array(df[col]))
|
||||
tiledb.from_pandas(uri, df, sparse=True)
|
||||
else:
|
||||
uri = ""
|
||||
|
||||
annotation = Annotation(
|
||||
tiledb_uri=uri,
|
||||
user_id=user_id,
|
||||
dataset_id=str(dataset_id),
|
||||
schema_hints=json.dumps(dataframe_schema_type_hints),
|
||||
)
|
||||
self.db.session.add(annotation)
|
||||
self.db.session.commit()
|
||||
|
||||
def update_parameters(self, parameters, data_adaptor):
|
||||
params = {}
|
||||
params["annotations"] = True
|
||||
params["user_annotation_collection_name_enabled"] = False
|
||||
|
||||
if self.ontology_data:
|
||||
params["annotations_cell_ontology_enabled"] = True
|
||||
params["annotations_cell_ontology_terms"] = self.ontology_data
|
||||
else:
|
||||
params["annotations_cell_ontology_enabled"] = False
|
||||
|
||||
parameters.update(params)
|
||||
@@ -1,196 +0,0 @@
|
||||
import base64
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from hashlib import blake2b
|
||||
|
||||
import pandas as pd
|
||||
from flask import session, has_request_context, current_app
|
||||
|
||||
from server import __version__ as cellxgene_version
|
||||
from server.common.annotations.annotations import Annotations
|
||||
from server.common.errors import AnnotationsError
|
||||
|
||||
|
||||
class AnnotationsLocalFile(Annotations):
|
||||
CXG_ANNO_COLLECTION = "cxg_anno_collection"
|
||||
|
||||
def __init__(self, output_dir, output_file):
|
||||
super().__init__()
|
||||
self.output_dir = output_dir
|
||||
self.output_file = output_file
|
||||
# lock used to protect label file write ops
|
||||
self.label_lock = threading.RLock()
|
||||
|
||||
# cache the most recent annotations
|
||||
self.last_fname = None
|
||||
self.last_labels = None
|
||||
|
||||
def is_safe_collection_name(self, name):
|
||||
"""
|
||||
return true if this is a safe collection name
|
||||
this is ultra conservative. If we want to allow full legal file name syntax,
|
||||
we could look at modules like `pathvalidate`
|
||||
"""
|
||||
if name is None:
|
||||
return False
|
||||
return re.match(r"^[\w\-]+$", name) is not None
|
||||
|
||||
def set_collection(self, name):
|
||||
session[self.CXG_ANNO_COLLECTION] = name
|
||||
session.permanent = True
|
||||
|
||||
def get_collection(self):
|
||||
if session is None:
|
||||
return None
|
||||
return session.get(self.CXG_ANNO_COLLECTION)
|
||||
|
||||
def read_labels(self, data_adaptor):
|
||||
if has_request_context():
|
||||
if not current_app.auth.is_user_authenticated():
|
||||
return pd.DataFrame()
|
||||
|
||||
fname = self._get_filename(data_adaptor)
|
||||
with self.label_lock:
|
||||
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
|
||||
# returned the cached labels if possible, otherwise read them from the file
|
||||
if fname == self.last_fname:
|
||||
return self.last_labels
|
||||
else:
|
||||
labels = pd.read_csv(
|
||||
fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
|
||||
)
|
||||
# update the cache
|
||||
self.last_fname = fname
|
||||
self.last_labels = labels
|
||||
return labels
|
||||
else:
|
||||
return pd.DataFrame()
|
||||
|
||||
def write_labels(self, df, data_adaptor):
|
||||
# update our internal state and save it. Multi-threading often enabled,
|
||||
# so treat this as a critical section.
|
||||
with self.label_lock:
|
||||
lastmod = data_adaptor.get_last_mod_time()
|
||||
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
|
||||
header = (
|
||||
f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
|
||||
f"using cellxgene version {cellxgene_version}\n"
|
||||
f"# Input data file was {data_adaptor.get_location()}, "
|
||||
f"which was last modified on {lastmodstr}\n"
|
||||
)
|
||||
|
||||
fname = self._get_filename(data_adaptor)
|
||||
self._backup(fname)
|
||||
if not df.empty:
|
||||
with open(fname, "w", newline="") as f:
|
||||
if header is not None:
|
||||
f.write(header)
|
||||
df.to_csv(f)
|
||||
else:
|
||||
open(fname, "w").close()
|
||||
|
||||
# update the cache
|
||||
self.last_fname = fname
|
||||
self.last_labels = df
|
||||
|
||||
def _get_userdata_idhash(self, data_adaptor):
|
||||
"""
|
||||
Return a short hash that weakly identifies the user and dataset.
|
||||
Used to create safe annotations output file names.
|
||||
"""
|
||||
uid = current_app.auth.get_user_id()
|
||||
id = (uid + data_adaptor.get_location()).encode()
|
||||
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
|
||||
return idhash
|
||||
|
||||
def _get_output_dir(self):
|
||||
if self.output_dir:
|
||||
return self.output_dir
|
||||
|
||||
if self.output_file:
|
||||
return os.path.dirname(self.path.abspath(self.output_dir))
|
||||
|
||||
return os.getcwd()
|
||||
|
||||
def _get_filename(self, data_adaptor):
|
||||
""" return the current annotation file name """
|
||||
if self.output_file:
|
||||
return self.output_file
|
||||
|
||||
# we need to generate a file name, which we can only do if we have a UID and collection name
|
||||
if session is None:
|
||||
raise AnnotationsError("unable to determine file name for annotations")
|
||||
|
||||
collection = self.get_collection()
|
||||
if collection is None:
|
||||
return None
|
||||
|
||||
if data_adaptor is None:
|
||||
raise AnnotationsError("unable to determine file name for annotations")
|
||||
|
||||
idhash = self._get_userdata_idhash(data_adaptor)
|
||||
return os.path.join(self._get_output_dir(), f"{collection}-{idhash}.csv")
|
||||
|
||||
def _backup(self, fname, max_backups=9):
|
||||
"""
|
||||
save N backups of file to backup_dir.
|
||||
1. fname -> backup_dir/fname-TIME
|
||||
2. delete excess files in backup_dir
|
||||
"""
|
||||
root, ext = os.path.splitext(fname)
|
||||
backup_dir = f"{root}-backups"
|
||||
|
||||
# Make sure there is work to do
|
||||
if not os.path.exists(fname):
|
||||
return
|
||||
|
||||
# Ensure backup_dir exists
|
||||
if not os.path.exists(backup_dir):
|
||||
os.mkdir(backup_dir)
|
||||
|
||||
# Save current file to backup_dir
|
||||
fname_base = os.path.basename(fname)
|
||||
fname_base_root, fname_base_ext = os.path.splitext(fname_base)
|
||||
# don't use ISO standard time format, as it contains characters illegal on some filesytems.
|
||||
nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
|
||||
backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
|
||||
if os.path.exists(backup_fname):
|
||||
os.remove(backup_fname)
|
||||
os.rename(fname, backup_fname)
|
||||
|
||||
# prune the backup_dir to max number of backup files, keeping the most recent backups
|
||||
backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
|
||||
excess_count = len(backups) - max_backups
|
||||
if excess_count > 0:
|
||||
backups.sort()
|
||||
for bu in backups[0:excess_count]:
|
||||
os.remove(os.path.join(backup_dir, bu))
|
||||
|
||||
def update_parameters(self, parameters, data_adaptor):
|
||||
params = {}
|
||||
params["annotations"] = True
|
||||
params["user_annotation_collection_name_enabled"] = True
|
||||
|
||||
if self.ontology_data:
|
||||
params["annotations_cell_ontology_enabled"] = True
|
||||
params["annotations_cell_ontology_terms"] = self.ontology_data
|
||||
else:
|
||||
params["annotations_cell_ontology_enabled"] = False
|
||||
|
||||
if self.output_file is not None:
|
||||
# user has hard-wired the name of the annotation data collection
|
||||
fname = os.path.basename(self.output_file)
|
||||
collection_fname = os.path.splitext(fname)[0]
|
||||
params["annotations-data-collection-is-read-only"] = True
|
||||
params["annotations-data-collection-name"] = collection_fname
|
||||
|
||||
elif session is not None:
|
||||
collection = self.get_collection()
|
||||
if current_app.auth.is_user_authenticated():
|
||||
params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
|
||||
params["annotations-data-collection-is-read-only"] = False
|
||||
params["annotations-data-collection-name"] = collection
|
||||
|
||||
parameters.update(params)
|
||||
Reference in New Issue
Block a user