diff --git a/server/common/annotations.py b/server/common/annotations.py index 592e4805..c1a0f735 100644 --- a/server/common/annotations.py +++ b/server/common/annotations.py @@ -1,3 +1,6 @@ +import json +import uuid +import time from datetime import datetime import re import os @@ -13,6 +16,9 @@ import fastobo from flask import session, current_app, has_request_context from abc import ABCMeta, abstractmethod +from server.db.cellxgene_orm import CellxGeneDataset, Annotation +from server.db.db_utils import DbUtils + class Annotations(metaclass=ABCMeta): """ baseclass for annotations, including ontologies""" @@ -259,3 +265,55 @@ class AnnotationsLocalFile(Annotations): params["annotations-data-collection-name"] = collection parameters.update(params) + + +class AnnotationsHostedTileDB(Annotations): + def __init__(self, directory_path: str, db: DbUtils): + super().__init__() + self.db = db + self.directory_path = directory_path + + def set_collection(self, name): + pass + + def read_labels(self, data_adaptor): + uid = current_app.auth.get_user_id() + dataset_name = data_adaptor.get_location() + dataset = self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]) + # Todo @madison retrieve latest based on timestamp + annotation_object = self.db.query_for_most_recent( # noqa F841 + Annotation, [Annotation.user_id == uid, Annotation.dataset == dataset] + ) + # Todo in future pr, retrieve dataframe from tiledb uri + + def write_labels(self, df, data_adaptor): + uid = current_app.auth.get_user_id() + timestamp = time.time() + dataset_name = data_adaptor.get_location() + try: + dataset_id = self.db.query( + table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name] + )[0].id + except IndexError: + dataset_id = uuid.uuid4() + dataset = CellxGeneDataset(id=dataset_id, name=dataset_name) + self.db.session.add(dataset) + + uri = f"{self.directory_path}/{dataset_name}/{uid}/{timestamp}" + if "s3" in uri: + pass + else: + os.makedirs(uri, exist_ok=True) + schema_hints = {} + annotation = Annotation( + tiledb_uri=uri, + user_id=uid, + dataset_id=str(dataset_id), + schema_hints=json.dumps(schema_hints) + ) + # todo in future pr -- write df to tiledb, store at uri + self.db.session.add(annotation) + self.db.session.commit() + + def update_parameters(self, parameters, data_adaptor): + pass diff --git a/server/db/cellxgene_orm.py b/server/db/cellxgene_orm.py index 27823bc2..f2209b63 100644 --- a/server/db/cellxgene_orm.py +++ b/server/db/cellxgene_orm.py @@ -1,11 +1,12 @@ -from datetime import datetime +import uuid from sqlalchemy import ( Column, DateTime, ForeignKey, String, -) + func, JSON) +from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship @@ -21,8 +22,8 @@ class CellxGeneUser(Base): __tablename__ = "cxguser" id = Column(String, primary_key=True) - created_at = Column(DateTime, nullable=False, default=datetime.utcnow) - updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + updated_at = Column(DateTime, nullable=False, server_default=func.now(), onupdate=func.now()) # Relationships annotations = relationship("Annotation", back_populates="cxguser") @@ -36,12 +37,13 @@ class Annotation(Base): __tablename__ = "annotation" - id = Column(String, primary_key=True) + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False) tiledb_uri = Column(String) user_id = Column(String, ForeignKey("cxguser.id"), nullable=False) - dataset_id = Column(String, ForeignKey("cxgdataset.id"), nullable=False) - created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + dataset_id = Column(UUID, ForeignKey("cxgdataset.id"), nullable=False) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + schema_hints = Column(JSON) # Relationships cxguser = relationship("CellxGeneUser", back_populates="annotations") dataset = relationship("CellxGeneDataset", back_populates="annotations") @@ -49,12 +51,13 @@ class Annotation(Base): class CellxGeneDataset(Base): """ - Datasets refer to cellxgene datasets stored in tiledb + Datasets refer to datasets stored by cellxgene """ __tablename__ = "cxgdataset" - id = Column(String, primary_key=True) - name = Column(String) - created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False) + name = Column(String, unique=True, index=True) + + created_at = Column(DateTime, nullable=False, server_default=func.now()) annotations = relationship("Annotation", back_populates="dataset") diff --git a/server/db/db_utils.py b/server/db/db_utils.py index 2cf3f59e..9cae9ea2 100644 --- a/server/db/db_utils.py +++ b/server/db/db_utils.py @@ -33,6 +33,9 @@ class DbUtils: else self.session.query(*table_args).all() ) + def query_for_most_recent(self, table: Base, filter_args: typing.List[bool] = None) -> Base: + return self.session.query(table).filter(*filter_args).order_by(table.created_at.desc()).limit(1).all()[0] + class DBSessionMaker: def __init__(self, database_uri): diff --git a/server/test/fixtures/database/__init__.py b/server/test/fixtures/database/__init__.py index 82b17b58..5a7fa984 100644 --- a/server/test/fixtures/database/__init__.py +++ b/server/test/fixtures/database/__init__.py @@ -29,23 +29,26 @@ class TestDatabase: def _create_test_user(self): user = CellxGeneUser(id="test_user_id") + user2 = CellxGeneUser(id='1234') self.db.session.add(user) + self.db.session.add(user2) self.db.session.commit() def _create_test_dataset(self): dataset = CellxGeneDataset( - id="test_dataset_id", name="test_dataset", ) self.db.session.add(dataset) self.db.session.commit() def _create_test_annotation(self): + dataset = self.db.query([CellxGeneDataset], + [CellxGeneDataset.name == "test_dataset"], + )[0] annotation = Annotation( - id="test_annotation_id", tiledb_uri="tiledb_uri", user_id="test_user_id", - dataset_id="test_dataset_id" + dataset_id=str(dataset.id) ) self.db.session.add(annotation) self.db.session.commit() @@ -65,7 +68,7 @@ class TestDatabase: def _create_test_datasets(self, dataset_count: int = 10): datasets = [] for i in range(dataset_count): - datasets.append(CellxGeneDataset(id=self.get_random_string(), name=self.get_random_string())) + datasets.append(CellxGeneDataset(name=self.get_random_string())) self.db.session.add_all(datasets) self.db.session.commit() @@ -78,10 +81,9 @@ class TestDatabase: dataset = self.order_by_random(CellxGeneDataset) user = self.order_by_random(CellxGeneUser) annotations.append(Annotation( - id=self.get_random_string(), tiledb_uri=self.get_random_string(), user_id=user.id, - dataset_id=dataset.id + dataset_id=str(dataset.id) )) self.db.session.add_all(annotations) self.db.session.commit() diff --git a/server/test/test_database/test_database.py b/server/test/test_database/test_database.py index 706d2fdc..4ecb4f27 100644 --- a/server/test/test_database/test_database.py +++ b/server/test/test_database/test_database.py @@ -4,7 +4,7 @@ from server.db.db_utils import DbUtils from server.test.fixtures.database import TestDatabase -class AppConfigTest(unittest.TestCase): +class DatabaseTest(unittest.TestCase): db = DbUtils("postgresql://postgres:test_pw@localhost:5432") @classmethod @@ -22,13 +22,39 @@ class AppConfigTest(unittest.TestCase): self.assertGreater(user_count, 10) def test_dataset_creation(self): - one_dataset = self.db.get(table=CellxGeneDataset, entity_id='test_dataset_id') - self.assertEqual(one_dataset.id, 'test_dataset_id') + one_dataset = self.db.query(table_args=[CellxGeneDataset], + filter_args=[CellxGeneDataset.name == 'test_dataset']) + self.assertEqual(one_dataset[0].name, 'test_dataset') dataset_count = self.db.session.query(CellxGeneDataset).count() self.assertGreater(dataset_count, 10) def test_annotation_creation(self): - one_annotation = self.db.get(table=Annotation, entity_id='test_annotation_id') - self.assertEqual(one_annotation.id, 'test_annotation_id') + one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0] + self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri') annotation_count = self.db.session.query(Annotation).count() self.assertGreater(annotation_count, 10) + + def test_get_most_recent_annotation_for_user_dataset(self): + dataset_id = str(self.db.query(table_args=[CellxGeneDataset], + filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id) + + # have to commit separately because created_at time written on the db server + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0')) + self.db.session.commit() + + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1')) + self.db.session.commit() + + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2')) + self.db.session.commit() + + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3')) + self.db.session.commit() + + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4')) + self.db.session.commit() + + most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id, + Annotation.user_id == 'test_user_id']) + + self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4')