fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-28 17:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

retrieve latest annotation from db (#1723)

Browse Source 
* add function to retrieve latest annotation from db, db updates

* dont create directory in s3
This commit is contained in:
Madison Dunitz
2020-08-11 15:48:12 -05:00
committed by GitHub
parent 49b89ba7af
commit 80f6137528
5 changed files with 114 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/common/annotations.py
+58
View File
@@ -1,3 +1,6 @@
import json
import uuid
import time
from datetime import datetime
import re
import os
@@ -13,6 +16,9 @@ import fastobo
from flask import session, current_app, has_request_context
from abc import ABCMeta, abstractmethod
from server.db.cellxgene_orm import CellxGeneDataset, Annotation
from server.db.db_utils import DbUtils
class Annotations(metaclass=ABCMeta):
""" baseclass for annotations, including ontologies"""
@@ -259,3 +265,55 @@ class AnnotationsLocalFile(Annotations):
params["annotations-data-collection-name"] = collection
parameters.update(params)
class AnnotationsHostedTileDB(Annotations):
def __init__(self, directory_path: str, db: DbUtils):
super().__init__()
self.db = db
self.directory_path = directory_path
def set_collection(self, name):
pass
def read_labels(self, data_adaptor):
uid = current_app.auth.get_user_id()
dataset_name = data_adaptor.get_location()
dataset = self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])
# Todo @madison retrieve latest based on timestamp
annotation_object = self.db.query_for_most_recent( # noqa F841
Annotation, [Annotation.user_id == uid, Annotation.dataset == dataset]
)
# Todo in future pr, retrieve dataframe from tiledb uri
def write_labels(self, df, data_adaptor):
uid = current_app.auth.get_user_id()
timestamp = time.time()
dataset_name = data_adaptor.get_location()
try:
dataset_id = self.db.query(
table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]
)[0].id
except IndexError:
dataset_id = uuid.uuid4()
dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
self.db.session.add(dataset)
uri = f"{self.directory_path}/{dataset_name}/{uid}/{timestamp}"
if "s3" in uri:
pass
else:
os.makedirs(uri, exist_ok=True)
schema_hints = {}
annotation = Annotation(
tiledb_uri=uri,
user_id=uid,
dataset_id=str(dataset_id),
schema_hints=json.dumps(schema_hints)
)
# todo in future pr -- write df to tiledb, store at uri
self.db.session.add(annotation)
self.db.session.commit()
def update_parameters(self, parameters, data_adaptor):
pass
server/db/cellxgene_orm.py
+14 -11
View File
@@ -1,11 +1,12 @@
from datetime import datetime
import uuid
from sqlalchemy import (
Column,
DateTime,
ForeignKey,
String,
)
func, JSON)
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
@@ -21,8 +22,8 @@ class CellxGeneUser(Base):
__tablename__ = "cxguser"
id = Column(String, primary_key=True)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
created_at = Column(DateTime, nullable=False, server_default=func.now())
updated_at = Column(DateTime, nullable=False, server_default=func.now(), onupdate=func.now())
# Relationships
annotations = relationship("Annotation", back_populates="cxguser")
@@ -36,12 +37,13 @@ class Annotation(Base):
__tablename__ = "annotation"
id = Column(String, primary_key=True)
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False)
tiledb_uri = Column(String)
user_id = Column(String, ForeignKey("cxguser.id"), nullable=False)
dataset_id = Column(String, ForeignKey("cxgdataset.id"), nullable=False)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
dataset_id = Column(UUID, ForeignKey("cxgdataset.id"), nullable=False)
created_at = Column(DateTime, nullable=False, server_default=func.now())
schema_hints = Column(JSON)
# Relationships
cxguser = relationship("CellxGeneUser", back_populates="annotations")
dataset = relationship("CellxGeneDataset", back_populates="annotations")
@@ -49,12 +51,13 @@ class Annotation(Base):
class CellxGeneDataset(Base):
"""
Datasets refer to cellxgene datasets stored in tiledb
Datasets refer to datasets stored by cellxgene
"""
__tablename__ = "cxgdataset"
id = Column(String, primary_key=True)
name = Column(String)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False)
name = Column(String, unique=True, index=True)
created_at = Column(DateTime, nullable=False, server_default=func.now())
annotations = relationship("Annotation", back_populates="dataset")
server/db/db_utils.py
+3
View File
@@ -33,6 +33,9 @@ class DbUtils:
else self.session.query(*table_args).all()
)
def query_for_most_recent(self, table: Base, filter_args: typing.List[bool] = None) -> Base:
return self.session.query(table).filter(*filter_args).order_by(table.created_at.desc()).limit(1).all()[0]
class DBSessionMaker:
def __init__(self, database_uri):
server/test/fixtures/database/__init__.py
Vendored
+8 -6
View File
@@ -29,23 +29,26 @@ class TestDatabase:
def _create_test_user(self):
user = CellxGeneUser(id="test_user_id")
user2 = CellxGeneUser(id='1234')
self.db.session.add(user)
self.db.session.add(user2)
self.db.session.commit()
def _create_test_dataset(self):
dataset = CellxGeneDataset(
id="test_dataset_id",
name="test_dataset",
)
self.db.session.add(dataset)
self.db.session.commit()
def _create_test_annotation(self):
dataset = self.db.query([CellxGeneDataset],
[CellxGeneDataset.name == "test_dataset"],
)[0]
annotation = Annotation(
id="test_annotation_id",
tiledb_uri="tiledb_uri",
user_id="test_user_id",
dataset_id="test_dataset_id"
dataset_id=str(dataset.id)
)
self.db.session.add(annotation)
self.db.session.commit()
@@ -65,7 +68,7 @@ class TestDatabase:
def _create_test_datasets(self, dataset_count: int = 10):
datasets = []
for i in range(dataset_count):
datasets.append(CellxGeneDataset(id=self.get_random_string(), name=self.get_random_string()))
datasets.append(CellxGeneDataset(name=self.get_random_string()))
self.db.session.add_all(datasets)
self.db.session.commit()
@@ -78,10 +81,9 @@ class TestDatabase:
dataset = self.order_by_random(CellxGeneDataset)
user = self.order_by_random(CellxGeneUser)
annotations.append(Annotation(
id=self.get_random_string(),
tiledb_uri=self.get_random_string(),
user_id=user.id,
dataset_id=dataset.id
dataset_id=str(dataset.id)
))
self.db.session.add_all(annotations)
self.db.session.commit()
server/test/test_database/test_database.py
+31 -5
View File
@@ -4,7 +4,7 @@ from server.db.db_utils import DbUtils
from server.test.fixtures.database import TestDatabase
class AppConfigTest(unittest.TestCase):
class DatabaseTest(unittest.TestCase):
db = DbUtils("postgresql://postgres:test_pw@localhost:5432")
@classmethod
@@ -22,13 +22,39 @@ class AppConfigTest(unittest.TestCase):
self.assertGreater(user_count, 10)
def test_dataset_creation(self):
one_dataset = self.db.get(table=CellxGeneDataset, entity_id='test_dataset_id')
self.assertEqual(one_dataset.id, 'test_dataset_id')
one_dataset = self.db.query(table_args=[CellxGeneDataset],
filter_args=[CellxGeneDataset.name == 'test_dataset'])
self.assertEqual(one_dataset[0].name, 'test_dataset')
dataset_count = self.db.session.query(CellxGeneDataset).count()
self.assertGreater(dataset_count, 10)
def test_annotation_creation(self):
one_annotation = self.db.get(table=Annotation, entity_id='test_annotation_id')
self.assertEqual(one_annotation.id, 'test_annotation_id')
one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0]
self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri')
annotation_count = self.db.session.query(Annotation).count()
self.assertGreater(annotation_count, 10)
def test_get_most_recent_annotation_for_user_dataset(self):
dataset_id = str(self.db.query(table_args=[CellxGeneDataset],
filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id)
# have to commit separately because created_at time written on the db server
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0'))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1'))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2'))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3'))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4'))
self.db.session.commit()
most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id,
Annotation.user_id == 'test_user_id'])
self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4')