Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands

backend/czi_hosted/db/cellxgene_orm.py

@@ -0,0 +1,58 @@
+import uuid
+
+from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+Base = declarative_base()
+
+
+class CellxGeneUser(Base):
+    """
+    A registered CellxGene user.
+    Links a user to their annotations
+    """
+
+    __tablename__ = "cxguser"
+
+    id = Column(String, primary_key=True)
+    created_at = Column(DateTime, nullable=False, server_default=func.now())
+    updated_at = Column(DateTime, nullable=False, server_default=func.now(), onupdate=func.now())
+
+    # Relationships
+    annotations = relationship("Annotation", back_populates="cxguser")
+
+
+class Annotation(Base):
+    """
+    An annotation is a link between a user, a dataset and tiledb dataframe. A user can have multiple annotations for a
+    dataset, the most recent annotation (based on created_at) will be the default returned when queried
+    """
+
+    __tablename__ = "annotation"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False)
+    tiledb_uri = Column(String)
+    user_id = Column(String, ForeignKey("cxguser.id"), nullable=False)
+    dataset_id = Column(UUID, ForeignKey("cxgdataset.id"), nullable=False)
+
+    created_at = Column(DateTime, nullable=False, server_default=func.now())
+    schema_hints = Column(JSON)
+    # Relationships
+    cxguser = relationship("CellxGeneUser", back_populates="annotations")
+    dataset = relationship("CellxGeneDataset", back_populates="annotations")
+
+
+class CellxGeneDataset(Base):
+    """
+    Datasets refer to datasets stored by cellxgene
+    """
+
+    __tablename__ = "cxgdataset"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False)
+    name = Column(String, unique=True, index=True)
+
+    created_at = Column(DateTime, nullable=False, server_default=func.now())
+    annotations = relationship("Annotation", back_populates="dataset")

backend/czi_hosted/db/create_db.py

@@ -0,0 +1,18 @@
+"""
+Drops and recreates all tables for local testing according to cellxgene_orm.py
+"""
+from sqlalchemy import create_engine
+
+from backend.czi_hosted.db.cellxgene_orm import Base
+
+
+def create_db(database_uri: str = "postgresql://postgres:test_pw@localhost:5432"):
+    engine = create_engine(database_uri)
+    print("Dropping tables")
+    Base.metadata.drop_all(engine)
+    print("Recreating tables")
+    Base.metadata.create_all(engine)
+
+
+if __name__ == "__main__":
+    create_db()

backend/czi_hosted/db/db_utils.py

@@ -0,0 +1,71 @@
+import typing
+import uuid
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from backend.czi_hosted.db.cellxgene_orm import Base, CellxGeneDataset, CellxGeneUser
+
+
+class DbUtils:
+    def __init__(self, database_uri: str = "postgresql://postgres:test_pw@localhost:5432"):
+        self.session = DBSessionMaker(database_uri).session()
+        self.engine = self.session.get_bind()
+
+    def get(self, table: Base, entity_id: typing.Union[str, typing.Tuple[str]]) -> typing.Union[Base, None]:
+        """
+        Query a table row by its primary key
+        :param table: SQLAlchemy Table to query
+        :param entity_id: Primary key of desired row
+        :return: SQLAlchemy Table object, None if not found
+        """
+        return self.session.query(table).get(entity_id)
+
+    def query(self, table_args: typing.List[Base], filter_args: typing.List[bool] = None) -> typing.List[Base]:
+        """
+        Query the database using the current DB session
+        :param table_args: List of SQLAlchemy Tables to query/join
+        :param filter_args: List of SQLAlchemy filter conditions
+        :return: List of SQLAlchemy query response objects
+        """
+        return (
+            self.session.query(*table_args).filter(*filter_args).all()
+            if filter_args
+            else self.session.query(*table_args).all()
+        )
+
+    def query_for_most_recent(self, table: Base, filter_args: typing.List[bool] = None) -> Base:
+        try:
+            return self.session.query(table).filter(*filter_args).order_by(table.created_at.desc()).limit(1).all()[0]
+        except IndexError:
+            return None
+
+    def get_or_create_dataset(self, dataset_name):
+        try:
+            dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[
+                0
+            ].id
+        except IndexError:
+            dataset_id = uuid.uuid4()
+            dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
+            self.session.add(dataset)
+            self.session.commit()
+        return str(dataset_id)
+
+    def get_or_create_user(self, user_id):
+        try:
+            user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id
+        except IndexError:
+            user = CellxGeneUser(id=user_id)
+            self.session.add(user)
+            self.session.commit()
+        return str(user_id)
+
+
+class DBSessionMaker:
+    def __init__(self, database_uri):
+        self.engine = create_engine(database_uri, connect_args={"connect_timeout": 5})
+        self.session_maker = sessionmaker(bind=self.engine)
+
+    def session(self, **kwargs):
+        return self.session_maker(**kwargs)