move common code into server, update tests and makefile (#2425)

* move common code into server, update tests and makefile

remove backend directory, refactor

update smoke tests

test/fixtures/dataset_config_outline.py

@@ -0,0 +1,30 @@
+f"""
+dataset:
+  app:
+    scripts: {scripts} #list of strs (filenames) or dicts containing keys
+    inline_scripts: {inline_scripts} #list of strs (filenames)
+
+  presentation:
+    max_categories: {max_categories}
+    custom_colors: {custom_colors}
+
+  user_annotations:
+    enable: {enable_users_annotations}
+    type: {annotation_type}
+    local_file_csv:
+      directory: {local_file_csv_directory}
+      file: {local_file_csv_file}
+      gene_sets_file: {local_file_csv_gene_sets_file}
+    gene_sets:
+      readonly: {gene_sets_readonly}
+
+  embeddings:
+    names: {embedding_names}
+
+  diffexp:
+    enable: {enable_difexp}
+    lfc_cutoff: {lfc_cutoff}
+    top_n: {top_n}
+
+  X_approximate_distribution: {X_approximate_distribution}
+"""

test/fixtures/fixtures.py

@@ -0,0 +1,12 @@
+pbmc3k_colors = {
+    "louvain": {
+        "B cells": "#2ca02c",
+        "CD14+ Monocytes": "#ff7f0e",
+        "CD4 T cells": "#1f77b4",
+        "CD8 T cells": "#d62728",
+        "Dendritic cells": "#e377c2",
+        "FCGR3A+ Monocytes": "#8c564b",
+        "Megakaryocytes": "#bcbd22",
+        "NK cells": "#9467bd",
+    }
+}

test/fixtures/pbmc3k-genesets.csv

@@ -0,0 +1,21 @@
+# Test fixture
+gene_set_name,gene_set_description,gene_symbol,gene_description
+first gene set name,,F5, a gene_description
+first gene set name,a description,NO_SUCH_GENE, non-existent gene
+first gene set name,a description,F5, duplicate gene
+first gene set name, a description,SUMO3,
+first gene set name,,SRM,
+second_gene_set,,RER1
+second_gene_set,,SIK1
+third gene set,,NO_SUCH_GENE
+fourth_gene_set,fourth description,,gene intentionally missing
+fifth_dataset,,,
+summary test,,ACD,
+summary test,,AATF,
+summary test,,F5,
+summary test,,PIGU,
+geneset_to_delete,,,
+geneset_to_edit,,,
+fill_this_geneset,,,
+empty_this_geneset,,SIK1,
+brush_this_gene,,SIK1,

test/fixtures/schema.json

@@ -0,0 +1,83 @@
+{
+  "dataframe": {
+    "nObs": 2638,
+    "nVar": 1838,
+    "type": "float32"
+  },
+  "annotations": {
+    "obs": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string",
+          "writable": false
+        },
+        {
+          "name": "n_genes",
+          "type": "int32",
+          "writable": false
+        },
+        {
+          "name": "percent_mito",
+          "type": "float32",
+          "writable": false
+        },
+        {
+          "name": "n_counts",
+          "type": "float32",
+          "writable": false
+        },
+        {
+          "name": "louvain",
+          "type": "categorical",
+          "categories": [
+            "CD4 T cells",
+            "CD14+ Monocytes",
+            "B cells",
+            "CD8 T cells",
+            "NK cells",
+            "FCGR3A+ Monocytes",
+            "Dendritic cells",
+            "Megakaryocytes"
+          ],
+          "writable": false
+        }
+      ]
+    },
+    "var": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string",
+          "writable": false
+        },
+        {
+          "name": "n_cells",
+          "type": "int32",
+          "writable": false
+        }
+      ]
+    }
+  },
+  "layout": {
+    "obs": [
+      {
+        "name": "umap",
+        "type": "float32",
+        "dims": ["umap_0", "umap_1"]
+      },
+      {
+        "name": "tsne",
+        "type": "float32",
+        "dims": ["tsne_0", "tsne_1"]
+      },
+      {
+        "name": "pca",
+        "type": "float32",
+        "dims": ["pca_0", "pca_1"]
+      }
+    ]
+  }
+}

test/fixtures/server_config_outline.py

@@ -0,0 +1,30 @@
+f"""server:
+  app:
+    verbose: {verbose}
+    debug: {debug}
+    host: {host}
+    port: {port}
+    open_browser: {open_browser}
+    force_https: {force_https}
+    flask_secret_key: {flask_secret_key}
+    generate_cache_control_headers: {generate_cache_control_headers}
+
+  single_dataset:
+    datapath: {dataset_datapath}
+    obs_names: {obs_names}
+    var_names: {var_names}
+    about: {about}
+    title: {title}
+
+  data_locator:
+    s3:
+      region_name: {data_locater_region_name}
+
+  adaptor:
+    anndata_adaptor:
+      backed: {anndata_backed}
+
+  limits:
+    column_request_max: {column_request_max}
+    diffexp_cellcount_max: {diffexp_cellcount_max}
+"""

test/fixtures/test_bad_config.yaml

@@ -0,0 +1,34 @@
+fixup_gene_symbols:
+    X: log1p
+obs:
+    cell_type_ontology_term_id:
+        louvain:
+            CD4 T cells: CL:00001
+            B cells: CL:00002
+            CD14+ Monocytes: CL:00003
+            NK cells: CL:00004
+            CD8 T cells: CL:00005
+            FCGR3A+ Monocytes: CL:00006
+            Dendritic cells: CL:00007
+            Megakaryocytes: CL:00008
+    tissue_ontology_term_id: UBERON:12345
+    assay_ontology_term_id: EFO:12345
+    disease_ontology_term_id: MONDO:12345
+    ethnicity_ontology_term_id: MANCESTRO:12345
+    development_stage_ontology_term_id: HsapDv:12345
+    sex: other
+uns:
+    version:
+        corpora_schema_version: 1.0.0
+        corpora_encoding_version: 0.1.0
+    organism_ontology_term_id: NCBITaxon:9606
+    title: Test dataset
+    contributors:
+        - name: Marcus
+          institution: CZI
+    layer_descriptions:
+        X: raw
+    project_links:
+        - link_url: https://chanzuckerberg.com/
+          link_name: CZI
+          link_type: SUMMARY