move common code into server, update tests and makefile (#2425)

* move common code into server, update tests and makefile

remove backend directory, refactor

update smoke tests

test/__init__.py

@@ -0,0 +1,11 @@
+import random
+import string
+from os import popen
+
+PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
+FIXTURES_ROOT = PROJECT_ROOT + "/test/fixtures"
+H5AD_FIXTURE = FIXTURES_ROOT + "/pbmc3k-CSC-gz.h5ad"
+
+
+def random_string(n):
+    return "".join(random.choice(string.ascii_letters) for _ in range(n))

test/decode_fbs.py

@@ -0,0 +1,31 @@
+"""
+Code to decode, for testing purposes, the flatbuffer encoded blobs.
+
+This code will need to be updated if fbs/matrix.fbs changes. For more information, see fbs/matrix.fbs and
+server/data_common/fbs/
+"""
+
+import server.common.fbs.NetEncoding.Matrix as Matrix
+from server.common.fbs.matrix import deserialize_typed_array
+
+
+def decode_matrix_FBS(buf):
+    """
+    Given a FBS Matrix, return an decoded Python dict containing same info in native format.
+    NOTE / TODO: row_idx not currently implemented
+    """
+    df = Matrix.Matrix.GetRootAsMatrix(buf, 0)
+    n_rows = df.NRows()
+    n_cols = df.NCols()
+
+    columns_length = df.ColumnsLength()
+
+    decoded_columns = []
+    for col_idx in range(0, columns_length):
+        col = df.Columns(col_idx)
+        tarr = (col.UType(), col.U())
+        decoded_columns.append(deserialize_typed_array(tarr))
+
+    cidx = deserialize_typed_array((df.ColIndexType(), df.ColIndex()))
+
+    return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None}

test/fixtures/dataset_config_outline.py

@@ -0,0 +1,30 @@
+f"""
+dataset:
+  app:
+    scripts: {scripts} #list of strs (filenames) or dicts containing keys
+    inline_scripts: {inline_scripts} #list of strs (filenames)
+
+  presentation:
+    max_categories: {max_categories}
+    custom_colors: {custom_colors}
+
+  user_annotations:
+    enable: {enable_users_annotations}
+    type: {annotation_type}
+    local_file_csv:
+      directory: {local_file_csv_directory}
+      file: {local_file_csv_file}
+      gene_sets_file: {local_file_csv_gene_sets_file}
+    gene_sets:
+      readonly: {gene_sets_readonly}
+
+  embeddings:
+    names: {embedding_names}
+
+  diffexp:
+    enable: {enable_difexp}
+    lfc_cutoff: {lfc_cutoff}
+    top_n: {top_n}
+
+  X_approximate_distribution: {X_approximate_distribution}
+"""

test/fixtures/fixtures.py

@@ -0,0 +1,12 @@
+pbmc3k_colors = {
+    "louvain": {
+        "B cells": "#2ca02c",
+        "CD14+ Monocytes": "#ff7f0e",
+        "CD4 T cells": "#1f77b4",
+        "CD8 T cells": "#d62728",
+        "Dendritic cells": "#e377c2",
+        "FCGR3A+ Monocytes": "#8c564b",
+        "Megakaryocytes": "#bcbd22",
+        "NK cells": "#9467bd",
+    }
+}

test/fixtures/pbmc3k-genesets.csv

@@ -0,0 +1,21 @@
+# Test fixture
+gene_set_name,gene_set_description,gene_symbol,gene_description
+first gene set name,,F5, a gene_description
+first gene set name,a description,NO_SUCH_GENE, non-existent gene
+first gene set name,a description,F5, duplicate gene
+first gene set name, a description,SUMO3,
+first gene set name,,SRM,
+second_gene_set,,RER1
+second_gene_set,,SIK1
+third gene set,,NO_SUCH_GENE
+fourth_gene_set,fourth description,,gene intentionally missing
+fifth_dataset,,,
+summary test,,ACD,
+summary test,,AATF,
+summary test,,F5,
+summary test,,PIGU,
+geneset_to_delete,,,
+geneset_to_edit,,,
+fill_this_geneset,,,
+empty_this_geneset,,SIK1,
+brush_this_gene,,SIK1,

test/fixtures/schema.json

@@ -0,0 +1,83 @@
+{
+  "dataframe": {
+    "nObs": 2638,
+    "nVar": 1838,
+    "type": "float32"
+  },
+  "annotations": {
+    "obs": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string",
+          "writable": false
+        },
+        {
+          "name": "n_genes",
+          "type": "int32",
+          "writable": false
+        },
+        {
+          "name": "percent_mito",
+          "type": "float32",
+          "writable": false
+        },
+        {
+          "name": "n_counts",
+          "type": "float32",
+          "writable": false
+        },
+        {
+          "name": "louvain",
+          "type": "categorical",
+          "categories": [
+            "CD4 T cells",
+            "CD14+ Monocytes",
+            "B cells",
+            "CD8 T cells",
+            "NK cells",
+            "FCGR3A+ Monocytes",
+            "Dendritic cells",
+            "Megakaryocytes"
+          ],
+          "writable": false
+        }
+      ]
+    },
+    "var": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string",
+          "writable": false
+        },
+        {
+          "name": "n_cells",
+          "type": "int32",
+          "writable": false
+        }
+      ]
+    }
+  },
+  "layout": {
+    "obs": [
+      {
+        "name": "umap",
+        "type": "float32",
+        "dims": ["umap_0", "umap_1"]
+      },
+      {
+        "name": "tsne",
+        "type": "float32",
+        "dims": ["tsne_0", "tsne_1"]
+      },
+      {
+        "name": "pca",
+        "type": "float32",
+        "dims": ["pca_0", "pca_1"]
+      }
+    ]
+  }
+}