genesets route for local server (#2079)

* first cut at GET /genesets route

* update existing tests to match code changes

* more GET /genesets and initial tests

* add missing test fixture

* geneset validation accepts OTA format

* genesets route: better error handling, more tests

* lint

local_server/test/unit/common/test_api.py

@@ -3,6 +3,8 @@ import time
 import unittest
 import zlib
 from http import HTTPStatus
+import tempfile
+from os import path
 
 import pandas as pd
 import requests
@@ -13,6 +15,7 @@ from local_server.test import (
     data_with_tmp_annotations,
     make_fbs,
     PROJECT_ROOT,
+    FIXTURES_ROOT,
     start_test_server,
     stop_test_server,
 )
@@ -26,6 +29,7 @@ BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
 
 class EndPoints(object):
     ANNOTATIONS_ENABLED = True
+    GENESETS_READONLY = False
 
     def test_initialize(self):
         endpoint = "schema"
@@ -49,6 +53,7 @@ class EndPoints(object):
         result_data = result.json()
         self.assertIn("library_versions", result_data["config"])
         self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
+        self.assertIsNotNone(result_data["config"]["parameters"])
 
     def test_get_layout_fbs(self):
         endpoint = "layout/obs"
@@ -286,6 +291,26 @@ class EndPoints(object):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
 
+    def test_genesets_config(self):
+        result = self.session.get(f"{self.URL_BASE}config")
+        config_data = result.json()
+        params = config_data["config"]["parameters"]
+        annotations_genesets = params["annotations_genesets"]
+        annotations_genesets_readonly = params["annotations_genesets_readonly"]
+        annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
+        self.assertTrue(annotations_genesets)
+        self.assertEqual(annotations_genesets_readonly, self.GENESETS_READONLY)
+        self.assertEqual(annotations_genesets_summary_methods, ["mean"])
+
+    def test_get_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+
     def _setupClass(child_class, command_line):
         child_class.ps, child_class.server = start_test_server(command_line)
         child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
@@ -304,7 +329,8 @@ class EndPointsAnnotations(EndPoints):
 
     def test_get_user_annotations_existing_obs_keys_fbs(self):
         self._test_get_user_annotations_obs_keys_fbs(
-            "cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
+            "cluster-test",
+            {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
         )
 
     def test_put_user_annotations_obs_fbs(self):
@@ -353,6 +379,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
     """Test Case for endpoints"""
 
     ANNOTATIONS_ENABLED = False
+    GENESETS_READONLY = True
 
     @classmethod
     def setUpClass(cls):
@@ -361,6 +388,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
             [
                 f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
                 "--disable-annotations",
+                "--disable-genesets-save",
                 "--experimental-enable-reembedding",
             ],
         )
@@ -408,15 +436,249 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
     """Test Case for endpoints"""
 
     ANNOTATIONS_ENABLED = True
+    GENESETS_READONLY = False
 
     @classmethod
     def setUpClass(cls):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
             MatrixDataType.H5AD, annotations_fixture=True
         )
-        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.label_output_file, cls.data.get_location()])
 
     @classmethod
     def tearDownClass(cls):
         shutil.rmtree(cls.tmp_dir)
         stop_test_server(cls.ps)
+
+
+class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
+    ANNOTATIONS_ENABLED = False
+    GENESETS_READONLY = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir = tempfile.mkdtemp()
+        genesets_file = path.join(cls.tmp_dir, "test_genesets.csv")
+        shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-genesets.csv", genesets_file)
+        cls._setupClass(
+            cls,
+            [
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                "--disable-annotations",
+                "--genesets-file",
+                genesets_file,
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)
+
+    def test_get_genesets_json(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+        self.assertIsNotNone(result_data["tid"])
+
+        self.assertEqual(
+            result_data,
+            {
+                "genesets": [
+                    {
+                        "genes": [
+                            {"gene_description": "a gene_description", "gene_symbol": "F5"},
+                            {"gene_description": "", "gene_symbol": "SUMO3"},
+                            {"gene_description": "", "gene_symbol": "SRM"},
+                        ],
+                        "geneset_description": "a description",
+                        "geneset_name": "first geneset name",
+                    },
+                    {
+                        "genes": [
+                            {"gene_description": "", "gene_symbol": "RER1"},
+                            {"gene_description": "", "gene_symbol": "SIK1"},
+                        ],
+                        "geneset_description": "",
+                        "geneset_name": "second geneset",
+                    },
+                    {"genes": [], "geneset_description": "", "geneset_name": "third geneset"},
+                    {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_geneset"},
+                    {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
+                ],
+                "tid": 0,
+            },
+        )
+
+    def test_get_genesets_csv(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "text/csv"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "text/csv")
+        self.assertEqual(
+            result.text,
+            """geneset_name,geneset_description,gene_symbol,gene_description\r
+first geneset name,a description,F5,a gene_description\r
+first geneset name,a description,SUMO3,\r
+first geneset name,a description,SRM,\r
+second geneset,,RER1,\r
+second geneset,,SIK1,\r
+third geneset,,,\r
+fourth_geneset,fourth description,,\r
+fifth_dataset,,,\r
+""",
+        )
+
+    def test_put_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        # assume we start with TID 0
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json()["tid"], 0)
+
+        test1 = {"tid": 3, "genesets": []}
+        result = self.session.put(url, json=test1)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test1)
+
+        # stale TID
+        result = self.session.put(url, json=test1)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
+
+        test2 = {"tid": 4, "genesets": [{"geneset_name": "foobar", "genes": []}]}
+        test2_response = {"tid": 4, "genesets": [{"geneset_name": "foobar", "geneset_description": "", "genes": []}]}
+        result = self.session.put(url, json=test2)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test2_response)
+
+        test3 = {
+            "tid": 5,
+            "genesets": [
+                {
+                    "geneset_name": "foobar",
+                    "geneset_description": "",
+                    "genes": [
+                        {
+                            "gene_symbol": "F5",
+                            "gene_description": "",
+                        }
+                    ],
+                }
+            ],
+        }
+        result = self.session.put(url, json=test3)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test3)
+
+    def test_put_genesets_malformed(self):
+        """ test malformed submissions that we expect the backend to catch/tolerate """
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        original_data = result.json()
+        tid = original_data["tid"]
+
+        def test_case(test, expected_code, original_data):
+            """ check for expected error AND that no change was made to the original state """
+            result = self.session.put(url, json=test)
+            self.assertEqual(result.status_code, expected_code)
+            result = self.session.get(url, headers={"Accept": "application/json"})
+            self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.json(), original_data)
+
+        # missing or malformed genesets
+        test_case(
+            {"tid": tid + 1},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": 99},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # illegal geneset_name
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": """, "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # duplicate geneset_name
+        test_case(
+            {
+                "tid": tid + 1,
+                "genesets": [
+                    {"geneset_name": "foo", "genes": []},
+                    {"geneset_name": "foo", "genes": []},
+                ],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # missing geneset_name
+        test_case(
+            {"tid": tid + 1, "genesets": [{"genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # non-numeric TID
+        test_case(
+            {"tid": [], "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": None, "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": "not a number", "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # duplicate gene_symbol
+        test_case(
+            {
+                "tid": "not a number",
+                "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": "SIK1"}, {"gene_symbol": "SIK1"}]}],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # gene_symbol is not a string
+        test_case(
+            {
+                "tid": "not a number",
+                "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": 99}]}],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+    """
+    TODO once we have some code to support it:
+    1. GET genesets_summary
+    2. genesets_summary obeys tid
+    """