From a5c9ffa88075aeb505298c353650a5b176565fbf Mon Sep 17 00:00:00 2001 From: maniarathi Date: Sat, 22 Aug 2020 09:42:11 -0700 Subject: [PATCH] When reading annotations from tiledb, check if the values are byte literals and if so, decode them. Also pin s3f3 to 0.4.2. (#1788) --- server/common/annotations/hosted_tiledb.py | 4 ++++ server/requirements.txt | 2 +- .../test/unit/common/test_writable_annotation.py | 15 ++++++--------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/server/common/annotations/hosted_tiledb.py b/server/common/annotations/hosted_tiledb.py index 16ac7b29..f65a6654 100644 --- a/server/common/annotations/hosted_tiledb.py +++ b/server/common/annotations/hosted_tiledb.py @@ -74,6 +74,10 @@ class AnnotationsHostedTileDB(Annotations): indexes = list() for col_name, col_val in data.items(): + # If the column values are byte literals, decode them + if isinstance(col_val[0], bytes): + col_val = [value.decode('utf-8') for value in col_val] + if repr_meta and col_name in repr_meta: new_col = pd.Series(col_val, dtype=repr_meta[col_name]) data[col_name] = new_col diff --git a/server/requirements.txt b/server/requirements.txt index ac08deed..d188764b 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -20,5 +20,5 @@ scipy>=1.3.0 requests>=2.22.0 sqlalchemy>=1.3.18 tiledb>=0.5.9,>=0.6.2 -s3fs>=0.4.2 +s3fs==0.4.2 gunicorn>=20.0.4 diff --git a/server/test/unit/common/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py index 45e09de3..a13c7569 100644 --- a/server/test/unit/common/test_writable_annotation.py +++ b/server/test/unit/common/test_writable_annotation.py @@ -1,22 +1,20 @@ import json -from os import path, listdir +import shutil import unittest +from os import path, listdir from unittest.mock import MagicMock, patch +import numpy as np +import pandas as pd import tiledb from flask import Flask import server.test.unit.decode_fbs as decode_fbs -import shutil - -import numpy as np -import pandas as pd - +from server.common.errors import AnnotationCategoryNameError from server.common.rest import schema_get_helper, annotations_put_fbs_helper +from server.data_common.matrix_loader import MatrixDataType from server.db.cellxgene_orm import CellxGeneDataset, Annotation from server.test import data_with_tmp_annotations, make_fbs, data_with_tmp_tiledb_annotations -from server.data_common.matrix_loader import MatrixDataType -from server.common.errors import AnnotationCategoryNameError class auth(object): @@ -77,7 +75,6 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self): with self.app.test_request_context(): - new_name = 'new_dataset/location' self.data.get_location = MagicMock(return_value=new_name) num_datasets = len(self.db.query([CellxGeneDataset]))