thuang-compress-annotation (#1980)

* thuang-compress-annotation

* compress test

* use zlib.decompress directly

client/package-lock.json

@@ -15483,8 +15483,7 @@
     "pako": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
-      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
+      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
-      "dev": true
     },
     "parallel-transform": {
       "version": "1.2.0",

client/package.json

@@ -54,6 +54,7 @@
     "is-number": "^7.0.0",
     "lodash": "^4.17.20",
     "memoize-one": "^5.1.1",
+    "pako": "^1.0.11",
     "react": "^16.13.1",
     "react-async": "^10.0.1",
     "react-dom": "^16.13.1",

client/src/actions/annotation.js

@@ -2,6 +2,7 @@
 Action creators for user annotation
 */
 import _ from "lodash";
+import pako from "pako";
 import * as globals from "../globals";
 import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
 
@@ -347,6 +348,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
 
   const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
   const matrix = MatrixFBS.encodeMatrixFBS(df);
+  const compressedMatrix = pako.deflate(matrix);
   try {
     const queryString =
       !dataCollectionNameIsReadOnly && !!dataCollectionName
@@ -358,7 +360,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
       `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
       {
         method: "PUT",
-        body: matrix,
+        body: compressedMatrix,
         headers: new Headers({
           "Content-Type": "application/octet-stream",
         }),

server/common/rest.py

@@ -2,6 +2,7 @@ import copy
 import logging
 import sys
 from http import HTTPStatus
+import zlib
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
@@ -159,13 +160,17 @@ def annotations_put_fbs_helper(data_adaptor, fbs):
     annotations.write_labels(new_label_df, data_adaptor)
 
 
+def inflate(data):
+    return zlib.decompress(data)
+
+
 def annotations_obs_put(request, data_adaptor):
     annotations = data_adaptor.dataset_config.user_annotations
     if annotations is None:
         return abort(HTTPStatus.NOT_IMPLEMENTED)
 
     anno_collection = request.args.get("annotation-collection-name", default=None)
-    fbs = request.get_data()
+    fbs = inflate(request.get_data())
 
     if anno_collection is not None:
         if not annotations.is_safe_collection_name(anno_collection):

server/test/unit/common/test_api.py

@@ -1,6 +1,7 @@
 import shutil
 import time
 import unittest
+import zlib
 from http import HTTPStatus
 
 import pandas as pd
@@ -343,7 +344,7 @@ class EndPointsAnnotations(EndPoints):
         url = f"{self.URL_BASE}{endpoint}?{query}"
         n_rows = self.data.get_shape()[0]
         fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
-        result = self.session.put(url, data=fbs)
+        result = self.session.put(url, data=zlib.compress(fbs))
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         self.assertEqual(result.json(), {"status": "OK"})