mirror of
https://github.com/Novartis/cellxgene-gateway.git
synced 2026-10-08 23:48:12 +08:00
rebased "Introduction of ItemSource interface" patch
This commit is contained in:
Alok Saldanha
@@ -0,0 +1,27 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from cellxgene_gateway.items.item import Item
|
||||
|
||||
|
||||
class FileItem(Item):
|
||||
"""e.g. FileItem(subpath = subpath, name = filename, type = ItemType.h5ad)
|
||||
|
||||
The Item superclass expects a 'name' and 'type'.
|
||||
"""
|
||||
|
||||
def __init__(self, subpath: str, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.subpath = subpath
|
||||
|
||||
@property
|
||||
def descriptor(self) -> str:
|
||||
return os.path.join(self.subpath, self.name).strip("/")
|
||||
@@ -0,0 +1,175 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
from cellxgene_gateway import dir_util
|
||||
from cellxgene_gateway.items.file.fileitem import FileItem
|
||||
from cellxgene_gateway.items.item import ItemTree, ItemType
|
||||
from cellxgene_gateway.items.item_source import ItemSource, LookupResult
|
||||
|
||||
|
||||
class FileItemSource(ItemSource):
|
||||
def __init__(
|
||||
self,
|
||||
base_path,
|
||||
name=None,
|
||||
h5ad_suffix=dir_util.h5ad_suffix,
|
||||
annotation_dir_suffix=dir_util.annotations_suffix,
|
||||
annotation_file_suffix=".csv",
|
||||
):
|
||||
self._name = name
|
||||
self.base_path = base_path
|
||||
self.h5ad_suffix = h5ad_suffix
|
||||
self.annotation_dir_suffix = annotation_dir_suffix
|
||||
self.annotation_file_suffix = annotation_file_suffix
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self._name or f"Files:{self.base_path}"
|
||||
|
||||
def is_h5ad_file(self, path: str) -> bool:
|
||||
return path.endswith(self.h5ad_suffix) and os.path.isfile(path)
|
||||
|
||||
def convert_annotation_path_to_h5ad(self, path):
|
||||
return path[: -len(self.annotation_dir_suffix)] + self.h5ad_suffix
|
||||
|
||||
def convert_h5ad_path_to_annotation(self, path):
|
||||
return path[: -len(self.h5ad_suffix)] + self.annotation_dir_suffix
|
||||
|
||||
def get_local_path(self, item: FileItem) -> str:
|
||||
return os.path.join(self.base_path, item.descriptor)
|
||||
|
||||
def get_annotations_subpath(self, item) -> str:
|
||||
return self.convert_h5ad_path_to_annotation(item.descriptor)
|
||||
|
||||
def list_items(self, filter: str = None) -> ItemTree:
|
||||
item_tree = self.scan_directory()
|
||||
|
||||
"""def get_items(dir):
|
||||
if dir.branches:
|
||||
return [*dir.items, *[item for subdir in dir.branches for item in get_items(subdir)]]
|
||||
else:
|
||||
return dir.items
|
||||
|
||||
return get_items(self.item_tree)"""
|
||||
|
||||
return item_tree
|
||||
|
||||
def scan_directory(self, subpath="") -> dict:
|
||||
base_path = os.path.join(self.base_path, subpath)
|
||||
|
||||
if not os.path.exists(base_path):
|
||||
raise Exception(f"Path for local files '{base_path}' does not exist.")
|
||||
|
||||
filepath_map = dict(
|
||||
(filepath, os.path.join(base_path, filepath))
|
||||
for filepath in sorted(os.listdir(base_path))
|
||||
)
|
||||
|
||||
def is_annotation_dir(dir):
|
||||
return (
|
||||
dir.endswith(self.annotation_dir_suffix)
|
||||
and self.convert_annotation_path_to_h5ad(dir) in h5ad_paths
|
||||
)
|
||||
|
||||
h5ad_paths = [
|
||||
filepath
|
||||
for filepath, full_path in filepath_map.items()
|
||||
if self.is_h5ad_file(full_path)
|
||||
]
|
||||
|
||||
subdirs = [
|
||||
filepath
|
||||
for filepath, full_path in filepath_map.items()
|
||||
if os.path.isdir(full_path) and not is_annotation_dir(filepath)
|
||||
]
|
||||
|
||||
items = [
|
||||
self.make_fileitem_from_path(filename, subpath) for filename in h5ad_paths
|
||||
]
|
||||
branches = None
|
||||
if len(subdirs) > 0:
|
||||
branches = [
|
||||
self.scan_directory(os.path.join(subpath, subdir)) for subdir in subdirs
|
||||
]
|
||||
|
||||
return ItemTree(subpath, items, branches)
|
||||
|
||||
def create_annotation(self, item: FileItem, name: str) -> FileItem:
|
||||
annotation = self.make_fileitem_from_path(
|
||||
name, self.get_annotations_subpath(item), is_annotation=True
|
||||
)
|
||||
item.annotations = (item.annotations or []).append(annotation)
|
||||
return annotation
|
||||
|
||||
def update(self, item: FileItem) -> None:
|
||||
pass
|
||||
|
||||
def full_path(self, p):
|
||||
return os.path.join(self.base_path, p)
|
||||
|
||||
def lookup_item(self, descriptor):
|
||||
full_path = self.full_path(descriptor)
|
||||
if self.is_h5ad_file(full_path):
|
||||
return self.shallowitem_from_descriptor(descriptor)
|
||||
|
||||
def lookup(self, indescriptor: str) -> LookupResult:
|
||||
descriptor = indescriptor.strip("/")
|
||||
if descriptor.endswith(self.annotation_file_suffix):
|
||||
annotation_item = self.shallowitem_from_descriptor(descriptor, True)
|
||||
h5ad_descriptor = self.convert_annotation_path_to_h5ad(
|
||||
annotation_item.subpath
|
||||
)
|
||||
item = self.lookup_item(h5ad_descriptor)
|
||||
if item is not None:
|
||||
return LookupResult(item, annotation_item)
|
||||
else:
|
||||
item = self.lookup_item(descriptor)
|
||||
if item is not None:
|
||||
return LookupResult(item)
|
||||
|
||||
def shallowitem_from_descriptor(self, descriptor, is_annotation=False):
|
||||
filename = os.path.basename(descriptor)
|
||||
subpath = os.path.dirname(descriptor)
|
||||
return self.make_fileitem_from_path(
|
||||
filename,
|
||||
subpath,
|
||||
is_annotation,
|
||||
True,
|
||||
)
|
||||
|
||||
def make_fileitem_from_path(
|
||||
self, filename, subpath, is_annotation=False, is_shallow=False
|
||||
) -> FileItem:
|
||||
item = FileItem(
|
||||
subpath=subpath,
|
||||
name=filename,
|
||||
type=ItemType.annotation if is_annotation else ItemType.h5ad,
|
||||
)
|
||||
|
||||
if not is_annotation and not is_shallow:
|
||||
annotations = self.make_annotations_for_fileitem(item)
|
||||
item.annotations = annotations
|
||||
|
||||
return item
|
||||
|
||||
def make_annotations_for_fileitem(self, item: FileItem) -> List[FileItem]:
|
||||
annotations_subpath = self.get_annotations_subpath(item)
|
||||
annotations_fullpath = self.full_path(annotations_subpath)
|
||||
if os.path.isdir(annotations_fullpath):
|
||||
return [
|
||||
self.make_fileitem_from_path(annotation, annotations_subpath, True)
|
||||
for annotation in sorted(os.listdir(annotations_fullpath))
|
||||
if annotation.endswith(self.annotation_file_suffix)
|
||||
and os.path.isfile(os.path.join(annotations_fullpath, annotation))
|
||||
]
|
||||
else:
|
||||
return None
|
||||
@@ -0,0 +1,41 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from enum import Enum
|
||||
from typing import List
|
||||
|
||||
|
||||
class ItemType(Enum):
|
||||
annotation = "annotation"
|
||||
h5ad = "h5ad"
|
||||
|
||||
|
||||
class Item(ABC):
|
||||
def __init__(self, name: str, type: ItemType, annotations: List["Item"] = None):
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.annotations = annotations
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def descriptor(self):
|
||||
raise Exception('"descriptor" not implemented')
|
||||
|
||||
|
||||
class ItemTree:
|
||||
def __init__(
|
||||
self,
|
||||
descriptor: str,
|
||||
items: List[Item] = None,
|
||||
branches: List["ItemTree"] = None,
|
||||
):
|
||||
self.descriptor = descriptor
|
||||
self.items = items
|
||||
self.branches = branches
|
||||
@@ -0,0 +1,50 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List
|
||||
|
||||
from cellxgene_gateway.items.item import Item
|
||||
|
||||
|
||||
class LookupResult:
|
||||
def __init__(self, h5ad_item: Item, annotation_item: Item = None):
|
||||
self.h5ad_item = h5ad_item
|
||||
self.annotation_item = annotation_item
|
||||
|
||||
|
||||
class ItemSource(ABC):
|
||||
@abstractmethod
|
||||
def list_items(self, filter: str = None) -> List[Item]:
|
||||
raise Exception('"list_items" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def get_local_path(self, item: Item) -> str:
|
||||
raise Exception('"local_path" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def get_annotations_subpath(self, item) -> str:
|
||||
raise Exception('"annotations_path" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def create_annotation(self, item: Item, name: str) -> Item:
|
||||
raise Exception('"annotation" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def update(self, item: Item) -> None:
|
||||
raise Exception('"update" unimplemented')
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, descriptor: str) -> LookupResult:
|
||||
raise Exception('"lookup" unimplemented')
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self):
|
||||
pass
|
||||
@@ -0,0 +1,27 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from cellxgene_gateway.items.item import Item
|
||||
|
||||
|
||||
class S3Item(Item):
|
||||
"""e.g. FileItem(subpath = subpath, name = filename, type = ItemType.h5ad)
|
||||
|
||||
The Item superclass expects a 'name' and 'type'.
|
||||
"""
|
||||
|
||||
def __init__(self, s3key: str, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.s3key = s3key
|
||||
|
||||
@property
|
||||
def descriptor(self) -> str:
|
||||
return self.s3key
|
||||
@@ -0,0 +1,169 @@
|
||||
# Copyright 2019 Novartis Institutes for BioMedical Research Inc. Licensed
|
||||
# under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
# this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless
|
||||
# required by applicable law or agreed to in writing, software distributed
|
||||
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
|
||||
# OR CONDITIONS OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing permissions and limitations under the License.
|
||||
|
||||
from os.path import basename, dirname, join
|
||||
from typing import List
|
||||
|
||||
import s3fs
|
||||
|
||||
from cellxgene_gateway import dir_util
|
||||
from cellxgene_gateway.items.item import ItemTree, ItemType
|
||||
from cellxgene_gateway.items.item_source import ItemSource, LookupResult
|
||||
from cellxgene_gateway.items.s3.s3item import S3Item
|
||||
|
||||
|
||||
class S3ItemSource(ItemSource):
|
||||
def __init__(
|
||||
self,
|
||||
bucket,
|
||||
name=None,
|
||||
h5ad_suffix=dir_util.h5ad_suffix,
|
||||
annotation_dir_suffix=dir_util.annotations_suffix,
|
||||
annotation_file_suffix=".csv",
|
||||
):
|
||||
self._name = name
|
||||
self.s3 = s3fs.S3FileSystem()
|
||||
self.bucket = bucket
|
||||
self.h5ad_suffix = h5ad_suffix
|
||||
self.annotation_dir_suffix = annotation_dir_suffix
|
||||
self.annotation_file_suffix = annotation_file_suffix
|
||||
|
||||
def url(self, path):
|
||||
return "s3://" + join(self.bucket, path)
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self._name or f"Items:{self.url('')}"
|
||||
|
||||
def is_h5ad_url(self, s3url: str) -> bool:
|
||||
return s3url.endswith(self.h5ad_suffix) and self.s3.exists(s3url)
|
||||
|
||||
def convert_annotation_key_to_h5ad(self, s3key):
|
||||
return s3key[: -len(self.annotation_dir_suffix)] + self.h5ad_suffix
|
||||
|
||||
def convert_h5ad_key_to_annotation(self, s3key):
|
||||
return s3key[: -len(self.h5ad_suffix)] + self.annotation_dir_suffix
|
||||
|
||||
def get_local_path(self, item: S3Item) -> str:
|
||||
return self.url(item.descriptor)
|
||||
|
||||
def get_annotations_subpath(self, item) -> str:
|
||||
return self.convert_h5ad_key_to_annotation(item.descriptor)
|
||||
|
||||
def list_items(self, filter: str = None) -> ItemTree:
|
||||
item_tree = self.scan_directory()
|
||||
return item_tree
|
||||
|
||||
def scan_directory(self, subpath="") -> dict:
|
||||
url = self.url(subpath)
|
||||
|
||||
if not self.s3.exists(url):
|
||||
raise Exception(f"S3 url '{url}' does not exist.")
|
||||
|
||||
s3key_map = dict(
|
||||
(filepath[len(self.bucket) :], "s3://" + filepath)
|
||||
for filepath in sorted(self.s3.ls(url))
|
||||
)
|
||||
|
||||
def is_annotation_dir(dir_s3key):
|
||||
return (
|
||||
dir_s3key.endswith(self.annotation_dir_suffix)
|
||||
and self.convert_annotation_key_to_h5ad(dir_s3key) in h5ad_paths
|
||||
)
|
||||
|
||||
h5ad_paths = [
|
||||
filepath
|
||||
for filepath, item_url in s3key_map.items()
|
||||
if self.is_h5ad_url(item_url)
|
||||
]
|
||||
|
||||
subdirs = [
|
||||
filepath
|
||||
for filepath, item_url in s3key_map.items()
|
||||
if self.s3.isdir(item_url) and not is_annotation_dir(filepath)
|
||||
]
|
||||
|
||||
items = [
|
||||
self.make_s3item_from_key(filename, join(subpath, filename))
|
||||
for filename in h5ad_paths
|
||||
]
|
||||
branches = None
|
||||
if len(subdirs) > 0:
|
||||
branches = [
|
||||
self.scan_directory(join(subpath, subdir)) for subdir in subdirs
|
||||
]
|
||||
|
||||
return ItemTree(subpath, items, branches)
|
||||
|
||||
def create_annotation(self, item: S3Item, name: str) -> S3Item:
|
||||
annotation = self.make_s3item_from_key(
|
||||
name, self.get_annotations_subpath(item), is_annotation=True
|
||||
)
|
||||
item.annotations = (item.annotations or []).append(annotation)
|
||||
return annotation
|
||||
|
||||
def update(self, item: S3Item) -> None:
|
||||
pass
|
||||
|
||||
def lookup_item(self, descriptor):
|
||||
full_path = self.url(descriptor)
|
||||
if self.is_h5ad_url(full_path):
|
||||
return self.shallowitem_from_descriptor(descriptor)
|
||||
|
||||
def lookup(self, indescriptor: str) -> LookupResult:
|
||||
descriptor = indescriptor.strip("/")
|
||||
if descriptor.endswith(self.annotation_file_suffix):
|
||||
annotation_item = self.shallowitem_from_descriptor(descriptor, True)
|
||||
if not self.s3.exists(self.url(annotation_item.s3key)):
|
||||
with self.s3.open(self.url(annotation_item.s3key), "w") as f:
|
||||
f.write("")
|
||||
h5ad_descriptor = self.convert_annotation_key_to_h5ad(
|
||||
dirname(annotation_item.s3key)
|
||||
)
|
||||
item = self.shallowitem_from_descriptor(h5ad_descriptor)
|
||||
return LookupResult(item, annotation_item)
|
||||
else:
|
||||
item = self.lookup_item(descriptor)
|
||||
if item is not None:
|
||||
return LookupResult(item)
|
||||
|
||||
def shallowitem_from_descriptor(self, descriptor, is_annotation=False):
|
||||
return self.make_s3item_from_key(
|
||||
basename(descriptor), descriptor, is_annotation, True
|
||||
)
|
||||
|
||||
def make_s3item_from_key(
|
||||
self, name, s3key, is_annotation=False, is_shallow=False
|
||||
) -> S3Item:
|
||||
item = S3Item(
|
||||
s3key=s3key,
|
||||
name=name,
|
||||
type=ItemType.annotation if is_annotation else ItemType.h5ad,
|
||||
)
|
||||
|
||||
if not is_annotation and not is_shallow:
|
||||
annotations = self.make_annotations_for_fileitem(item)
|
||||
item.annotations = annotations
|
||||
|
||||
return item
|
||||
|
||||
def make_annotations_for_fileitem(self, item: S3Item) -> List[S3Item]:
|
||||
annotations_subpath = self.get_annotations_subpath(item)
|
||||
annotations_fullpath = self.url(annotations_subpath)
|
||||
if self.s3.isdir(annotations_fullpath):
|
||||
return [
|
||||
self.make_s3item_from_key(
|
||||
annotation, join(annotations_subpath, annotation), True
|
||||
)
|
||||
for annotation in sorted(self.s3.ls(annotations_fullpath))
|
||||
if annotation.endswith(self.annotation_file_suffix)
|
||||
and self.s3.isfile(join(annotations_fullpath, annotation))
|
||||
]
|
||||
else:
|
||||
return None
|
||||
Reference in New Issue
Block a user