fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-05 14:28:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add schema subcommand (#1939)

Browse Source 
Add the `cellxgene schema apply` and `cellxgene schema validate` subcommands.

The first takes an h5ad file and a yaml with config information and produces a new h5ad that follows the cellxgene data integration schema.

The second takes an h5ad and checks if it follows the schema version written into its metadata.

Both are currently marked as "experimental" as the primary intended users are still at CZI.
This commit is contained in:
Marcus Kinsella
2020-11-02 08:26:37 -08:00
committed by GitHub
parent b9e132a00c
commit 78176f9711
20 changed files with 2224 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/test/fixtures/hgnc_example.txt.gz
Vendored
BIN
View File
Binary file not shown.
server/test/fixtures/schema_test_data/generate_test_data.sh
Vendored Executable
+139
View File
@@ -0,0 +1,139 @@
#!/bin/bash
wget "https://s3-us-west-2.amazonaws.com/10x.files/samples/cell/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz"
tar xf "pbmc3k_filtered_gene_bc_matrices.tar.gz"
python3 - <<MERGE_GENES
import os
from scipy.io import mmread, mmwrite
import scipy.sparse
import pandas as pd
from server.converters.schema import gene_symbol
mat = mmread("filtered_gene_bc_matrices/hg19/matrix.mtx").todense()
genes = pd.read_csv("filtered_gene_bc_matrices/hg19/genes.tsv", sep='\t', names=["gene_id", "gene_symbol"])
upgraded_genes = gene_symbol.get_upgraded_var_index(pd.DataFrame(index=genes["gene_symbol"]))
df = pd.DataFrame(data=mat, index=upgraded_genes).T
merged = df.sum(axis=1, level=0, skipna=False)
os.makedirs("merged")
merged.columns.to_frame().to_csv("merged/genes.tsv", index=False, header=False)
mmwrite("merged/matrix.mtx", scipy.sparse.coo_matrix(merged).T)
MERGE_GENES
cp "filtered_gene_bc_matrices/hg19/barcodes.tsv" "merged/barcodes.tsv"
awk '{print $1"\t"$1}' merged/genes.tsv > genes_tmp.tsv; mv genes_tmp.tsv merged/genes.tsv
echo -e "\n\n\nRunning tutorial on original\n\n\n"
Rscript - <<TUTORIAL
library(Seurat)
pbmc.data <- Read10X(data.dir = "filtered_gene_bc_matrices/hg19/")
pbmc <- CreateSeuratObject(counts = pbmc.data, project = "pbmc3k", min.features = 200)
pbmc <- NormalizeData(pbmc, normalization.method = "LogNormalize", scale.factor = 10000)
pbmc <- FindVariableFeatures(pbmc, selection.method = "vst", nfeatures = 2000)
pbmc[["percent.mt"]] <- PercentageFeatureSet(pbmc, pattern = "^MT-")
all.genes <- rownames(pbmc)
pbmc <- ScaleData(pbmc, features = all.genes)
pbmc <- RunPCA(pbmc, features = VariableFeatures(object = pbmc))
pbmc <- FindNeighbors(pbmc, dims = 1:10)
pbmc <- FindClusters(pbmc, resolution = 0.5)
pbmc <- RunUMAP(pbmc, dims = 1:10)
saveRDS(pbmc, file = "./seurat_tutorial.rds")
TUTORIAL
echo -e "\n\n\nRunning tutorial on merged\n\n\n"
Rscript - <<TUTORIAL_MERGED
library(Seurat)
pbmc.data <- Read10X(data.dir = "merged/")
pbmc <- CreateSeuratObject(counts = pbmc.data, project = "pbmc3k", min.features = 200)
pbmc <- NormalizeData(pbmc, normalization.method = "LogNormalize", scale.factor = 10000)
pbmc <- FindVariableFeatures(pbmc, selection.method = "vst", nfeatures = 2000)
pbmc[["percent.mt"]] <- PercentageFeatureSet(pbmc, pattern = "^MT-")
all.genes <- rownames(pbmc)
pbmc <- ScaleData(pbmc, features = all.genes)
pbmc <- RunPCA(pbmc, features = VariableFeatures(object = pbmc))
pbmc <- FindNeighbors(pbmc, dims = 1:10)
pbmc <- FindClusters(pbmc, resolution = 0.5)
pbmc <- RunUMAP(pbmc, dims = 1:10)
saveRDS(pbmc, file = "./seurat_tutorial_merged.rds")
TUTORIAL_MERGED
echo -e "\n\n\nRunning SCTransform on original\n\n\n"
Rscript - <<SCTRANSFORM
library(Seurat)
library(sctransform)
pbmc.data <- Read10X(data.dir = "filtered_gene_bc_matrices/hg19/")
pbmc <- CreateSeuratObject(counts = pbmc.data)
pbmc <- PercentageFeatureSet(pbmc, pattern = "^MT-", col.name = "percent.mt")
pbmc <- SCTransform(pbmc, vars.to.regress = "percent.mt", verbose = FALSE)
pbmc <- RunPCA(pbmc, verbose = FALSE)
pbmc <- RunUMAP(pbmc, dims = 1:30, verbose = FALSE)
pbmc <- FindNeighbors(pbmc, dims = 1:30, verbose = FALSE)
pbmc <- FindClusters(pbmc, verbose = FALSE)
saveRDS(pbmc, file = "./sctransform.rds")
SCTRANSFORM
echo -e "\n\n\nRunning SCTransform on merged\n\n\n"
Rscript - <<SCTRANSFORM_MERGED
library(Seurat)
library(sctransform)
pbmc.data <- Read10X(data.dir = "merged/")
pbmc <- CreateSeuratObject(counts = pbmc.data)
pbmc <- PercentageFeatureSet(pbmc, pattern = "^MT-", col.name = "percent.mt")
pbmc <- SCTransform(pbmc, vars.to.regress = "percent.mt", verbose = FALSE)
pbmc <- RunPCA(pbmc, verbose = FALSE)
pbmc <- RunUMAP(pbmc, dims = 1:30, verbose = FALSE)
pbmc <- FindNeighbors(pbmc, dims = 1:30, verbose = FALSE)
pbmc <- FindClusters(pbmc, verbose = FALSE)
saveRDS(pbmc, file = "./sctransform_merged.rds")
SCTRANSFORM_MERGED
echo -e "\n\n\nConverting\n\n\n"
Rscript - <<SCEASY
library(sceasy)
srt <- readRDS("seurat_tutorial.rds")
sceasy::convertFormat(srt,
outFile = "seurat_tutorial.h5ad",
from = "seurat",
to = "anndata",
assay = "RNA",
main_layer = "data",
transfer_layers = c("data", "counts", "scale.data"),
drop_single_values = FALSE)
srt <- readRDS("seurat_tutorial_merged.rds")
sceasy::convertFormat(srt,
outFile = "seurat_tutorial_merged.h5ad",
from = "seurat",
to = "anndata",
assay = "RNA",
main_layer = "data",
transfer_layers = c("data", "counts", "scale.data"),
drop_single_values = FALSE)
srt <- readRDS("sctransform.rds")
sceasy::convertFormat(srt,
outFile = "sctransform.h5ad",
from = "seurat",
to = "anndata",
assay = "SCT",
main_layer = "data",
transfer_layers = c("data", "counts", "scale.data"),
drop_single_values = FALSE)
srt <- readRDS("sctransform_merged.rds")
sceasy::convertFormat(srt,
outFile = "sctransform_merged.h5ad",
from = "seurat",
to = "anndata",
assay = "SCT",
main_layer = "data",
transfer_layers = c("data", "counts", "scale.data"),
drop_single_values = FALSE)
SCEASY
server/test/fixtures/test_bad_config.yaml
Vendored
+34
View File
@@ -0,0 +1,34 @@
fixup_gene_symbols:
X: log1p
obs:
cell_type_ontology_term_id:
louvain:
CD4 T cells: CL:00001
B cells: CL:00002
CD14+ Monocytes: CL:00003
NK cells: CL:00004
CD8 T cells: CL:00005
FCGR3A+ Monocytes: CL:00006
Dendritic cells: CL:00007
Megakaryocytes: CL:00008
tissue_ontology_term_id: UBERON:12345
assay_ontology_term_id: EFO:12345
disease_ontology_term_id: MONDO:12345
ethnicity_ontology_term_id: MANCESTRO:12345
development_stage_ontology_term_id: HsapDv:12345
sex: other
uns:
version:
corpora_schema_version: 1.0.0
corpora_encoding_version: 0.1.0
organism_ontology_term_id: NCBITaxon:9606
title: Test dataset
contributors:
- name: Marcus
institution: CZI
layer_descriptions:
X: raw
project_links:
- link_url: https://chanzuckerberg.com/
link_name: CZI
link_type: SUMMARY
server/test/fixtures/test_config.yaml
Vendored
+34
View File
@@ -0,0 +1,34 @@
fixup_gene_symbols:
X: log1p
obs:
cell_type_ontology_term_id:
louvain:
CD4 T cells: CL:00001
B cells: CL:00002
CD14+ Monocytes: CL:00003
NK cells: CL:00004
CD8 T cells: CL:00005
FCGR3A+ Monocytes: CL:00006
Dendritic cells: CL:00007
Megakaryocytes: CL:00008
tissue_ontology_term_id: UBERON:12345
assay_ontology_term_id: EFO:12345
disease_ontology_term_id: MONDO:12345
ethnicity_ontology_term_id: HANCESTRO:12345
development_stage_ontology_term_id: HsapDv:12345
sex: other
uns:
version:
corpora_schema_version: 1.0.0
corpora_encoding_version: 0.1.0
organism_ontology_term_id: NCBITaxon:9606
title: Test dataset
contributors:
- name: Marcus
institution: CZI
layer_descriptions:
X: raw
project_links:
- link_url: https://chanzuckerberg.com/
link_name: CZI
link_type: SUMMARY