fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-02 10:28:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Split out the local backend (#2052)

Browse Source 
This splits the backend into two parts: the local backend for desktop cellxgene and the AWS backend for hosted cellxgene. The local backend is in local_server while the hosted remains in server. The general idea is to copy everything from server to local_server, pull unneeded stuff out of local_server, and keep server as-is for this PR. Not touching server means all the infra and deployment code will continue working just as it did before so we can make those changes incrementally.
This commit is contained in:
Marcus Kinsella
2021-02-18 12:58:22 -08:00
committed by GitHub
parent 036b5f8c0f
commit fb61bd6e9c
153 changed files with 14027 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
local_server/test/fixtures/schema_test_data/generate_test_data.sh
Vendored Executable
+139
View File
@@ -0,0 +1,139 @@
#!/bin/bash
wget "https://s3-us-west-2.amazonaws.com/10x.files/samples/cell/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz"
tar xf "pbmc3k_filtered_gene_bc_matrices.tar.gz"
python3 - <<MERGE_GENES
import os
from scipy.io import mmread, mmwrite
import scipy.sparse
import pandas as pd
from local_server.converters.schema import gene_symbol
mat = mmread("filtered_gene_bc_matrices/hg19/matrix.mtx").todense()
genes = pd.read_csv("filtered_gene_bc_matrices/hg19/genes.tsv", sep='\t', names=["gene_id", "gene_symbol"])
upgraded_genes = gene_symbol.get_upgraded_var_index(pd.DataFrame(index=genes["gene_symbol"]))
df = pd.DataFrame(data=mat, index=upgraded_genes).T
merged = df.sum(axis=1, level=0, skipna=False)
os.makedirs("merged")
merged.columns.to_frame().to_csv("merged/genes.tsv", index=False, header=False)
mmwrite("merged/matrix.mtx", scipy.sparse.coo_matrix(merged).T)
MERGE_GENES
cp "filtered_gene_bc_matrices/hg19/barcodes.tsv" "merged/barcodes.tsv"
awk '{print $1"\t"$1}' merged/genes.tsv > genes_tmp.tsv; mv genes_tmp.tsv merged/genes.tsv
echo -e "\n\n\nRunning tutorial on original\n\n\n"
Rscript - <<TUTORIAL
library(Seurat)
pbmc.data <- Read10X(data.dir = "filtered_gene_bc_matrices/hg19/")
pbmc <- CreateSeuratObject(counts = pbmc.data, project = "pbmc3k", min.features = 200)
pbmc <- NormalizeData(pbmc, normalization.method = "LogNormalize", scale.factor = 10000)
pbmc <- FindVariableFeatures(pbmc, selection.method = "vst", nfeatures = 2000)
pbmc[["percent.mt"]] <- PercentageFeatureSet(pbmc, pattern = "^MT-")
all.genes <- rownames(pbmc)
pbmc <- ScaleData(pbmc, features = all.genes)
pbmc <- RunPCA(pbmc, features = VariableFeatures(object = pbmc))
pbmc <- FindNeighbors(pbmc, dims = 1:10)
pbmc <- FindClusters(pbmc, resolution = 0.5)
pbmc <- RunUMAP(pbmc, dims = 1:10)
saveRDS(pbmc, file = "./seurat_tutorial.rds")
TUTORIAL
echo -e "\n\n\nRunning tutorial on merged\n\n\n"
Rscript - <<TUTORIAL_MERGED
library(Seurat)
pbmc.data <- Read10X(data.dir = "merged/")
pbmc <- CreateSeuratObject(counts = pbmc.data, project = "pbmc3k", min.features = 200)
pbmc <- NormalizeData(pbmc, normalization.method = "LogNormalize", scale.factor = 10000)
pbmc <- FindVariableFeatures(pbmc, selection.method = "vst", nfeatures = 2000)
pbmc[["percent.mt"]] <- PercentageFeatureSet(pbmc, pattern = "^MT-")
all.genes <- rownames(pbmc)
pbmc <- ScaleData(pbmc, features = all.genes)
pbmc <- RunPCA(pbmc, features = VariableFeatures(object = pbmc))
pbmc <- FindNeighbors(pbmc, dims = 1:10)
pbmc <- FindClusters(pbmc, resolution = 0.5)
pbmc <- RunUMAP(pbmc, dims = 1:10)
saveRDS(pbmc, file = "./seurat_tutorial_merged.rds")
TUTORIAL_MERGED
echo -e "\n\n\nRunning SCTransform on original\n\n\n"
Rscript - <<SCTRANSFORM
library(Seurat)
library(sctransform)
pbmc.data <- Read10X(data.dir = "filtered_gene_bc_matrices/hg19/")
pbmc <- CreateSeuratObject(counts = pbmc.data)
pbmc <- PercentageFeatureSet(pbmc, pattern = "^MT-", col.name = "percent.mt")
pbmc <- SCTransform(pbmc, vars.to.regress = "percent.mt", verbose = FALSE)
pbmc <- RunPCA(pbmc, verbose = FALSE)
pbmc <- RunUMAP(pbmc, dims = 1:30, verbose = FALSE)
pbmc <- FindNeighbors(pbmc, dims = 1:30, verbose = FALSE)
pbmc <- FindClusters(pbmc, verbose = FALSE)
saveRDS(pbmc, file = "./sctransform.rds")
SCTRANSFORM
echo -e "\n\n\nRunning SCTransform on merged\n\n\n"
Rscript - <<SCTRANSFORM_MERGED
library(Seurat)
library(sctransform)
pbmc.data <- Read10X(data.dir = "merged/")
pbmc <- CreateSeuratObject(counts = pbmc.data)
pbmc <- PercentageFeatureSet(pbmc, pattern = "^MT-", col.name = "percent.mt")
pbmc <- SCTransform(pbmc, vars.to.regress = "percent.mt", verbose = FALSE)
pbmc <- RunPCA(pbmc, verbose = FALSE)
pbmc <- RunUMAP(pbmc, dims = 1:30, verbose = FALSE)
pbmc <- FindNeighbors(pbmc, dims = 1:30, verbose = FALSE)
pbmc <- FindClusters(pbmc, verbose = FALSE)
saveRDS(pbmc, file = "./sctransform_merged.rds")
SCTRANSFORM_MERGED
echo -e "\n\n\nConverting\n\n\n"
Rscript - <<SCEASY
library(sceasy)
srt <- readRDS("seurat_tutorial.rds")
sceasy::convertFormat(srt,
outFile = "seurat_tutorial.h5ad",
from = "seurat",
to = "anndata",
assay = "RNA",
main_layer = "data",
transfer_layers = c("data", "counts", "scale.data"),
drop_single_values = FALSE)
srt <- readRDS("seurat_tutorial_merged.rds")
sceasy::convertFormat(srt,
outFile = "seurat_tutorial_merged.h5ad",
from = "seurat",
to = "anndata",
assay = "RNA",
main_layer = "data",
transfer_layers = c("data", "counts", "scale.data"),
drop_single_values = FALSE)
srt <- readRDS("sctransform.rds")
sceasy::convertFormat(srt,
outFile = "sctransform.h5ad",
from = "seurat",
to = "anndata",
assay = "SCT",
main_layer = "data",
transfer_layers = c("data", "counts", "scale.data"),
drop_single_values = FALSE)
srt <- readRDS("sctransform_merged.rds")
sceasy::convertFormat(srt,
outFile = "sctransform_merged.h5ad",
from = "seurat",
to = "anndata",
assay = "SCT",
main_layer = "data",
transfer_layers = c("data", "counts", "scale.data"),
drop_single_values = FALSE)
SCEASY