mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-16 05:07:55 +08:00
5c0b8c6296
* Improve diffexp for tiledb - The rows from the A and B sets are gathered and processed at the same time. In this way the matrix is only accessed once instead of twice for each tile. - There is now a single thread queue that gets shared between all callers of the diffexp. This will slow down work if diffexp gets too busy. - There is a target_workunit amount of work given to each thread. Previously the workunit was (rows selected * width of tile), which could be small. Now multiple column tiles can be combined into one workunit. If the target is too small then thread and other overheads may reduce performance. If target_workunit is too large then the size of the gathered sub matrix may take up too much memory. - add configuration parameters (max_workers, cpu_multiplier, and target_workunit)
87 lines
2.8 KiB
Python
87 lines
2.8 KiB
Python
import sys
|
|
import argparse
|
|
import random
|
|
import time
|
|
import numpy as np
|
|
|
|
import server.compute.diffexp_cxg as diffexp_cxg
|
|
import server.compute.diffexp_generic as diffexp_generic
|
|
|
|
from server.common.app_config import AppConfig
|
|
from server.data_common.matrix_loader import MatrixDataLoader
|
|
from server.data_cxg.cxg_adaptor import CxgAdaptor
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser("A command to test diffexp")
|
|
parser.add_argument("dataset", help="name of a dataset to load")
|
|
parser.add_argument("-na", "--numA", type=int, required=True, help="number of rows in group A")
|
|
parser.add_argument("-nb", "--numB", type=int, required=True, help="number of rows in group B")
|
|
parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials")
|
|
parser.add_argument(
|
|
"-a", "--alg", choices=("default", "generic", "cxg"), default="default", help="algorithm to use"
|
|
)
|
|
parser.add_argument("-s", "--show", default=False, action="store_true", help="show the results")
|
|
parser.add_argument(
|
|
"-n", "--new-selection", default=False, action="store_true", help="change the selection between each trial"
|
|
)
|
|
parser.add_argument("--seed", default=1, type=int, help="set the random seed")
|
|
|
|
args = parser.parse_args()
|
|
|
|
app_config = AppConfig()
|
|
app_config.single_dataset__datapath = args.dataset
|
|
app_config.server__verbose = True
|
|
app_config.complete_config()
|
|
|
|
loader = MatrixDataLoader(args.dataset)
|
|
adaptor = loader.open(app_config)
|
|
|
|
if args.show:
|
|
if isinstance(adaptor, CxgAdaptor):
|
|
adaptor.open_array("X").schema.dump()
|
|
|
|
numA = args.numA
|
|
numB = args.numB
|
|
rows = adaptor.get_shape()[0]
|
|
|
|
random.seed(args.seed)
|
|
|
|
if not args.new_selection:
|
|
samples = random.sample(range(rows), numA + numB)
|
|
filterA = samples[:numA]
|
|
filterB = samples[numA:]
|
|
|
|
for i in range(args.trials):
|
|
if args.new_selection:
|
|
samples = random.sample(range(rows), numA + numB)
|
|
filterA = samples[:numA]
|
|
filterB = samples[numA:]
|
|
|
|
maskA = np.zeros(rows, dtype=bool)
|
|
maskA[filterA] = True
|
|
maskB = np.zeros(rows, dtype=bool)
|
|
maskB[filterB] = True
|
|
|
|
t1 = time.time()
|
|
if args.alg == "default":
|
|
results = adaptor.compute_diffexp_ttest(maskA, maskB)
|
|
elif args.alg == "generic":
|
|
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB)
|
|
elif args.alg == "cxg":
|
|
if not isinstance(adaptor, CxgAdaptor):
|
|
print("cxg only works with CxgAdaptor")
|
|
sys.exit(1)
|
|
results = diffexp_cxg.diffexp_ttest(adaptor, maskA, maskB)
|
|
|
|
t2 = time.time()
|
|
print("TIME=", t2 - t1)
|
|
|
|
if args.show:
|
|
for res in results:
|
|
print(res)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|