Compare commits

..
20 Commits
Author SHA1 Message Date
Heng Li dd3d637c20 Merge branch 'master' into avx 2020-01-10 15:23:52 -05:00
Heng Li cdb7857841 r963: --junc-bonus not working; resolves #513 2020-01-06 22:03:50 -05:00
Heng Li 3c0d05d272 r962: abort given wrong RG line; resolves #541 2020-01-06 21:53:21 -05:00
Heng Li 47b646acbf r961: print indexed length 2020-01-06 21:13:33 -05:00
Heng Li a79cb3e991 Merge remote-tracking branch 'origin/master' 2019-12-23 17:33:56 -05:00
Heng Li 367aed4271 added the asan and tsan targets to Makefile 2019-12-23 17:33:10 -05:00
xdudiagnoa 081df6ac7d Fix example.c seq read logic
for every idx should map all input seqs
2019-11-11 00:46:07 -05:00
Torsten Seemann a3e7a575fb Add splice:hq to --help 2019-11-11 00:45:13 -05:00
Heng Li c18cd3ad2d Merge branch 'master' into avx 2019-05-24 23:52:56 -04:00
Heng Li e60d78e0b1 Merge branch 'master' into avx 2019-05-01 10:55:18 -04:00
Heng Li e9a45a4e1c Merge branch 'master' into avx 2019-03-04 11:15:56 -05:00
Heng Li f5e2176bc5 r903: merged extd2_avx512.c into extd2_sse.c 2019-01-01 13:46:30 -05:00
Heng Li 0b4be2996e avx512 working on MT 2019-01-01 13:05:30 -05:00
Heng Li feca68c71d fixed two bugs when computing full score 2019-01-01 11:46:01 -05:00
Heng Li 6d9ce56721 blend was reversed 2019-01-01 10:27:59 -05:00
Heng Li f2f425890d can be compiled on Linux 2019-01-01 10:04:03 -05:00
Heng Li 58f4210dea compiled, but not working 2019-01-01 01:32:43 -05:00
Heng Li a4782c7d7a r897: avx2 working on test/MT-*.fa
srli/slli behaves differently between SSE2 and AVX2, which is very annoying.
2018-12-31 10:26:19 -05:00
Heng Li 2a7d071e8b minor equivalent changes 2018-12-30 23:15:07 -05:00
Heng Li 9462da5159 initial avx2 support; not working properly yet 2018-12-30 11:09:44 -05:00
11 changed files with 397 additions and 252 deletions
+32 -4
View File
@@ -2,13 +2,25 @@ CFLAGS= -g -Wall -O2 -Wc++-compat #-Wextra
CPPFLAGS= -DHAVE_KALLOC CPPFLAGS= -DHAVE_KALLOC
INCLUDES= INCLUDES=
OBJS= kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o options.o index.o chain.o align.o hit.o map.o format.o pe.o esterr.o splitidx.o ksw2_ll_sse.o OBJS= kthread.o kalloc.o misc.o bseq.o sketch.o sdust.o options.o index.o chain.o align.o hit.o map.o format.o pe.o esterr.o splitidx.o ksw2_ll_sse.o
OBJS_SSE= ksw2_extz2_sse41.o ksw2_extd2_sse41.o ksw2_exts2_sse41.o ksw2_extz2_sse2.o ksw2_extd2_sse2.o ksw2_exts2_sse2.o
DISPATCH_FLAG=-msse4.1
PROG= minimap2 PROG= minimap2
PROG_EXTRA= sdust minimap2-lite PROG_EXTRA= sdust minimap2-lite
LIBS= -lm -lz -lpthread LIBS= -lm -lz -lpthread
ifeq ($(arm_neon),) # if arm_neon is not defined ifeq ($(arm_neon),) # if arm_neon is not defined
ifeq ($(sse2only),) # if sse2only is not defined ifeq ($(sse2only),) # if sse2only is not defined
OBJS+=ksw2_extz2_sse41.o ksw2_extd2_sse41.o ksw2_exts2_sse41.o ksw2_extz2_sse2.o ksw2_extd2_sse2.o ksw2_exts2_sse2.o ksw2_dispatch.o ifeq ($(avx512),)
ifeq ($(avx2),)
OBJS+=$(OBJS_SSE) ksw2_dispatch.o
else
OBJS+=ksw2_extd2_avx2.o $(OBJS_SSE) ksw2_dispatch.o
DISPATCH_FLAG=-mavx2
endif
else
OBJS+=ksw2_extd2_avx512.o ksw2_extd2_avx2.o $(OBJS_SSE) ksw2_dispatch.o
DISPATCH_FLAG=-mavx512bw
endif
else # if sse2only is defined else # if sse2only is defined
OBJS+=ksw2_extz2_sse.o ksw2_extd2_sse.o ksw2_exts2_sse.o OBJS+=ksw2_extz2_sse.o ksw2_extd2_sse.o ksw2_exts2_sse.o
endif endif
@@ -22,6 +34,16 @@ else #if aarch64 is defined
endif endif
endif endif
ifneq ($(asan),)
CFLAGS+=-fsanitize=address
LIBS+=-fsanitize=address
endif
ifneq ($(tsan),)
CFLAGS+=-fsanitize=thread
LIBS+=-fsanitize=thread
endif
.PHONY:all extra clean depend .PHONY:all extra clean depend
.SUFFIXES:.c .o .SUFFIXES:.c .o
@@ -57,11 +79,17 @@ ksw2_extz2_sse41.o:ksw2_extz2_sse.c ksw2.h kalloc.h
ksw2_extz2_sse2.o:ksw2_extz2_sse.c ksw2.h kalloc.h ksw2_extz2_sse2.o:ksw2_extz2_sse.c ksw2.h kalloc.h
$(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@ $(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
ksw2_extd2_avx2.o:ksw2_extd2_sse.c ksw2.h kalloc.h
$(CC) -c $(CFLAGS) -mavx2 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
ksw2_extd2_avx512.o:ksw2_extd2_sse.c ksw2.h kalloc.h
$(CC) -c $(CFLAGS) -mavx512bw $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
ksw2_extd2_sse41.o:ksw2_extd2_sse.c ksw2.h kalloc.h ksw2_extd2_sse41.o:ksw2_extd2_sse.c ksw2.h kalloc.h
$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ $(CC) -c $(CFLAGS) -msse4.1 -mno-avx2 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
ksw2_extd2_sse2.o:ksw2_extd2_sse.c ksw2.h kalloc.h ksw2_extd2_sse2.o:ksw2_extd2_sse.c ksw2.h kalloc.h
$(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@ $(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 -mno-avx2 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
ksw2_exts2_sse41.o:ksw2_exts2_sse.c ksw2.h kalloc.h ksw2_exts2_sse41.o:ksw2_exts2_sse.c ksw2.h kalloc.h
$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ $(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
@@ -70,7 +98,7 @@ ksw2_exts2_sse2.o:ksw2_exts2_sse.c ksw2.h kalloc.h
$(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@ $(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@
ksw2_dispatch.o:ksw2_dispatch.c ksw2.h ksw2_dispatch.o:ksw2_dispatch.c ksw2.h
$(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ $(CC) -c $(CFLAGS) $(DISPATCH_FLAG) $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@
# NEON-specific targets on ARM # NEON-specific targets on ARM
+2
View File
@@ -35,6 +35,8 @@ int main(int argc, char *argv[])
while ((mi = mm_idx_reader_read(r, n_threads)) != 0) { // traverse each part of the index while ((mi = mm_idx_reader_read(r, n_threads)) != 0) { // traverse each part of the index
mm_mapopt_update(&mopt, mi); // this sets the maximum minimizer occurrence; TODO: set a better default in mm_mapopt_init()! mm_mapopt_update(&mopt, mi); // this sets the maximum minimizer occurrence; TODO: set a better default in mm_mapopt_init()!
mm_tbuf_t *tbuf = mm_tbuf_init(); // thread buffer; for multi-threading, allocate one tbuf for each thread mm_tbuf_t *tbuf = mm_tbuf_init(); // thread buffer; for multi-threading, allocate one tbuf for each thread
gzrewind(f);
kseq_rewind(ks);
while (kseq_read(ks) >= 0) { // each kseq_read() call reads one query sequence while (kseq_read(ks) >= 0) { // each kseq_read() call reads one query sequence
mm_reg1_t *reg; mm_reg1_t *reg;
int j, i, n_reg; int j, i, n_reg;
+8 -4
View File
@@ -79,11 +79,11 @@ static char *mm_escape(char *s)
return s; return s;
} }
static void sam_write_rg_line(kstring_t *str, const char *s) static int sam_write_rg_line(kstring_t *str, const char *s)
{ {
char *p, *q, *r, *rg_line = 0; char *p, *q, *r, *rg_line = 0;
memset(mm_rg_id, 0, 256); memset(mm_rg_id, 0, 256);
if (s == 0) return; if (s == 0) return 0;
if (strstr(s, "@RG") != s) { if (strstr(s, "@RG") != s) {
if (mm_verbose >= 1) fprintf(stderr, "[ERROR] the read group line is not started with @RG\n"); if (mm_verbose >= 1) fprintf(stderr, "[ERROR] the read group line is not started with @RG\n");
goto err_set_rg; goto err_set_rg;
@@ -108,20 +108,23 @@ static void sam_write_rg_line(kstring_t *str, const char *s)
for (q = p, r = mm_rg_id; *q && *q != '\t' && *q != '\n'; ++q) for (q = p, r = mm_rg_id; *q && *q != '\t' && *q != '\n'; ++q)
*r++ = *q; *r++ = *q;
mm_sprintf_lite(str, "%s\n", rg_line); mm_sprintf_lite(str, "%s\n", rg_line);
return 0;
err_set_rg: err_set_rg:
free(rg_line); free(rg_line);
return -1;
} }
void mm_write_sam_hdr(const mm_idx_t *idx, const char *rg, const char *ver, int argc, char *argv[]) int mm_write_sam_hdr(const mm_idx_t *idx, const char *rg, const char *ver, int argc, char *argv[])
{ {
kstring_t str = {0,0,0}; kstring_t str = {0,0,0};
int ret = 0;
if (idx) { if (idx) {
uint32_t i; uint32_t i;
for (i = 0; i < idx->n_seq; ++i) for (i = 0; i < idx->n_seq; ++i)
mm_sprintf_lite(&str, "@SQ\tSN:%s\tLN:%d\n", idx->seq[i].name, idx->seq[i].len); mm_sprintf_lite(&str, "@SQ\tSN:%s\tLN:%d\n", idx->seq[i].name, idx->seq[i].len);
} }
if (rg) sam_write_rg_line(&str, rg); if (rg) ret = sam_write_rg_line(&str, rg);
mm_sprintf_lite(&str, "@PG\tID:minimap2\tPN:minimap2"); mm_sprintf_lite(&str, "@PG\tID:minimap2\tPN:minimap2");
if (ver) mm_sprintf_lite(&str, "\tVN:%s", ver); if (ver) mm_sprintf_lite(&str, "\tVN:%s", ver);
if (argc > 1) { if (argc > 1) {
@@ -132,6 +135,7 @@ void mm_write_sam_hdr(const mm_idx_t *idx, const char *rg, const char *ver, int
} }
mm_err_puts(str.s); mm_err_puts(str.s);
free(str.s); free(str.s);
return ret;
} }
static void write_cs_core(kstring_t *s, const uint8_t *tseq, const uint8_t *qseq, const mm_reg1_t *r, char *tmp, int no_iden, int write_tag) static void write_cs_core(kstring_t *s, const uint8_t *tseq, const uint8_t *qseq, const mm_reg1_t *r, char *tmp, int no_iden, int write_tag)
+23 -42
View File
@@ -1,5 +1,4 @@
#include <stdlib.h> #include <stdlib.h>
#include <limits.h>
#include <assert.h> #include <assert.h>
#if defined(WIN32) || defined(_WIN32) #if defined(WIN32) || defined(_WIN32)
#include <io.h> // for open(2) #include <io.h> // for open(2)
@@ -118,8 +117,8 @@ void mm_idx_stat(const mm_idx_t *mi)
if (kh_key(h, k)&1) ++n1; if (kh_key(h, k)&1) ++n1;
} }
} }
fprintf(stderr, "[M::%s::%.3f*%.2f] distinct minimizers: %d (%.2f%% are singletons); average occurrences: %.3lf; average spacing: %.3lf\n", fprintf(stderr, "[M::%s::%.3f*%.2f] distinct minimizers: %d (%.2f%% are singletons); average occurrences: %.3lf; average spacing: %.3lf; total length: %ld\n",
__func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), n, 100.0*n1/n, (double)sum / n, (double)len / sum); __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0), n, 100.0*n1/n, (double)sum / n, (double)len / sum, (long)len);
} }
int mm_idx_index_name(mm_idx_t *mi) int mm_idx_index_name(mm_idx_t *mi)
@@ -189,18 +188,12 @@ int32_t mm_idx_cal_max_occ(const mm_idx_t *mi, float f)
* Sort and generate hash tables * * Sort and generate hash tables *
*********************************/ *********************************/
typedef struct {
mm_idx_t *mi;
int min_occ, max_occ;
} idx_post_t;
static void worker_post(void *g, long i, int tid) static void worker_post(void *g, long i, int tid)
{ {
int n, n_keys; int n, n_keys;
size_t j, start_a, start_p; size_t j, start_a, start_p;
idxhash_t *h; idxhash_t *h;
idx_post_t *o = (idx_post_t*)g; mm_idx_t *mi = (mm_idx_t*)g;
mm_idx_t *mi = o->mi;
mm_idx_bucket_t *b = &mi->B[i]; mm_idx_bucket_t *b = &mi->B[i];
if (b->a.n == 0) return; if (b->a.n == 0) return;
@@ -210,10 +203,8 @@ static void worker_post(void *g, long i, int tid)
// count and preallocate // count and preallocate
for (j = 1, n = 1, n_keys = 0, b->n = 0; j <= b->a.n; ++j) { for (j = 1, n = 1, n_keys = 0, b->n = 0; j <= b->a.n; ++j) {
if (j == b->a.n || b->a.a[j].x>>8 != b->a.a[j-1].x>>8) { if (j == b->a.n || b->a.a[j].x>>8 != b->a.a[j-1].x>>8) {
if (n >= o->min_occ && n <= o->max_occ) { ++n_keys;
++n_keys; if (n > 1) b->n += n;
if (n > 1) b->n += n;
}
n = 1; n = 1;
} else ++n; } else ++n;
} }
@@ -227,20 +218,18 @@ static void worker_post(void *g, long i, int tid)
khint_t itr; khint_t itr;
int absent; int absent;
mm128_t *p = &b->a.a[j-1]; mm128_t *p = &b->a.a[j-1];
if (n >= o->min_occ && n <= o->max_occ) { itr = kh_put(idx, h, p->x>>8>>mi->b<<1, &absent);
itr = kh_put(idx, h, p->x>>8>>mi->b<<1, &absent); assert(absent && j == start_a + n);
assert(absent && j == start_a + n); if (n == 1) {
if (n == 1) { kh_key(h, itr) |= 1;
kh_key(h, itr) |= 1; kh_val(h, itr) = p->y;
kh_val(h, itr) = p->y; } else {
} else { int k;
int k; for (k = 0; k < n; ++k)
for (k = 0; k < n; ++k) b->p[start_p + k] = b->a.a[start_a + k].y;
b->p[start_p + k] = b->a.a[start_a + k].y; radix_sort_64(&b->p[start_p], &b->p[start_p + n]); // sort by position; needed as in-place radix_sort_128x() is not stable
radix_sort_64(&b->p[start_p], &b->p[start_p + n]); // sort by position; needed as in-place radix_sort_128x() is not stable kh_val(h, itr) = (uint64_t)start_p<<32 | n;
kh_val(h, itr) = (uint64_t)start_p<<32 | n; start_p += n;
start_p += n;
}
} }
start_a = j, n = 1; start_a = j, n = 1;
} else ++n; } else ++n;
@@ -253,12 +242,9 @@ static void worker_post(void *g, long i, int tid)
b->a.n = b->a.m = 0, b->a.a = 0; b->a.n = b->a.m = 0, b->a.a = 0;
} }
static void mm_idx_post(mm_idx_t *mi, int n_threads, int min_occ, int max_occ) static void mm_idx_post(mm_idx_t *mi, int n_threads)
{ {
idx_post_t t; kt_for(n_threads, worker_post, mi, 1<<mi->b);
if (max_occ <= 0 || max_occ < min_occ) max_occ = INT_MAX;
t.mi = mi, t.min_occ = min_occ, t.max_occ = max_occ;
kt_for(n_threads, worker_post, &t, 1<<mi->b);
} }
/****************** /******************
@@ -364,7 +350,7 @@ static void *worker_pipeline(void *shared, int step, void *in)
return 0; return 0;
} }
mm_idx_t *mm_idx_gen2(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini_batch_size, int n_threads, uint64_t batch_size, int min_occ, int max_occ) mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini_batch_size, int n_threads, uint64_t batch_size)
{ {
pipeline_t pl; pipeline_t pl;
if (fp == 0 || mm_bseq_eof(fp)) return 0; if (fp == 0 || mm_bseq_eof(fp)) return 0;
@@ -378,18 +364,13 @@ mm_idx_t *mm_idx_gen2(mm_bseq_file_t *fp, int w, int k, int b, int flag, int min
if (mm_verbose >= 3) if (mm_verbose >= 3)
fprintf(stderr, "[M::%s::%.3f*%.2f] collected minimizers\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0)); fprintf(stderr, "[M::%s::%.3f*%.2f] collected minimizers\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
mm_idx_post(pl.mi, n_threads, min_occ, max_occ); mm_idx_post(pl.mi, n_threads);
if (mm_verbose >= 3) if (mm_verbose >= 3)
fprintf(stderr, "[M::%s::%.3f*%.2f] sorted minimizers\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0)); fprintf(stderr, "[M::%s::%.3f*%.2f] sorted minimizers\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
return pl.mi; return pl.mi;
} }
mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int flag, int mini_batch_size, int n_threads, uint64_t batch_size)
{
return mm_idx_gen2(fp, w, k, b, flag, mini_batch_size, n_threads, batch_size, 0, INT_MAX);
}
mm_idx_t *mm_idx_build(const char *fn, int w, int k, int flag, int n_threads) // a simpler interface; deprecated mm_idx_t *mm_idx_build(const char *fn, int w, int k, int flag, int n_threads) // a simpler interface; deprecated
{ {
mm_bseq_file_t *fp; mm_bseq_file_t *fp;
@@ -446,7 +427,7 @@ mm_idx_t *mm_idx_str(int w, int k, int is_hpc, int bucket_bits, int n, const cha
} }
} }
free(a.a); free(a.a);
mm_idx_post(mi, 1, 0, 0); mm_idx_post(mi, 1);
return mi; return mi;
} }
@@ -607,7 +588,7 @@ mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
if (mi && mm_verbose >= 2 && (mi->k != r->opt.k || mi->w != r->opt.w || (mi->flag&MM_I_HPC) != (r->opt.flag&MM_I_HPC))) if (mi && mm_verbose >= 2 && (mi->k != r->opt.k || mi->w != r->opt.w || (mi->flag&MM_I_HPC) != (r->opt.flag&MM_I_HPC)))
fprintf(stderr, "[WARNING]\033[1;31m Indexing parameters (-k, -w or -H) overridden by parameters used in the prebuilt index.\033[0m\n"); fprintf(stderr, "[WARNING]\033[1;31m Indexing parameters (-k, -w or -H) overridden by parameters used in the prebuilt index.\033[0m\n");
} else } else
mi = mm_idx_gen2(r->fp.seq, r->opt.w, r->opt.k, r->opt.bucket_bits, r->opt.flag, r->opt.mini_batch_size, n_threads, r->opt.batch_size, r->opt.min_occ, r->opt.max_occ); mi = mm_idx_gen(r->fp.seq, r->opt.w, r->opt.k, r->opt.bucket_bits, r->opt.flag, r->opt.mini_batch_size, n_threads, r->opt.batch_size);
if (mi) { if (mi) {
if (r->fp_out) mm_idx_dump(r->fp_out, mi); if (r->fp_out) mm_idx_dump(r->fp_out, mi);
mi->index = r->n_parts++; mi->index = r->n_parts++;
+25 -9
View File
@@ -2,15 +2,16 @@
#include <stdlib.h> #include <stdlib.h>
#include "ksw2.h" #include "ksw2.h"
#define SIMD_SSE 0x1 #define SIMD_SSE 0x1
#define SIMD_SSE2 0x2 #define SIMD_SSE2 0x2
#define SIMD_SSE3 0x4 #define SIMD_SSE3 0x4
#define SIMD_SSSE3 0x8 #define SIMD_SSSE3 0x8
#define SIMD_SSE4_1 0x10 #define SIMD_SSE4_1 0x10
#define SIMD_SSE4_2 0x20 #define SIMD_SSE4_2 0x20
#define SIMD_AVX 0x40 #define SIMD_AVX 0x40
#define SIMD_AVX2 0x80 #define SIMD_AVX2 0x80
#define SIMD_AVX512F 0x100 #define SIMD_AVX512F 0x100
#define SIMD_AVX512BW 0x200
#ifndef _MSC_VER #ifndef _MSC_VER
// adapted from https://github.com/01org/linux-sgx/blob/master/common/inc/internal/linux/cpuid_gnu.h // adapted from https://github.com/01org/linux-sgx/blob/master/common/inc/internal/linux/cpuid_gnu.h
@@ -48,6 +49,7 @@ static int x86_simd(void)
__cpuidex(cpuid, 7, 0); __cpuidex(cpuid, 7, 0);
if (cpuid[1]>>5 &1) flag |= SIMD_AVX2; if (cpuid[1]>>5 &1) flag |= SIMD_AVX2;
if (cpuid[1]>>16&1) flag |= SIMD_AVX512F; if (cpuid[1]>>16&1) flag |= SIMD_AVX512F;
if (cpuid[1]>>30&1) flag |= SIMD_AVX512BW;
} }
return flag; return flag;
} }
@@ -71,7 +73,21 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez); int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
extern void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, extern void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez); int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
extern void ksw_extd2_avx2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
extern void ksw_extd2_avx512(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez);
if (ksw_simd < 0) ksw_simd = x86_simd(); if (ksw_simd < 0) ksw_simd = x86_simd();
#if defined(__AVX512BW__)
if (ksw_simd & SIMD_AVX512BW)
ksw_extd2_avx512(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
else
#endif
#if defined(__AVX2__)
if (ksw_simd & SIMD_AVX2)
ksw_extd2_avx2(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
else
#endif
if (ksw_simd & SIMD_SSE4_1) if (ksw_simd & SIMD_SSE4_1)
ksw_extd2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez); ksw_extd2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez);
else if (ksw_simd & SIMD_SSE2) else if (ksw_simd & SIMD_SSE2)
+291 -158
View File
@@ -4,7 +4,27 @@
#include "ksw2.h" #include "ksw2.h"
#ifdef __SSE2__ #ifdef __SSE2__
#if defined(__AVX512BW__)
#include <immintrin.h>
#define SIMD_INT __m512i
#define SIMD_SHIFT 6
#define simd_func(func) _mm512_##func
#define simd_funcw(func) _mm512_##func##_si512
#elif defined(__AVX2__)
#include <immintrin.h>
#define SIMD_INT __m256i
#define SIMD_SHIFT 5
#define simd_func(func) _mm256_##func
#define simd_funcw(func) _mm256_##func##_si256
#elif defined(__SSE2__)
#include <emmintrin.h> #include <emmintrin.h>
#define SIMD_INT __m128i
#define SIMD_SHIFT 4
#define simd_func(func) _mm_##func
#define simd_funcw(func) _mm_##func##_si128
#ifdef KSW_SSE2_ONLY #ifdef KSW_SSE2_ONLY
#undef __SSE4_1__ #undef __SSE4_1__
@@ -13,12 +33,39 @@
#ifdef __SSE4_1__ #ifdef __SSE4_1__
#include <smmintrin.h> #include <smmintrin.h>
#endif #endif
#endif // defined(__SSE2__)
#define SIMD_WIDTH (1<<SIMD_SHIFT)
#if !defined(__AVX512BW__)
#if defined(__AVX2__)
static inline __m256i simd_slli_1(__m256i x)
{
return _mm256_insert_epi8(_mm256_slli_si256(x, 1), _mm256_extract_epi8(x, 15), 16);
}
static inline __m256i simd_srli_last(__m256i x)
{
return _mm256_insert_epi8(_mm256_setzero_si256(), _mm256_extract_epi8(x, 31), 0);
}
#elif defined(__SSE2__)
static inline __m128i simd_slli_1(__m128i x) { return _mm_slli_si128(x, 1); }
static inline __m128i simd_srli_last(__m128i x) { return _mm_srli_si128(x, 15); }
#endif
#endif // ~__AVX512BW__
#ifdef KSW_CPU_DISPATCH #ifdef KSW_CPU_DISPATCH
#ifdef __SSE4_1__ #if defined(__AVX512BW__)
void ksw_extd2_avx512(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
#elif defined(__AVX2__)
void ksw_extd2_avx2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
#elif defined(__SSE4_1__)
void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez) int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
#else #elif defined(__SSE2__)
void ksw_extd2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, void ksw_extd2_sse2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat,
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez) int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
#endif #endif
@@ -27,64 +74,91 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez) int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez)
#endif // ~KSW_CPU_DISPATCH #endif // ~KSW_CPU_DISPATCH
{ {
#if defined(__AVX512BW__)
#define __dp_code_block1 \ #define __dp_code_block1 \
z = _mm_load_si128(&s[t]); \ z = _mm512_load_si512(&s[t]); \
xt1 = _mm_load_si128(&x[t]); /* xt1 <- x[r-1][t..t+15] */ \ tmp = _mm512_loadu_si512((uint8_t*)&x[t] - 1); \
tmp = _mm_srli_si128(xt1, 15); /* tmp <- x[r-1][t+15] */ \ xt1 = _mm512_mask_blend_epi8(1, tmp, x1_); \
xt1 = _mm_or_si128(_mm_slli_si128(xt1, 1), x1_); /* xt1 <- x[r-1][t-1..t+14] */ \ x1_ = _mm512_maskz_set1_epi8(1, *((uint8_t*)&x[t] + 63)); \
tmp = _mm512_loadu_si512((uint8_t*)&v[t] - 1); \
vt1 = _mm512_mask_blend_epi8(1, tmp, v1_); \
v1_ = _mm512_maskz_set1_epi8(1, *((uint8_t*)&v[t] + 63)); \
a = _mm512_add_epi8(xt1, vt1); \
ut = _mm512_load_si512(&u[t]); \
b = _mm512_add_epi8(_mm512_load_si512(&y[t]), ut); \
tmp = _mm512_loadu_si512((uint8_t*)&x2[t] - 1); \
x2t1 = _mm512_mask_blend_epi8(1, tmp, x21_); \
x21_ = _mm512_maskz_set1_epi8(1, *((uint8_t*)&x2[t] + 63)); \
a2= _mm512_add_epi8(x2t1, vt1); \
b2= _mm512_add_epi8(_mm512_load_si512(&y2[t]), ut);
#else
#define __dp_code_block1 \
z = simd_funcw(load)(&s[t]); \
xt1 = simd_funcw(load)(&x[t]); /* xt1 <- x[r-1][t..t+15] */ \
tmp = simd_srli_last(xt1); /* tmp <- x[r-1][t+15] */ \
xt1 = simd_funcw(or)(simd_slli_1(xt1), x1_); /* xt1 <- x[r-1][t-1..t+14] */ \
x1_ = tmp; \ x1_ = tmp; \
vt1 = _mm_load_si128(&v[t]); /* vt1 <- v[r-1][t..t+15] */ \ vt1 = simd_funcw(load)(&v[t]); /* vt1 <- v[r-1][t..t+15] */ \
tmp = _mm_srli_si128(vt1, 15); /* tmp <- v[r-1][t+15] */ \ tmp = simd_srli_last(vt1); /* tmp <- v[r-1][t+15] */ \
vt1 = _mm_or_si128(_mm_slli_si128(vt1, 1), v1_); /* vt1 <- v[r-1][t-1..t+14] */ \ vt1 = simd_funcw(or)(simd_slli_1(vt1), v1_); /* vt1 <- v[r-1][t-1..t+14] */ \
v1_ = tmp; \ v1_ = tmp; \
a = _mm_add_epi8(xt1, vt1); /* a <- x[r-1][t-1..t+14] + v[r-1][t-1..t+14] */ \ a = simd_func(add_epi8)(xt1, vt1); /* a <- x[r-1][t-1..t+14] + v[r-1][t-1..t+14] */ \
ut = _mm_load_si128(&u[t]); /* ut <- u[t..t+15] */ \ ut = simd_funcw(load)(&u[t]); /* ut <- u[t..t+15] */ \
b = _mm_add_epi8(_mm_load_si128(&y[t]), ut); /* b <- y[r-1][t..t+15] + u[r-1][t..t+15] */ \ b = simd_func(add_epi8)(simd_funcw(load)(&y[t]), ut); /* b <- y[r-1][t..t+15] + u[r-1][t..t+15] */ \
x2t1= _mm_load_si128(&x2[t]); \ x2t1= simd_funcw(load)(&x2[t]); \
tmp = _mm_srli_si128(x2t1, 15); \ tmp = simd_srli_last(x2t1); \
x2t1= _mm_or_si128(_mm_slli_si128(x2t1, 1), x21_); \ x2t1= simd_funcw(or)(simd_slli_1(x2t1), x21_); \
x21_= tmp; \ x21_= tmp; \
a2= _mm_add_epi8(x2t1, vt1); \ a2= simd_func(add_epi8)(x2t1, vt1); \
b2= _mm_add_epi8(_mm_load_si128(&y2[t]), ut); b2= simd_func(add_epi8)(simd_funcw(load)(&y2[t]), ut);
#endif // ~__AVX512BW__
#define __dp_code_block2 \ #define __dp_code_block2 \
_mm_store_si128(&u[t], _mm_sub_epi8(z, vt1)); /* u[r][t..t+15] <- z - v[r-1][t-1..t+14] */ \ simd_funcw(store)(&u[t], simd_func(sub_epi8)(z, vt1));/* u[r][t..t+15] <- z - v[r-1][t-1..t+14] */ \
_mm_store_si128(&v[t], _mm_sub_epi8(z, ut)); /* v[r][t..t+15] <- z - u[r-1][t..t+15] */ \ simd_funcw(store)(&v[t], simd_func(sub_epi8)(z, ut)); /* v[r][t..t+15] <- z - u[r-1][t..t+15] */ \
tmp = _mm_sub_epi8(z, q_); \ tmp = simd_func(sub_epi8)(z, q_); \
a = _mm_sub_epi8(a, tmp); \ a = simd_func(sub_epi8)(a, tmp); \
b = _mm_sub_epi8(b, tmp); \ b = simd_func(sub_epi8)(b, tmp); \
tmp = _mm_sub_epi8(z, q2_); \ tmp = simd_func(sub_epi8)(z, q2_); \
a2= _mm_sub_epi8(a2, tmp); \ a2= simd_func(sub_epi8)(a2, tmp); \
b2= _mm_sub_epi8(b2, tmp); b2= simd_func(sub_epi8)(b2, tmp);
int r, t, qe = q + e, n_col_, *off = 0, *off_end = 0, tlen_, qlen_, last_st, last_en, wl, wr, max_sc, min_sc, long_thres, long_diff; int r, t, qe = q + e, n_col_, *off = 0, *off_end = 0, tlen_, qlen_, last_st, last_en, wl, wr, max_sc, min_sc, long_thres, long_diff;
int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX); int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX);
int32_t *H = 0, H0 = 0, last_H0_t = 0; int32_t *H = 0, H0 = 0, last_H0_t = 0;
uint8_t *qr, *sf, *mem, *mem2 = 0; uint8_t *qr, *sf, *mem, *mem2 = 0;
__m128i q_, q2_, qe_, qe2_, zero_, sc_mch_, sc_mis_, m1_, sc_N_; SIMD_INT q_, q2_, qe_, qe2_, zero_, sc_mch_, sc_mis_, m1_, sc_N_, mask1_;
__m128i *u, *v, *x, *y, *x2, *y2, *s, *p = 0; SIMD_INT *u, *v, *x, *y, *x2, *y2, *s, *p = 0;
ksw_reset_extz(ez); ksw_reset_extz(ez);
if (m <= 1 || qlen <= 0 || tlen <= 0) return; if (m <= 1 || qlen <= 0 || tlen <= 0) return;
if (q2 + e2 < q + e) t = q, q = q2, q2 = t, t = e, e = e2, e2 = t; // make sure q+e no larger than q2+e2 if (q2 + e2 < q + e) t = q, q = q2, q2 = t, t = e, e = e2, e2 = t; // make sure q+e no larger than q2+e2
zero_ = _mm_set1_epi8(0); zero_ = simd_func(set1_epi8)(0);
q_ = _mm_set1_epi8(q); q_ = simd_func(set1_epi8)(q);
q2_ = _mm_set1_epi8(q2); q2_ = simd_func(set1_epi8)(q2);
qe_ = _mm_set1_epi8(q + e); qe_ = simd_func(set1_epi8)(q + e);
qe2_ = _mm_set1_epi8(q2 + e2); qe2_ = simd_func(set1_epi8)(q2 + e2);
sc_mch_ = _mm_set1_epi8(mat[0]); sc_mch_ = simd_func(set1_epi8)(mat[0]);
sc_mis_ = _mm_set1_epi8(mat[1]); sc_mis_ = simd_func(set1_epi8)(mat[1]);
sc_N_ = mat[m*m-1] == 0? _mm_set1_epi8(-e2) : _mm_set1_epi8(mat[m*m-1]); sc_N_ = mat[m*m-1] == 0? simd_func(set1_epi8)(-e2) : simd_func(set1_epi8)(mat[m*m-1]);
m1_ = _mm_set1_epi8(m - 1); // wildcard m1_ = simd_func(set1_epi8)(m - 1); // wildcard
#if defined(__AVX512BW__)
mask1_ = _mm512_maskz_set1_epi8(1, 0xff);
#elif defined(__AVX2__)
mask1_ = _mm256_setr_epi32(0xff, 0, 0, 0, 0, 0, 0, 0);
#elif defined(__SSE2__)
mask1_ = _mm_setr_epi32(0xff, 0, 0, 0);
#endif
if (w < 0) w = tlen > qlen? tlen : qlen; if (w < 0) w = tlen > qlen? tlen : qlen;
wl = wr = w; wl = wr = w;
tlen_ = (tlen + 15) / 16; tlen_ = (tlen + SIMD_WIDTH - 1) / SIMD_WIDTH;
n_col_ = qlen < tlen? qlen : tlen; n_col_ = qlen < tlen? qlen : tlen;
n_col_ = ((n_col_ < w + 1? n_col_ : w + 1) + 15) / 16 + 1; n_col_ = ((n_col_ < w + 1? n_col_ : w + 1) + SIMD_WIDTH - 1) / SIMD_WIDTH + 1;
qlen_ = (qlen + 15) / 16; qlen_ = (qlen + SIMD_WIDTH - 1) / SIMD_WIDTH;
for (t = 1, max_sc = mat[0], min_sc = mat[1]; t < m * m; ++t) { for (t = 1, max_sc = mat[0], min_sc = mat[1]; t < m * m; ++t) {
max_sc = max_sc > mat[t]? max_sc : mat[t]; max_sc = max_sc > mat[t]? max_sc : mat[t];
min_sc = min_sc < mat[t]? min_sc : mat[t]; min_sc = min_sc < mat[t]? min_sc : mat[t];
@@ -96,23 +170,23 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
++long_thres; ++long_thres;
long_diff = long_thres * (e - e2) - (q2 - q) - e2; long_diff = long_thres * (e - e2) - (q2 - q) - e2;
mem = (uint8_t*)kcalloc(km, tlen_ * 8 + qlen_ + 1, 16); mem = (uint8_t*)kcalloc(km, tlen_ * 8 + qlen_ + 1, SIMD_WIDTH);
u = (__m128i*)(((size_t)mem + 15) >> 4 << 4); // 16-byte aligned u = (SIMD_INT*)(((size_t)mem + SIMD_WIDTH - 1) >> SIMD_SHIFT << SIMD_SHIFT); // 16-byte aligned
v = u + tlen_, x = v + tlen_, y = x + tlen_, x2 = y + tlen_, y2 = x2 + tlen_; v = u + tlen_, x = v + tlen_, y = x + tlen_, x2 = y + tlen_, y2 = x2 + tlen_;
s = y2 + tlen_, sf = (uint8_t*)(s + tlen_), qr = sf + tlen_ * 16; s = y2 + tlen_, sf = (uint8_t*)(s + tlen_), qr = sf + tlen_ * SIMD_WIDTH;
memset(u, -q - e, tlen_ * 16); memset(u, -q - e, tlen_ * SIMD_WIDTH);
memset(v, -q - e, tlen_ * 16); memset(v, -q - e, tlen_ * SIMD_WIDTH);
memset(x, -q - e, tlen_ * 16); memset(x, -q - e, tlen_ * SIMD_WIDTH);
memset(y, -q - e, tlen_ * 16); memset(y, -q - e, tlen_ * SIMD_WIDTH);
memset(x2, -q2 - e2, tlen_ * 16); memset(x2, -q2 - e2, tlen_ * SIMD_WIDTH);
memset(y2, -q2 - e2, tlen_ * 16); memset(y2, -q2 - e2, tlen_ * SIMD_WIDTH);
if (!approx_max) { if (!approx_max) {
H = (int32_t*)kmalloc(km, tlen_ * 16 * 4); H = (int32_t*)kmalloc(km, tlen_ * SIMD_WIDTH * 4);
for (t = 0; t < tlen_ * 16; ++t) H[t] = KSW_NEG_INF; for (t = 0; t < tlen_ * SIMD_WIDTH; ++t) H[t] = KSW_NEG_INF;
} }
if (with_cigar) { if (with_cigar) {
mem2 = (uint8_t*)kmalloc(km, ((size_t)(qlen + tlen - 1) * n_col_ + 1) * 16); mem2 = (uint8_t*)kmalloc(km, ((size_t)(qlen + tlen - 1) * n_col_ + 1) * SIMD_WIDTH);
p = (__m128i*)(((size_t)mem2 + 15) >> 4 << 4); p = (SIMD_INT*)(((size_t)mem2 + SIMD_WIDTH - 1) >> SIMD_SHIFT << SIMD_SHIFT);
off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int) * 2); off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int) * 2);
off_end = off + qlen + tlen - 1; off_end = off + qlen + tlen - 1;
} }
@@ -125,7 +199,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
int8_t x1, x21, v1; int8_t x1, x21, v1;
uint8_t *qrr = qr + (qlen - 1 - r); uint8_t *qrr = qr + (qlen - 1 - r);
int8_t *u8 = (int8_t*)u, *v8 = (int8_t*)v, *x8 = (int8_t*)x, *x28 = (int8_t*)x2; int8_t *u8 = (int8_t*)u, *v8 = (int8_t*)v, *x8 = (int8_t*)x, *x28 = (int8_t*)x2;
__m128i x1_, x21_, v1_; SIMD_INT x1_, x21_, v1_;
// find the boundaries // find the boundaries
if (st < r - qlen + 1) st = r - qlen + 1; if (st < r - qlen + 1) st = r - qlen + 1;
if (en > r) en = r; if (en > r) en = r;
@@ -136,7 +210,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
break; break;
} }
st0 = st, en0 = en; st0 = st, en0 = en;
st = st / 16 * 16, en = (en + 16) / 16 * 16 - 1; st = st / SIMD_WIDTH * SIMD_WIDTH, en = (en + SIMD_WIDTH) / SIMD_WIDTH * SIMD_WIDTH - 1;
// set boundary conditions // set boundary conditions
if (st > 0) { if (st > 0) {
if (st - 1 >= last_st && st - 1 <= last_en) { if (st - 1 >= last_st && st - 1 <= last_en) {
@@ -155,47 +229,53 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
} }
// loop fission: set scores first // loop fission: set scores first
if (!(flag & KSW_EZ_GENERIC_SC)) { if (!(flag & KSW_EZ_GENERIC_SC)) {
for (t = st0; t <= en0; t += 16) { for (t = st0; t <= en0; t += SIMD_WIDTH) {
__m128i sq, st, tmp, mask; SIMD_INT sq, st, tmp;
sq = _mm_loadu_si128((__m128i*)&sf[t]); sq = simd_funcw(loadu)((SIMD_INT*)&sf[t]);
st = _mm_loadu_si128((__m128i*)&qrr[t]); st = simd_funcw(loadu)((SIMD_INT*)&qrr[t]);
mask = _mm_or_si128(_mm_cmpeq_epi8(sq, m1_), _mm_cmpeq_epi8(st, m1_)); #if defined(__AVX512BW__)
tmp = _mm_cmpeq_epi8(sq, st); __mmask64 mask = _mm512_cmpeq_epi8_mask(sq, m1_) | _mm512_cmpeq_epi8_mask(st, m1_);
#ifdef __SSE4_1__ tmp = _mm512_mask_blend_epi8(_mm512_cmpeq_epi8_mask(sq, st), sc_mis_, sc_mch_);
tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp); tmp = _mm512_mask_blend_epi8(mask, tmp, sc_N_);
tmp = _mm_blendv_epi8(tmp, sc_N_, mask); #elif defined(__SSE4_1__) || defined(__AVX2__)
#else SIMD_INT mask = simd_funcw(or)(simd_func(cmpeq_epi8)(sq, m1_), simd_func(cmpeq_epi8)(st, m1_));
tmp = simd_func(cmpeq_epi8)(sq, st);
tmp = simd_func(blendv_epi8)(sc_mis_, sc_mch_, tmp);
tmp = simd_func(blendv_epi8)(tmp, sc_N_, mask);
#elif defined(__SSE2__) // emulate blendv
SIMD_INT mask = simd_funcw(or)(simd_func(cmpeq_epi8)(sq, m1_), simd_func(cmpeq_epi8)(st, m1_));
tmp = simd_func(cmpeq_epi8)(sq, st);
tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_)); tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_));
tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp), _mm_and_si128(mask, sc_N_)); tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp), _mm_and_si128(mask, sc_N_));
#endif #endif
_mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp); simd_funcw(storeu)((SIMD_INT*)((int8_t*)s + t), tmp);
} }
} else { } else {
for (t = st0; t <= en0; ++t) for (t = st0; t <= en0; ++t)
((uint8_t*)s)[t] = mat[sf[t] * m + qrr[t]]; ((uint8_t*)s)[t] = mat[sf[t] * m + qrr[t]];
} }
// core loop // core loop
x1_ = _mm_cvtsi32_si128((uint8_t)x1); x1_ = simd_funcw(and)(simd_func(set1_epi8)((uint8_t)x1), mask1_);
x21_ = _mm_cvtsi32_si128((uint8_t)x21); x21_ = simd_funcw(and)(simd_func(set1_epi8)((uint8_t)x21), mask1_);
v1_ = _mm_cvtsi32_si128((uint8_t)v1); v1_ = simd_funcw(and)(simd_func(set1_epi8)((uint8_t)v1), mask1_);
st_ = st / 16, en_ = en / 16; st_ = st / SIMD_WIDTH, en_ = en / SIMD_WIDTH;
assert(en_ - st_ + 1 <= n_col_); assert(en_ - st_ + 1 <= n_col_);
if (!with_cigar) { // score only if (!with_cigar) { // score only
for (t = st_; t <= en_; ++t) { for (t = st_; t <= en_; ++t) {
__m128i z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; SIMD_INT z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp;
__dp_code_block1; __dp_code_block1;
#ifdef __SSE4_1__ #if defined(__SSE4_1__) || defined(__AVX2__) || defined(__AVX512BW__)
z = _mm_max_epi8(z, a); z = simd_func(max_epi8)(z, a);
z = _mm_max_epi8(z, b); z = simd_func(max_epi8)(z, b);
z = _mm_max_epi8(z, a2); z = simd_func(max_epi8)(z, a2);
z = _mm_max_epi8(z, b2); z = simd_func(max_epi8)(z, b2);
z = _mm_min_epi8(z, sc_mch_); z = simd_func(min_epi8)(z, sc_mch_);
__dp_code_block2; // save u[] and v[]; update a, b, a2 and b2 __dp_code_block2; // save u[] and v[]; update a, b, a2 and b2
_mm_store_si128(&x[t], _mm_sub_epi8(_mm_max_epi8(a, zero_), qe_)); simd_funcw(store)(&x[t], simd_func(sub_epi8)(simd_func(max_epi8)(a, zero_), qe_));
_mm_store_si128(&y[t], _mm_sub_epi8(_mm_max_epi8(b, zero_), qe_)); simd_funcw(store)(&y[t], simd_func(sub_epi8)(simd_func(max_epi8)(b, zero_), qe_));
_mm_store_si128(&x2[t], _mm_sub_epi8(_mm_max_epi8(a2, zero_), qe2_)); simd_funcw(store)(&x2[t], simd_func(sub_epi8)(simd_func(max_epi8)(a2, zero_), qe2_));
_mm_store_si128(&y2[t], _mm_sub_epi8(_mm_max_epi8(b2, zero_), qe2_)); simd_funcw(store)(&y2[t], simd_func(sub_epi8)(simd_func(max_epi8)(b2, zero_), qe2_));
#else #elif defined(__SSE2__)
tmp = _mm_cmpgt_epi8(a, z); tmp = _mm_cmpgt_epi8(a, z);
z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a)); z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a));
tmp = _mm_cmpgt_epi8(b, z); tmp = _mm_cmpgt_epi8(b, z);
@@ -218,22 +298,42 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
#endif #endif
} }
} else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment } else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment
__m128i *pr = p + (size_t)r * n_col_ - st_; SIMD_INT *pr = p + (size_t)r * n_col_ - st_;
off[r] = st, off_end[r] = en; off[r] = st, off_end[r] = en;
for (t = st_; t <= en_; ++t) { for (t = st_; t <= en_; ++t) {
__m128i d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; SIMD_INT d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp;
__dp_code_block1; __dp_code_block1;
#ifdef __SSE4_1__ #if defined(__AVX512BW__)
d = _mm_and_si128(_mm_cmpgt_epi8(a, z), _mm_set1_epi8(1)); // d = a > z? 1 : 0 d = _mm512_maskz_set1_epi8(_mm512_cmpgt_epi8_mask(a, z), 1);
z = _mm_max_epi8(z, a); z = _mm512_max_epi8(z, a);
d = _mm_blendv_epi8(d, _mm_set1_epi8(2), _mm_cmpgt_epi8(b, z)); // d = b > z? 2 : d d = _mm512_mask_blend_epi8(_mm512_cmpgt_epi8_mask(b, z), d, _mm512_set1_epi8(2));
z = _mm_max_epi8(z, b); z = _mm512_max_epi8(z, b);
d = _mm_blendv_epi8(d, _mm_set1_epi8(3), _mm_cmpgt_epi8(a2, z)); // d = a2 > z? 3 : d d = _mm512_mask_blend_epi8(_mm512_cmpgt_epi8_mask(a2, z), d, _mm512_set1_epi8(3));
z = _mm_max_epi8(z, a2); z = _mm512_max_epi8(z, a2);
d = _mm_blendv_epi8(d, _mm_set1_epi8(4), _mm_cmpgt_epi8(b2, z)); // d = a2 > z? 3 : d d = _mm512_mask_blend_epi8(_mm512_cmpgt_epi8_mask(b2, z), d, _mm512_set1_epi8(4));
z = _mm_max_epi8(z, b2); z = _mm512_max_epi8(z, b2);
z = _mm_min_epi8(z, sc_mch_); z = _mm512_min_epi8(z, sc_mch_);
#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() __dp_code_block2;
d = _mm512_or_si512(d, _mm512_maskz_set1_epi8(_mm512_cmpgt_epi8_mask(a, zero_), 0x08)); // d = a > 0? 1<<3 : 0
_mm512_store_si512(&x[t], _mm512_sub_epi8(_mm512_max_epi8(a, zero_), qe_));
d = _mm512_or_si512(d, _mm512_maskz_set1_epi8(_mm512_cmpgt_epi8_mask(b, zero_), 0x10)); // d = b > 0? 1<<4 : 0
_mm512_store_si512(&y[t], _mm512_sub_epi8(_mm512_max_epi8(b, zero_), qe_));
d = _mm512_or_si512(d, _mm512_maskz_set1_epi8(_mm512_cmpgt_epi8_mask(a2, zero_), 0x20)); // d = a2 > 0? 1<<5 : 0
_mm512_store_si512(&x2[t], _mm512_sub_epi8(_mm512_max_epi8(a2, zero_), qe2_));
d = _mm512_or_si512(d, _mm512_maskz_set1_epi8(_mm512_cmpgt_epi8_mask(b2, zero_), 0x40)); // d = b2 > 0? 1<<6 : 0
_mm512_store_si512(&y2[t], _mm512_sub_epi8(_mm512_max_epi8(b2, zero_), qe2_));
#else
#if defined(__SSE4_1__) || defined(__AVX2__)
d = simd_funcw(and)(simd_func(cmpgt_epi8)(a, z), simd_func(set1_epi8)(1)); // d = a > z? 1 : 0
z = simd_func(max_epi8)(z, a);
d = simd_func(blendv_epi8)(d, simd_func(set1_epi8)(2), simd_func(cmpgt_epi8)(b, z)); // d = b > z? 2 : d
z = simd_func(max_epi8)(z, b);
d = simd_func(blendv_epi8)(d, simd_func(set1_epi8)(3), simd_func(cmpgt_epi8)(a2, z)); // d = a2 > z? 3 : d
z = simd_func(max_epi8)(z, a2);
d = simd_func(blendv_epi8)(d, simd_func(set1_epi8)(4), simd_func(cmpgt_epi8)(b2, z)); // d = a2 > z? 3 : d
z = simd_func(max_epi8)(z, b2);
z = simd_func(min_epi8)(z, sc_mch_);
#elif defined(__SSE2__) // emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8()
tmp = _mm_cmpgt_epi8(a, z); tmp = _mm_cmpgt_epi8(a, z);
d = _mm_and_si128(tmp, _mm_set1_epi8(1)); d = _mm_and_si128(tmp, _mm_set1_epi8(1));
z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a)); z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a));
@@ -248,39 +348,60 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, b2)); z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, b2));
tmp = _mm_cmplt_epi8(sc_mch_, z); tmp = _mm_cmplt_epi8(sc_mch_, z);
z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z));
#endif #endif // ~__SSE2__
__dp_code_block2; __dp_code_block2;
tmp = _mm_cmpgt_epi8(a, zero_); tmp = simd_func(cmpgt_epi8)(a, zero_);
_mm_store_si128(&x[t], _mm_sub_epi8(_mm_and_si128(tmp, a), qe_)); simd_funcw(store)(&x[t], simd_func(sub_epi8)(simd_funcw(and)(tmp, a), qe_));
d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x08))); // d = a > 0? 1<<3 : 0 d = simd_funcw(or)(d, simd_funcw(and)(tmp, simd_func(set1_epi8)(0x08))); // d = a > 0? 1<<3 : 0
tmp = _mm_cmpgt_epi8(b, zero_); tmp = simd_func(cmpgt_epi8)(b, zero_);
_mm_store_si128(&y[t], _mm_sub_epi8(_mm_and_si128(tmp, b), qe_)); simd_funcw(store)(&y[t], simd_func(sub_epi8)(simd_funcw(and)(tmp, b), qe_));
d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x10))); // d = b > 0? 1<<4 : 0 d = simd_funcw(or)(d, simd_funcw(and)(tmp, simd_func(set1_epi8)(0x10))); // d = b > 0? 1<<4 : 0
tmp = _mm_cmpgt_epi8(a2, zero_); tmp = simd_func(cmpgt_epi8)(a2, zero_);
_mm_store_si128(&x2[t], _mm_sub_epi8(_mm_and_si128(tmp, a2), qe2_)); simd_funcw(store)(&x2[t], simd_func(sub_epi8)(simd_funcw(and)(tmp, a2), qe2_));
d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0 d = simd_funcw(or)(d, simd_funcw(and)(tmp, simd_func(set1_epi8)(0x20))); // d = a > 0? 1<<5 : 0
tmp = _mm_cmpgt_epi8(b2, zero_); tmp = simd_func(cmpgt_epi8)(b2, zero_);
_mm_store_si128(&y2[t], _mm_sub_epi8(_mm_and_si128(tmp, b2), qe2_)); simd_funcw(store)(&y2[t], simd_func(sub_epi8)(simd_funcw(and)(tmp, b2), qe2_));
d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x40))); // d = b > 0? 1<<6 : 0 d = simd_funcw(or)(d, simd_funcw(and)(tmp, simd_func(set1_epi8)(0x40))); // d = b > 0? 1<<6 : 0
_mm_store_si128(&pr[t], d); #endif // ~__AVX512BW__
simd_funcw(store)(&pr[t], d);
} }
} else { // gap right-alignment } else { // gap right-alignment
__m128i *pr = p + (size_t)r * n_col_ - st_; SIMD_INT *pr = p + (size_t)r * n_col_ - st_;
off[r] = st, off_end[r] = en; off[r] = st, off_end[r] = en;
for (t = st_; t <= en_; ++t) { for (t = st_; t <= en_; ++t) {
__m128i d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; SIMD_INT d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp;
__dp_code_block1; __dp_code_block1;
#ifdef __SSE4_1__ #if defined(__AVX512BW__)
d = _mm_andnot_si128(_mm_cmpgt_epi8(z, a), _mm_set1_epi8(1)); // d = z > a? 0 : 1 d = _mm512_maskz_set1_epi8(_mm512_cmpge_epi8_mask(a, z), 1);
z = _mm_max_epi8(z, a); z = _mm512_max_epi8(z, a);
d = _mm_blendv_epi8(_mm_set1_epi8(2), d, _mm_cmpgt_epi8(z, b)); // d = z > b? d : 2 d = _mm512_mask_blend_epi8(_mm512_cmpge_epi8_mask(b, z), d, _mm512_set1_epi8(2));
z = _mm_max_epi8(z, b); z = _mm512_max_epi8(z, b);
d = _mm_blendv_epi8(_mm_set1_epi8(3), d, _mm_cmpgt_epi8(z, a2)); // d = z > a2? d : 3 d = _mm512_mask_blend_epi8(_mm512_cmpge_epi8_mask(a2, z), d, _mm512_set1_epi8(3));
z = _mm_max_epi8(z, a2); z = _mm512_max_epi8(z, a2);
d = _mm_blendv_epi8(_mm_set1_epi8(4), d, _mm_cmpgt_epi8(z, b2)); // d = z > b2? d : 4 d = _mm512_mask_blend_epi8(_mm512_cmpge_epi8_mask(b2, z), d, _mm512_set1_epi8(4));
z = _mm_max_epi8(z, b2); z = _mm512_max_epi8(z, b2);
z = _mm_min_epi8(z, sc_mch_); z = _mm512_min_epi8(z, sc_mch_);
#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() __dp_code_block2;
d = _mm512_or_si512(d, _mm512_maskz_set1_epi8(_mm512_cmpge_epi8_mask(a, zero_), 0x08)); // d = a >= 0? 1<<3 : 0
_mm512_store_si512(&x[t], _mm512_sub_epi8(_mm512_max_epi8(a, zero_), qe_));
d = _mm512_or_si512(d, _mm512_maskz_set1_epi8(_mm512_cmpge_epi8_mask(b, zero_), 0x10)); // d = b >= 0? 1<<4 : 0
_mm512_store_si512(&y[t], _mm512_sub_epi8(_mm512_max_epi8(b, zero_), qe_));
d = _mm512_or_si512(d, _mm512_maskz_set1_epi8(_mm512_cmpge_epi8_mask(a2, zero_), 0x20)); // d = a2 >= 0? 1<<5 : 0
_mm512_store_si512(&x2[t], _mm512_sub_epi8(_mm512_max_epi8(a2, zero_), qe2_));
d = _mm512_or_si512(d, _mm512_maskz_set1_epi8(_mm512_cmpge_epi8_mask(b2, zero_), 0x40)); // d = b2 >= 0? 1<<6 : 0
_mm512_store_si512(&y2[t], _mm512_sub_epi8(_mm512_max_epi8(b2, zero_), qe2_));
#else
#if defined(__SSE4_1__) || defined(__AVX2__)
d = simd_funcw(andnot)(simd_func(cmpgt_epi8)(z, a), simd_func(set1_epi8)(1)); // d = z > a? 0 : 1
z = simd_func(max_epi8)(z, a);
d = simd_func(blendv_epi8)(simd_func(set1_epi8)(2), d, simd_func(cmpgt_epi8)(z, b)); // d = z > b? d : 2
z = simd_func(max_epi8)(z, b);
d = simd_func(blendv_epi8)(simd_func(set1_epi8)(3), d, simd_func(cmpgt_epi8)(z, a2)); // d = z > a2? d : 3
z = simd_func(max_epi8)(z, a2);
d = simd_func(blendv_epi8)(simd_func(set1_epi8)(4), d, simd_func(cmpgt_epi8)(z, b2)); // d = z > b2? d : 4
z = simd_func(max_epi8)(z, b2);
z = simd_func(min_epi8)(z, sc_mch_);
#elif defined(__SSE2__)
tmp = _mm_cmpgt_epi8(z, a); tmp = _mm_cmpgt_epi8(z, a);
d = _mm_andnot_si128(tmp, _mm_set1_epi8(1)); d = _mm_andnot_si128(tmp, _mm_set1_epi8(1));
z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, a)); z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, a));
@@ -295,52 +416,64 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, b2)); z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, b2));
tmp = _mm_cmplt_epi8(sc_mch_, z); tmp = _mm_cmplt_epi8(sc_mch_, z);
z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z));
#endif #endif // ~__SSE2__
__dp_code_block2; __dp_code_block2;
tmp = _mm_cmpgt_epi8(zero_, a); tmp = simd_func(cmpgt_epi8)(zero_, a);
_mm_store_si128(&x[t], _mm_sub_epi8(_mm_andnot_si128(tmp, a), qe_)); simd_funcw(store)(&x[t], simd_func(sub_epi8)(simd_funcw(andnot)(tmp, a), qe_));
d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x08))); // d = a > 0? 1<<3 : 0 d = simd_funcw(or)(d, simd_funcw(andnot)(tmp, simd_func(set1_epi8)(0x08))); // d = a > 0? 1<<3 : 0
tmp = _mm_cmpgt_epi8(zero_, b); tmp = simd_func(cmpgt_epi8)(zero_, b);
_mm_store_si128(&y[t], _mm_sub_epi8(_mm_andnot_si128(tmp, b), qe_)); simd_funcw(store)(&y[t], simd_func(sub_epi8)(simd_funcw(andnot)(tmp, b), qe_));
d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x10))); // d = b > 0? 1<<4 : 0 d = simd_funcw(or)(d, simd_funcw(andnot)(tmp, simd_func(set1_epi8)(0x10))); // d = b > 0? 1<<4 : 0
tmp = _mm_cmpgt_epi8(zero_, a2); tmp = simd_func(cmpgt_epi8)(zero_, a2);
_mm_store_si128(&x2[t], _mm_sub_epi8(_mm_andnot_si128(tmp, a2), qe2_)); simd_funcw(store)(&x2[t], simd_func(sub_epi8)(simd_funcw(andnot)(tmp, a2), qe2_));
d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0 d = simd_funcw(or)(d, simd_funcw(andnot)(tmp, simd_func(set1_epi8)(0x20))); // d = a > 0? 1<<5 : 0
tmp = _mm_cmpgt_epi8(zero_, b2); tmp = simd_func(cmpgt_epi8)(zero_, b2);
_mm_store_si128(&y2[t], _mm_sub_epi8(_mm_andnot_si128(tmp, b2), qe2_)); simd_funcw(store)(&y2[t], simd_func(sub_epi8)(simd_funcw(andnot)(tmp, b2), qe2_));
d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x40))); // d = b > 0? 1<<6 : 0 d = simd_funcw(or)(d, simd_funcw(andnot)(tmp, simd_func(set1_epi8)(0x40))); // d = b > 0? 1<<6 : 0
_mm_store_si128(&pr[t], d); #endif // ~__AVX512BW__
simd_funcw(store)(&pr[t], d);
} }
} }
if (!approx_max) { // find the exact max with a 32-bit score array if (!approx_max) { // find the exact max with a 32-bit score array
int32_t max_H, max_t; int32_t max_H, max_t;
// compute H[], max_H and max_t // compute H[], max_H and max_t
if (r > 0) { if (r > 0) {
int32_t HH[4], tt[4], en1 = st0 + (en0 - st0) / 4 * 4, i; int32_t HH[SIMD_WIDTH/4], tt[SIMD_WIDTH/4], en1 = st0 + (en0 - st0) / (SIMD_WIDTH/4) * (SIMD_WIDTH/4), i;
__m128i max_H_, max_t_; SIMD_INT max_H_, max_t_;
max_H = H[en0] = en0 > 0? H[en0-1] + u8[en0] : H[en0] + v8[en0]; // special casing the last element max_H = H[en0] = en0 > 0? H[en0-1] + u8[en0] : H[en0] + v8[en0]; // special casing the last element
max_t = en0; max_t = en0;
max_H_ = _mm_set1_epi32(max_H); max_H_ = simd_func(set1_epi32)(max_H);
max_t_ = _mm_set1_epi32(max_t); max_t_ = simd_func(set1_epi32)(max_t);
for (t = st0; t < en1; t += 4) { // this implements: H[t]+=v8[t]-qe; if(H[t]>max_H) max_H=H[t],max_t=t; for (t = st0; t < en1; t += SIMD_WIDTH/4) { // this implements: H[t]+=v8[t]; if(H[t]>max_H) max_H=H[t],max_t=t;
__m128i H1, tmp, t_; SIMD_INT H1, t_;
H1 = _mm_loadu_si128((__m128i*)&H[t]); H1 = simd_funcw(loadu)((SIMD_INT*)&H[t]);
#if defined(__AVX512BW__)
t_ = _mm512_cvtepi8_epi32(_mm_loadu_si128((__m128i*)&v8[t]));
#elif defined(__AVX2__)
t_ = _mm256_setr_epi32(v8[t], v8[t+1], v8[t+2], v8[t+3], v8[t+4], v8[t+5], v8[t+6], v8[t+7]);
#elif defined(__SSE2__)
t_ = _mm_setr_epi32(v8[t], v8[t+1], v8[t+2], v8[t+3]); t_ = _mm_setr_epi32(v8[t], v8[t+1], v8[t+2], v8[t+3]);
H1 = _mm_add_epi32(H1, t_); #endif
_mm_storeu_si128((__m128i*)&H[t], H1); H1 = simd_func(add_epi32)(H1, t_);
t_ = _mm_set1_epi32(t); simd_funcw(storeu)((SIMD_INT*)&H[t], H1);
tmp = _mm_cmpgt_epi32(H1, max_H_); t_ = simd_func(set1_epi32)(t);
#ifdef __SSE4_1__ #if defined(__AVX512BW__)
max_H_ = _mm_blendv_epi8(max_H_, H1, tmp); __mmask64 tmp = _mm512_cmpgt_epi32_mask(H1, max_H_);
max_t_ = _mm_blendv_epi8(max_t_, t_, tmp); max_H_ = _mm512_mask_blend_epi32(tmp, max_H_, H1);
#else max_t_ = _mm512_mask_blend_epi32(tmp, max_t_, t_);
max_H_ = _mm_or_si128(_mm_and_si128(tmp, H1), _mm_andnot_si128(tmp, max_H_)); #elif defined(__SSE4_1__) || defined(__AVX2__)
max_t_ = _mm_or_si128(_mm_and_si128(tmp, t_), _mm_andnot_si128(tmp, max_t_)); SIMD_INT tmp = simd_func(cmpgt_epi32)(H1, max_H_);
max_H_ = simd_func(blendv_epi8)(max_H_, H1, tmp);
max_t_ = simd_func(blendv_epi8)(max_t_, t_, tmp);
#elif defined(__SSE2__)
SIMD_INT tmp = simd_func(cmpgt_epi32)(H1, max_H_);
max_H_ = simd_funcw(or)(simd_funcw(and)(tmp, H1), simd_funcw(andnot)(tmp, max_H_));
max_t_ = simd_funcw(or)(simd_funcw(and)(tmp, t_), simd_funcw(andnot)(tmp, max_t_));
#endif #endif
} }
_mm_storeu_si128((__m128i*)HH, max_H_); simd_funcw(storeu)((SIMD_INT*)HH, max_H_);
_mm_storeu_si128((__m128i*)tt, max_t_); simd_funcw(storeu)((SIMD_INT*)tt, max_t_);
for (i = 0; i < 4; ++i) for (i = 0; i < SIMD_WIDTH/4; ++i)
if (max_H < HH[i]) max_H = HH[i], max_t = tt[i] + i; if (max_H < HH[i]) max_H = HH[i], max_t = tt[i] + i;
for (; t < en0; ++t) { // for the rest of values that haven't been computed with SSE for (; t < en0; ++t) { // for the rest of values that haven't been computed with SSE
H[t] += (int32_t)v8[t]; H[t] += (int32_t)v8[t];
@@ -381,12 +514,12 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin
if (with_cigar) { // backtrack if (with_cigar) { // backtrack
int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR); int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR);
if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) { if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) {
ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar); ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*SIMD_WIDTH, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
} else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > (int)ez->max) { } else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > (int)ez->max) {
ez->reach_end = 1; ez->reach_end = 1;
ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar); ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*SIMD_WIDTH, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
} else if (ez->max_t >= 0 && ez->max_q >= 0) { } else if (ez->max_t >= 0 && ez->max_q >= 0) {
ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar); ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*SIMD_WIDTH, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar);
} }
kfree(km, mem2); kfree(km, off); kfree(km, mem2); kfree(km, off);
} }
+15 -16
View File
@@ -7,7 +7,7 @@
#include "mmpriv.h" #include "mmpriv.h"
#include "ketopt.h" #include "ketopt.h"
#define MM_VERSION "2.17-r954-dirty" #define MM_VERSION "2.17-r963-dirty"
#ifdef __linux__ #ifdef __linux__
#include <sys/resource.h> #include <sys/resource.h>
@@ -67,10 +67,6 @@ static ko_longopt_t long_options[] = {
{ "junc-bed", ko_required_argument, 340 }, { "junc-bed", ko_required_argument, 340 },
{ "junc-bonus", ko_required_argument, 341 }, { "junc-bonus", ko_required_argument, 341 },
{ "sam-hit-only", ko_no_argument, 342 }, { "sam-hit-only", ko_no_argument, 342 },
{ "idx-min-occ", ko_required_argument, 343 },
{ "idx-max-occ", ko_required_argument, 344 },
{ "flt-max-dv", ko_required_argument, 345 },
{ "flt-min-blen", ko_required_argument, 346 },
{ "help", ko_no_argument, 'h' }, { "help", ko_no_argument, 'h' },
{ "max-intron-len", ko_required_argument, 'G' }, { "max-intron-len", ko_required_argument, 'G' },
{ "version", ko_no_argument, 'V' }, { "version", ko_no_argument, 'V' },
@@ -213,11 +209,8 @@ int main(int argc, char *argv[])
else if (c == 337) opt.max_sw_mat = mm_parse_num(o.arg); // --cap-sw-mat else if (c == 337) opt.max_sw_mat = mm_parse_num(o.arg); // --cap-sw-mat
else if (c == 338) opt.max_qlen = mm_parse_num(o.arg); // --max-qlen else if (c == 338) opt.max_qlen = mm_parse_num(o.arg); // --max-qlen
else if (c == 340) junc_bed = o.arg; // --junc-bed else if (c == 340) junc_bed = o.arg; // --junc-bed
else if (c == 341) opt.junc_bonus = atoi(o.arg); // --junc-bonus
else if (c == 342) opt.flag |= MM_F_SAM_HIT_ONLY; // --sam-hit-only else if (c == 342) opt.flag |= MM_F_SAM_HIT_ONLY; // --sam-hit-only
else if (c == 343) ipt.min_occ = mm_parse_num(o.arg); // --idx-min-occ
else if (c == 344) ipt.max_occ = mm_parse_num(o.arg); // --idx-max-occ
else if (c == 345) opt.flt_max_dv = atof(o.arg); // --flt-max-dv
else if (c == 346) opt.flt_min_blen = mm_parse_num(o.arg); // --flt-min-blen
else if (c == 314) { // --frag else if (c == 314) { // --frag
yes_or_no(&opt, MM_F_FRAG_MODE, o.longidx, o.arg, 1); yes_or_no(&opt, MM_F_FRAG_MODE, o.longidx, o.arg, 1);
} else if (c == 315) { // --secondary } else if (c == 315) { // --secondary
@@ -331,11 +324,11 @@ int main(int argc, char *argv[])
fprintf(fp_help, " --version show version number\n"); fprintf(fp_help, " --version show version number\n");
fprintf(fp_help, " Preset:\n"); fprintf(fp_help, " Preset:\n");
fprintf(fp_help, " -x STR preset (always applied before other options; see minimap2.1 for details) []\n"); fprintf(fp_help, " -x STR preset (always applied before other options; see minimap2.1 for details) []\n");
fprintf(fp_help, " - map-pb/map-ont: PacBio/Nanopore vs reference mapping\n"); fprintf(fp_help, " - map-pb/map-ont - PacBio/Nanopore vs reference mapping\n");
fprintf(fp_help, " - ava-pb/ava-ont: PacBio/Nanopore read overlap\n"); fprintf(fp_help, " - ava-pb/ava-ont - PacBio/Nanopore read overlap\n");
fprintf(fp_help, " - asm5/asm10/asm20: asm-to-ref mapping, for ~0.1/1/5%% sequence divergence\n"); fprintf(fp_help, " - asm5/asm10/asm20 - asm-to-ref mapping, for ~0.1/1/5%% sequence divergence\n");
fprintf(fp_help, " - splice: long-read spliced alignment\n"); fprintf(fp_help, " - splice/splice:hq - long-read/Pacbio-CCS spliced alignment\n");
fprintf(fp_help, " - sr: genomic short-read mapping\n"); fprintf(fp_help, " - sr - genomic short-read mapping\n");
fprintf(fp_help, "\nSee `man ./minimap2.1' for detailed description of these and other advanced command-line options.\n"); fprintf(fp_help, "\nSee `man ./minimap2.1' for detailed description of these and other advanced command-line options.\n");
return fp_help == stdout? 0 : 1; return fp_help == stdout? 0 : 1;
} }
@@ -364,13 +357,19 @@ int main(int argc, char *argv[])
return 1; return 1;
} }
if ((opt.flag & MM_F_OUT_SAM) && idx_rdr->n_parts == 1) { if ((opt.flag & MM_F_OUT_SAM) && idx_rdr->n_parts == 1) {
int ret;
if (mm_idx_reader_eof(idx_rdr)) { if (mm_idx_reader_eof(idx_rdr)) {
mm_write_sam_hdr(mi, rg, MM_VERSION, argc, argv); ret = mm_write_sam_hdr(mi, rg, MM_VERSION, argc, argv);
} else { } else {
mm_write_sam_hdr(0, rg, MM_VERSION, argc, argv); ret = mm_write_sam_hdr(0, rg, MM_VERSION, argc, argv);
if (opt.split_prefix == 0 && mm_verbose >= 2) if (opt.split_prefix == 0 && mm_verbose >= 2)
fprintf(stderr, "[WARNING]\033[1;31m For a multi-part index, no @SQ lines will be outputted. Please use --split-prefix.\033[0m\n"); fprintf(stderr, "[WARNING]\033[1;31m For a multi-part index, no @SQ lines will be outputted. Please use --split-prefix.\033[0m\n");
} }
if (ret != 0) {
mm_idx_destroy(mi);
mm_idx_reader_close(idx_rdr);
return 1;
}
} }
if (mm_verbose >= 3) if (mm_verbose >= 3)
fprintf(stderr, "[M::%s::%.3f*%.2f] loaded/built the index for %d target sequence(s)\n", fprintf(stderr, "[M::%s::%.3f*%.2f] loaded/built the index for %d target sequence(s)\n",
-9
View File
@@ -351,15 +351,6 @@ void mm_map_frag(const mm_idx_t *mi, int n_segs, const int *qlens, const char **
chain_post(opt, max_chain_gap_ref, mi, b->km, qlen_sum, n_segs, qlens, &n_regs0, regs0, a); chain_post(opt, max_chain_gap_ref, mi, b->km, qlen_sum, n_segs, qlens, &n_regs0, regs0, a);
if (!is_sr) mm_est_err(mi, qlen_sum, n_regs0, regs0, a, n_mini_pos, mini_pos); if (!is_sr) mm_est_err(mi, qlen_sum, n_regs0, regs0, a, n_mini_pos, mini_pos);
if (!is_sr && n_segs == 1) {
for (i = j = 0; i < n_regs0; ++i) {
mm_reg1_t *r = &regs0[i];
if (r->div > opt->flt_max_dv) continue;
if (r->blen < opt->flt_min_blen) continue;
regs0[j++] = regs0[i];
}
n_regs0 = j;
}
if (n_segs == 1) { // uni-segment if (n_segs == 1) { // uni-segment
regs0 = align_regs(opt, mi, b->km, qlens[0], seqs[0], &n_regs0, regs0, a); regs0 = align_regs(opt, mi, b->km, qlens[0], seqs[0], &n_regs0, regs0, a);
-4
View File
@@ -101,7 +101,6 @@ typedef struct {
typedef struct { typedef struct {
short k, w, flag, bucket_bits; short k, w, flag, bucket_bits;
int mini_batch_size; int mini_batch_size;
int min_occ, max_occ;
uint64_t batch_size; uint64_t batch_size;
} mm_idxopt_t; } mm_idxopt_t;
@@ -119,9 +118,6 @@ typedef struct {
int min_cnt; // min number of minimizers on each chain int min_cnt; // min number of minimizers on each chain
int min_chain_score; // min chaining score int min_chain_score; // min chaining score
float flt_max_dv;
int flt_min_blen;
float mask_level; float mask_level;
float pri_ratio; float pri_ratio;
int best_n; // top best_n chains are subjected to DP alignment int best_n; // top best_n chains are subjected to DP alignment
+1 -1
View File
@@ -59,7 +59,7 @@ uint32_t ks_ksmall_uint32_t(size_t n, uint32_t arr[], size_t kk);
void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, int is_hpc, mm128_v *p); void mm_sketch(void *km, const char *str, int len, int w, int k, uint32_t rid, int is_hpc, mm128_v *p);
void mm_write_sam_hdr(const mm_idx_t *mi, const char *rg, const char *ver, int argc, char *argv[]); int mm_write_sam_hdr(const mm_idx_t *mi, const char *rg, const char *ver, int argc, char *argv[]);
void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int opt_flag); void mm_write_paf(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int opt_flag);
void mm_write_paf3(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int opt_flag, int rep_len); void mm_write_paf3(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, void *km, int opt_flag, int rep_len);
void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs); void mm_write_sam(kstring_t *s, const mm_idx_t *mi, const mm_bseq1_t *t, const mm_reg1_t *r, int n_regs, const mm_reg1_t *regs);
-5
View File
@@ -1,5 +1,4 @@
#include <stdio.h> #include <stdio.h>
#include <limits.h>
#include "mmpriv.h" #include "mmpriv.h"
void mm_idxopt_init(mm_idxopt_t *opt) void mm_idxopt_init(mm_idxopt_t *opt)
@@ -9,7 +8,6 @@ void mm_idxopt_init(mm_idxopt_t *opt)
opt->bucket_bits = 14; opt->bucket_bits = 14;
opt->mini_batch_size = 50000000; opt->mini_batch_size = 50000000;
opt->batch_size = 4000000000ULL; opt->batch_size = 4000000000ULL;
opt->min_occ = 0, opt->max_occ = INT_MAX;
} }
void mm_mapopt_init(mm_mapopt_t *opt) void mm_mapopt_init(mm_mapopt_t *opt)
@@ -27,9 +25,6 @@ void mm_mapopt_init(mm_mapopt_t *opt)
opt->max_chain_skip = 25; opt->max_chain_skip = 25;
opt->max_chain_iter = 5000; opt->max_chain_iter = 5000;
opt->flt_max_dv = 1.0f;
opt->flt_min_blen = 0;
opt->mask_level = 0.5f; opt->mask_level = 0.5f;
opt->pri_ratio = 0.8f; opt->pri_ratio = 0.8f;
opt->best_n = 5; opt->best_n = 5;