diff --git a/Makefile b/Makefile index ed341f6..ee7bd8e 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,11 @@ LIBS= -lm -lz -lpthread ifeq ($(arm_neon),) # if arm_neon is not defined ifeq ($(sse2only),) # if sse2only is not defined +ifeq ($(avx2),) OBJS+=ksw2_extz2_sse41.o ksw2_extd2_sse41.o ksw2_exts2_sse41.o ksw2_extz2_sse2.o ksw2_extd2_sse2.o ksw2_exts2_sse2.o ksw2_dispatch.o +else + OBJS+=ksw2_extd2_avx2.o ksw2_extz2_sse41.o ksw2_extd2_sse41.o ksw2_exts2_sse41.o ksw2_extz2_sse2.o ksw2_extd2_sse2.o ksw2_exts2_sse2.o ksw2_dispatch.o +endif else # if sse2only is defined OBJS+=ksw2_extz2_sse.o ksw2_extd2_sse.o ksw2_exts2_sse.o endif @@ -57,11 +61,14 @@ ksw2_extz2_sse41.o:ksw2_extz2_sse.c ksw2.h kalloc.h ksw2_extz2_sse2.o:ksw2_extz2_sse.c ksw2.h kalloc.h $(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@ +ksw2_extd2_avx2.o:ksw2_extd2_sse.c ksw2.h kalloc.h + $(CC) -c $(CFLAGS) -mavx2 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ + ksw2_extd2_sse41.o:ksw2_extd2_sse.c ksw2.h kalloc.h - $(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ + $(CC) -c $(CFLAGS) -msse4.1 -mno-avx2 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ ksw2_extd2_sse2.o:ksw2_extd2_sse.c ksw2.h kalloc.h - $(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@ + $(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 -mno-avx2 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@ ksw2_exts2_sse41.o:ksw2_exts2_sse.c ksw2.h kalloc.h $(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ @@ -69,8 +76,13 @@ ksw2_exts2_sse41.o:ksw2_exts2_sse.c ksw2.h kalloc.h ksw2_exts2_sse2.o:ksw2_exts2_sse.c ksw2.h kalloc.h $(CC) -c $(CFLAGS) -msse2 -mno-sse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH -DKSW_SSE2_ONLY $(INCLUDES) $< -o $@ +ifeq ($(avx2),) ksw2_dispatch.o:ksw2_dispatch.c ksw2.h $(CC) -c $(CFLAGS) -msse4.1 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ +else +ksw2_dispatch.o:ksw2_dispatch.c ksw2.h + $(CC) -c $(CFLAGS) -mavx2 $(CPPFLAGS) -DKSW_CPU_DISPATCH $(INCLUDES) $< -o $@ +endif # NEON-specific targets on ARM diff --git a/ksw2_dispatch.c b/ksw2_dispatch.c index 22ca9bc..79bf191 100644 --- a/ksw2_dispatch.c +++ b/ksw2_dispatch.c @@ -71,7 +71,14 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez); extern void ksw_extd2_sse41(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez); + extern void ksw_extd2_avx2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, + int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int end_bonus, int flag, ksw_extz_t *ez); if (ksw_simd < 0) ksw_simd = x86_simd(); +#ifdef __AVX2__ + if (ksw_simd & SIMD_AVX2) + ksw_extd2_avx2(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez); + else +#endif if (ksw_simd & SIMD_SSE4_1) ksw_extd2_sse41(km, qlen, query, tlen, target, m, mat, q, e, q2, e2, w, zdrop, end_bonus, flag, ez); else if (ksw_simd & SIMD_SSE2) diff --git a/ksw2_extd2_sse.c b/ksw2_extd2_sse.c index b578274..f19a51d 100644 --- a/ksw2_extd2_sse.c +++ b/ksw2_extd2_sse.c @@ -4,7 +4,20 @@ #include "ksw2.h" #ifdef __SSE2__ + +#if defined(__AVX2__) +#include +#define SIMD_INT __m256i +#define SIMD_SHIFT 5 +#define simd_func(func) _mm256_##func +#define simd_funcw(func) _mm256_##func##_si256 + +#elif defined(__SSE2__) #include +#define SIMD_INT __m128i +#define SIMD_SHIFT 4 +#define simd_func(func) _mm_##func +#define simd_funcw(func) _mm_##func##_si128 #ifdef KSW_SSE2_ONLY #undef __SSE4_1__ @@ -13,12 +26,19 @@ #ifdef __SSE4_1__ #include #endif +#endif // defined(__SSE2__) + +#define SIMD_WIDTH (1< qlen? tlen : qlen; wl = wr = w; - tlen_ = (tlen + 15) / 16; + tlen_ = (tlen + SIMD_WIDTH - 1) / SIMD_WIDTH; n_col_ = qlen < tlen? qlen : tlen; - n_col_ = ((n_col_ < w + 1? n_col_ : w + 1) + 15) / 16 + 1; - qlen_ = (qlen + 15) / 16; + n_col_ = ((n_col_ < w + 1? n_col_ : w + 1) + SIMD_WIDTH - 1) / SIMD_WIDTH + 1; + qlen_ = (qlen + SIMD_WIDTH - 1) / SIMD_WIDTH; for (t = 1, max_sc = mat[0], min_sc = mat[1]; t < m * m; ++t) { max_sc = max_sc > mat[t]? max_sc : mat[t]; min_sc = min_sc < mat[t]? min_sc : mat[t]; @@ -96,23 +116,23 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin ++long_thres; long_diff = long_thres * (e - e2) - (q2 - q) - e2; - mem = (uint8_t*)kcalloc(km, tlen_ * 8 + qlen_ + 1, 16); - u = (__m128i*)(((size_t)mem + 15) >> 4 << 4); // 16-byte aligned + mem = (uint8_t*)kcalloc(km, tlen_ * 8 + qlen_ + 1, SIMD_WIDTH); + u = (SIMD_INT*)(((size_t)mem + SIMD_WIDTH - 1) >> SIMD_SHIFT << SIMD_SHIFT); // 16-byte aligned v = u + tlen_, x = v + tlen_, y = x + tlen_, x2 = y + tlen_, y2 = x2 + tlen_; - s = y2 + tlen_, sf = (uint8_t*)(s + tlen_), qr = sf + tlen_ * 16; - memset(u, -q - e, tlen_ * 16); - memset(v, -q - e, tlen_ * 16); - memset(x, -q - e, tlen_ * 16); - memset(y, -q - e, tlen_ * 16); - memset(x2, -q2 - e2, tlen_ * 16); - memset(y2, -q2 - e2, tlen_ * 16); + s = y2 + tlen_, sf = (uint8_t*)(s + tlen_), qr = sf + tlen_ * SIMD_WIDTH; + memset(u, -q - e, tlen_ * SIMD_WIDTH); + memset(v, -q - e, tlen_ * SIMD_WIDTH); + memset(x, -q - e, tlen_ * SIMD_WIDTH); + memset(y, -q - e, tlen_ * SIMD_WIDTH); + memset(x2, -q2 - e2, tlen_ * SIMD_WIDTH); + memset(y2, -q2 - e2, tlen_ * SIMD_WIDTH); if (!approx_max) { - H = (int32_t*)kmalloc(km, tlen_ * 16 * 4); - for (t = 0; t < tlen_ * 16; ++t) H[t] = KSW_NEG_INF; + H = (int32_t*)kmalloc(km, tlen_ * SIMD_WIDTH * 4); + for (t = 0; t < tlen_ * SIMD_WIDTH; ++t) H[t] = KSW_NEG_INF; } if (with_cigar) { - mem2 = (uint8_t*)kmalloc(km, ((size_t)(qlen + tlen - 1) * n_col_ + 1) * 16); - p = (__m128i*)(((size_t)mem2 + 15) >> 4 << 4); + mem2 = (uint8_t*)kmalloc(km, ((size_t)(qlen + tlen - 1) * n_col_ + 1) * SIMD_WIDTH); + p = (SIMD_INT*)(((size_t)mem2 + SIMD_WIDTH - 1) >> SIMD_SHIFT << SIMD_SHIFT); off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int) * 2); off_end = off + qlen + tlen - 1; } @@ -125,7 +145,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin int8_t x1, x21, v1; uint8_t *qrr = qr + (qlen - 1 - r); int8_t *u8 = (int8_t*)u, *v8 = (int8_t*)v, *x8 = (int8_t*)x, *x28 = (int8_t*)x2; - __m128i x1_, x21_, v1_; + SIMD_INT x1_, x21_, v1_; // find the boundaries if (st < r - qlen + 1) st = r - qlen + 1; if (en > r) en = r; @@ -136,7 +156,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin break; } st0 = st, en0 = en; - st = st / 16 * 16, en = (en + 16) / 16 * 16 - 1; + st = st / SIMD_WIDTH * SIMD_WIDTH, en = (en + SIMD_WIDTH) / SIMD_WIDTH * SIMD_WIDTH - 1; // set boundary conditions if (st > 0) { if (st - 1 >= last_st && st - 1 <= last_en) { @@ -155,47 +175,53 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin } // loop fission: set scores first if (!(flag & KSW_EZ_GENERIC_SC)) { - for (t = st0; t <= en0; t += 16) { - __m128i sq, st, tmp, mask; - sq = _mm_loadu_si128((__m128i*)&sf[t]); - st = _mm_loadu_si128((__m128i*)&qrr[t]); - mask = _mm_or_si128(_mm_cmpeq_epi8(sq, m1_), _mm_cmpeq_epi8(st, m1_)); - tmp = _mm_cmpeq_epi8(sq, st); -#ifdef __SSE4_1__ - tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp); - tmp = _mm_blendv_epi8(tmp, sc_N_, mask); -#else + for (t = st0; t <= en0; t += SIMD_WIDTH) { + SIMD_INT sq, st, tmp, mask; + sq = simd_funcw(loadu)((SIMD_INT*)&sf[t]); + st = simd_funcw(loadu)((SIMD_INT*)&qrr[t]); + mask = simd_funcw(or)(simd_func(cmpeq_epi8)(sq, m1_), simd_func(cmpeq_epi8)(st, m1_)); + tmp = simd_func(cmpeq_epi8)(sq, st); +#if defined(__SSE4_1__) || defined(__AVX2__) + tmp = simd_func(blendv_epi8)(sc_mis_, sc_mch_, tmp); + tmp = simd_func(blendv_epi8)(tmp, sc_N_, mask); +#elif defined(__SSE2__) // emulate blendv tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_)); tmp = _mm_or_si128(_mm_andnot_si128(mask, tmp), _mm_and_si128(mask, sc_N_)); #endif - _mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp); + simd_funcw(storeu)((SIMD_INT*)((int8_t*)s + t), tmp); } } else { for (t = st0; t <= en0; ++t) ((uint8_t*)s)[t] = mat[sf[t] * m + qrr[t]]; } // core loop - x1_ = _mm_cvtsi32_si128((uint8_t)x1); - x21_ = _mm_cvtsi32_si128((uint8_t)x21); - v1_ = _mm_cvtsi32_si128((uint8_t)v1); - st_ = st / 16, en_ = en / 16; +#if defined(__AVX2__) + x1_ = _mm256_setr_epi32((uint8_t)x1, 0, 0, 0, 0, 0, 0, 0); + x21_ = _mm256_setr_epi32((uint8_t)x21, 0, 0, 0, 0, 0, 0, 0); + v1_ = _mm256_setr_epi32((uint8_t)v1, 0, 0, 0, 0, 0, 0, 0); +#elif defined(__SSE2__) + x1_ = _mm_setr_epi32((uint8_t)x1, 0, 0, 0); + x21_ = _mm_setr_epi32((uint8_t)x21, 0, 0, 0); + v1_ = _mm_setr_epi32((uint8_t)v1, 0, 0, 0); +#endif + st_ = st / SIMD_WIDTH, en_ = en / SIMD_WIDTH; assert(en_ - st_ + 1 <= n_col_); if (!with_cigar) { // score only for (t = st_; t <= en_; ++t) { - __m128i z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; + SIMD_INT z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; __dp_code_block1; -#ifdef __SSE4_1__ - z = _mm_max_epi8(z, a); - z = _mm_max_epi8(z, b); - z = _mm_max_epi8(z, a2); - z = _mm_max_epi8(z, b2); - z = _mm_min_epi8(z, sc_mch_); +#if defined(__SSE4_1__) || defined(__AVX2__) + z = simd_func(max_epi8)(z, a); + z = simd_func(max_epi8)(z, b); + z = simd_func(max_epi8)(z, a2); + z = simd_func(max_epi8)(z, b2); + z = simd_func(min_epi8)(z, sc_mch_); __dp_code_block2; // save u[] and v[]; update a, b, a2 and b2 - _mm_store_si128(&x[t], _mm_sub_epi8(_mm_max_epi8(a, zero_), qe_)); - _mm_store_si128(&y[t], _mm_sub_epi8(_mm_max_epi8(b, zero_), qe_)); - _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_max_epi8(a2, zero_), qe2_)); - _mm_store_si128(&y2[t], _mm_sub_epi8(_mm_max_epi8(b2, zero_), qe2_)); -#else + simd_funcw(store)(&x[t], simd_func(sub_epi8)(simd_func(max_epi8)(a, zero_), qe_)); + simd_funcw(store)(&y[t], simd_func(sub_epi8)(simd_func(max_epi8)(b, zero_), qe_)); + simd_funcw(store)(&x2[t], simd_func(sub_epi8)(simd_func(max_epi8)(a2, zero_), qe2_)); + simd_funcw(store)(&y2[t], simd_func(sub_epi8)(simd_func(max_epi8)(b2, zero_), qe2_)); +#elif defined(__SSE2__) tmp = _mm_cmpgt_epi8(a, z); z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a)); tmp = _mm_cmpgt_epi8(b, z); @@ -218,22 +244,22 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin #endif } } else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment - __m128i *pr = p + (size_t)r * n_col_ - st_; + SIMD_INT *pr = p + (size_t)r * n_col_ - st_; off[r] = st, off_end[r] = en; for (t = st_; t <= en_; ++t) { - __m128i d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; + SIMD_INT d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; __dp_code_block1; -#ifdef __SSE4_1__ - d = _mm_and_si128(_mm_cmpgt_epi8(a, z), _mm_set1_epi8(1)); // d = a > z? 1 : 0 - z = _mm_max_epi8(z, a); - d = _mm_blendv_epi8(d, _mm_set1_epi8(2), _mm_cmpgt_epi8(b, z)); // d = b > z? 2 : d - z = _mm_max_epi8(z, b); - d = _mm_blendv_epi8(d, _mm_set1_epi8(3), _mm_cmpgt_epi8(a2, z)); // d = a2 > z? 3 : d - z = _mm_max_epi8(z, a2); - d = _mm_blendv_epi8(d, _mm_set1_epi8(4), _mm_cmpgt_epi8(b2, z)); // d = a2 > z? 3 : d - z = _mm_max_epi8(z, b2); - z = _mm_min_epi8(z, sc_mch_); -#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() +#if defined(__SSE4_1__) || defined(__AVX2__) + d = simd_funcw(and)(simd_func(cmpgt_epi8)(a, z), simd_func(set1_epi8)(1)); // d = a > z? 1 : 0 + z = simd_func(max_epi8)(z, a); + d = simd_func(blendv_epi8)(d, simd_func(set1_epi8)(2), simd_func(cmpgt_epi8)(b, z)); // d = b > z? 2 : d + z = simd_func(max_epi8)(z, b); + d = simd_func(blendv_epi8)(d, simd_func(set1_epi8)(3), simd_func(cmpgt_epi8)(a2, z)); // d = a2 > z? 3 : d + z = simd_func(max_epi8)(z, a2); + d = simd_func(blendv_epi8)(d, simd_func(set1_epi8)(4), simd_func(cmpgt_epi8)(b2, z)); // d = a2 > z? 3 : d + z = simd_func(max_epi8)(z, b2); + z = simd_func(min_epi8)(z, sc_mch_); +#elif defined(__SSE2__) // emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() tmp = _mm_cmpgt_epi8(a, z); d = _mm_and_si128(tmp, _mm_set1_epi8(1)); z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a)); @@ -250,37 +276,37 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); #endif __dp_code_block2; - tmp = _mm_cmpgt_epi8(a, zero_); - _mm_store_si128(&x[t], _mm_sub_epi8(_mm_and_si128(tmp, a), qe_)); - d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x08))); // d = a > 0? 1<<3 : 0 - tmp = _mm_cmpgt_epi8(b, zero_); - _mm_store_si128(&y[t], _mm_sub_epi8(_mm_and_si128(tmp, b), qe_)); - d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x10))); // d = b > 0? 1<<4 : 0 - tmp = _mm_cmpgt_epi8(a2, zero_); - _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_and_si128(tmp, a2), qe2_)); - d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0 - tmp = _mm_cmpgt_epi8(b2, zero_); - _mm_store_si128(&y2[t], _mm_sub_epi8(_mm_and_si128(tmp, b2), qe2_)); - d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x40))); // d = b > 0? 1<<6 : 0 - _mm_store_si128(&pr[t], d); + tmp = simd_func(cmpgt_epi8)(a, zero_); + simd_funcw(store)(&x[t], simd_func(sub_epi8)(simd_funcw(and)(tmp, a), qe_)); + d = simd_funcw(or)(d, simd_funcw(and)(tmp, simd_func(set1_epi8)(0x08))); // d = a > 0? 1<<3 : 0 + tmp = simd_func(cmpgt_epi8)(b, zero_); + simd_funcw(store)(&y[t], simd_func(sub_epi8)(simd_funcw(and)(tmp, b), qe_)); + d = simd_funcw(or)(d, simd_funcw(and)(tmp, simd_func(set1_epi8)(0x10))); // d = b > 0? 1<<4 : 0 + tmp = simd_func(cmpgt_epi8)(a2, zero_); + simd_funcw(store)(&x2[t], simd_func(sub_epi8)(simd_funcw(and)(tmp, a2), qe2_)); + d = simd_funcw(or)(d, simd_funcw(and)(tmp, simd_func(set1_epi8)(0x20))); // d = a > 0? 1<<5 : 0 + tmp = simd_func(cmpgt_epi8)(b2, zero_); + simd_funcw(store)(&y2[t], simd_func(sub_epi8)(simd_funcw(and)(tmp, b2), qe2_)); + d = simd_funcw(or)(d, simd_funcw(and)(tmp, simd_func(set1_epi8)(0x40))); // d = b > 0? 1<<6 : 0 + simd_funcw(store)(&pr[t], d); } } else { // gap right-alignment - __m128i *pr = p + (size_t)r * n_col_ - st_; + SIMD_INT *pr = p + (size_t)r * n_col_ - st_; off[r] = st, off_end[r] = en; for (t = st_; t <= en_; ++t) { - __m128i d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; + SIMD_INT d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; __dp_code_block1; -#ifdef __SSE4_1__ - d = _mm_andnot_si128(_mm_cmpgt_epi8(z, a), _mm_set1_epi8(1)); // d = z > a? 0 : 1 - z = _mm_max_epi8(z, a); - d = _mm_blendv_epi8(_mm_set1_epi8(2), d, _mm_cmpgt_epi8(z, b)); // d = z > b? d : 2 - z = _mm_max_epi8(z, b); - d = _mm_blendv_epi8(_mm_set1_epi8(3), d, _mm_cmpgt_epi8(z, a2)); // d = z > a2? d : 3 - z = _mm_max_epi8(z, a2); - d = _mm_blendv_epi8(_mm_set1_epi8(4), d, _mm_cmpgt_epi8(z, b2)); // d = z > b2? d : 4 - z = _mm_max_epi8(z, b2); - z = _mm_min_epi8(z, sc_mch_); -#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() +#if defined(__SSE4_1__) || defined(__AVX2__) + d = simd_funcw(andnot)(simd_func(cmpgt_epi8)(z, a), simd_func(set1_epi8)(1)); // d = z > a? 0 : 1 + z = simd_func(max_epi8)(z, a); + d = simd_func(blendv_epi8)(simd_func(set1_epi8)(2), d, simd_func(cmpgt_epi8)(z, b)); // d = z > b? d : 2 + z = simd_func(max_epi8)(z, b); + d = simd_func(blendv_epi8)(simd_func(set1_epi8)(3), d, simd_func(cmpgt_epi8)(z, a2)); // d = z > a2? d : 3 + z = simd_func(max_epi8)(z, a2); + d = simd_func(blendv_epi8)(simd_func(set1_epi8)(4), d, simd_func(cmpgt_epi8)(z, b2)); // d = z > b2? d : 4 + z = simd_func(max_epi8)(z, b2); + z = simd_func(min_epi8)(z, sc_mch_); +#elif defined(__SSE2__) tmp = _mm_cmpgt_epi8(z, a); d = _mm_andnot_si128(tmp, _mm_set1_epi8(1)); z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, a)); @@ -297,50 +323,54 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); #endif __dp_code_block2; - tmp = _mm_cmpgt_epi8(zero_, a); - _mm_store_si128(&x[t], _mm_sub_epi8(_mm_andnot_si128(tmp, a), qe_)); - d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x08))); // d = a > 0? 1<<3 : 0 - tmp = _mm_cmpgt_epi8(zero_, b); - _mm_store_si128(&y[t], _mm_sub_epi8(_mm_andnot_si128(tmp, b), qe_)); - d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x10))); // d = b > 0? 1<<4 : 0 - tmp = _mm_cmpgt_epi8(zero_, a2); - _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_andnot_si128(tmp, a2), qe2_)); - d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0 - tmp = _mm_cmpgt_epi8(zero_, b2); - _mm_store_si128(&y2[t], _mm_sub_epi8(_mm_andnot_si128(tmp, b2), qe2_)); - d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x40))); // d = b > 0? 1<<6 : 0 - _mm_store_si128(&pr[t], d); + tmp = simd_func(cmpgt_epi8)(zero_, a); + simd_funcw(store)(&x[t], simd_func(sub_epi8)(simd_funcw(andnot)(tmp, a), qe_)); + d = simd_funcw(or)(d, simd_funcw(andnot)(tmp, simd_func(set1_epi8)(0x08))); // d = a > 0? 1<<3 : 0 + tmp = simd_func(cmpgt_epi8)(zero_, b); + simd_funcw(store)(&y[t], simd_func(sub_epi8)(simd_funcw(andnot)(tmp, b), qe_)); + d = simd_funcw(or)(d, simd_funcw(andnot)(tmp, simd_func(set1_epi8)(0x10))); // d = b > 0? 1<<4 : 0 + tmp = simd_func(cmpgt_epi8)(zero_, a2); + simd_funcw(store)(&x2[t], simd_func(sub_epi8)(simd_funcw(andnot)(tmp, a2), qe2_)); + d = simd_funcw(or)(d, simd_funcw(andnot)(tmp, simd_func(set1_epi8)(0x20))); // d = a > 0? 1<<5 : 0 + tmp = simd_func(cmpgt_epi8)(zero_, b2); + simd_funcw(store)(&y2[t], simd_func(sub_epi8)(simd_funcw(andnot)(tmp, b2), qe2_)); + d = simd_funcw(or)(d, simd_funcw(andnot)(tmp, simd_func(set1_epi8)(0x40))); // d = b > 0? 1<<6 : 0 + simd_funcw(store)(&pr[t], d); } } if (!approx_max) { // find the exact max with a 32-bit score array int32_t max_H, max_t; // compute H[], max_H and max_t if (r > 0) { - int32_t HH[4], tt[4], en1 = st0 + (en0 - st0) / 4 * 4, i; - __m128i max_H_, max_t_; + int32_t HH[SIMD_WIDTH/4], tt[SIMD_WIDTH/4], en1 = st0 + (en0 - st0) / (SIMD_WIDTH/4) * (SIMD_WIDTH/4), i; + SIMD_INT max_H_, max_t_; max_H = H[en0] = en0 > 0? H[en0-1] + u8[en0] : H[en0] + v8[en0]; // special casing the last element max_t = en0; - max_H_ = _mm_set1_epi32(max_H); - max_t_ = _mm_set1_epi32(max_t); - for (t = st0; t < en1; t += 4) { // this implements: H[t]+=v8[t]-qe; if(H[t]>max_H) max_H=H[t],max_t=t; - __m128i H1, tmp, t_; - H1 = _mm_loadu_si128((__m128i*)&H[t]); + max_H_ = simd_func(set1_epi32)(max_H); + max_t_ = simd_func(set1_epi32)(max_t); + for (t = st0; t < en1; t += SIMD_WIDTH/4) { // this implements: H[t]+=v8[t]-qe; if(H[t]>max_H) max_H=H[t],max_t=t; + SIMD_INT H1, tmp, t_; + H1 = simd_funcw(loadu)((SIMD_INT*)&H[t]); +#if defined(__AVX2__) + t_ = _mm256_setr_epi32(v8[t], v8[t+1], v8[t+2], v8[t+3], v8[t+4], v8[t+5], v8[t+6], v8[t+7]); +#elif defined(__SSE2__) t_ = _mm_setr_epi32(v8[t], v8[t+1], v8[t+2], v8[t+3]); - H1 = _mm_add_epi32(H1, t_); - _mm_storeu_si128((__m128i*)&H[t], H1); - t_ = _mm_set1_epi32(t); - tmp = _mm_cmpgt_epi32(H1, max_H_); -#ifdef __SSE4_1__ - max_H_ = _mm_blendv_epi8(max_H_, H1, tmp); - max_t_ = _mm_blendv_epi8(max_t_, t_, tmp); -#else - max_H_ = _mm_or_si128(_mm_and_si128(tmp, H1), _mm_andnot_si128(tmp, max_H_)); - max_t_ = _mm_or_si128(_mm_and_si128(tmp, t_), _mm_andnot_si128(tmp, max_t_)); +#endif + H1 = simd_func(add_epi32)(H1, t_); + simd_funcw(storeu)((SIMD_INT*)&H[t], H1); + t_ = simd_func(set1_epi32)(t); + tmp = simd_func(cmpgt_epi32)(H1, max_H_); +#if defined(__SSE4_1__) || defined(__AVX2__) + max_H_ = simd_func(blendv_epi8)(max_H_, H1, tmp); + max_t_ = simd_func(blendv_epi8)(max_t_, t_, tmp); +#elif defined(__SSE2__) + max_H_ = simd_funcw(or)(simd_funcw(and)(tmp, H1), simd_funcw(andnot)(tmp, max_H_)); + max_t_ = simd_funcw(or)(simd_funcw(and)(tmp, t_), simd_funcw(andnot)(tmp, max_t_)); #endif } - _mm_storeu_si128((__m128i*)HH, max_H_); - _mm_storeu_si128((__m128i*)tt, max_t_); - for (i = 0; i < 4; ++i) + simd_funcw(storeu)((SIMD_INT*)HH, max_H_); + simd_funcw(storeu)((SIMD_INT*)tt, max_t_); + for (i = 0; i < SIMD_WIDTH/4; ++i) if (max_H < HH[i]) max_H = HH[i], max_t = tt[i] + i; for (; t < en0; ++t) { // for the rest of values that haven't been computed with SSE H[t] += (int32_t)v8[t]; @@ -381,12 +411,12 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin if (with_cigar) { // backtrack int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR); if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) { - ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar); + ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*SIMD_WIDTH, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar); } else if (!ez->zdropped && (flag&KSW_EZ_EXTZ_ONLY) && ez->mqe + end_bonus > (int)ez->max) { ez->reach_end = 1; - ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar); + ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*SIMD_WIDTH, ez->mqe_t, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar); } else if (ez->max_t >= 0 && ez->max_q >= 0) { - ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar); + ksw_backtrack(km, 1, rev_cigar, 0, (uint8_t*)p, off, off_end, n_col_*SIMD_WIDTH, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar); } kfree(km, mem2); kfree(km, off); }