diff --git a/CommandLines.cpp b/CommandLines.cpp index 2d27297..bb828cd 100644 --- a/CommandLines.cpp +++ b/CommandLines.cpp @@ -102,6 +102,7 @@ void init_opt(hifiasm_opt_t* asm_opt) asm_opt->k_mer_length = 51; asm_opt->mz_win = 51; asm_opt->bf_shift = 37; + asm_opt->max_kmer_cnt = 4000; asm_opt->high_factor = 5.0; asm_opt->max_ov_diff_ec = 0.04; asm_opt->max_ov_diff_final = 0.03; diff --git a/CommandLines.h b/CommandLines.h index 86b351b..cdff95e 100644 --- a/CommandLines.h +++ b/CommandLines.h @@ -35,6 +35,7 @@ typedef struct { int k_mer_length; int mz_win; int bf_shift; + int max_kmer_cnt; double high_factor; // coverage cutoff set to high_factor*hom_cov double max_ov_diff_ec; double max_ov_diff_final; diff --git a/anchor.cpp b/anchor.cpp index ddf2589..6888be0 100644 --- a/anchor.cpp +++ b/anchor.cpp @@ -73,7 +73,7 @@ void ha_get_new_candidates(ha_abuf_t *ab, int64_t rid, UC_Read *ucr, overlap_reg rlen = Get_READ_LENGTH(R_INF, rid); // read length // get the list of anchors - ha_sketch_query(ucr->seq, ucr->length, asm_opt.mz_win, asm_opt.k_mer_length, 0, !(asm_opt.flag & HA_F_NO_HPC), &ab->mz, ha_flt_tab, k_flag, dbg_ct); + ha_sketch(ucr->seq, ucr->length, asm_opt.mz_win, asm_opt.k_mer_length, 0, !(asm_opt.flag & HA_F_NO_HPC), &ab->mz, ha_flt_tab, k_flag, dbg_ct); // minimizer of queried read if (ab->mz.m > ab->old_mz_m) { ab->old_mz_m = ab->mz.m; diff --git a/htab.cpp b/htab.cpp index 2194d23..d92b097 100644 --- a/htab.cpp +++ b/htab.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -557,7 +556,7 @@ static void worker_for_mz(void *data, long i, int tid) ha_mz1_v *b = &s->mz_buf[tid]; s->mz_buf[tid].n = 0; ///s->p->opt->w = 51, s->p->opt->k - ha_sketch_query(s->seq[i], s->len[i], s->p->opt->w, s->p->opt->k, s->n_seq0 + i, s->p->opt->is_HPC, b, s->p->flt_tab, 0, 0); + ha_sketch(s->seq[i], s->len[i], s->p->opt->w, s->p->opt->k, s->n_seq0 + i, s->p->opt->is_HPC, b, s->p->flt_tab, 0, 0); s->mz[i].n = s->mz[i].m = b->n; MALLOC(s->mz[i].a, b->n); memcpy(s->mz[i].a, b->a, b->n * sizeof(ha_mz1_t)); @@ -803,12 +802,15 @@ ha_ct_t *ha_count(const hifiasm_opt_t *asm_opt, int flag, ha_pt_t *p0, const voi * High count filter table * ***************************/ +// Warning: the max count is 32767 KHASHL_MAP_INIT(static klib_unused, yak_ft_t, yak_ft, uint64_t, int16_t, kh_hash_dummy, kh_eq_generic) -static yak_ft_t *gen_hh(const ha_ct_t *h) +static yak_ft_t *gen_hh(const ha_ct_t *h, int max_cnt) { int i; yak_ft_t *hh; + if (max_cnt > YAK_MAX_COUNT - 1) max_cnt = YAK_MAX_COUNT - 1; + if (max_cnt > INT16_MAX - 1) max_cnt = INT16_MAX - 1; hh = yak_ft_init(); yak_ft_resize(hh, h->tot * 2); for (i = 0; i < 1<pre; ++i) { @@ -819,28 +821,22 @@ static yak_ft_t *gen_hh(const ha_ct_t *h) uint64_t y = kh_key(ht, k) >> h->pre << YAK_COUNTER_BITS | i; int absent; l = yak_ft_put(hh, y, &absent); - if (absent) - kh_val(hh, l) = kh_key(ht, k)&YAK_MAX_COUNT; + if (absent) { + int cnt = kh_key(ht, k) & YAK_MAX_COUNT; + kh_val(hh, l) = cnt > max_cnt? INT16_MAX : cnt; + } } } } return hh; } -int ha_ft_isflt(const void *hh, uint64_t y) +int32_t ha_ft_cnt(const void *hh, uint64_t y) { yak_ft_t *h = (yak_ft_t*)hh; khint_t k; k = yak_ft_get(h, y); - return k == kh_end(h)? 0 : 1; -} - -int ha_ft_cnt(const void *hh, uint64_t y) -{ - yak_ft_t *h = (yak_ft_t*)hh; - khint_t k; - k = yak_ft_get(h, y); - return k == kh_end(h)? 0 : kh_val(h, k); + return k == kh_end(h)? 0 : kh_val(h, k) == INT16_MAX? INT32_MAX : kh_val(h, k); } void ha_ft_destroy(void *h) @@ -906,7 +902,7 @@ void *ha_ft_gen(const hifiasm_opt_t *asm_opt, All_reads *rs, int *hom_cov, int i if (cutoff > YAK_MAX_COUNT - 1) cutoff = YAK_MAX_COUNT - 1; } ha_ct_shrink(h, cutoff, YAK_MAX_COUNT, asm_opt->thread_num); - flt_tab = gen_hh(h); + flt_tab = gen_hh(h, asm_opt->max_kmer_cnt); ha_ct_destroy(h); fprintf(stderr, "[M::%s::%.3f*%.2f@%.3fGB] ==> filtered out %ld k-mers occurring %d or more times\n", __func__, yak_realtime(), yak_cpu_usage(), yak_peakrss_in_gb(), (long)kh_size(flt_tab), cutoff); @@ -970,7 +966,6 @@ int query_ct_index(void* ct_idx, uint64_t hash) return kh_key(g->h, k)&YAK_MAX_COUNT; } - int write_ct_index(void *i_ct_idx, char* file_name) { char* gfa_name = (char*)malloc(strlen(file_name)+25); diff --git a/htab.h b/htab.h index c703cb8..4a8de45 100644 --- a/htab.h +++ b/htab.h @@ -31,9 +31,8 @@ extern void *ha_flt_tab_hp; extern ha_pt_t *ha_idx_hp; extern void *ha_ct_table; - void *ha_ft_gen(const hifiasm_opt_t *asm_opt, All_reads *rs, int *hom_cov, int is_hp_mode); -int ha_ft_cnt(const void *hh, uint64_t y); +int32_t ha_ft_cnt(const void *hh, uint64_t y); void ha_ft_destroy(void *h); ha_pt_t *ha_pt_gen(const hifiasm_opt_t *asm_opt, const void *flt_tab, int read_from_store, int is_hp_mode, All_reads *rs, int *hom_cov, int *het_cov); @@ -59,8 +58,7 @@ double yak_cpu_usage(void); void ha_triobin(const hifiasm_opt_t *opt); -void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf); -void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct); +void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct); int ha_analyze_count(int n_cnt, int start_cnt, const int64_t *cnt, int *peak_het); static inline uint64_t yak_hash64(uint64_t key, uint64_t mask) // invertible integer hash function diff --git a/sketch.cpp b/sketch.cpp index 2fe0b22..b9ff3bd 100644 --- a/sketch.cpp +++ b/sketch.cpp @@ -36,7 +36,7 @@ static inline int tq_shift(tiny_queue_t *q) * @param is_hpc homopolymer-compressed or not * @param p minimizers */ -void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct) +void ha_sketch(const char *str, int len, int w, int k, uint32_t rid, int is_hpc, ha_mz1_v *p, const void *hf, kvec_t_u8_warp* k_flag, kvec_t_u64_warp* dbg_ct) { ///in default, w = 51, k = 51, is_hpc = 1 /** uint64_t x; @@ -57,8 +57,7 @@ void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int i memset(k_flag->a.a, 0, k_flag->a.n); } - ///sizeof(ha_mz1_t) = 16 - memset(buf, 0xff, w * 16); + memset(buf, 0xff, w * sizeof(ha_mz1_t)); memset(&tq, 0, sizeof(tiny_queue_t)); ///len/w is the evaluated minimizer numbers kv_resize(ha_mz1_t, *p, p->n + len/w); @@ -97,12 +96,13 @@ void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int i ++l; if (l >= k && kmer_span < 256) { uint64_t y; - int filtered = 0; + int32_t cnt = 0, filtered = 0; y = yak_hash64_64(kmer[z<<1|0]) + yak_hash64_64(kmer[z<<1|1]); - if (hf != 0) filtered = (ha_ft_cnt(hf, y) > 0); + if (hf != 0) cnt = ha_ft_cnt(hf, y); + filtered = (cnt > 0); if (dbg_ct != NULL) kv_push(uint64_t, dbg_ct->a, ((((uint64_t)(query_ct_index(ha_ct_table, y))<<1)|filtered)<<32)|(uint64_t)(i)); - if (filtered == 0) - info.x = y, info.rid = rid, info.pos = i, info.rev = z, info.span = kmer_span; + if (cnt < 1<<28 && filtered == 0) + info.x = y, info.rid = cnt, info.pos = i, info.rev = z, info.span = kmer_span; if (k_flag != NULL) k_flag->a.a[i]++; if (k_flag != NULL && filtered > 0) k_flag->a.a[i]++; } @@ -150,4 +150,6 @@ void ha_sketch_query(const char *str, int len, int w, int k, uint32_t rid, int i } if (min.x != UINT64_MAX) kv_push(ha_mz1_t, *p, min); + for (i = 0; i < (int)p->n; ++i) // populate .rid as this was keeping counts + p->a[i].rid = rid; }